ShuriZma
/
dolphin
mirror of https://github.com/dolphin-emu/dolphin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JIT64: optimize carry handling 

Carries are rather common and unpredictable, so do them branchlessly wherever
we can.
This commit is contained in:
Fiora 2014-08-24 11:35:57 -07:00
parent a570c6b4a4
commit a95d8cbcb4
6 changed files with 84 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Source/Core/Core/PowerPC/Gekko.h
View File

@ -331,9 +331,12 @@ union UFPR
float f[2];
};
#define XER_CA_MASK 0x20000000
#define XER_OV_MASK 0x40000000
#define XER_SO_MASK 0x80000000
#define XER_CA_SHIFT 29
#define XER_OV_SHIFT 30
#define XER_SO_SHIFT 31
#define XER_CA_MASK (1U << XER_CA_SHIFT)
#define XER_OV_MASK (1U << XER_OV_SHIFT)
#define XER_SO_MASK (1U << XER_SO_SHIFT)
// XER
union UReg_XER
{

4
Source/Core/Core/PowerPC/Jit64/Jit.h
View File

@ -101,10 +101,6 @@ public:
void GenerateConstantOverflow(s64 val);
void GenerateOverflow();
void FinalizeCarryOverflow(bool oe, bool inv = false);
void GetCarryRSCRATCHAndClear();
void FinalizeCarryGenerateOverflowRSCRATCH(bool oe, bool inv = false);
void GenerateCarry();
void GenerateRC();
void ComputeRC(const Gen::OpArg & arg);
// Use to extract bytes from a register using the regcache. offset is in bytes.

260
Source/Core/Core/PowerPC/Jit64/Jit_Integer.cpp
View File

@ -31,6 +31,7 @@ void Jit64::GenerateConstantOverflow(bool overflow)
}
}
// We could do overflow branchlessly, but unlike carry it seems to be quite a bit rarer.
void Jit64::GenerateOverflow()
{
FixupBranch jno = J_CC(CC_NO);
@ -49,82 +50,24 @@ void Jit64::FinalizeCarryOverflow(bool oe, bool inv)
// USES_XER
if (oe)
{
// this is slightly messy because JitSetCAIf modifies x86 flags, so we have to do it in both
// sides of the branch.
FixupBranch jno = J_CC(CC_NO);
// Do carry
FixupBranch carry1 = J_CC(inv ? CC_C : CC_NC);
JitSetCA();
SetJumpTarget(carry1);
JitSetCAIf(inv ? CC_NC : CC_C);
//XER[OV/SO] = 1
OR(32, PPCSTATE(spr[SPR_XER]), Imm32(XER_SO_MASK | XER_OV_MASK));
FixupBranch exit = J();
SetJumpTarget(jno);
// Do carry
FixupBranch carry2 = J_CC(inv ? CC_C : CC_NC);
JitSetCA();
SetJumpTarget(carry2);
JitSetCAIf(inv ? CC_NC : CC_C);
SetJumpTarget(exit);
}
else
{
// Do carry
FixupBranch carry1 = J_CC(inv ? CC_C : CC_NC);
JitSetCA();
SetJumpTarget(carry1);
JitSetCAIf(inv ? CC_NC : CC_C);
}
}
void Jit64::GetCarryRSCRATCHAndClear()
{
MOV(32, R(RSCRATCH), PPCSTATE(spr[SPR_XER]));
BTR(32, R(RSCRATCH), Imm8(29));
}
// Assumes that XER is in RSCRATCH and that the CA bit is clear.
void Jit64::FinalizeCarryGenerateOverflowRSCRATCH(bool oe, bool inv)
{
// USES_XER
if (oe)
{
FixupBranch jno = J_CC(CC_NO);
// Do carry
FixupBranch carry1 = J_CC(inv ? CC_C : CC_NC);
OR(32, R(RSCRATCH), Imm32(XER_CA_MASK));
SetJumpTarget(carry1);
//XER[OV/SO] = 1
OR(32, R(RSCRATCH), Imm32(XER_SO_MASK | XER_OV_MASK));
FixupBranch exit = J();
SetJumpTarget(jno);
// Do carry
FixupBranch carry2 = J_CC(inv ? CC_C : CC_NC);
OR(32, R(RSCRATCH), Imm32(XER_CA_MASK));
SetJumpTarget(carry2);
//XER[OV] = 0
AND(32, R(RSCRATCH), Imm32(~XER_OV_MASK));
SetJumpTarget(exit);
}
else
{
// Do carry
FixupBranch carry1 = J_CC(inv ? CC_C : CC_NC);
OR(32, R(RSCRATCH), Imm32(XER_CA_MASK));
SetJumpTarget(carry1);
}
// Dump RSCRATCH back into XER
MOV(32, PPCSTATE(spr[SPR_XER]), R(RSCRATCH));
}
// Assumes that the flags were just set through an addition.
void Jit64::GenerateCarry()
{
// USES_XER
FixupBranch pNoCarry = J_CC(CC_NC);
OR(32, PPCSTATE(spr[SPR_XER]), Imm32(XER_CA_MASK));
FixupBranch pContinue = J();
SetJumpTarget(pNoCarry);
AND(32, PPCSTATE(spr[SPR_XER]), Imm32(~(XER_CA_MASK)));
SetJumpTarget(pContinue);
}
void Jit64::ComputeRC(const Gen::OpArg & arg)
{
if (arg.IsImm())
@ -153,12 +96,12 @@ OpArg Jit64::ExtractFromReg(int reg, int offset)
// we can't do this optimization in the emitter because MOVZX and AND have different effects on flags.
void Jit64::AndWithMask(X64Reg reg, u32 mask)
{
{
if (mask == 0xff)
MOVZX(32, 8, reg, R(reg));
else if (mask == 0xffff)
MOVZX(32, 16, reg, R(reg));
else
else
AND(32, R(reg), Imm32(mask));
}
@ -188,22 +131,16 @@ void Jit64::regimmop(int d, int a, bool binary, u32 value, Operation doop, void
gpr.Lock(d, a);
if (a || binary || carry) // yeh nasty special case addic
{
if (carry)
JitClearCAOV(false);
if (gpr.R(a).IsImm() && !carry)
{
gpr.SetImmediate32(d, doop((u32)gpr.R(a).offset, value));
if (Rc)
{
ComputeRC(gpr.R(d));
}
}
else if (a == d)
{
gpr.KillImmediate(d, true, true);
(this->*op)(32, gpr.R(d), Imm32(value)); //m_GPR[d] = m_GPR[_inst.RA] + _inst.SIMM_16;
if (carry)
GenerateCarry();
if (Rc)
ComputeRC(gpr.R(d));
}
else
{
@ -217,11 +154,11 @@ void Jit64::regimmop(int d, int a, bool binary, u32 value, Operation doop, void
MOV(32, gpr.R(d), gpr.R(a));
(this->*op)(32, gpr.R(d), Imm32(value)); //m_GPR[d] = m_GPR[_inst.RA] + _inst.SIMM_16;
}
if (carry)
GenerateCarry();
if (Rc)
ComputeRC(gpr.R(d));
}
if (carry)
JitSetCAIf(CC_C);
if (Rc)
ComputeRC(gpr.R(d));
}
else if (doop == Add)
{
@ -848,13 +785,11 @@ void Jit64::subfic(UGeckoInstruction inst)
{
if (imm == 0)
{
JitClearCA();
JitClearCAOV(false);
// Flags act exactly like subtracting from 0
NEG(32, gpr.R(d));
// Output carry is inverted
FixupBranch carry1 = J_CC(CC_C);
JitSetCA();
SetJumpTarget(carry1);
JitSetCAIf(CC_NC);
}
else if (imm == -1)
{
@ -864,24 +799,20 @@ void Jit64::subfic(UGeckoInstruction inst)
}
else
{
JitClearCA();
JitClearCAOV(false);
NOT(32, gpr.R(d));
ADD(32, gpr.R(d), Imm32(imm+1));
// Output carry is normal
FixupBranch carry1 = J_CC(CC_NC);
JitSetCA();
SetJumpTarget(carry1);
JitSetCAIf(CC_C);
}
}
else
{
JitClearCA();
JitClearCAOV(false);
MOV(32, gpr.R(d), Imm32(imm));
SUB(32, gpr.R(d), gpr.R(a));
// Output carry is inverted
FixupBranch carry1 = J_CC(CC_C);
JitSetCA();
SetJumpTarget(carry1);
JitSetCAIf(CC_NC);
}
gpr.UnlockAll();
// This instruction has no RC flag
@ -926,7 +857,7 @@ void Jit64::subfex(UGeckoInstruction inst)
gpr.Lock(a, b, d);
gpr.BindToRegister(d, (d == a || d == b), true);
GetCarryRSCRATCHAndClear();
JitGetAndClearCAOV(inst.OE);
bool invertedCarry = false;
if (d == b)
@ -947,7 +878,7 @@ void Jit64::subfex(UGeckoInstruction inst)
NOT(32, gpr.R(d));
ADC(32, gpr.R(d), gpr.R(b));
}
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE, invertedCarry);
FinalizeCarryOverflow(inst.OE, invertedCarry);
if (inst.Rc)
ComputeRC(gpr.R(d));
@ -963,14 +894,12 @@ void Jit64::subfmex(UGeckoInstruction inst)
gpr.Lock(a, d);
gpr.BindToRegister(d, d == a);
GetCarryRSCRATCHAndClear();
JitGetAndClearCAOV(inst.OE);
if (d != a)
{
MOV(32, gpr.R(d), gpr.R(a));
}
NOT(32, gpr.R(d));
ADC(32, gpr.R(d), Imm32(0xFFFFFFFF));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
FinalizeCarryOverflow(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
@ -986,14 +915,12 @@ void Jit64::subfzex(UGeckoInstruction inst)
gpr.Lock(a, d);
gpr.BindToRegister(d, d == a);
GetCarryRSCRATCHAndClear();
JitGetAndClearCAOV(inst.OE);
if (d != a)
{
MOV(32, gpr.R(d), gpr.R(a));
}
NOT(32, gpr.R(d));
ADC(32, gpr.R(d), Imm8(0));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
FinalizeCarryOverflow(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
@ -1011,13 +938,9 @@ void Jit64::subfx(UGeckoInstruction inst)
s32 i = (s32)gpr.R(b).offset, j = (s32)gpr.R(a).offset;
gpr.SetImmediate32(d, i - j);
if (inst.Rc)
{
ComputeRC(gpr.R(d));
}
if (inst.OE)
{
GenerateConstantOverflow((s64)i - (s64)j);
}
}
else
{
@ -1477,31 +1400,22 @@ void Jit64::addex(UGeckoInstruction inst)
JITDISABLE(bJITIntegerOff);
int a = inst.RA, b = inst.RB, d = inst.RD;
gpr.Lock(a, b, d);
gpr.BindToRegister(d, (d == a) || (d == b));
JitGetAndClearCAOV(inst.OE);
if ((d == a) || (d == b))
{
gpr.Lock(a, b, d);
gpr.BindToRegister(d, true);
GetCarryRSCRATCHAndClear();
ADC(32, gpr.R(d), gpr.R((d == a) ? b : a));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
else
{
gpr.Lock(a, b, d);
gpr.BindToRegister(d, false);
GetCarryRSCRATCHAndClear();
MOV(32, gpr.R(d), gpr.R(a));
ADC(32, gpr.R(d), gpr.R(b));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
FinalizeCarryOverflow(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
void Jit64::addcx(UGeckoInstruction inst)
@ -1543,31 +1457,16 @@ void Jit64::addmex(UGeckoInstruction inst)
JITDISABLE(bJITIntegerOff);
int a = inst.RA, d = inst.RD;
if (d == a)
{
gpr.Lock(d);
gpr.BindToRegister(d, true);
GetCarryRSCRATCHAndClear();
ADC(32, gpr.R(d), Imm32(0xFFFFFFFF));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
else
{
gpr.Lock(a, d);
gpr.BindToRegister(d, false);
GetCarryRSCRATCHAndClear();
gpr.Lock(d);
gpr.BindToRegister(d, d == a);
JitGetAndClearCAOV(inst.OE);
if (d != a)
MOV(32, gpr.R(d), gpr.R(a));
ADC(32, gpr.R(d), Imm32(0xFFFFFFFF));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
ADC(32, gpr.R(d), Imm32(0xFFFFFFFF));
FinalizeCarryOverflow(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
void Jit64::addzex(UGeckoInstruction inst)
@ -1577,31 +1476,16 @@ void Jit64::addzex(UGeckoInstruction inst)
JITDISABLE(bJITIntegerOff);
int a = inst.RA, d = inst.RD;
if (d == a)
{
gpr.Lock(d);
gpr.BindToRegister(d, true);
GetCarryRSCRATCHAndClear();
ADC(32, gpr.R(d), Imm8(0));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
else
{
gpr.Lock(a, d);
gpr.BindToRegister(d, false);
GetCarryRSCRATCHAndClear();
gpr.Lock(d);
gpr.BindToRegister(d, d == a);
JitGetAndClearCAOV(inst.OE);
if (d != a)
MOV(32, gpr.R(d), gpr.R(a));
ADC(32, gpr.R(d), Imm8(0));
FinalizeCarryGenerateOverflowRSCRATCH(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
ADC(32, gpr.R(d), Imm8(0));
FinalizeCarryOverflow(inst.OE);
if (inst.Rc)
ComputeRC(gpr.R(d));
gpr.UnlockAll();
}
void Jit64::rlwinmx(UGeckoInstruction inst)
@ -1793,8 +1677,8 @@ void Jit64::rlwnmx(UGeckoInstruction inst)
// no register choice
gpr.FlushLockX(ECX);
gpr.Lock(a, b, s);
gpr.BindToRegister(a, (a == b || a == s), true);
MOV(32, R(ECX), gpr.R(b));
gpr.BindToRegister(a, (a == s), true);
if (a != s)
{
MOV(32, gpr.R(a), gpr.R(s));
@ -1903,9 +1787,7 @@ void Jit64::slwx(UGeckoInstruction inst)
MOV(32, R(ECX), gpr.R(b));
gpr.BindToRegister(a, a == s, true);
if (a != s)
{
MOV(32, gpr.R(a), gpr.R(s));
}
SHL(64, gpr.R(a), R(ECX));
if (inst.Rc)
{
@ -1932,7 +1814,7 @@ void Jit64::srawx(UGeckoInstruction inst)
gpr.FlushLockX(ECX);
gpr.Lock(a, s, b);
gpr.BindToRegister(a, (a == s || a == b), true);
JitClearCA();
JitClearCAOV(false);
MOV(32, R(ECX), gpr.R(b));
if (a != s)
MOV(32, gpr.R(a), gpr.R(s));
@ -1941,16 +1823,11 @@ void Jit64::srawx(UGeckoInstruction inst)
MOV(32, R(RSCRATCH), gpr.R(a));
SHR(64, gpr.R(a), Imm8(32));
TEST(32, gpr.R(a), R(RSCRATCH));
FixupBranch nocarry = J_CC(CC_Z);
JitSetCA();
SetJumpTarget(nocarry);
JitSetCAIf(CC_NZ);
gpr.UnlockAll();
gpr.UnlockAllX();
if (inst.Rc)
{
ComputeRC(gpr.R(a));
}
}
void Jit64::srawix(UGeckoInstruction inst)
@ -1964,39 +1841,27 @@ void Jit64::srawix(UGeckoInstruction inst)
{
gpr.Lock(a, s);
gpr.BindToRegister(a, a == s, true);
JitClearCA();
JitClearCAOV(false);
MOV(32, R(RSCRATCH), gpr.R(s));
if (a != s)
{
MOV(32, gpr.R(a), R(RSCRATCH));
}
SAR(32, gpr.R(a), Imm8(amount));
if (inst.Rc)
ComputeRC(gpr.R(a));
SHL(32, R(RSCRATCH), Imm8(32-amount));
SHL(32, R(RSCRATCH), Imm8(32 - amount));
TEST(32, R(RSCRATCH), gpr.R(a));
FixupBranch nocarry = J_CC(CC_Z);
JitSetCA();
SetJumpTarget(nocarry);
gpr.UnlockAll();
JitSetCAIf(CC_NZ);
}
else
{
gpr.Lock(a, s);
JitClearCA();
JitClearCAOV(false);
gpr.BindToRegister(a, a == s, true);
if (a != s)
{
MOV(32, gpr.R(a), gpr.R(s));
}
if (inst.Rc)
{
ComputeRC(gpr.R(a));
}
gpr.UnlockAll();
}
if (inst.Rc)
ComputeRC(gpr.R(a));
gpr.UnlockAll();
}
// count leading zeroes
@ -2032,10 +1897,7 @@ void Jit64::cntlzwx(UGeckoInstruction inst)
}
if (inst.Rc)
{
ComputeRC(gpr.R(a));
// TODO: Check PPC manual too
}
}
void Jit64::twx(UGeckoInstruction inst)

2
Source/Core/Core/PowerPC/Jit64IL/IR_X86.cpp
View File

@ -1106,7 +1106,7 @@ static void DoWriteCode(IRBuilder* ibuild, JitIL* Jit, u32 exitAddress)
Jit->JitSetCA();
FixupBranch cont = Jit->J();
Jit->SetJumpTarget(nocarry);
Jit->JitClearCA();
Jit->JitClearCAOV(false);
Jit->SetJumpTarget(cont);
regNormalRegClear(RI, I);
break;

17
Source/Core/Core/PowerPC/JitCommon/Jit_Util.cpp
View File

@ -803,10 +803,11 @@ void EmuCodeBlock::SetFPRF(Gen::X64Reg xmm)
OR(32, PPCSTATE(fpscr), R(RSCRATCH));
}
void EmuCodeBlock::JitClearCA()
void EmuCodeBlock::JitGetAndClearCAOV(bool oe)
{
AND(32, PPCSTATE(spr[SPR_XER]), Imm32(~XER_CA_MASK)); //XER.CA = 0
if (oe)
AND(32, PPCSTATE(spr[SPR_XER]), Imm32(~XER_OV_MASK)); //XER.OV = 0
BTR(32, PPCSTATE(spr[SPR_XER]), Imm8(29)); //carry = XER.CA, XER.CA = 0
}
void EmuCodeBlock::JitSetCA()
@ -814,6 +815,16 @@ void EmuCodeBlock::JitSetCA()
OR(32, PPCSTATE(spr[SPR_XER]), Imm32(XER_CA_MASK)); //XER.CA = 1
}
// Some testing shows CA is set roughly ~1/3 of the time (relative to clears), so
// branchless calculation of CA is probably faster in general.
void EmuCodeBlock::JitSetCAIf(CCFlags conditionCode)
{
SETcc(conditionCode, R(RSCRATCH));
MOVZX(32, 8, RSCRATCH, R(RSCRATCH));
SHL(32, R(RSCRATCH), Imm8(XER_CA_SHIFT));
OR(32, PPCSTATE(spr[SPR_XER]), R(RSCRATCH)); //XER.CA = 1
}
void EmuCodeBlock::JitClearCAOV(bool oe)
{
if (oe)

3
Source/Core/Core/PowerPC/JitCommon/Jit_Util.h
View File

@ -71,8 +71,9 @@ public:
void SafeWriteF32ToReg(Gen::X64Reg xmm_value, Gen::X64Reg reg_addr, s32 offset, u32 registersInUse, int flags = 0);
void WriteToConstRamAddress(int accessSize, Gen::X64Reg arg, u32 address, bool swap = false);
void JitClearCA();
void JitGetAndClearCAOV(bool oe);
void JitSetCA();
void JitSetCAIf(Gen::CCFlags conditionCode);
void JitClearCAOV(bool oe);
void ForceSinglePrecisionS(Gen::X64Reg xmm);