JIT: implement frsqrte

Mostly a straightforward translation of the interpreter code, with a few
tricksy optimizations and fallbacks for rare paths.

Source/Core/Common/MathUtil.cpp

@@ -90,6 +90,84 @@ u32 ClassifyFloat(float fvalue)
 	}
 }
 
+const int frsqrte_expected_base[] =
+{
+	0x3ffa000, 0x3c29000, 0x38aa000, 0x3572000,
+	0x3279000, 0x2fb7000, 0x2d26000, 0x2ac0000,
+	0x2881000, 0x2665000, 0x2468000, 0x2287000,
+	0x20c1000, 0x1f12000, 0x1d79000, 0x1bf4000,
+	0x1a7e800, 0x17cb800, 0x1552800, 0x130c000,
+	0x10f2000, 0x0eff000, 0x0d2e000, 0x0b7c000,
+	0x09e5000, 0x0867000, 0x06ff000, 0x05ab800,
+	0x046a000, 0x0339800, 0x0218800, 0x0105800,
+};
+const int frsqrte_expected_dec[] =
+{
+	0x7a4, 0x700, 0x670, 0x5f2,
+	0x584, 0x524, 0x4cc, 0x47e,
+	0x43a, 0x3fa, 0x3c2, 0x38e,
+	0x35e, 0x332, 0x30a, 0x2e6,
+	0x568, 0x4f3, 0x48d, 0x435,
+	0x3e7, 0x3a2, 0x365, 0x32e,
+	0x2fc, 0x2d0, 0x2a8, 0x283,
+	0x261, 0x243, 0x226, 0x20b,
+};
+
+double ApproximateReciprocalSquareRoot(double val)
+{
+	union
+	{
+		double valf;
+		s64 vali;
+	};
+	valf = val;
+	s64 mantissa = vali & ((1LL << 52) - 1);
+	s64 sign = vali & (1ULL << 63);
+	s64 exponent = vali & (0x7FFLL << 52);
+
+	// Special case 0
+	if (mantissa == 0 && exponent == 0)
+		return sign ? -std::numeric_limits<double>::infinity() :
+		std::numeric_limits<double>::infinity();
+	// Special case NaN-ish numbers
+	if (exponent == (0x7FFLL << 52))
+	{
+		if (mantissa == 0)
+		{
+			if (sign)
+				return std::numeric_limits<double>::quiet_NaN();
+
+			return 0.0;
+		}
+
+		return 0.0 + valf;
+	}
+
+	// Negative numbers return NaN
+	if (sign)
+		return std::numeric_limits<double>::quiet_NaN();
+
+	if (!exponent)
+	{
+		// "Normalize" denormal values
+		do
+		{
+			exponent -= 1LL << 52;
+			mantissa <<= 1;
+		} while (!(mantissa & (1LL << 52)));
+		mantissa &= (1LL << 52) - 1;
+		exponent += 1LL << 52;
+	}
+
+	bool odd_exponent = !(exponent & (1LL << 52));
+	exponent = ((0x3FFLL << 52) - ((exponent - (0x3FELL << 52)) / 2)) & (0x7FFLL << 52);
+
+	int i = (int)(mantissa >> 37);
+	vali = sign | exponent;
+	int index = i / 2048 + (odd_exponent ? 16 : 0);
+	vali |= (s64)(frsqrte_expected_base[index] - frsqrte_expected_dec[index] * (i % 2048)) << 26;
+	return valf;
+}
 
 }  // namespace
 

Source/Core/Common/MathUtil.h

@@ -123,6 +123,12 @@ u32 ClassifyDouble(double dvalue);
 // More efficient float version.
 u32 ClassifyFloat(float fvalue);
 
+extern const int frsqrte_expected_base[];
+extern const int frsqrte_expected_dec[];
+
+// The PowerPC approximate square root algorithm
+double ApproximateReciprocalSquareRoot(double val);
+
 template<class T>
 struct Rectangle
 {

Source/Core/Core/PowerPC/Gekko.h

@@ -386,6 +386,29 @@ union UReg_MSR
 #define FPRF_SHIFT 12
 #define FPRF_MASK (0x1F << FPRF_SHIFT)
 
+// FPSCR exception flags
+const u32 FPSCR_FX     = 1U << (31 -  0);
+const u32 FPSCR_FEX    = 1U << (31 -  1);
+const u32 FPSCR_VX     = 1U << (31 -  2);
+const u32 FPSCR_OX     = 1U << (31 -  3);
+const u32 FPSCR_UX     = 1U << (31 -  4);
+const u32 FPSCR_ZX     = 1U << (31 -  5);
+const u32 FPSCR_XX     = 1U << (31 -  6);
+const u32 FPSCR_VXSNAN = 1U << (31 -  7);
+const u32 FPSCR_VXISI  = 1U << (31 -  8);
+const u32 FPSCR_VXIDI  = 1U << (31 -  9);
+const u32 FPSCR_VXZDZ  = 1U << (31 - 10);
+const u32 FPSCR_VXIMZ  = 1U << (31 - 11);
+const u32 FPSCR_VXVC   = 1U << (31 - 12);
+const u32 FPSCR_VXSOFT = 1U << (31 - 21);
+const u32 FPSCR_VXSQRT = 1U << (31 - 22);
+const u32 FPSCR_VXCVI  = 1U << (31 - 23);
+
+const u32 FPSCR_VX_ANY = FPSCR_VXSNAN | FPSCR_VXISI | FPSCR_VXIDI | FPSCR_VXZDZ | FPSCR_VXIMZ |
+                         FPSCR_VXVC | FPSCR_VXSOFT | FPSCR_VXSQRT | FPSCR_VXCVI;
+
+const u32 FPSCR_ANY_X = FPSCR_OX | FPSCR_UX | FPSCR_ZX | FPSCR_XX | FPSCR_VX_ANY;
+
 // Floating Point Status and Control Register
 union UReg_FPSCR
 {

Source/Core/Core/PowerPC/Interpreter/Interpreter_FPUtils.h

@@ -16,27 +16,6 @@
 #define MIN_SINGLE 0xc7efffffe0000000ull
 #define MAX_SINGLE 0x47efffffe0000000ull
 
-// FPSCR exception flags
-const u32 FPSCR_OX         = (u32)1 << (31 - 3);
-const u32 FPSCR_UX         = (u32)1 << (31 - 4);
-const u32 FPSCR_ZX         = (u32)1 << (31 - 5);
-// ! XX shouldn't be accessed directly to set 1. Use SetFI() instead !
-const u32 FPSCR_XX         = (u32)1 << (31 - 6);
-const u32 FPSCR_VXSNAN     = (u32)1 << (31 - 7);
-const u32 FPSCR_VXISI      = (u32)1 << (31 - 8);
-const u32 FPSCR_VXIDI      = (u32)1 << (31 - 9);
-const u32 FPSCR_VXZDZ      = (u32)1 << (31 - 10);
-const u32 FPSCR_VXIMZ      = (u32)1 << (31 - 11);
-const u32 FPSCR_VXVC       = (u32)1 << (31 - 12);
-const u32 FPSCR_VXSOFT     = (u32)1 << (31 - 21);
-const u32 FPSCR_VXSQRT     = (u32)1 << (31 - 22);
-const u32 FPSCR_VXCVI      = (u32)1 << (31 - 23);
-
-const u32 FPSCR_VX_ANY     = FPSCR_VXSNAN | FPSCR_VXISI | FPSCR_VXIDI | FPSCR_VXZDZ |
-                             FPSCR_VXIMZ | FPSCR_VXVC | FPSCR_VXSOFT | FPSCR_VXSQRT | FPSCR_VXCVI;
-
-const u32 FPSCR_ANY_X      = FPSCR_OX | FPSCR_UX | FPSCR_ZX | FPSCR_XX | FPSCR_VX_ANY;
-
 const u64 PPC_NAN_U64      = 0x7ff8000000000000ull;
 const double PPC_NAN       = *(double* const)&PPC_NAN_U64;
 
@@ -346,79 +325,3 @@ inline double ApproximateReciprocal(double val)
 	return valf;
 }
 
-inline double ApproximateReciprocalSquareRoot(double val)
-{
-	static const int expected_base[] = {
-		0x3ffa000, 0x3c29000, 0x38aa000, 0x3572000,
-		0x3279000, 0x2fb7000, 0x2d26000, 0x2ac0000,
-		0x2881000, 0x2665000, 0x2468000, 0x2287000,
-		0x20c1000, 0x1f12000, 0x1d79000, 0x1bf4000,
-		0x1a7e800, 0x17cb800, 0x1552800, 0x130c000,
-		0x10f2000, 0x0eff000, 0x0d2e000, 0x0b7c000,
-		0x09e5000, 0x0867000, 0x06ff000, 0x05ab800,
-		0x046a000, 0x0339800, 0x0218800, 0x0105800,
-	};
-	static const int expected_dec[] = {
-		0x7a4, 0x700, 0x670, 0x5f2,
-		0x584, 0x524, 0x4cc, 0x47e,
-		0x43a, 0x3fa, 0x3c2, 0x38e,
-		0x35e, 0x332, 0x30a, 0x2e6,
-		0x568, 0x4f3, 0x48d, 0x435,
-		0x3e7, 0x3a2, 0x365, 0x32e,
-		0x2fc, 0x2d0, 0x2a8, 0x283,
-		0x261, 0x243, 0x226, 0x20b,
-	};
-
-	union
-	{
-		double valf;
-		s64 vali;
-	};
-	valf = val;
-	s64 mantissa = vali & ((1LL << 52) - 1);
-	s64 sign = vali & (1ULL << 63);
-	s64 exponent = vali & (0x7FFLL << 52);
-
-	// Special case 0
-	if (mantissa == 0 && exponent == 0)
-		return sign ? -std::numeric_limits<double>::infinity() :
-		               std::numeric_limits<double>::infinity();
-	// Special case NaN-ish numbers
-	if (exponent == (0x7FFLL << 52))
-	{
-		if (mantissa == 0)
-		{
-			if (sign)
-				return std::numeric_limits<double>::quiet_NaN();
-
-			return 0.0;
-		}
-
-		return 0.0 + valf;
-	}
-
-	// Negative numbers return NaN
-	if (sign)
-		return std::numeric_limits<double>::quiet_NaN();
-
-	if (!exponent)
-	{
-		// "Normalize" denormal values
-		do
-		{
-			exponent -= 1LL << 52;
-			mantissa <<= 1;
-		} while (!(mantissa & (1LL << 52)));
-		mantissa &= (1LL << 52) - 1;
-		exponent += 1LL << 52;
-	}
-
-	bool odd_exponent = !(exponent & (1LL << 52));
-	exponent = ((0x3FFLL << 52) - ((exponent - (0x3FELL << 52)) / 2)) & (0x7FFLL << 52);
-
-	int i = (int)(mantissa >> 37);
-	vali = sign | exponent;
-	int index = i / 2048 + (odd_exponent ? 16 : 0);
-	vali |= (s64)(expected_base[index] - expected_dec[index] * (i % 2048)) << 26;
-	return valf;
-}

Source/Core/Core/PowerPC/Jit64/Jit.h

@@ -189,6 +189,7 @@ public:
 	void fctiwx(UGeckoInstruction inst);
 	void fmrx(UGeckoInstruction inst);
 	void frspx(UGeckoInstruction inst);
+	void frsqrtex(UGeckoInstruction inst);
 
 	void cmpXX(UGeckoInstruction inst);
 

Source/Core/Core/PowerPC/Jit64/Jit64_Tables.cpp

@@ -360,7 +360,7 @@ static GekkoOPTemplate table63_2[] =
 	{22, &Jit64::FallBackToInterpreter}, //"fsqrtx",   OPTYPE_FPU, FL_RC_BIT_F}},
 	{23, &Jit64::FallBackToInterpreter}, //"fselx",    OPTYPE_FPU, FL_RC_BIT_F}},
 	{25, &Jit64::fp_arith},              //"fmulx",    OPTYPE_FPU, FL_RC_BIT_F}},
-	{26, &Jit64::FallBackToInterpreter}, //"frsqrtex", OPTYPE_FPU, FL_RC_BIT_F}},
+	{26, &Jit64::frsqrtex},              //"frsqrtex", OPTYPE_FPU, FL_RC_BIT_F}},
 	{28, &Jit64::fmaddXX},               //"fmsubx",   OPTYPE_FPU, FL_RC_BIT_F}},
 	{29, &Jit64::fmaddXX},               //"fmaddx",   OPTYPE_FPU, FL_RC_BIT_F}},
 	{30, &Jit64::fmaddXX},               //"fnmsubx",  OPTYPE_FPU, FL_RC_BIT_F}},

Source/Core/Core/PowerPC/Jit64/JitAsm.cpp

@@ -149,6 +149,8 @@ void Jit64AsmRoutineManager::GenerateCommon()
 	GenFifoWrite(32);
 	fifoDirectWriteFloat = AlignCode4();
 	GenFifoFloatWrite();
+	frsqrte = AlignCode4();
+	GenFrsqrte();
 
 	GenQuantizedLoads();
 	GenQuantizedStores();

Source/Core/Core/PowerPC/Jit64/Jit_FloatingPoint.cpp

@@ -366,3 +366,23 @@ void Jit64::frspx(UGeckoInstruction inst)
 	SetFPRFIfNeeded(inst, fpr.RX(d));
 	fpr.UnlockAll();
 }
+
+void Jit64::frsqrtex(UGeckoInstruction inst)
+{
+	INSTRUCTION_START
+	JITDISABLE(bJITFloatingPointOff);
+	FALLBACK_IF(inst.Rc);
+	int b = inst.FB;
+	int d = inst.FD;
+
+	// rsqrtex requires ECX and EDX free
+	gpr.FlushLockX(ECX, EDX);
+	fpr.Lock(b, d);
+	fpr.BindToRegister(d, d == b);
+	MOVSD(XMM0, fpr.R(b));
+	CALL((void *)asm_routines.frsqrte);
+	MOVSD(fpr.R(d), XMM0);
+	SetFPRFIfNeeded(inst, fpr.RX(d));
+	fpr.UnlockAll();
+	gpr.UnlockAllX();
+}

Source/Core/Core/PowerPC/JitCommon/JitAsmCommon.cpp

@@ -3,6 +3,7 @@
 // Refer to the license.txt file included.
 
 #include "Common/CPUDetect.h"
+#include "Common/MathUtil.h"
 #include "Common/MemoryUtil.h"
 
 #include "Core/PowerPC/JitCommon/JitAsmCommon.h"
@@ -51,6 +52,71 @@ void CommonAsmRoutines::GenFifoFloatWrite()
 	RET();
 }
 
+void CommonAsmRoutines::GenFrsqrte()
+{
+	// Assume input in XMM0.
+	// This function clobbers EAX, ECX, and EDX.
+	MOVQ_xmm(R(RAX), XMM0);
+
+	// Negative and zero inputs set an exception and take the complex path.
+	TEST(64, R(RAX), R(RAX));
+	FixupBranch zero = J_CC(CC_Z, true);
+	FixupBranch negative = J_CC(CC_S, true);
+	MOV(64, R(RCX), R(RAX));
+	SHR(64, R(RCX), Imm8(52));
+
+	// Zero and max exponents (non-normal floats) take the complex path.
+	FixupBranch complex1 = J_CC(CC_Z, true);
+	CMP(32, R(ECX), Imm32(0x7FF));
+	FixupBranch complex2 = J_CC(CC_E, true);
+
+	SUB(32, R(ECX), Imm32(0x3FD));
+	SAR(32, R(ECX), Imm8(1));
+	MOV(32, R(EDX), Imm32(0x3FF));
+	SUB(32, R(EDX), R(ECX));
+	SHL(64, R(RDX), Imm8(52));   // exponent = ((0x3FFLL << 52) - ((exponent - (0x3FELL << 52)) / 2)) & (0x7FFLL << 52);
+
+	MOV(64, R(RCX), R(RAX));
+	SHR(64, R(RCX), Imm8(48));
+	AND(32, R(ECX), Imm8(0x1F));
+	XOR(32, R(ECX), Imm8(0x10)); // int index = i / 2048 + (odd_exponent ? 16 : 0);
+
+	SHR(64, R(RAX), Imm8(37));
+	AND(32, R(EAX), Imm32(0x7FF));
+	IMUL(32, EAX, MScaled(RCX, SCALE_4, (u32)(u64)MathUtil::frsqrte_expected_dec));
+	MOV(32, R(ECX), MScaled(RCX, SCALE_4, (u32)(u64)MathUtil::frsqrte_expected_base));
+	SUB(32, R(ECX), R(EAX));
+	SHL(64, R(RCX), Imm8(26));
+	OR(64, R(RDX), R(RCX));     // vali |= (s64)(frsqrte_expected_base[index] - frsqrte_expected_dec[index] * (i % 2048)) << 26;
+	MOVQ_xmm(XMM0, R(RDX));
+	RET();
+
+	// Exception flags for zero input.
+	SetJumpTarget(zero);
+	TEST(32, M(&FPSCR), Imm32(FPSCR_ZX));
+	FixupBranch skip_set_fx1 = J_CC(CC_NZ);
+	OR(32, M(&FPSCR), Imm32(FPSCR_FX));
+	SetJumpTarget(skip_set_fx1);
+	OR(32, M(&FPSCR), Imm32(FPSCR_ZX));
+	FixupBranch complex3 = J();
+
+	// Exception flags for negative input.
+	SetJumpTarget(negative);
+	TEST(32, M(&FPSCR), Imm32(FPSCR_VXSQRT));
+	FixupBranch skip_set_fx2 = J_CC(CC_NZ);
+	OR(32, M(&FPSCR), Imm32(FPSCR_FX));
+	SetJumpTarget(skip_set_fx2);
+	OR(32, M(&FPSCR), Imm32(FPSCR_VXSQRT));
+
+	SetJumpTarget(complex1);
+	SetJumpTarget(complex2);
+	SetJumpTarget(complex3);
+	ABI_PushRegistersAndAdjustStack(QUANTIZED_REGS_TO_SAVE, false);
+	ABI_CallFunction((void *)&MathUtil::ApproximateReciprocalSquareRoot);
+	ABI_PopRegistersAndAdjustStack(QUANTIZED_REGS_TO_SAVE, false);
+	RET();
+}
+
 // Safe + Fast Quantizers, originally from JITIL by magumagu
 
 static const u8 GC_ALIGNED16(pbswapShuffle1x4[16]) = {3, 2, 1, 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};

Source/Core/Core/PowerPC/JitCommon/JitAsmCommon.h

@@ -24,6 +24,8 @@ public:
 	const u8 *dispatchPcInEAX;
 	const u8 *doTiming;
 
+	const u8 *frsqrte;
+
 	// In: array index: GQR to use.
 	// In: ECX: Address to read from.
 	// Out: XMM0: Bottom two 32-bit slots hold the read value,
@@ -56,5 +58,6 @@ public:
 	void GenFifoWrite(int size);
 	void GenFifoXmm64Write();
 	void GenFifoFloatWrite();
+	void GenFrsqrte();
 
 };