Set the value for NaN according to the SH4 specs

The SH4 sets the signaling bit to 0 for qNaN: 7fbfffff instead of the
usual 7fffffff. Same games seem to rely on this.
Fixes Fur Fighters freeze and missing geometry in game.

core/hw/sh4/dyna/shil_canonical.h

@@ -140,9 +140,9 @@ shil_compile \
 #define BIN_OP_I2(tp,z) BIN_OP_I_BASE( return ((tp) r1) z ((tp) r2); ,u32,u32)
 #define BIN_OP_I3(z,w) BIN_OP_I_BASE( return (r1 z r2) w; ,u32,u32)
 #define BIN_OP_I4(tp,z,rt,pt) BIN_OP_I_BASE( return ((tp)(pt)r1) z ((tp)(pt)r2); ,u32,rt)
-	
-#define BIN_OP_F(z) BIN_OP_I_BASE( return r1 z r2; ,f32,f32)
-#define BIN_OP_FU(z) BIN_OP_I_BASE( return r1 z r2; ,f32,u32)
+
+#define BIN_OP_F(z) BIN_OP_I_BASE( return fixNaN(r1 z r2); ,f32,f32)
+#define BIN_OP_FU(z) BIN_OP_I_BASE( return fixNaN(r1 z r2); ,f32,u32)
 
 #define UN_OP_I(z) UN_OP_I_BASE( return z (r1); ,u32)
 #define UN_OP_F(z) UN_OP_I_BASE( return z (r1); ,f32)
@@ -681,6 +681,21 @@ u32,f1,(f32 f1),
 		return 0x7fffffff;
 	else
 		return (s32)f1;
+
+	// No fast-math
+//	if (f1 != f1)					// NaN
+//		return 0x80000000;
+//	else if (f1 > 2147483520.0f)	// IEEE 754: 0x4effffff
+//		return 0x7fffffff;
+//	else
+//	{
+//		u32 res = (s32)f1;
+//		// Fix result sign for Intel CPUs
+//		if (res == 0x80000000 && *(s32 *)&f1 > 0)
+//			res = 0x7fffffff;
+//
+//		return res;
+//	}
 )
 
 shil_compile
@@ -862,7 +877,7 @@ f32,f1,(float* fn, float* fm),
 	idp+=fn[2]*fm[2];
 	idp+=fn[3]*fm[3];
 
-	return idp;
+	return fixNaN(idp);
 )
 
 shil_compile
@@ -871,7 +886,6 @@ shil_compile
 	shil_cf_arg_ptr(rs1);
 	shil_cf(fipr_impl);
 	shil_cf_rv_f32(rd);
-	//die();
 )
 
 shil_opc_end()
@@ -908,10 +922,10 @@ void,f1,(float* fd,float* fn, float* fm),
 		 fm[11] * fn[2] +
 		 fm[15] * fn[3];
 
-	fd[0] = v1;
-	fd[1] = v2;
-	fd[2] = v3;
-	fd[3] = v4;
+	fd[0] = fixNaN(v1);
+	fd[1] = fixNaN(v2);
+	fd[2] = fixNaN(v3);
+	fd[3] = fixNaN(v4);
 )
 shil_compile
 (
@@ -919,7 +933,6 @@ shil_compile
 	shil_cf_arg_ptr(rs1);
 	shil_cf_arg_ptr(rd);
 	shil_cf(ftrv_impl);
-	//die();
 )
 shil_opc_end()
 
@@ -928,7 +941,7 @@ shil_opc(fmac)
 shil_canonical
 (
 f32,f1,(float fn, float f0,float fm),
-	return fn + f0 * fm;
+	return fixNaN(fn + f0 * fm);
 )
 shil_compile
 (
@@ -937,7 +950,6 @@ shil_compile
 	shil_cf_arg_f32(rs1);
 	shil_cf(f1);
 	shil_cf_rv_f32(rd);
-	//die();
 )
 shil_opc_end()
 

core/hw/sh4/interpr/sh4_fpu.cpp

@@ -72,7 +72,7 @@ INLINE void Denorm32(float &value)
 }
 
 
-#define CHECK_FPU_32(v) //Denorm32(v)
+#define CHECK_FPU_32(v) v = fixNaN(v)
 #define CHECK_FPU_64(v)
 
 
@@ -84,7 +84,7 @@ sh4op(i1111_nnnn_mmmm_0000)
 		u32 n = GetN(op);
 		u32 m = GetM(op);
 		fr[n] += fr[m];
-		//CHECK_FPU_32(fr[n]);
+		CHECK_FPU_32(fr[n]);
 	}
 	else
 	{

core/hw/sh4/sh4_core.h

@@ -107,4 +107,13 @@ struct SH4ThrownException {
 	u32 epc;
 	u32 expEvn;
 	u32 callVect;
-};
+};
+
+// The SH4 sets the signaling bit to 0 for qNaN (unlike all recent CPUs). Some games relies on this.
+static INLINE float fixNaN(f32 f)
+{
+	// no fast-math
+//	return f == f ? f : 0x7fbfffff;
+	// fast-math
+	return (*(u32 *)&f & 0x7fffffff) <= 0x7f800000 ? f :  0x7fbfffff;
+}