nv2a: Fix -0.0 clamping of RCC instruction and vertex shader W-output

Xbox rounds -0.0 to the negative range and 0.0 to the positive range. This
commit also restores RCC instruction clamping to be done on the output of
reciprocal calculation (which current Xemu release does) with fix for the
input=Infinity case.

hw/xbox/nv2a/pgraph/glsl/vsh-ff.c

@@ -422,7 +422,7 @@ GLSL_DEFINE(materialEmissionColor, GLSL_LTCTXA(NV_IGRAPH_XF_LTCTXA_CM_COL) ".xyz
 
     mstring_append(body,
     "   oPos = tPosition * compositeMat;\n"
-    "   oPos.w = (2.0f * step(0.0f, oPos.w) - 1.0f) * clamp(abs(oPos.w), 5.421011e-20, 1.8446744e19);\n"
+    "   oPos.w = clampAwayZeroInf(oPos.w);\n"
     "   oPos = invViewport * oPos;\n"
     );
 

hw/xbox/nv2a/pgraph/glsl/vsh-prog.c

@@ -735,8 +735,8 @@ static const char* vsh_header =
     "#define RCC(dest, mask, src) dest.mask = _RCC(_in(src).x).mask\n"
     "vec4 _RCC(float src)\n"
     "{\n"
-    "  src = (2.0f * step(0.0f, src) - 1.0f) * clamp(abs(src), 5.421011e-20, 1.8446744e19);\n"
-    "  return vec4(1.0 / src);\n"
+    "  float t = clampAwayZeroInf(1.0 / src);\n"
+    "  return vec4(t);\n"
     "}\n"
     "\n"
     "#define RSQ(dest, mask, src) dest.mask = _RSQ(_in(src).x).mask\n"
@@ -838,7 +838,7 @@ void pgraph_gen_vsh_prog_glsl(uint16_t version,
 
     mstring_append(body,
         "  oPos.z = oPos.z / clipRange.y;\n"
-        "  oPos.w = (2.0f * step(0.0f, oPos.w) - 1.0f) * clamp(abs(oPos.w), 5.421011e-20, 1.8446744e19);\n"
+        "  oPos.w = clampAwayZeroInf(oPos.w);\n"
 
         /* Undo perspective divide by w.
          * Note that games may also have vertex shaders that do

hw/xbox/nv2a/pgraph/glsl/vsh.c

@@ -72,6 +72,15 @@ MString *pgraph_gen_vsh_glsl(const ShaderState *state, bool prefix_outputs)
         "    float y = float(bitfieldExtract(cmp, 11, 11)) / 1023.0;\n"
         "    float z = float(bitfieldExtract(cmp, 22, 10)) / 511.0;\n"
         "    return vec4(x, y, z, 1);\n"
+        "}\n"
+        "\n"
+        "float clampAwayZeroInf(float t) {\n"
+        "  if (t > 0.0 || floatBitsToUint(t) == 0) {\n"
+        "    t = clamp(t, 5.421011e-20, 1.8446744e19);\n"
+        "  } else {\n"
+        "    t = clamp(t, -1.8446744e19, -5.421011e-20);\n"
+        "  }\n"
+        "  return t;\n"
         "}\n");
 
     pgraph_get_glsl_vtx_header(header, state->vulkan, state->smooth_shading,