From c720af00bb9db153fc26e982ae379006fa7dbbc2 Mon Sep 17 00:00:00 2001
From: Erik Abair <abaire@users.noreply.github.com>
Date: Thu, 15 May 2025 12:54:56 -0700
Subject: [PATCH] nv2a/vsh: Replace NaN with 1.0 for Bx, Dx, Fog outputs and
 MUL zero-check

---
 hw/xbox/nv2a/pgraph/glsl/vsh-prog.c |  2 +-
 hw/xbox/nv2a/pgraph/glsl/vsh.c      | 17 +++++++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/hw/xbox/nv2a/pgraph/glsl/vsh-prog.c b/hw/xbox/nv2a/pgraph/glsl/vsh-prog.c
index 0530e7ea7b..3e0ab5fbba 100644
--- a/hw/xbox/nv2a/pgraph/glsl/vsh-prog.c
+++ b/hw/xbox/nv2a/pgraph/glsl/vsh-prog.c
@@ -639,7 +639,7 @@ static const char* vsh_header =
     // Unfortunately mix() falls victim to the same handling of exceptional
     // (inf/NaN) handling as a multiply, so per-component comparisons are used
     // to guarantee HW behavior (anything * 0 must == 0).
-    "  vec4 zero_components = sign(src0) * sign(src1);\n"
+    "  vec4 zero_components = sign(NaNToOne(src0)) * sign(NaNToOne(src1));\n"
     "  vec4 ret = src0 * src1;\n"
     "  if (zero_components.x == 0.0) { ret.x = 0.0; }\n"
     "  if (zero_components.y == 0.0) { ret.y = 0.0; }\n"
diff --git a/hw/xbox/nv2a/pgraph/glsl/vsh.c b/hw/xbox/nv2a/pgraph/glsl/vsh.c
index 25c846bbde..f3c5dd5a43 100644
--- a/hw/xbox/nv2a/pgraph/glsl/vsh.c
+++ b/hw/xbox/nv2a/pgraph/glsl/vsh.c
@@ -81,6 +81,10 @@ MString *pgraph_gen_vsh_glsl(const ShaderState *state, bool prefix_outputs)
         "    t = clamp(t, uintBitsToFloat(0xDF800000), uintBitsToFloat(0x9F800000));\n"
         "  }\n"
         "  return t;\n"
+        "}\n"
+        "\n"
+        "vec4 NaNToOne(vec4 src) {\n"
+        "  return mix(src, vec4(1.0), isnan(src));\n"
         "}\n");
 
     pgraph_get_glsl_vtx_header(header, state->vulkan, state->smooth_shading,
@@ -128,6 +132,7 @@ MString *pgraph_gen_vsh_glsl(const ShaderState *state, bool prefix_outputs)
             }
         }
     }
+
     mstring_append(header, "\n");
 
     MString *body = mstring_from_str("void main() {\n");
@@ -232,17 +237,17 @@ MString *pgraph_gen_vsh_glsl(const ShaderState *state, bool prefix_outputs)
             break;
         }
 
-        mstring_append(body, "  oFog.xyzw = vec4(fogFactor);\n");
+        mstring_append(body, "  oFog = NaNToOne(vec4(fogFactor));\n");
     } else {
         /* FIXME: Is the fog still calculated / passed somehow?!
          */
-        mstring_append(body, "  oFog.xyzw = vec4(1.0);\n");
+        mstring_append(body, "  oFog = vec4(1.0);\n");
     }
 
     /* Set outputs */
     mstring_append(body, "\n"
-                   "  vtxD0 = clamp(oD0, 0.0, 1.0);\n"
-                   "  vtxB0 = clamp(oB0, 0.0, 1.0);\n"
+                   "  vtxD0 = clamp(NaNToOne(oD0), 0.0, 1.0);\n"
+                   "  vtxB0 = clamp(NaNToOne(oB0), 0.0, 1.0);\n"
                    "  vtxFog = oFog.x;\n"
                    "  vtxT0 = oT0;\n"
                    "  vtxT1 = oT1;\n"
@@ -253,8 +258,8 @@ MString *pgraph_gen_vsh_glsl(const ShaderState *state, bool prefix_outputs)
 
     if (state->specular_enable) {
         mstring_append(body,
-                       "  vtxD1 = clamp(oD1, 0.0, 1.0);\n"
-                       "  vtxB1 = clamp(oB1, 0.0, 1.0);\n"
+                       "  vtxD1 = clamp(NaNToOne(oD1), 0.0, 1.0);\n"
+                       "  vtxB1 = clamp(NaNToOne(oB1), 0.0, 1.0);\n"
         );
 
         if (state->ignore_specular_alpha) {