From 3a4d8379bf16e61348dafc1750a8e4b65394f7a9 Mon Sep 17 00:00:00 2001
From: Pokechu22 <Pokechu022@gmail.com>
Date: Tue, 20 Jul 2021 19:00:29 -0700
Subject: [PATCH] Convert NaN to 1 when generating texture coordinates

This fixes eyelids in Shadow the Hedgehog during cutscenes (https://bugs.dolphin-emu.org/issues/11458)
---
 Source/Core/VideoBackends/Software/TransformUnit.cpp | 9 +++++++++
 Source/Core/VideoCommon/UberShaderVertex.cpp         | 7 +++++++
 Source/Core/VideoCommon/VertexShaderGen.cpp          | 7 +++++++
 3 files changed, 23 insertions(+)

diff --git a/Source/Core/VideoBackends/Software/TransformUnit.cpp b/Source/Core/VideoBackends/Software/TransformUnit.cpp
index 71d33f33ed..bf56307025 100644
--- a/Source/Core/VideoBackends/Software/TransformUnit.cpp
+++ b/Source/Core/VideoBackends/Software/TransformUnit.cpp
@@ -137,6 +137,15 @@ static void TransformTexCoordRegular(const TexMtxInfo& texinfo, int coordNum,
   }
   }
 
+  // Convert NaNs to 1 - needed to fix eyelids in Shadow the Hedgehog during cutscenes
+  // See https://bugs.dolphin-emu.org/issues/11458
+  if (std::isnan(src.x))
+    src.x = 1;
+  if (std::isnan(src.y))
+    src.y = 1;
+  if (std::isnan(src.z))
+    src.z = 1;
+
   const float* mat = &xfmem.posMatrices[srcVertex->texMtx[coordNum] * 4];
   Vec3* dst = &dstVertex->texCoords[coordNum];
 
diff --git a/Source/Core/VideoCommon/UberShaderVertex.cpp b/Source/Core/VideoCommon/UberShaderVertex.cpp
index c5a20f033c..5b02b1f989 100644
--- a/Source/Core/VideoCommon/UberShaderVertex.cpp
+++ b/Source/Core/VideoCommon/UberShaderVertex.cpp
@@ -435,6 +435,13 @@ static void GenVertexShaderTexGens(APIType api_type, u32 num_texgen, ShaderCode&
   out.Write("    coord.z = 1.0f;\n"
             "\n");
 
+  // Convert NaNs to 1 - needed to fix eyelids in Shadow the Hedgehog during cutscenes
+  // See https://bugs.dolphin-emu.org/issues/11458
+  out.Write("  // Convert NaN to 1\n");
+  out.Write("  if (isnan(coord.x)) coord.x = 1.0;\n");
+  out.Write("  if (isnan(coord.y)) coord.y = 1.0;\n");
+  out.Write("  if (isnan(coord.z)) coord.z = 1.0;\n");
+
   out.Write("  // first transformation\n");
   out.Write("  uint texgentype = {};\n", BitfieldExtract<&TexMtxInfo::texgentype>("texMtxInfo"));
   out.Write("  float3 output_tex;\n"
diff --git a/Source/Core/VideoCommon/VertexShaderGen.cpp b/Source/Core/VideoCommon/VertexShaderGen.cpp
index aa28c0f896..4e4121b564 100644
--- a/Source/Core/VideoCommon/VertexShaderGen.cpp
+++ b/Source/Core/VideoCommon/VertexShaderGen.cpp
@@ -332,6 +332,13 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
     if (texinfo.inputform == TexInputForm::AB11)
       out.Write("coord.z = 1.0;\n");
 
+    // Convert NaNs to 1 - needed to fix eyelids in Shadow the Hedgehog during cutscenes
+    // See https://bugs.dolphin-emu.org/issues/11458
+    out.Write("// Convert NaN to 1\n");
+    out.Write("if (isnan(coord.x)) coord.x = 1.0;\n");
+    out.Write("if (isnan(coord.y)) coord.y = 1.0;\n");
+    out.Write("if (isnan(coord.z)) coord.z = 1.0;\n");
+
     // first transformation
     switch (texinfo.texgentype)
     {