Use custom isnan implementation to avoid HLSL optimizer issues

This adjusts the NaN replacement logic introduced in #9928 to work around the HLSL compiler optimizing away calls to isnan, which caused that functionality to not work with ubershaders on D3D11 and D3D12 (it did work with specialized shaders, despite a warning being logged for both; that warning is also now gone).  Note that the `D3DCOMPILE_IEEE_STRICTNESS` flag did not solve this issue, despite the warning suggesting that it might.

Suggested by @kayru and @jamiehayes.

Source/Core/VideoCommon/ShaderGenCommon.cpp

@@ -87,6 +87,24 @@ std::string GetDiskShaderCacheFileName(APIType api_type, const char* type, bool
   return filename;
 }
 
+void WriteIsNanHeader(ShaderCode& out, APIType api_type)
+{
+  if (api_type == APIType::D3D)
+  {
+    out.Write("bool dolphin_isnan(float f) {{\n"
+              "  // Workaround for the HLSL compiler deciding that isnan can never be true and\n"
+              "  // optimising away the call, even though the value can actually be NaN\n"
+              "  // Just look for the bit pattern that indicates NaN instead\n"
+              "  return (asint(f) & 0x7FFFFFFF) > 0x7F800000;\n"
+              "}}\n\n");
+    // If isfinite is needed, (asint(f) & 0x7F800000) != 0x7F800000 can be used
+  }
+  else
+  {
+    out.Write("#define dolphin_isnan(f) isnan(f)\n");
+  }
+}
+
 static void DefineOutputMember(ShaderCode& object, APIType api_type, std::string_view qualifier,
                                std::string_view type, std::string_view name, int var_index,
                                std::string_view semantic = {}, int semantic_index = -1)

Source/Core/VideoCommon/ShaderGenCommon.h

@@ -176,6 +176,8 @@ union ShaderHostConfig
 std::string GetDiskShaderCacheFileName(APIType api_type, const char* type, bool include_gameid,
                                        bool include_host_config, bool include_api = true);
 
+void WriteIsNanHeader(ShaderCode& out, APIType api_type);
+
 void GenerateVSOutputMembers(ShaderCode& object, APIType api_type, u32 texgens,
                              const ShaderHostConfig& host_config, std::string_view qualifier);
 

Source/Core/VideoCommon/UberShaderVertex.cpp

@@ -50,6 +50,7 @@ ShaderCode GenVertexShader(APIType api_type, const ShaderHostConfig& host_config
   out.Write("}};\n\n");
 
   WriteUberShaderCommonHeader(out, api_type, host_config);
+  WriteIsNanHeader(out, api_type);
   WriteLightingFunction(out);
 
   if (api_type == APIType::OpenGL || api_type == APIType::Vulkan)
@@ -438,9 +439,9 @@ static void GenVertexShaderTexGens(APIType api_type, u32 num_texgen, ShaderCode&
   // Convert NaNs to 1 - needed to fix eyelids in Shadow the Hedgehog during cutscenes
   // See https://bugs.dolphin-emu.org/issues/11458
   out.Write("  // Convert NaN to 1\n");
-  out.Write("  if (isnan(coord.x)) coord.x = 1.0;\n");
+  out.Write("  if (dolphin_isnan(coord.x)) coord.x = 1.0;\n");
-  out.Write("  if (isnan(coord.y)) coord.y = 1.0;\n");
+  out.Write("  if (dolphin_isnan(coord.y)) coord.y = 1.0;\n");
-  out.Write("  if (isnan(coord.z)) coord.z = 1.0;\n");
+  out.Write("  if (dolphin_isnan(coord.z)) coord.z = 1.0;\n");
 
   out.Write("  // first transformation\n");
   out.Write("  uint texgentype = {};\n", BitfieldExtract<&TexMtxInfo::texgentype>("texMtxInfo"));

Source/Core/VideoCommon/VertexShaderGen.cpp

@@ -96,7 +96,9 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
 
   out.Write("struct VS_OUTPUT {{\n");
   GenerateVSOutputMembers(out, api_type, uid_data->numTexGens, host_config, "");
-  out.Write("}};\n");
+  out.Write("}};\n\n");
+
+  WriteIsNanHeader(out, api_type);
 
   if (api_type == APIType::OpenGL || api_type == APIType::Vulkan)
   {
@@ -335,9 +337,9 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
     // Convert NaNs to 1 - needed to fix eyelids in Shadow the Hedgehog during cutscenes
     // See https://bugs.dolphin-emu.org/issues/11458
     out.Write("// Convert NaN to 1\n");
-    out.Write("if (isnan(coord.x)) coord.x = 1.0;\n");
+    out.Write("if (dolphin_isnan(coord.x)) coord.x = 1.0;\n");
-    out.Write("if (isnan(coord.y)) coord.y = 1.0;\n");
+    out.Write("if (dolphin_isnan(coord.y)) coord.y = 1.0;\n");
-    out.Write("if (isnan(coord.z)) coord.z = 1.0;\n");
+    out.Write("if (dolphin_isnan(coord.z)) coord.z = 1.0;\n");
 
     // first transformation
     switch (texinfo.texgentype)