PixelShaderGen: Clamp texture layer when using manual texture sampling with stereoscopic 3D

Otherwise, texelFetch() will use an out-of-bounds layer for game textures (that have 1 layer; EFB copies have 2 layers in stereoscopic 3D mode), which is undefined behavior (often resulting in a black image). The fast texture sampling path uses texture(), which always clamps (see https://www.khronos.org/opengl/wiki/Array_Texture#Access_in_shaders), so it was unaffected by this difference.

Source/Core/VideoCommon/PixelShaderGen.cpp

@@ -615,6 +615,7 @@ uint WrapCoord(int coord, uint wrap, int size) {{
   int3 size = textureSize(tex, 0);
   int size_s = size.x;
   int size_t = size.y;
+  int num_layers = size.z;
 )");
       if (g_ActiveConfig.backend_info.bSupportsTextureQueryLevels)
       {
@@ -633,6 +634,8 @@ uint WrapCoord(int coord, uint wrap, int size) {{
   // Rescale uv to account for the new texture size
   uv.x = (uv.x * size_s) / native_size_s;
   uv.y = (uv.y * size_t) / native_size_t;
+  // Clamp layer as well (texture() automatically clamps, but texelFetch() doesn't)
+  layer = clamp(layer, 0, num_layers - 1);
 )");
     }
     else

Source/Core/VideoCommon/ShaderGenCommon.cpp

@@ -42,7 +42,7 @@ ShaderHostConfig ShaderHostConfig::GetCurrent()
   bits.enable_validation_layer = g_ActiveConfig.bEnableValidationLayer;
   bits.manual_texture_sampling = !g_ActiveConfig.bFastTextureSampling;
   bits.manual_texture_sampling_custom_texture_sizes =
-      g_ActiveConfig.ManualTextureSamplingWithHiResTextures();
+      g_ActiveConfig.ManualTextureSamplingWithCustomTextureSizes();
   bits.backend_sampler_lod_bias = g_ActiveConfig.backend_info.bSupportsLodBiasInSampler;
   bits.backend_dynamic_vertex_loader = g_ActiveConfig.backend_info.bSupportsDynamicVertexLoader;
   bits.backend_vs_point_line_expand = g_ActiveConfig.UseVSForLinePointExpand();

Source/Core/VideoCommon/VideoConfig.h

@@ -274,15 +274,25 @@ struct VideoConfig final
     return backend_info.bSupportsGPUTextureDecoding && bEnableGPUTextureDecoding;
   }
   bool UseVertexRounding() const { return bVertexRounding && iEFBScale != 1; }
-  bool ManualTextureSamplingWithHiResTextures() const
+  bool ManualTextureSamplingWithCustomTextureSizes() const
   {
-    // Hi-res textures (including hi-res EFB copies, but not native-resolution EFB copies at higher
-    // internal resolutions) breaks the wrapping logic used by manual texture sampling.
+    // If manual texture sampling is disabled, we don't need to do anything.
     if (bFastTextureSampling)
       return false;
+    // Hi-res textures break the wrapping logic used by manual texture sampling, as a texture's
+    // size won't match the size the game sets.
+    if (bHiresTextures)
+      return true;
+    // Hi-res EFB copies (but not native-resolution EFB copies at higher internal resolutions)
+    // also result in different texture sizes that need special handling.
     if (iEFBScale != 1 && bCopyEFBScaled)
       return true;
-    return bHiresTextures;
+    // Stereoscopic 3D changes the number of layers some textures have (EFB copies have 2 layers,
+    // while game textures still have 1), meaning bounds checks need to be added.
+    if (stereo_mode != StereoMode::Off)
+      return true;
+    // Otherwise, manual texture sampling can use the sizes games specify directly.
+    return false;
   }
   bool UsingUberShaders() const;
   u32 GetShaderCompilerThreads() const;