From f25a0b43b6d3b17ca0d86bde19b646b1fc4a91a4 Mon Sep 17 00:00:00 2001
From: TellowKrinkle <tellowkrinkle@gmail.com>
Date: Sun, 11 Dec 2022 01:15:15 -0600
Subject: [PATCH] VideoCommon: Fix stereoscopic 3D on OpenGL < 4.3 (macOS)

---
 Source/Core/VideoBackends/OGL/OGLConfig.cpp   | 4 ++++
 Source/Core/VideoCommon/GeometryShaderGen.cpp | 4 ++++
 Source/Core/VideoCommon/PixelShaderGen.cpp    | 5 ++++-
 Source/Core/VideoCommon/ShaderGenCommon.cpp   | 1 +
 Source/Core/VideoCommon/ShaderGenCommon.h     | 1 +
 Source/Core/VideoCommon/UberShaderPixel.cpp   | 5 ++++-
 Source/Core/VideoCommon/VideoConfig.h         | 1 +
 7 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/Source/Core/VideoBackends/OGL/OGLConfig.cpp b/Source/Core/VideoBackends/OGL/OGLConfig.cpp
index 10722e0526..ab31592bd2 100644
--- a/Source/Core/VideoBackends/OGL/OGLConfig.cpp
+++ b/Source/Core/VideoBackends/OGL/OGLConfig.cpp
@@ -528,6 +528,10 @@ bool PopulateConfig(GLContext* m_main_gl_context)
     glEnable(GL_PROGRAM_POINT_SIZE);
   }
 
+  // Supported by all GS-supporting ES and 4.3+
+  g_Config.backend_info.bSupportsGLLayerInFS = g_Config.backend_info.bSupportsGeometryShaders &&
+                                               g_ogl_config.eSupportedGLSLVersion >= Glsl430;
+
   g_Config.backend_info.bSupportsBBox = g_Config.backend_info.bSupportsFragmentStoresAndAtomics;
 
   // Either method can do early-z tests. See PixelShaderGen for details.
diff --git a/Source/Core/VideoCommon/GeometryShaderGen.cpp b/Source/Core/VideoCommon/GeometryShaderGen.cpp
index 2db201c636..5b552f9298 100644
--- a/Source/Core/VideoCommon/GeometryShaderGen.cpp
+++ b/Source/Core/VideoCommon/GeometryShaderGen.cpp
@@ -122,6 +122,8 @@ ShaderCode GenerateGeometryShaderCode(APIType api_type, const ShaderHostConfig&
                             ShaderStage::Geometry);
 
     out.Write("}} ps;\n");
+    if (stereo && !host_config.backend_gl_layer_in_fs)
+      out.Write("flat out int layer;");
 
     out.Write("void main()\n{{\n");
   }
@@ -348,6 +350,8 @@ static void EmitVertex(ShaderCode& out, const ShaderHostConfig& host_config,
     {
       out.Write("\tps.layer = eye;\n");
     }
+    if (!host_config.backend_gl_layer_in_fs)
+      out.Write("\tlayer = eye;\n");
   }
 
   if (api_type == APIType::OpenGL || api_type == APIType::Vulkan)
diff --git a/Source/Core/VideoCommon/PixelShaderGen.cpp b/Source/Core/VideoCommon/PixelShaderGen.cpp
index 4dcd681185..57b2113bc3 100644
--- a/Source/Core/VideoCommon/PixelShaderGen.cpp
+++ b/Source/Core/VideoCommon/PixelShaderGen.cpp
@@ -866,6 +866,8 @@ ShaderCode GeneratePixelShaderCode(APIType api_type, const ShaderHostConfig& hos
                             GetInterpolationQualifier(msaa, ssaa, true, true), ShaderStage::Pixel);
 
     out.Write("}};\n");
+    if (stereo && !host_config.backend_gl_layer_in_fs)
+      out.Write("flat in int layer;");
   }
   else
   {
@@ -923,7 +925,8 @@ ShaderCode GeneratePixelShaderCode(APIType api_type, const ShaderHostConfig& hos
 
   if (host_config.backend_geometry_shaders && stereo)
   {
-    out.Write("\tint layer = gl_Layer;\n");
+    if (host_config.backend_gl_layer_in_fs)
+      out.Write("\tint layer = gl_Layer;\n");
   }
   else
   {
diff --git a/Source/Core/VideoCommon/ShaderGenCommon.cpp b/Source/Core/VideoCommon/ShaderGenCommon.cpp
index 8253679ef9..44037d01dc 100644
--- a/Source/Core/VideoCommon/ShaderGenCommon.cpp
+++ b/Source/Core/VideoCommon/ShaderGenCommon.cpp
@@ -46,6 +46,7 @@ ShaderHostConfig ShaderHostConfig::GetCurrent()
   bits.backend_sampler_lod_bias = g_ActiveConfig.backend_info.bSupportsLodBiasInSampler;
   bits.backend_dynamic_vertex_loader = g_ActiveConfig.backend_info.bSupportsDynamicVertexLoader;
   bits.backend_vs_point_line_expand = g_ActiveConfig.UseVSForLinePointExpand();
+  bits.backend_gl_layer_in_fs = g_ActiveConfig.backend_info.bSupportsGLLayerInFS;
   return bits;
 }
 
diff --git a/Source/Core/VideoCommon/ShaderGenCommon.h b/Source/Core/VideoCommon/ShaderGenCommon.h
index 25b7e32eb6..acbed7bb2f 100644
--- a/Source/Core/VideoCommon/ShaderGenCommon.h
+++ b/Source/Core/VideoCommon/ShaderGenCommon.h
@@ -179,6 +179,7 @@ union ShaderHostConfig
   BitField<26, 1, bool, u32> backend_sampler_lod_bias;
   BitField<27, 1, bool, u32> backend_dynamic_vertex_loader;
   BitField<28, 1, bool, u32> backend_vs_point_line_expand;
+  BitField<29, 1, bool, u32> backend_gl_layer_in_fs;
 
   static ShaderHostConfig GetCurrent();
 };
diff --git a/Source/Core/VideoCommon/UberShaderPixel.cpp b/Source/Core/VideoCommon/UberShaderPixel.cpp
index 21309bc8b3..0bf4b3072c 100644
--- a/Source/Core/VideoCommon/UberShaderPixel.cpp
+++ b/Source/Core/VideoCommon/UberShaderPixel.cpp
@@ -135,6 +135,8 @@ ShaderCode GenPixelShader(APIType api_type, const ShaderHostConfig& host_config,
                             GetInterpolationQualifier(msaa, ssaa, true, true), ShaderStage::Pixel);
 
     out.Write("}};\n\n");
+    if (stereo && !host_config.backend_gl_layer_in_fs)
+      out.Write("flat in int layer;");
   }
   else
   {
@@ -532,7 +534,8 @@ ShaderCode GenPixelShader(APIType api_type, const ShaderHostConfig& host_config,
 
   if (host_config.backend_geometry_shaders && stereo)
   {
-    out.Write("\tint layer = gl_Layer;\n");
+    if (host_config.backend_gl_layer_in_fs)
+      out.Write("\tint layer = gl_Layer;\n");
   }
   else
   {
diff --git a/Source/Core/VideoCommon/VideoConfig.h b/Source/Core/VideoCommon/VideoConfig.h
index 1e5e2f683e..a479276e6c 100644
--- a/Source/Core/VideoCommon/VideoConfig.h
+++ b/Source/Core/VideoCommon/VideoConfig.h
@@ -271,6 +271,7 @@ struct VideoConfig final
     bool bSupportsPartialMultisampleResolve = false;
     bool bSupportsDynamicVertexLoader = false;
     bool bSupportsVSLinePointExpand = false;
+    bool bSupportsGLLayerInFS = true;
   } backend_info;
 
   // Utility