GPU/Vulkan: Possible workaround for Adreno GS shader compiler bug

src/core/gpu_hw_d3d11.cpp

@@ -273,7 +273,7 @@ bool GPU_HW_D3D11::CreateBatchInputLayout()
   GPU_HW_ShaderGen shadergen(m_host_display->GetRenderAPI(), m_resolution_scale, m_true_color, m_scaled_dithering,
                              m_texture_filtering, m_supports_dual_source_blend);
   ComPtr<ID3DBlob> vs_bytecode =
-    m_shader_cache.GetShaderBlob(D3D11::ShaderCompiler::Type::Vertex, shadergen.GenerateBatchVertexShader(true));
+    m_shader_cache.GetShaderBlob(D3D11::ShaderCompiler::Type::Vertex, shadergen.GenerateBatchVertexShader(true, false));
   if (!vs_bytecode)
     return false;
 
@@ -379,7 +379,7 @@ bool GPU_HW_D3D11::CompileShaders()
 
   for (u8 textured = 0; textured < 2; textured++)
   {
-    const std::string vs = shadergen.GenerateBatchVertexShader(ConvertToBoolUnchecked(textured));
+    const std::string vs = shadergen.GenerateBatchVertexShader(ConvertToBoolUnchecked(textured), false);
     m_batch_vertex_shaders[textured] = m_shader_cache.GetVertexShader(m_device.Get(), vs);
     if (!m_batch_vertex_shaders[textured])
       return false;

src/core/gpu_hw_opengl.cpp

@@ -348,7 +348,7 @@ bool GPU_HW_OpenGL::CompilePrograms()
         for (u8 interlacing = 0; interlacing < 2; interlacing++)
         {
           const bool textured = (static_cast<TextureMode>(texture_mode) != TextureMode::Disabled);
-          const std::string batch_vs = shadergen.GenerateBatchVertexShader(textured);
+          const std::string batch_vs = shadergen.GenerateBatchVertexShader(textured, false);
           const std::string fs = shadergen.GenerateBatchFragmentShader(
             static_cast<BatchRenderMode>(render_mode), static_cast<TextureMode>(texture_mode),
             ConvertToBoolUnchecked(dithering), ConvertToBoolUnchecked(interlacing));
@@ -397,9 +397,10 @@ bool GPU_HW_OpenGL::CompilePrograms()
 
           if (!textured && m_supports_geometry_shaders)
           {
+            const std::string line_expand_vs = shadergen.GenerateBatchVertexShader(textured, true);
             const std::string line_expand_gs = shadergen.GenerateBatchLineExpandGeometryShader();
 
-            prog = m_shader_cache.GetProgram(batch_vs, line_expand_gs, fs, link_callback);
+            prog = m_shader_cache.GetProgram(line_expand_vs, line_expand_gs, fs, link_callback);
             if (!prog)
               return false;
 

src/core/gpu_hw_shadergen.cpp

@@ -295,7 +295,7 @@ void GPU_HW_ShaderGen::DeclareTextureBuffer(std::stringstream& ss, const char* n
 void GPU_HW_ShaderGen::DeclareVertexEntryPoint(
   std::stringstream& ss, const std::initializer_list<const char*>& attributes, u32 num_color_outputs,
   u32 num_texcoord_outputs, const std::initializer_list<std::pair<const char*, const char*>>& additional_outputs,
-  bool declare_vertex_id)
+  bool declare_vertex_id, const char* output_block_suffix)
 {
   if (m_glsl)
   {
@@ -319,7 +319,7 @@ void GPU_HW_ShaderGen::DeclareVertexEntryPoint(
       if (IsVulkan())
         ss << "layout(location = 0) ";
 
-      ss << "out VertexData {\n";
+      ss << "out VertexData" << output_block_suffix << " {\n";
       for (u32 i = 0; i < num_color_outputs; i++)
         ss << "  float4 v_col" << i << ";\n";
 
@@ -502,7 +502,7 @@ void GPU_HW_ShaderGen::WriteBatchUniformBuffer(std::stringstream& ss)
                        false);
 }
 
-std::string GPU_HW_ShaderGen::GenerateBatchVertexShader(bool textured)
+std::string GPU_HW_ShaderGen::GenerateBatchVertexShader(bool textured, bool upscaled_lines)
 {
   std::stringstream ss;
   WriteHeader(ss);
@@ -513,14 +513,15 @@ std::string GPU_HW_ShaderGen::GenerateBatchVertexShader(bool textured)
 
   ss << "CONSTANT float EPSILON = 0.00001;\n";
 
+  const char* output_block_suffix = upscaled_lines ? "VS" : "";
   if (textured)
   {
     DeclareVertexEntryPoint(ss, {"int3 a_pos", "float4 a_col0", "uint a_texcoord", "uint a_texpage"}, 1, 1,
-                            {{"nointerpolation", "uint4 v_texpage"}}, false);
+                            {{"nointerpolation", "uint4 v_texpage"}}, false, output_block_suffix);
   }
   else
   {
-    DeclareVertexEntryPoint(ss, {"int3 a_pos", "float4 a_col0"}, 1, 0, {}, false);
+    DeclareVertexEntryPoint(ss, {"int3 a_pos", "float4 a_col0"}, 1, 0, {}, false, output_block_suffix);
   }
 
   ss << R"(
@@ -899,7 +900,7 @@ CONSTANT float2 WIDTH = (float(RESOLUTION_SCALE * 2u) / float2(VRAM_SIZE));
     if (IsVulkan())
       ss << "layout(location = 0) ";
 
-    ss << R"(in VertexData {
+    ss << R"(in VertexDataVS {
   float4 v_col0;
 } in_data[];)";
 

src/core/gpu_hw_shadergen.h

@@ -13,7 +13,7 @@ public:
 
   static bool UseGLSLBindingLayout();
 
-  std::string GenerateBatchVertexShader(bool textured);
+  std::string GenerateBatchVertexShader(bool textured, bool upscaled_lines);
   std::string GenerateBatchFragmentShader(GPU_HW::BatchRenderMode transparency, GPU::TextureMode texture_mode,
                                           bool dithering, bool interlacing);
   std::string GenerateBatchLineExpandGeometryShader();
@@ -39,7 +39,7 @@ private:
   void DeclareVertexEntryPoint(std::stringstream& ss, const std::initializer_list<const char*>& attributes,
                                u32 num_color_outputs, u32 num_texcoord_outputs,
                                const std::initializer_list<std::pair<const char*, const char*>>& additional_outputs,
-                               bool declare_vertex_id = false);
+                               bool declare_vertex_id = false, const char* output_block_suffix = "");
   void DeclareFragmentEntryPoint(std::stringstream& ss, u32 num_color_inputs, u32 num_texcoord_inputs,
                                  const std::initializer_list<std::pair<const char*, const char*>>& additional_inputs,
                                  bool declare_fragcoord = false, u32 num_color_outputs = 1, bool depth_output = false);

src/core/gpu_hw_vulkan.cpp

@@ -578,7 +578,7 @@ bool GPU_HW_Vulkan::CompilePipelines()
 
   // vertex shaders - [textured]
   // fragment shaders - [render_mode][texture_mode][dithering][interlacing]
-  DimensionalArray<VkShaderModule, 2> batch_vertex_shaders{};
+  DimensionalArray<VkShaderModule, 2, 2> batch_vertex_shaders{};
   DimensionalArray<VkShaderModule, 2, 2, 9, 4> batch_fragment_shaders{};
   VkShaderModule batch_line_geometry_shader = VK_NULL_HANDLE;
   Common::ScopeGuard batch_shader_guard(
@@ -590,12 +590,12 @@ bool GPU_HW_Vulkan::CompilePipelines()
 
   for (u8 textured = 0; textured < 2; textured++)
   {
-    const std::string vs = shadergen.GenerateBatchVertexShader(ConvertToBoolUnchecked(textured));
+    const std::string vs = shadergen.GenerateBatchVertexShader(ConvertToBoolUnchecked(textured), false);
     VkShaderModule shader = g_vulkan_shader_cache->GetVertexShader(vs);
     if (shader == VK_NULL_HANDLE)
       return false;
 
-    batch_vertex_shaders[textured] = shader;
+    batch_vertex_shaders[textured][0] = shader;
   }
 
   for (u8 render_mode = 0; render_mode < 4; render_mode++)
@@ -628,6 +628,16 @@ bool GPU_HW_Vulkan::CompilePipelines()
       batch_line_geometry_shader = g_vulkan_shader_cache->GetGeometryShader(gs);
       if (batch_line_geometry_shader == VK_NULL_HANDLE)
         return false;
+
+      for (u8 textured = 0; textured < 2; textured++)
+      {
+        const std::string vs = shadergen.GenerateBatchVertexShader(ConvertToBoolUnchecked(textured), true);
+        VkShaderModule shader = g_vulkan_shader_cache->GetVertexShader(vs);
+        if (shader == VK_NULL_HANDLE)
+          return false;
+
+        batch_vertex_shaders[textured][1] = shader;
+      }
     }
     else
     {
@@ -667,17 +677,18 @@ bool GPU_HW_Vulkan::CompilePipelines()
                 }
 
                 gpbuilder.SetPrimitiveTopology(primitive_mapping[primitive]);
-                gpbuilder.SetVertexShader(batch_vertex_shaders[BoolToUInt8(textured)]);
                 gpbuilder.SetFragmentShader(batch_fragment_shaders[render_mode][texture_mode][dithering][interlacing]);
                 if (static_cast<BatchPrimitive>(primitive) == BatchPrimitive::Lines &&
                     batch_line_geometry_shader != VK_NULL_HANDLE)
                 {
+                  gpbuilder.SetVertexShader(batch_vertex_shaders[BoolToUInt8(textured)][1]);
                   gpbuilder.SetGeometryShader(batch_line_geometry_shader);
                   gpbuilder.SetRasterizationState(polygon_mode_mapping[static_cast<u8>(BatchPrimitive::Triangles)],
                                                   VK_CULL_MODE_NONE, VK_FRONT_FACE_CLOCKWISE);
                 }
                 else
                 {
+                  gpbuilder.SetVertexShader(batch_vertex_shaders[BoolToUInt8(textured)][0]);
                   gpbuilder.SetRasterizationState(polygon_mode_mapping[primitive], VK_CULL_MODE_NONE,
                                                   VK_FRONT_FACE_CLOCKWISE);
                 }