PostProcessing/ReShade: Fix up for GL/Vulkan

dep/reshadefx/src/effect_codegen_glsl.cpp

@@ -99,9 +99,16 @@ private:
 				"uvec4 compCond(bvec4 cond, uvec4 a, uvec4 b) { return uvec4(cond.x ? a.x : b.x, cond.y ? a.y : b.y, cond.z ? a.z : b.z, cond.w ? a.w : b.w); }\n";
 
 		if (!_ubo_block.empty())
-			// Read matrices in column major layout, even though they are actually row major, to avoid transposing them on every access (since GLSL uses column matrices)
-			// TODO: This technically only works with square matrices
-			preamble += "layout(std140, column_major, binding = 0) uniform _Globals {\n" + _ubo_block + "};\n";
+		{
+			if (_vulkan_semantics)
+			{
+				preamble += "layout(std140, set = 0, binding = 0) uniform _Globals {\n" + _ubo_block + "};\n";
+			}
+			else
+			{
+				preamble += "layout(std140, binding = 1) uniform _Globals {\n" + _ubo_block + "};\n";
+			}
+		}
 
 		module.code.assign(preamble.begin(), preamble.end());
 
@@ -620,7 +627,14 @@ private:
 
 		write_location(code, loc);
 
-		code += "layout(binding = " + std::to_string(info.binding);
+		code += "layout(";
+		if (_vulkan_semantics)
+			code += "set = 1, ";
+#if 0
+		code += "binding = " + std::to_string(info.binding);
+#else
+		code += "binding = /*SAMPLER:" + info.unique_name + "*/0";
+#endif
 		code += ") uniform ";
 		write_type(code, info.type);
 		code += ' ' + id_to_name(info.id) + ";\n";

src/util/postprocessing_shader_fx.cpp

@@ -22,6 +22,7 @@
 
 #include "fmt/format.h"
 
+#include <bitset>
 #include <cctype>
 #include <cmath>
 #include <cstring>
@@ -41,25 +42,32 @@ static std::unique_ptr<reshadefx::codegen> CreateRFXCodegen()
 {
   const bool debug_info = g_gpu_device ? g_gpu_device->IsDebugDevice() : false;
   const bool uniforms_to_spec_constants = false;
+  const RenderAPI rapi = GetRenderAPI();
 
-  switch (GetRenderAPI())
+  switch (rapi)
   {
     case RenderAPI::None:
     case RenderAPI::D3D11:
     case RenderAPI::D3D12:
+    {
       return std::unique_ptr<reshadefx::codegen>(
         reshadefx::create_codegen_hlsl(50, debug_info, uniforms_to_spec_constants));
+    }
 
     case RenderAPI::Vulkan:
     case RenderAPI::Metal:
-      return std::unique_ptr<reshadefx::codegen>(
-        reshadefx::create_codegen_glsl(true, debug_info, uniforms_to_spec_constants));
+    {
+      return std::unique_ptr<reshadefx::codegen>(reshadefx::create_codegen_glsl(
+        true, debug_info, uniforms_to_spec_constants, false, (rapi == RenderAPI::Vulkan)));
+    }
 
     case RenderAPI::OpenGL:
     case RenderAPI::OpenGLES:
     default:
+    {
       return std::unique_ptr<reshadefx::codegen>(
-        reshadefx::create_codegen_glsl(false, debug_info, uniforms_to_spec_constants));
+        reshadefx::create_codegen_glsl(false, debug_info, uniforms_to_spec_constants, false, true));
+    }
   }
 }
 
@@ -989,12 +997,6 @@ bool PostProcessing::ReShadeFXShader::CompilePipeline(GPUTexture::Format format,
   const RenderAPI api = g_gpu_device->GetRenderAPI();
   const bool needs_main_defn = (api != RenderAPI::D3D11 && api != RenderAPI::D3D12);
 
-  if (api != RenderAPI::D3D11)
-  {
-    Log_ErrorPrintf("ReShade is currently only supported on Direct3D 11 (due to bugs)");
-    return false;
-  }
-
   m_valid = false;
   m_textures.clear();
   m_passes.clear();
@@ -1021,8 +1023,20 @@ bool PostProcessing::ReShadeFXShader::CompilePipeline(GPUTexture::Format format,
                                                    GPUShaderStage stage) {
     std::string real_code;
     if (needs_main_defn)
-      real_code = fmt::format("#version 460 core\n#define ENTRY_POINT_{}\n{}", name, code);
+    {
+      // dFdx/dFdy are not defined in the vertex shader.
+      const char* defns = (stage == GPUShaderStage::Vertex) ? "#define dFdx(x) x\n#define dFdy(x) x\n" : "";
+      real_code = fmt::format("#version 460 core\n#define ENTRY_POINT_{}\n{}\n{}", name, defns, code);
+
+      for (const Sampler& sampler : samplers)
+      {
+        std::string decl = fmt::format("binding = /*SAMPLER:{}*/0", sampler.reshade_name);
+        std::string replacement = fmt::format("binding = {}", sampler.slot);
+        StringUtil::ReplaceAll(&real_code, decl, replacement);
+      }
+    }
     else
+    {
       real_code = std::string(code);
 
       for (const Sampler& sampler : samplers)
@@ -1037,6 +1051,7 @@ bool PostProcessing::ReShadeFXShader::CompilePipeline(GPUTexture::Format format,
           fmt::format("__{}_s : register({}s{});", sampler.reshade_name, (sampler.slot < 10) ? " " : "", sampler.slot);
         StringUtil::ReplaceAll(&real_code, decl, replacement);
       }
+    }
 
     // FileSystem::WriteStringToFile("D:\\foo.txt", real_code);
 
@@ -1281,24 +1296,38 @@ bool PostProcessing::ReShadeFXShader::Apply(GPUTexture* input, GPUFramebuffer* f
     g_gpu_device->SetPipeline(pass.pipeline.get());
 
     // Set all inputs first, before the render pass starts.
+    std::bitset<GPUDevice::MAX_TEXTURE_SAMPLERS> bound_textures = {};
     for (const Sampler& sampler : pass.samplers)
     {
       GL_INS("Texture Sampler %u: ID %d [%s]", sampler.slot, sampler.texture_id,
              GetTextureNameForID(sampler.texture_id));
       g_gpu_device->SetTextureSampler(sampler.slot, GetTextureByID(sampler.texture_id, input, final_target),
                                       sampler.sampler);
+      bound_textures[sampler.slot] = true;
+    }
+
+    // Ensure RT wasn't left bound as a previous output, it breaks VK/DX12.
+    // TODO: Maybe move this into the backend? Not sure...
+    for (u32 i = 0; i < GPUDevice::MAX_TEXTURE_SAMPLERS; i++)
+    {
+      if (!bound_textures[i])
+        g_gpu_device->SetTextureSampler(i, nullptr, nullptr);
     }
 
     if (!output_fb)
     {
       // Drawing to final buffer.
       if (!g_gpu_device->BeginPresent(false))
+      {
+        GL_POP();
         return false;
       }
+    }
 
     g_gpu_device->Draw(pass.num_vertices, 0);
   }
 
+  GL_POP();
   m_frame_timer.Reset();
   return true;
 }