GS/Vulkan: Implement FXAA

bin/resources/shaders/common/fxaa.fx

@@ -1,8 +1,11 @@
-#if defined(SHADER_MODEL) || defined(FXAA_GLSL_130)
+#if defined(SHADER_MODEL) || defined(FXAA_GLSL_130) || defined(FXAA_GLSL_VK)
 
 #ifndef FXAA_GLSL_130
     #define FXAA_GLSL_130 0
 #endif
+#ifndef FXAA_GLSL_VK
+    #define FXAA_GLSL_VK 0
+#endif
 
 #define UHQ_FXAA 1          //High Quality Fast Approximate Anti Aliasing. Adapted for GS from Timothy Lottes FXAA 3.11.
 #define FxaaSubpixMax 0.0   //[0.00 to 1.00] Amount of subpixel aliasing removal. 0.00: Edge only antialiasing (no blurring)
@@ -17,6 +20,12 @@ in vec2 PSin_t;
 
 layout(location = 0) out vec4 SV_Target0;
 
+#elif (FXAA_GLSL_VK == 1)
+
+layout(location = 0) in vec2 PSin_t;
+layout(location = 0) out vec4 SV_Target0;
+layout(set = 0, binding = 0) uniform sampler2D TextureSampler;
+
 #elif (SHADER_MODEL >= 0x400)
 Texture2D Texture : register(t0);
 SamplerState TextureSampler : register(s0);
@@ -52,7 +61,7 @@ struct PS_OUTPUT
 #define FXAA_HLSL_4 1
 #define FXAA_GATHER4_ALPHA 0
 
-#elif (FXAA_GLSL_130 == 1)
+#elif (FXAA_GLSL_130 == 1 || FXAA_GLSL_VK == 1)
 #define FXAA_GATHER4_ALPHA 1
 #endif
 
@@ -72,7 +81,7 @@ struct FxaaTex { SamplerState smpl; Texture2D tex; };
 #define FxaaDiscard clip(-1)
 #define FxaaSat(x) saturate(x)
 
-#elif (FXAA_GLSL_130 == 1)
+#elif (FXAA_GLSL_130 == 1 || FXAA_GLSL_VK == 1)
 #define int2 ivec2
 #define float2 vec2
 #define float3 vec3
@@ -470,7 +479,7 @@ float4 FxaaPixelShader(float2 pos, FxaaTex tex, float2 fxaaRcpFrame, float fxaaS
 	return float4(FxaaTexTop(tex, posM).xyz, lumaM);
 }
 
-#if (FXAA_GLSL_130 == 1)
+#if (FXAA_GLSL_130 == 1 || FXAA_GLSL_VK == 1)
 float4 FxaaPass(float4 FxaaColor, float2 uv0)
 #elif (SHADER_MODEL >= 0x400)
 float4 FxaaPass(float4 FxaaColor : COLOR0, float2 uv0 : TEXCOORD0)
@@ -486,7 +495,7 @@ float4 FxaaPass(float4 FxaaColor : COLOR0, float2 uv0 : TEXCOORD0)
 	Texture.GetDimensions(PixelSize.x, PixelSize.y);
 	FxaaColor = FxaaPixelShader(uv0, tex, 1.0/PixelSize.xy, FxaaSubpixMax, FxaaEdgeThreshold, FxaaEdgeThresholdMin);
 
-	#elif (FXAA_GLSL_130 == 1)
+	#elif (FXAA_GLSL_130 == 1 || FXAA_GLSL_VK == 1)
 	vec2 PixelSize = textureSize(TextureSampler, 0);
 	FxaaColor = FxaaPixelShader(uv0, TextureSampler, 1.0/PixelSize.xy, FxaaSubpixMax, FxaaEdgeThreshold, FxaaEdgeThresholdMin);
 	#endif
@@ -497,9 +506,9 @@ float4 FxaaPass(float4 FxaaColor : COLOR0, float2 uv0 : TEXCOORD0)
 /*------------------------------------------------------------------------------
                       [MAIN() & COMBINE PASS CODE SECTION]
 ------------------------------------------------------------------------------*/
-#if (FXAA_GLSL_130 == 1)
+#if (FXAA_GLSL_130 == 1 || FXAA_GLSL_VK == 1)
 
-void ps_main()
+void main()
 {
 	vec4 color = texture(TextureSampler, PSin_t);
 	color      = PreGammaPass(color, PSin_t);

pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp

@@ -121,7 +121,7 @@ bool GSDeviceVK::Create(HostDisplay* display)
 		return false;
 
 	if (!CompileConvertPipelines() || !CompileInterlacePipelines() ||
-		!CompileMergePipelines() || !CompileShadeBoostPipeline())
+		!CompileMergePipelines() || !CompilePostProcessingPipelines())
 	{
 		Host::ReportErrorAsync("GS", "Failed to compile utility pipelines");
 		return false;
@@ -883,6 +883,21 @@ void GSDeviceVK::DoShadeBoost(GSTexture* sTex, GSTexture* dTex, const float para
 	static_cast<GSTextureVK*>(dTex)->TransitionToLayout(VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
 }
 
+void GSDeviceVK::DoFXAA(GSTexture* sTex, GSTexture* dTex)
+{
+	const GSVector4 sRect(0.0f, 0.0f, 1.0f, 1.0f);
+	const GSVector4i dRect(0, 0, dTex->GetWidth(), dTex->GetHeight());
+	EndRenderPass();
+	OMSetRenderTargets(dTex, nullptr, dRect, false);
+	SetUtilityTexture(sTex, m_linear_sampler);
+	BeginRenderPass(m_utility_color_render_pass_discard, dRect);
+	SetPipeline(m_fxaa_pipeline);
+	DrawStretchRect(sRect, GSVector4(dRect), dTex->GetSize());
+	EndRenderPass();
+
+	static_cast<GSTextureVK*>(dTex)->TransitionToLayout(VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+}
+
 void GSDeviceVK::IASetVertexBuffer(const void* vertex, size_t stride, size_t count)
 {
 	const u32 size = static_cast<u32>(stride) * static_cast<u32>(count);
@@ -1626,29 +1641,13 @@ bool GSDeviceVK::CompileMergePipelines()
 	return true;
 }
 
-bool GSDeviceVK::CompileShadeBoostPipeline()
+bool GSDeviceVK::CompilePostProcessingPipelines()
 {
-	std::optional<std::string> shader = Host::ReadResourceFileToString("shaders/vulkan/shadeboost.glsl");
-	if (!shader)
-	{
-		Host::ReportErrorAsync("GS", "Failed to read shaders/vulkan/shadeboost.glsl.");
-		return false;
-	}
-
 	VkRenderPass rp = g_vulkan_context->GetRenderPass(
 		LookupNativeFormat(GSTexture::Format::Color), VK_FORMAT_UNDEFINED, VK_ATTACHMENT_LOAD_OP_LOAD);
 	if (!rp)
 		return false;
 
-	VkShaderModule vs = GetUtilityVertexShader(*shader);
-	VkShaderModule ps = GetUtilityFragmentShader(*shader);
-	ScopedGuard shader_guard([&vs, &ps]() {
-		Vulkan::Util::SafeDestroyShaderModule(vs);
-		Vulkan::Util::SafeDestroyShaderModule(ps);
-	});
-	if (vs == VK_NULL_HANDLE || ps == VK_NULL_HANDLE)
-		return false;	
-
 	Vulkan::GraphicsPipelineBuilder gpb;
 	AddUtilityVertexAttributes(gpb);
 	gpb.SetPipelineLayout(m_utility_pipeline_layout);
@@ -1658,18 +1657,74 @@ bool GSDeviceVK::CompileShadeBoostPipeline()
 	gpb.SetNoDepthTestState();
 	gpb.SetNoBlendingState();
 	gpb.SetRenderPass(rp, 0);
-	gpb.SetVertexShader(vs);
-	gpb.SetFragmentShader(ps);
 
 	// we enable provoking vertex here anyway, in case it doesn't support multiple modes in the same pass
 	if (m_features.provoking_vertex_last)
 		gpb.SetProvokingVertex(VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT);
 
-	m_shadeboost_pipeline = gpb.Create(g_vulkan_context->GetDevice(), g_vulkan_shader_cache->GetPipelineCache(true), false);
+	{
-	if (!m_shadeboost_pipeline)
+		std::optional<std::string> vshader = Host::ReadResourceFileToString("shaders/vulkan/convert.glsl");
-		return false;
+		if (!vshader)
+		{
+			Host::ReportErrorAsync("GS", "Failed to read shaders/vulkan/convert.glsl.");
+			return false;
+		}
+
+		std::optional<std::string> pshader = Host::ReadResourceFileToString("shaders/common/fxaa.fx");
+		if (!pshader)
+		{
+			Host::ReportErrorAsync("GS", "Failed to read shaders/common/fxaa.fx.");
+			return false;
+		}
+
+		const std::string psource = "#define FXAA_GLSL_VK 1\n" + *pshader;
+
+		VkShaderModule vs = GetUtilityVertexShader(*vshader);
+		VkShaderModule ps = GetUtilityFragmentShader(psource, "ps_main");
+		ScopedGuard shader_guard([&vs, &ps]() {
+			Vulkan::Util::SafeDestroyShaderModule(vs);
+			Vulkan::Util::SafeDestroyShaderModule(ps);
+		});
+		if (vs == VK_NULL_HANDLE || ps == VK_NULL_HANDLE)
+			return false;
+
+		gpb.SetVertexShader(vs);
+		gpb.SetFragmentShader(ps);
+
+		m_fxaa_pipeline = gpb.Create(g_vulkan_context->GetDevice(), g_vulkan_shader_cache->GetPipelineCache(true), false);
+		if (!m_fxaa_pipeline)
+			return false;
+
+		Vulkan::Util::SetObjectName(g_vulkan_context->GetDevice(), m_fxaa_pipeline, "FXAA pipeline");
+	}
+
+	{
+		std::optional<std::string> shader = Host::ReadResourceFileToString("shaders/vulkan/shadeboost.glsl");
+		if (!shader)
+		{
+			Host::ReportErrorAsync("GS", "Failed to read shaders/vulkan/shadeboost.glsl.");
+			return false;
+		}
+
+		VkShaderModule vs = GetUtilityVertexShader(*shader);
+		VkShaderModule ps = GetUtilityFragmentShader(*shader);
+		ScopedGuard shader_guard([&vs, &ps]() {
+			Vulkan::Util::SafeDestroyShaderModule(vs);
+			Vulkan::Util::SafeDestroyShaderModule(ps);
+		});
+		if (vs == VK_NULL_HANDLE || ps == VK_NULL_HANDLE)
+			return false;
+
+		gpb.SetVertexShader(vs);
+		gpb.SetFragmentShader(ps);
+
+		m_shadeboost_pipeline = gpb.Create(g_vulkan_context->GetDevice(), g_vulkan_shader_cache->GetPipelineCache(true), false);
+		if (!m_shadeboost_pipeline)
+			return false;
+
+		Vulkan::Util::SetObjectName(g_vulkan_context->GetDevice(), m_shadeboost_pipeline, "Shadeboost pipeline");
+	}
 
-	Vulkan::Util::SetObjectName(g_vulkan_context->GetDevice(), m_shadeboost_pipeline, "Shadeboost pipeline");
 	return true;
 }
 
@@ -1755,6 +1810,7 @@ void GSDeviceVK::DestroyResources()
 			Vulkan::Util::SafeDestroyPipeline(m_date_image_setup_pipelines[ds][datm]);
 		}
 	}
+	Vulkan::Util::SafeDestroyPipeline(m_fxaa_pipeline);
 	Vulkan::Util::SafeDestroyPipeline(m_shadeboost_pipeline);
 
 	for (auto& it : m_samplers)

pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h

@@ -124,6 +124,7 @@ private:
 	VkPipeline m_hdr_finish_pipelines[2][2] = {}; // [depth][feedback_loop]
 	VkRenderPass m_date_image_setup_render_passes[2][2] = {}; // [depth][clear]
 	VkPipeline m_date_image_setup_pipelines[2][2] = {}; // [depth][datm]
+	VkPipeline m_fxaa_pipeline = {};
 	VkPipeline m_shadeboost_pipeline = {};
 
 	std::unordered_map<u32, VkShaderModule> m_tfx_vertex_shaders;
@@ -155,6 +156,7 @@ private:
 		const GSRegEXTBUF& EXTBUF, const GSVector4& c) final;
 	void DoInterlace(GSTexture* sTex, GSTexture* dTex, int shader, bool linear, float yoffset = 0) final;
 	void DoShadeBoost(GSTexture* sTex, GSTexture* dTex, const float params[4]) final;
+	void DoFXAA(GSTexture* sTex, GSTexture* dTex) final;
 
 	VkSampler GetSampler(GSHWDrawConfig::SamplerSelector ss);
 	void ClearSamplerCache() final;
@@ -177,7 +179,7 @@ private:
 	bool CompileConvertPipelines();
 	bool CompileInterlacePipelines();
 	bool CompileMergePipelines();
-	bool CompileShadeBoostPipeline();
+	bool CompilePostProcessingPipelines();
 
 	bool CheckStagingBufferSize(u32 required_size);
 	void DestroyStagingBuffer();

pcsx2/GS/Window/GSwxDialog.cpp

@@ -454,7 +454,7 @@ PostTab::PostTab(wxWindow* parent)
 	auto not_vk_prereq = [this] { return !m_is_vk_hw; };
 
 	m_ui.addCheckBox(shader_box.inner, "Texture Filtering of Display", "linear_present", IDC_LINEAR_PRESENT);
-	m_ui.addCheckBox(shader_box.inner, "FXAA Shader (PgUp)",           "fxaa",           IDC_FXAA, not_vk_prereq);
+	m_ui.addCheckBox(shader_box.inner, "FXAA Shader (PgUp)",           "fxaa",           IDC_FXAA);
 
 	CheckboxPrereq shade_boost_check(m_ui.addCheckBox(shader_box.inner, "Enable Shade Boost", "ShadeBoost", IDC_SHADEBOOST));
 
@@ -462,7 +462,7 @@ PostTab::PostTab(wxWindow* parent)
 	auto* shader_boost_grid = new wxFlexGridSizer(2, space, space);
 	shader_boost_grid->AddGrowableCol(1);
 
-	auto shader_boost_prereq = [shade_boost_check, this] { return !m_is_vk_hw && shade_boost_check.box->GetValue(); };
+	auto shader_boost_prereq = [shade_boost_check, this] { return shade_boost_check.box->GetValue(); };
 	m_ui.addSliderAndLabel(shader_boost_grid, "Brightness:", "ShadeBoost_Brightness", 0, 100, 50, -1, shader_boost_prereq);
 	m_ui.addSliderAndLabel(shader_boost_grid, "Contrast:",   "ShadeBoost_Contrast",   0, 100, 50, -1, shader_boost_prereq);
 	m_ui.addSliderAndLabel(shader_boost_grid, "Saturation:", "ShadeBoost_Saturation", 0, 100, 50, -1, shader_boost_prereq);