From 7f7950cd6bf85e520fa21c59c4ba45895dc28dcb Mon Sep 17 00:00:00 2001
From: lightningterror <18107717+lightningterror@users.noreply.github.com>
Date: Tue, 21 Feb 2023 17:49:46 +0100
Subject: [PATCH] GS-d3d: Check each channel individually if it overflows and
 do corrections.

---
 bin/resources/shaders/dx11/tfx.fx | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/bin/resources/shaders/dx11/tfx.fx b/bin/resources/shaders/dx11/tfx.fx
index 41dff19a79..4c167ea36c 100644
--- a/bin/resources/shaders/dx11/tfx.fx
+++ b/bin/resources/shaders/dx11/tfx.fx
@@ -763,8 +763,10 @@ void ps_color_clamp_wrap(inout float3 C)
 	}
 }
 
-void ps_blend(inout float4 Color, inout float As, float2 pos_xy)
+void ps_blend(inout float4 Color, inout float4 As_rgba, float2 pos_xy)
 {
+	float As = As_rgba.a;
+
 	if (SW_BLEND)
 	{
 		// PABE
@@ -813,15 +815,14 @@ void ps_blend(inout float4 Color, inout float As, float2 pos_xy)
 		{
 			// Replace Af with As so we can do proper compensation for Alpha.
 			if (PS_BLEND_C == 2)
-				As = Af;
+				As_rgba = (float4)Af;
 			// Subtract 1 for alpha to compensate for the changed equation,
 			// if c.rgb > 255.0f then we further need to adjust alpha accordingly,
 			// we pick the lowest overflow from all colors because it's the safest,
 			// we divide by 255 the color because we don't know Cd value,
 			// changed alpha should only be done for hw blend.
-			float min_color = min(min(Color.r, Color.g), Color.b);
-			float alpha_compensate = max(1.0f, min_color / 255.0f);
-			As -= alpha_compensate;
+			float3 alpha_compensate = max((float3)1.0f, Color.rgb / (float3)255.0f);
+			As_rgba.rgb -= alpha_compensate;
 		}
 		else if (PS_CLR_HW == 2)
 		{
@@ -920,15 +921,15 @@ PS_OUTPUT ps_main(PS_INPUT input)
 		C.a = 128.0f;
 	}
 
-	float alpha_blend;
+	float4 alpha_blend;
 	if (PS_BLEND_C == 1 && PS_CLR_HW > 3)
 	{
 		float4 RT = trunc(RtTexture.Load(int3(input.p.xy, 0)) * 255.0f + 0.1f);
-		alpha_blend = RT.a / 128.0f;
+		alpha_blend = (float4)(RT.a / 128.0f);
 	}
 	else
 	{
-		alpha_blend = C.a / 128.0f;
+		alpha_blend = (float4)(C.a / 128.0f);
 	}
 
 	// Alpha correction
@@ -978,12 +979,12 @@ PS_OUTPUT ps_main(PS_INPUT input)
 #if !PS_NO_COLOR
 	output.c0 = PS_HDR ? float4(C.rgb / 65535.0f, C.a / 255.0f) : C / 255.0f;
 #if !PS_NO_COLOR1
-	output.c1 = (float4)(alpha_blend);
+	output.c1 = alpha_blend;
 #endif
 
 #if PS_NO_ABLEND
 	// write alpha blend factor into col0
-	output.c0.a = alpha_blend;
+	output.c0.a = alpha_blend.a;
 #endif
 #if PS_ONLY_ALPHA
 	// rgb isn't used