GS-hw: Extend Blend mix to work when alpha is higher than 1.

Clamp the alpha(As/Af) to 1 in the shader, it is already clamped to 1 in hw blend unit.
It should allow to better render the effects, still not fully accurate but it's closer to what we want.
Hopefully it helps D3D11 more.

bin/resources/shaders/dx11/tfx.fx

@@ -47,6 +47,7 @@
 #define PS_BLEND_B 0
 #define PS_BLEND_C 0
 #define PS_BLEND_D 0
+#define PS_ALPHA_CLAMP 0
 #define PS_PABE 0
 #define PS_DITHER 0
 #define PS_ZCLAMP 0
@@ -704,9 +705,13 @@ void ps_blend(inout float4 Color, float As, float2 pos_xy)
 
 		float3 A = (PS_BLEND_A == 0) ? Cs : ((PS_BLEND_A == 1) ? Cd : (float3)0.0f);
 		float3 B = (PS_BLEND_B == 0) ? Cs : ((PS_BLEND_B == 1) ? Cd : (float3)0.0f);
-		float3 C = (PS_BLEND_C == 0) ? As : ((PS_BLEND_C == 1) ? Ad : Af);
+		float C = (PS_BLEND_C == 0) ? As : ((PS_BLEND_C == 1) ? Ad : Af);
 		float3 D = (PS_BLEND_D == 0) ? Cs : ((PS_BLEND_D == 1) ? Cd : (float3)0.0f);
 
+		// As/Af clamp alpha for Blend mix
+		if (PS_ALPHA_CLAMP)
+			C = min(C, (float)1.0f);
+
 		Color.rgb = (PS_BLEND_A == PS_BLEND_B) ? D : trunc(((A - B) * C) + D);
 
 		// PABE

bin/resources/shaders/opengl/tfx_fs.glsl

@@ -702,6 +702,11 @@ void ps_blend(inout vec4 Color, float As)
     vec3 D = vec3(0.0f);
 #endif
 
+    // As/Af clamp alpha for Blend mix
+#if PS_ALPHA_CLAMP
+    C = min(C, float(1.0f));
+#endif
+
 #if PS_BLEND_A == PS_BLEND_B
     Color.rgb = D;
 #else

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -213,14 +213,15 @@ struct alignas(16) GSHWDrawConfig
 
 				// *** Word 2
 				// Blend and Colclip
-				u32 blend_a : 2;
-				u32 blend_b : 2;
-				u32 blend_c : 2;
-				u32 blend_d : 2;
-				u32 clr1    : 1; // useful?
-				u32 hdr     : 1;
-				u32 colclip : 1;
-				u32 pabe    : 1;
+				u32 blend_a     : 2;
+				u32 blend_b     : 2;
+				u32 blend_c     : 2;
+				u32 blend_d     : 2;
+				u32 clr1        : 1; // useful?
+				u32 hdr         : 1;
+				u32 colclip     : 1;
+				u32 alpha_clamp : 1;
+				u32 pabe        : 1;
 
 				// Others ways to fetch the texture
 				u32 channel : 3;
@@ -244,7 +245,7 @@ struct alignas(16) GSHWDrawConfig
 				// Scan mask
 				u32 scanmsk : 2;
 
-				u32 _free2 : 4;
+				u32 _free2 : 3;
 			};
 
 			u64 key;

pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp

@@ -186,6 +186,7 @@ void GSDevice11::SetupPS(PSSelector sel, const GSHWDrawConfig::PSConstantBuffer*
 		sm.AddMacro("PS_BLEND_B", sel.blend_b);
 		sm.AddMacro("PS_BLEND_C", sel.blend_c);
 		sm.AddMacro("PS_BLEND_D", sel.blend_d);
+		sm.AddMacro("PS_ALPHA_CLAMP", sel.alpha_clamp);
 		sm.AddMacro("PS_PABE", sel.pabe);
 		sm.AddMacro("PS_DITHER", sel.dither);
 		sm.AddMacro("PS_ZCLAMP", sel.zclamp);

pcsx2/GS/Renderers/HW/GSRendererNew.cpp

@@ -588,8 +588,6 @@ void GSRendererNew::EmulateBlending(bool& DATE_GL42, bool& DATE_GL45)
 	if (m_sw_blending != AccBlendLevel::None)
 	{
 		blend_mix &= !sw_blending;
-		// Do not enable if As > 128 or F > 128, hw blend clamps to 1
-		blend_mix &= !((ALPHA.C == 0 && GetAlphaMinMax().max > 128) || (ALPHA.C == 2 && ALPHA.FIX > 128u));
 		sw_blending |= blend_mix;
 	}
 
@@ -709,6 +707,7 @@ void GSRendererNew::EmulateBlending(bool& DATE_GL42, bool& DATE_GL45)
 		else if (blend_mix)
 		{
 			m_conf.blend = {blend_index, ALPHA.FIX, ALPHA.C == 2, false, true};
+			m_conf.ps.alpha_clamp = 1;
 
 			if (blend_mix1)
 			{

pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp

@@ -1055,6 +1055,7 @@ std::string GSDeviceOGL::GetPSSource(PSSelector sel)
 		+ format("#define PS_HDR %d\n", sel.hdr)
 		+ format("#define PS_DITHER %d\n", sel.dither)
 		+ format("#define PS_ZCLAMP %d\n", sel.zclamp)
+		+ format("#define PS_ALPHA_CLAMP %d\n", sel.alpha_clamp)
 		+ format("#define PS_PABE %d\n", sel.pabe)
 		+ format("#define PS_SCANMSK %d\n", sel.scanmsk)
 		+ format("#define PS_SCALE_FACTOR %d\n", m_upscale_multiplier)