GS-HW: Use correct alphas when AA1 is enabled + ABE disabled.

bin/resources/shaders/dx11/tfx.fx

@@ -50,6 +50,7 @@
 #define PS_BLEND_C 0
 #define PS_BLEND_D 0
 #define PS_BLEND_MIX 0
+#define PS_FIXED_ONE_A 0
 #define PS_PABE 0
 #define PS_DITHER 0
 #define PS_ZCLAMP 0
@@ -847,6 +848,13 @@ PS_OUTPUT ps_main(PS_INPUT input)
 	}
 
 	// Must be done before alpha correction
+
+	// AA (Fixed one) will output a coverage of 1.0 as alpha
+	if (PS_FIXED_ONE_A)
+	{
+		C.a = 128.0f;
+	}
+
 	float alpha_blend;
 	if (PS_BLEND_C == 1 && PS_CLR_HW > 3)
 	{

bin/resources/shaders/opengl/tfx_fs.glsl

@@ -903,6 +903,12 @@ void ps_main()
 #endif
 
     // Must be done before alpha correction
+
+   // AA (Fixed one) will output a coverage of 1.0 as alpha
+#if PS_FIXED_ONE_A
+   C.a = 128.0f;
+#endif
+
 #if (PS_BLEND_C == 1 && PS_CLR_HW > 3)
 #if HAS_FRAMEBUFFER_FETCH
     vec4 RT = trunc(LAST_FRAG_COLOR * 255.0f + 0.1f);

bin/resources/shaders/vulkan/tfx.glsl

@@ -337,6 +337,7 @@ void main()
 #define PS_BLEND_B 0
 #define PS_BLEND_C 0
 #define PS_BLEND_D 0
+#define PS_FIXED_ONE_A 0
 #define PS_PABE 0
 #define PS_DITHER 0
 #define PS_ZCLAMP 0
@@ -1162,6 +1163,12 @@ void main()
 	#endif
 
   // Must be done before alpha correction
+
+  // AA (Fixed one) will output a coverage of 1.0 as alpha
+#if PS_FIXED_ONE_A
+   C.a = 128.0f;
+#endif
+
 #if (PS_BLEND_C == 1 && PS_CLR_HW > 3)
   vec4 RT = trunc(subpassLoad(RtSampler) * 255.0f + 0.1f);
   float alpha_blend = (PS_DFMT == FMT_24) ? 1.0f : RT.a / 128.0f;

pcsx2/GS/GSState.cpp

@@ -1721,10 +1721,7 @@ inline bool GSState::TestDrawChanged()
 			prim_mask &= ~0x7;
 		else
 			return true;
-
-		if (GSConfig.UseHardwareRenderer() && GSUtil::GetPrimClass(m_env.PRIM.PRIM) == GS_TRIANGLE_CLASS)
-			prim_mask &= ~0x80; // Mask out AA1.
-
+			
 		if ((m_env.PRIM.U32[0] ^ m_prev_env.PRIM.U32[0]) & prim_mask)
 			return true;
 

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -264,6 +264,7 @@ struct alignas(16) GSHWDrawConfig
 				u32 blend_b     : 2;
 				u32 blend_c     : 2;
 				u32 blend_d     : 2;
+				u32 fixed_one_a : 1;
 				u32 clr_hw      : 3;
 				u32 hdr         : 1;
 				u32 colclip     : 1;

pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp

@@ -188,6 +188,7 @@ void GSDevice11::SetupPS(const PSSelector& sel, const GSHWDrawConfig::PSConstant
 		sm.AddMacro("PS_BLEND_C", sel.blend_c);
 		sm.AddMacro("PS_BLEND_D", sel.blend_d);
 		sm.AddMacro("PS_BLEND_MIX", sel.blend_mix);
+		sm.AddMacro("PS_FIXED_ONE_A", sel.fixed_one_a);
 		sm.AddMacro("PS_PABE", sel.pabe);
 		sm.AddMacro("PS_DITHER", sel.dither);
 		sm.AddMacro("PS_ZCLAMP", sel.zclamp);

pcsx2/GS/Renderers/DX12/GSDevice12.cpp

@@ -1567,6 +1567,7 @@ const ID3DBlob* GSDevice12::GetTFXPixelShader(const GSHWDrawConfig::PSSelector&
 	sm.AddMacro("PS_BLEND_C", sel.blend_c);
 	sm.AddMacro("PS_BLEND_D", sel.blend_d);
 	sm.AddMacro("PS_BLEND_MIX", sel.blend_mix);
+	sm.AddMacro("PS_FIXED_ONE_A", sel.fixed_one_a);
 	sm.AddMacro("PS_PABE", sel.pabe);
 	sm.AddMacro("PS_DITHER", sel.dither);
 	sm.AddMacro("PS_ZCLAMP", sel.zclamp);

pcsx2/GS/Renderers/HW/GSRendererHW.cpp

@@ -1277,6 +1277,45 @@ void GSRendererHW::Draw()
 	//  Test if we can optimize Alpha Test as a NOP
 	context->TEST.ATE = context->TEST.ATE && !GSRenderer::TryAlphaTest(fm, fm_mask, zm);
 
+	const bool force_a_one = !PRIM->ABE && PRIM->AA1 && (m_vt.m_primclass == GS_LINE_CLASS || m_vt.m_primclass == GS_TRIANGLE_CLASS);
+
+	// Need to fix the alpha test, since the alpha will be fixed to 1.0 if ABE is disabled and AA1 is enabled
+	// So if it doesn't meet the condition, always fail, if it does, always pass (turn off the test).
+	if (force_a_one && context->TEST.ATE && context->TEST.ATST > 1)
+	{
+		const float aref = (float)context->TEST.AREF;
+		const int old_ATST = context->TEST.ATST;
+		context->TEST.ATST = 0;
+
+		switch (old_ATST)
+		{
+			case ATST_LESS:
+				if (128.0f < aref)
+					context->TEST.ATE = false;
+				break;
+			case ATST_LEQUAL:
+				if (128.0f <= aref)
+					context->TEST.ATE = false;
+				break;
+			case ATST_EQUAL:
+				if (128.0f == aref)
+					context->TEST.ATE = false;
+				break;
+			case ATST_GEQUAL:
+				if (128.0f >= aref)
+					context->TEST.ATE = false;
+				break;
+			case ATST_GREATER:
+				if (128.0f > aref)
+					context->TEST.ATE = false;
+				break;
+			case ATST_NOTEQUAL:
+				if (128.0f != aref)
+					context->TEST.ATE = false;
+				break;
+		}
+	}
+
 	context->FRAME.FBMSK = fm;
 	context->ZBUF.ZMSK = zm != 0;
 
@@ -2336,7 +2375,7 @@ void GSRendererHW::EmulateBlending(bool& DATE_PRIMID, bool& DATE_BARRIER, bool&
 {
 	// AA1: Don't enable blending on AA1, not yet implemented on hardware mode,
 	// it requires coverage sample so it's safer to turn it off instead.
-	const bool AA1 = PRIM->AA1 && (m_vt.m_primclass == GS_LINE_CLASS);
+	const bool AA1 = PRIM->AA1 && (m_vt.m_primclass == GS_LINE_CLASS || m_vt.m_primclass == GS_TRIANGLE_CLASS);
 	// PABE: Check condition early as an optimization.
 	const bool PABE = PRIM->ABE && m_env.PABE.PABE && (GetAlphaMinMax().max < 128);
 	// FBMASK: Color is not written, no need to do blending.
@@ -2361,10 +2400,19 @@ void GSRendererHW::EmulateBlending(bool& DATE_PRIMID, bool& DATE_BARRIER, bool&
 	m_conf.ps.blend_c = ALPHA.C;
 	m_conf.ps.blend_d = ALPHA.D;
 
+	// When AA1 is enabled and Alpha Blending is disabled, alpha blending done with coverage instead of alpha.
+	// We use a COV value of 128 (full coverage) in triangles (except the edge geometry, which we can't do easily).
+	const bool force_a_one = !PRIM->ABE && AA1;
+	if (force_a_one)
+	{
+		m_conf.ps.fixed_one_a = 1;
+		m_conf.ps.blend_c = 0;
+	}
+
 	// Get alpha value
-	const bool alpha_c0_zero = (m_conf.ps.blend_c == 0 && GetAlphaMinMax().max == 0);
-	const bool alpha_c0_one = (m_conf.ps.blend_c == 0 && (GetAlphaMinMax().min == 128) && (GetAlphaMinMax().max == 128));
-	const bool alpha_c0_high_max_one = (m_conf.ps.blend_c == 0 && GetAlphaMinMax().max > 128);
+	const bool alpha_c0_zero = (m_conf.ps.blend_c == 0 && GetAlphaMinMax().max == 0) && !force_a_one;
+	const bool alpha_c0_one = (m_conf.ps.blend_c == 0 && (GetAlphaMinMax().min == 128) && (GetAlphaMinMax().max == 128)) || force_a_one;
+	const bool alpha_c0_high_max_one = (m_conf.ps.blend_c == 0 && GetAlphaMinMax().max > 128) && !force_a_one;
 	const bool alpha_c2_zero = (m_conf.ps.blend_c == 2 && ALPHA.FIX == 0u);
 	const bool alpha_c2_one = (m_conf.ps.blend_c == 2 && ALPHA.FIX == 128u);
 	const bool alpha_c2_high_one = (m_conf.ps.blend_c == 2 && ALPHA.FIX > 128u);

pcsx2/GS/Renderers/Metal/GSDeviceMTL.mm

@@ -1212,6 +1212,7 @@ void GSDeviceMTL::MRESetHWPipelineState(GSHWDrawConfig::VSSelector vssel, GSHWDr
 		setFnConstantB(m_fn_constants, pssel.hdr,                GSMTLConstantIndex_PS_HDR);
 		setFnConstantB(m_fn_constants, pssel.colclip,            GSMTLConstantIndex_PS_COLCLIP);
 		setFnConstantB(m_fn_constants, pssel.blend_mix,          GSMTLConstantIndex_PS_BLEND_MIX);
+		setFnConstantB(m_fn_constants, pssel.fixed_one_a,        GSMTLConstantIndex_PS_FIXED_ONE_A);
 		setFnConstantB(m_fn_constants, pssel.pabe,               GSMTLConstantIndex_PS_PABE);
 		setFnConstantB(m_fn_constants, pssel.no_color,           GSMTLConstantIndex_PS_NO_COLOR);
 		setFnConstantB(m_fn_constants, pssel.no_color1,          GSMTLConstantIndex_PS_NO_COLOR1);

pcsx2/GS/Renderers/Metal/GSMTLSharedHeader.h

@@ -144,6 +144,7 @@ enum GSMTLFnConstants
 	GSMTLConstantIndex_PS_HDR,
 	GSMTLConstantIndex_PS_COLCLIP,
 	GSMTLConstantIndex_PS_BLEND_MIX,
+	GSMTLConstantIndex_PS_FIXED_ONE_A,
 	GSMTLConstantIndex_PS_PABE,
 	GSMTLConstantIndex_PS_NO_COLOR,
 	GSMTLConstantIndex_PS_NO_COLOR1,

pcsx2/GS/Renderers/Metal/tfx.metal

@@ -49,6 +49,7 @@ constant uint PS_CLR_HW             [[function_constant(GSMTLConstantIndex_PS_CL
 constant bool PS_HDR                [[function_constant(GSMTLConstantIndex_PS_HDR)]];
 constant bool PS_COLCLIP            [[function_constant(GSMTLConstantIndex_PS_COLCLIP)]];
 constant bool PS_BLEND_MIX          [[function_constant(GSMTLConstantIndex_PS_BLEND_MIX)]];
+constant bool PS_FIXED_ONE_A        [[function_constant(GSMTLConstantIndex_PS_FIXED_ONE_A)]];
 constant bool PS_PABE               [[function_constant(GSMTLConstantIndex_PS_PABE)]];
 constant bool PS_NO_COLOR           [[function_constant(GSMTLConstantIndex_PS_NO_COLOR)]];
 constant bool PS_NO_COLOR1          [[function_constant(GSMTLConstantIndex_PS_NO_COLOR1)]];
@@ -811,6 +812,13 @@ struct PSMain
 		}
 
 		// Must be done before alpha correction
+
+		// AA (Fixed one) will output a coverage of 1.0 as alpha
+		if (PS_FIXED_ONE_A)
+		{
+			C.a = 128.0f;
+		}
+
 		float alpha_blend = SW_AD_TO_HW ? (PS_DFMT == FMT_24 ? 1.f : trunc(current_color.a * 255.5f) / 128.f) : (C.a / 128.f);
 
 		if (PS_DFMT == FMT_16)

pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp

@@ -1124,6 +1124,7 @@ std::string GSDeviceOGL::GetPSSource(const PSSelector& sel)
 		+ StringUtil::StdStringFromFormat("#define PS_DITHER %d\n", sel.dither)
 		+ StringUtil::StdStringFromFormat("#define PS_ZCLAMP %d\n", sel.zclamp)
 		+ StringUtil::StdStringFromFormat("#define PS_BLEND_MIX %d\n", sel.blend_mix)
+		+ StringUtil::StdStringFromFormat("#define PS_FIXED_ONE_A %d\n", sel.fixed_one_a)
 		+ StringUtil::StdStringFromFormat("#define PS_PABE %d\n", sel.pabe)
 		+ StringUtil::StdStringFromFormat("#define PS_SCANMSK %d\n", sel.scanmsk)
 		+ StringUtil::StdStringFromFormat("#define PS_SCALE_FACTOR %d\n", GSConfig.UpscaleMultiplier)

pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp

@@ -1978,6 +1978,7 @@ VkShaderModule GSDeviceVK::GetTFXFragmentShader(const GSHWDrawConfig::PSSelector
 	AddMacro(ss, "PS_BLEND_C", sel.blend_c);
 	AddMacro(ss, "PS_BLEND_D", sel.blend_d);
 	AddMacro(ss, "PS_BLEND_MIX", sel.blend_mix);
+	AddMacro(ss, "PS_FIXED_ONE_A", sel.fixed_one_a);
 	AddMacro(ss, "PS_IIP", sel.iip);
 	AddMacro(ss, "PS_SHUFFLE", sel.shuffle);
 	AddMacro(ss, "PS_READ_BA", sel.read_ba);