GS: Fix edge case of broken alpha with no-DSB+ATST

pcsx2/GS/Renderers/Common/GSDevice.cpp

@@ -477,6 +477,33 @@ bool GSDevice::ResizeTarget(GSTexture** t)
 	return ResizeTexture(t, GSTexture::Type::RenderTarget, s.x, s.y);
 }
 
+void GSDevice::SetHWDrawConfigForAlphaPass(GSHWDrawConfig::PSSelector* ps,
+	GSHWDrawConfig::ColorMaskSelector* cms,
+	GSHWDrawConfig::BlendState* bs,
+	GSHWDrawConfig::DepthStencilSelector* dss)
+{
+	// only need to compute the alpha component (allow the shader to optimize better)
+	ps->no_ablend = false;
+	ps->only_alpha = true;
+
+	// definitely don't need to compute software blend (this may get rid of some barriers)
+	ps->blend_a = ps->blend_b = ps->blend_c = ps->blend_d = 0;
+
+	// only write alpha (RGB=0,A=1)
+	cms->wrgba = (1 << 3);
+
+	// no need for hardware blending, since we're not writing RGB
+	bs->enable = false;
+
+	// if depth writes are on, we can optimize to an EQUAL test, otherwise we leave the tests alone
+	// since the alpha channel isn't blended, the last fragment wins and this'll be okay
+	if (dss->zwe)
+	{
+		dss->zwe = false;
+		dss->ztst = ZTST_GEQUAL;
+	}
+}
+
 GSAdapter::operator std::string() const
 {
 	char buf[sizeof "12345678:12345678:12345678:12345678"];

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -557,6 +557,8 @@ struct alignas(16) GSHWDrawConfig
 	DestinationAlphaMode destination_alpha;
 	bool datm : 1;
 	bool line_expand : 1;
+	bool separate_alpha_pass : 1;
+	bool second_separate_alpha_pass : 1;
 
 	struct AlphaPass
 	{
@@ -569,7 +571,6 @@ struct alignas(16) GSHWDrawConfig
 	static_assert(sizeof(AlphaPass) == 24, "alpha pass is 24 bytes");
 
 	AlphaPass alpha_second_pass;
-	AlphaPass alpha_third_pass;
 
 	VSConstantBuffer cb_vs;
 	PSConstantBuffer cb_ps;
@@ -775,6 +776,12 @@ public:
 		return (IsDualSourceBlendFactor(m_blendMap[index].src) ||
 				IsDualSourceBlendFactor(m_blendMap[index].dst));
 	}
+
+	/// Alters the pipeline configuration for drawing the separate alpha pass.
+	static void SetHWDrawConfigForAlphaPass(GSHWDrawConfig::PSSelector* ps,
+		GSHWDrawConfig::ColorMaskSelector* cms,
+		GSHWDrawConfig::BlendState* bs,
+		GSHWDrawConfig::DepthStencilSelector* dss);
 };
 
 struct GSAdapter

pcsx2/GS/Renderers/DX11/GSDevice11.cpp

@@ -1348,6 +1348,16 @@ void GSDevice11::RenderHW(GSHWDrawConfig& config)
 
 	DrawIndexedPrimitive();
 
+	if (config.separate_alpha_pass)
+	{
+		GSHWDrawConfig::BlendState sap_blend = {};
+		SetHWDrawConfigForAlphaPass(&config.ps, &config.colormask, &sap_blend, &config.depth);
+		SetupOM(config.depth, convertSel(config.colormask, sap_blend), config.blend.constant);
+		SetupPS(config.ps, &config.cb_ps, config.sampler);
+
+		DrawIndexedPrimitive();
+	}
+
 	if (config.alpha_second_pass.enable)
 	{
 		preprocessSel(config.alpha_second_pass.ps);
@@ -1365,6 +1375,16 @@ void GSDevice11::RenderHW(GSHWDrawConfig& config)
 		SetupOM(config.alpha_second_pass.depth, convertSel(config.alpha_second_pass.colormask, config.blend), config.blend.constant);
 
 		DrawIndexedPrimitive();
+
+		if (config.second_separate_alpha_pass)
+		{
+			GSHWDrawConfig::BlendState sap_blend = {};
+			SetHWDrawConfigForAlphaPass(&config.alpha_second_pass.ps, &config.alpha_second_pass.colormask, &sap_blend, &config.alpha_second_pass.depth);
+			SetupOM(config.alpha_second_pass.depth, convertSel(config.alpha_second_pass.colormask, sap_blend), config.blend.constant);
+			SetupPS(config.alpha_second_pass.ps, &config.cb_ps, config.sampler);
+
+			DrawIndexedPrimitive();
+		}
 	}
 
 	EndScene();

pcsx2/GS/Renderers/HW/GSRendererNew.cpp

@@ -1821,41 +1821,30 @@ void GSRendererNew::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 	if (blending_alpha_pass)
 	{
-		// ensure alpha blend output is disabled on both passes
+		// write alpha blend as the single alpha output
 		m_conf.ps.no_ablend = true;
 
-		// if we're doing RGBA then Z alpha testing, we can't combine Z and A, and we need a third pass :(
+		// there's a case we can skip this: RGB_then_ZA alternate handling.
-		if (ate_RGBA_then_Z)
+		// but otherwise, we need to write alpha separately.
+		if (m_conf.colormask.wa)
 		{
-			// move second pass to third pass, since we want to write A first
-			std::memcpy(&m_conf.alpha_third_pass, &m_conf.alpha_second_pass, sizeof(m_conf.alpha_third_pass));
-			m_conf.alpha_third_pass.ps.no_ablend = true;
-			m_conf.alpha_second_pass.enable = false;
-		}
-
-		if (!m_conf.alpha_second_pass.enable)
-		{
-			m_conf.alpha_second_pass.enable = true;
-			memcpy(&m_conf.alpha_second_pass.ps, &m_conf.ps, sizeof(m_conf.ps));
-			memcpy(&m_conf.alpha_second_pass.colormask, &m_conf.colormask, sizeof(m_conf.colormask));
-			memcpy(&m_conf.alpha_second_pass.depth, &m_conf.depth, sizeof(m_conf.depth));
-
-			// disable alpha writes on first pass
 			m_conf.colormask.wa = false;
+			m_conf.separate_alpha_pass = true;
 		}
 
-		// only need to compute the alpha component (allow the shader to optimize better)
+		// do we need to do this for the failed alpha fragments?
-		m_conf.alpha_second_pass.ps.no_ablend = false;
+		if (m_conf.alpha_second_pass.enable)
-		m_conf.alpha_second_pass.ps.only_alpha = true;
-		m_conf.alpha_second_pass.colormask.wr = m_conf.alpha_second_pass.colormask.wg = m_conf.alpha_second_pass.colormask.wb = false;
-		m_conf.alpha_second_pass.colormask.wa = true;
-
-		// if depth writes are on, we can optimize to an EQUAL test, otherwise we leave the tests alone
-		// since the alpha channel isn't blended, the last fragment wins and this'll be okay
-		if (m_conf.alpha_second_pass.depth.zwe)
 		{
-			m_conf.alpha_second_pass.depth.zwe = false;
+			// there's also a case we can skip here: when we're not writing RGB, there's
-			m_conf.alpha_second_pass.depth.ztst = ZTST_GEQUAL;
+			// no blending, so we can just write the normal alpha!
+			const u8 second_pass_wrgba = m_conf.alpha_second_pass.colormask.wrgba;
+			if ((second_pass_wrgba & (1 << 3)) != 0 && second_pass_wrgba != (1 << 3))
+			{
+				// this sucks. potentially up to 4 passes. but no way around it when we don't have dual-source blend.
+				m_conf.alpha_second_pass.ps.no_ablend = true;
+				m_conf.alpha_second_pass.colormask.wa = false;
+				m_conf.second_separate_alpha_pass = true;
+			}
 		}
 	}
 

pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp

@@ -1961,6 +1961,25 @@ void GSDeviceOGL::RenderHW(GSHWDrawConfig& config)
 
 	SendHWDraw(config);
 
+	if (config.separate_alpha_pass)
+	{
+		GSHWDrawConfig::BlendState dummy_bs;
+		SetHWDrawConfigForAlphaPass(&psel.ps, &config.colormask, &dummy_bs, &config.depth);
+		SetupPipeline(psel);
+		OMSetColorMaskState(config.alpha_second_pass.colormask);
+		SetupOM(config.alpha_second_pass.depth);
+		OMSetBlendState();
+		SendHWDraw(config);
+
+		// restore blend state if we're doing a second pass
+		if (config.alpha_second_pass.enable)
+		{
+			OMSetBlendState(config.blend.enable, s_gl_blend_factors[config.blend.src_factor],
+				s_gl_blend_factors[config.blend.dst_factor], s_gl_blend_ops[config.blend.op],
+				config.blend.constant_enable, config.blend.constant);
+		}
+	}
+
 	if (config.alpha_second_pass.enable)
 	{
 		// cbuffer will definitely be dirty if aref changes, no need to check it
@@ -1975,25 +1994,18 @@ void GSDeviceOGL::RenderHW(GSHWDrawConfig& config)
 		SetupPipeline(psel);
 		OMSetColorMaskState(config.alpha_second_pass.colormask);
 		SetupOM(config.alpha_second_pass.depth);
-
 		SendHWDraw(config);
-	}
+
-	if (config.alpha_third_pass.enable)
+		if (config.second_separate_alpha_pass)
-	{
-		// cbuffer will definitely be dirty if aref changes, no need to check it
-		if (config.cb_ps.FogColor_AREF.a != config.alpha_third_pass.ps_aref)
 		{
-			config.cb_ps.FogColor_AREF.a = config.alpha_third_pass.ps_aref;
+			GSHWDrawConfig::BlendState dummy_bs;
-			WriteToStreamBuffer(m_fragment_uniform_stream_buffer.get(), g_ps_cb_index,
+			SetHWDrawConfigForAlphaPass(&psel.ps, &config.colormask, &dummy_bs, &config.depth);
-				m_uniform_buffer_alignment, &config.cb_ps, sizeof(config.cb_ps));
+			SetupPipeline(psel);
+			OMSetColorMaskState(config.alpha_second_pass.colormask);
+			SetupOM(config.alpha_second_pass.depth);
+			OMSetBlendState();
+			SendHWDraw(config);
 		}
-
-		psel.ps = config.alpha_third_pass.ps;
-		SetupPipeline(psel);
-		OMSetColorMaskState(config.alpha_third_pass.colormask);
-		SetupOM(config.alpha_third_pass.depth);
-
-		SendHWDraw(config);
 	}
 
 	if (config.destination_alpha == GSHWDrawConfig::DestinationAlphaMode::PrimIDTracking)

pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp

@@ -3026,7 +3026,15 @@ void GSDeviceVK::RenderHW(GSHWDrawConfig& config)
 
 	// now we can do the actual draw
 	if (BindDrawPipeline(pipe))
+	{
 		SendHWDraw(config, draw_rt);
+		if (config.separate_alpha_pass)
+		{
+			SetHWDrawConfigForAlphaPass(&pipe.ps, &pipe.cms, &pipe.bs, &pipe.dss);
+			if (BindDrawPipeline(pipe))
+				SendHWDraw(config, draw_rt);
+		}
+	}
 
 	// and the alpha pass
 	if (config.alpha_second_pass.enable)
@@ -3041,23 +3049,17 @@ void GSDeviceVK::RenderHW(GSHWDrawConfig& config)
 		pipe.ps = config.alpha_second_pass.ps;
 		pipe.cms = config.alpha_second_pass.colormask;
 		pipe.dss = config.alpha_second_pass.depth;
+		pipe.bs = config.blend;
 		if (BindDrawPipeline(pipe))
-			SendHWDraw(config, draw_rt);
-	}
-	if (config.alpha_third_pass.enable)
-	{
-		// cbuffer will definitely be dirty if aref changes, no need to check it
-		if (config.cb_ps.FogColor_AREF.a != config.alpha_third_pass.ps_aref)
 		{
-			config.cb_ps.FogColor_AREF.a = config.alpha_third_pass.ps_aref;
-			SetPSConstantBuffer(config.cb_ps);
-		}
-
-		pipe.ps = config.alpha_third_pass.ps;
-		pipe.cms = config.alpha_third_pass.colormask;
-		pipe.dss = config.alpha_third_pass.depth;
-		if (BindDrawPipeline(pipe))
 			SendHWDraw(config, draw_rt);
+			if (config.second_separate_alpha_pass)
+			{
+				SetHWDrawConfigForAlphaPass(&pipe.ps, &pipe.cms, &pipe.bs, &pipe.dss);
+				if (BindDrawPipeline(pipe))
+					SendHWDraw(config, draw_rt);
+			}
+		}
 	}
 
 	if (copy_ds)