From a0e323dae5914727ca2bab42359e85797fc42441 Mon Sep 17 00:00:00 2001
From: refractionpcsx2 <refraction@gmail.com>
Date: Tue, 23 Apr 2024 11:14:32 +0100
Subject: [PATCH] GS: Improve Autoflush detection with channel masks

---
 pcsx2/GS/GSState.cpp | 7 ++++++-
 pcsx2/GS/GSUtil.cpp  | 8 ++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/pcsx2/GS/GSState.cpp b/pcsx2/GS/GSState.cpp
index 8df2202fb2..041a5f1c74 100644
--- a/pcsx2/GS/GSState.cpp
+++ b/pcsx2/GS/GSState.cpp
@@ -3090,6 +3090,10 @@ __forceinline bool GSState::IsAutoFlushDraw(u32 prim)
 	if (!PRIM->TME || (GSConfig.UserHacks_AutoFlush == GSHWAutoFlushLevel::SpritesOnly && prim != GS_SPRITE))
 		return false;
 
+	// Not using the same channels.
+	if (!(GSUtil::GetChannelMask(m_context->TEX0.PSM) & GSUtil::GetChannelMask(m_context->FRAME.PSM, m_context->FRAME.FBMSK | ~(GSLocalMemory::m_psm[m_context->FRAME.PSM].fmsk))))
+		return false;
+
 	const u32 frame_mask = GSLocalMemory::m_psm[m_context->FRAME.PSM].fmsk;
 	const bool frame_hit = m_context->FRAME.Block() == m_context->TEX0.TBP0 && !(m_context->TEST.ATE && m_context->TEST.ATST == 0 && m_context->TEST.AFAIL == 2) && ((m_context->FRAME.FBMSK & frame_mask) != frame_mask);
 	// There's a strange behaviour we need to test on a PS2 here, if the FRAME is a Z format, like Powerdrome something swaps over, and it seems Alpha Fail of "FB Only" writes to the Z.. it's odd.
@@ -3272,8 +3276,9 @@ __forceinline void GSState::HandleAutoFlush()
 		if (tex_rect.y == tex_rect.w)
 			tex_rect += GSVector4i::cxpr(0, 0, 0, 1);
 
+		const bool swap_index = index_swap && GSUtil::GetPrimClass(m_prev_env.PRIM.PRIM) != GS_SPRITE_CLASS;
 		// Get the last texture position from the last draw.
-		const GSVertex* v = &m_vertex.buff[m_index.buff[m_index.tail - (index_swap ? n : 1)]];
+		const GSVertex* v = &m_vertex.buff[m_index.buff[m_index.tail - (swap_index ? n : 1)]];
 
 		if (PRIM->FST)
 		{
diff --git a/pcsx2/GS/GSUtil.cpp b/pcsx2/GS/GSUtil.cpp
index a6ae155564..a2a74be468 100644
--- a/pcsx2/GS/GSUtil.cpp
+++ b/pcsx2/GS/GSUtil.cpp
@@ -191,10 +191,10 @@ u32 GSUtil::GetChannelMask(u32 spsm)
 u32 GSUtil::GetChannelMask(u32 spsm, u32 fbmsk)
 {
 	u32 mask = GetChannelMask(spsm);
-	mask &= (fbmsk & 0xFF) ? (~0x1 & 0xf) : 0xf;
-	mask &= (fbmsk & 0xFF00) ? (~0x2 & 0xf) : 0xf;
-	mask &= (fbmsk & 0xFF0000) ? (~0x4 & 0xf) : 0xf;
-	mask &= (fbmsk & 0xFF000000) ? (~0x8 & 0xf) : 0xf;
+	mask &= ((fbmsk & 0xFF) == 0xFF) ? (~0x1 & 0xf) : 0xf;
+	mask &= ((fbmsk & 0xFF00) == 0xFF00) ? (~0x2 & 0xf) : 0xf;
+	mask &= ((fbmsk & 0xFF0000) == 0xFF0000) ? (~0x4 & 0xf) : 0xf;
+	mask &= ((fbmsk & 0xFF000000) == 0xFF000000) ? (~0x8 & 0xf) : 0xf;
 	return mask;
 }