From d29f843525284b3cab49978af8e6098062ce9d49 Mon Sep 17 00:00:00 2001
From: Connor McLaughlin <stenzek@gmail.com>
Date: Tue, 1 Feb 2022 19:33:07 +1000
Subject: [PATCH] GS/HW: Don't disable depth testing for channel shuffle

Mercenaries needs it. But we can skip it when z testing is off.
---
 pcsx2/GS/Renderers/HW/GSRendererNew.cpp   | 34 +++++++++++++----------
 pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp |  5 ++--
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/pcsx2/GS/Renderers/HW/GSRendererNew.cpp b/pcsx2/GS/Renderers/HW/GSRendererNew.cpp
index 1bf3c9618c..d8f0273be5 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererNew.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererNew.cpp
@@ -479,17 +479,23 @@ void GSRendererNew::EmulateChannelShuffle(const GSTextureCache::Source* tex)
 	if (m_channel_shuffle)
 	{
 		m_conf.tex = tex->m_from_target;
-		if (m_conf.tex == m_conf.rt)
+		if (m_conf.tex)
 		{
-			// sample from fb instead
-			m_conf.tex = nullptr;
-			m_conf.ps.tex_is_fb = true;
-			m_conf.require_one_barrier = true;
-		}
-		else if (m_conf.tex == m_conf.ds)
-		{
-			// using the current depth buffer. make sure it's not bound (needed for not-GL).
-			m_conf.ds = nullptr;
+			if (m_conf.tex == m_conf.rt)
+			{
+				// sample from fb instead
+				m_conf.tex = nullptr;
+				m_conf.ps.tex_is_fb = true;
+				m_conf.require_one_barrier = true;
+			}
+			else if (m_conf.tex == m_conf.ds)
+			{
+				// if depth testing is disabled, we don't need to copy, and can just unbind the depth buffer
+				// no need for a barrier for GL either, since it's not bound to depth and texture concurrently
+				// otherwise, the backend should recognise the hazard, and copy the buffer (D3D/Vulkan).
+				if (m_conf.depth.ztst == ZTST_ALWAYS)
+					m_conf.ds = nullptr;
+			}
 		}
 
 		// Replace current draw with a fullscreen sprite
@@ -1165,6 +1171,9 @@ void GSRendererNew::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 	ASSERT(g_gs_device != NULL);
 
+	// Z setup has to come before channel shuffle
+	EmulateZbuffer();
+
 	// HLE implementation of the channel selection effect
 	//
 	// Warning it must be done at the begining because it will change the
@@ -1315,11 +1324,6 @@ void GSRendererNew::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 	m_conf.datm = m_context->TEST.DATM;
 
-	// om
-
-	if (!m_channel_shuffle)
-		EmulateZbuffer(); // will update VS depth mask
-
 	// vs
 
 	m_conf.vs.tme = PRIM->TME;
diff --git a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
index 6d7f20c484..458094a9cc 100644
--- a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
+++ b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
@@ -2043,9 +2043,10 @@ void GSDeviceOGL::SendHWDraw(const GSHWDrawConfig& config)
 			DrawIndexedPrimitive(p, config.indices_per_prim);
 		}
 	}
-	else if (config.require_one_barrier)
+	else if (config.require_one_barrier || (config.tex && config.tex == config.ds))
 	{
-		// One barrier needed
+		// The common renderer code doesn't put a barrier here because D3D/VK need to copy the DS, so we need to check it.
+		// One barrier needed for non-overlapping draw.
 		glTextureBarrier();
 		DrawIndexedPrimitive();
 	}