diff --git a/pcsx2/GS/GSUtil.cpp b/pcsx2/GS/GSUtil.cpp
index 1f12d21f58..0140ecf973 100644
--- a/pcsx2/GS/GSUtil.cpp
+++ b/pcsx2/GS/GSUtil.cpp
@@ -196,6 +196,16 @@ u32 GSUtil::GetChannelMask(u32 spsm)
 	}
 }
 
+u32 GSUtil::GetChannelMask(u32 spsm, u32 fbmsk)
+{
+	u32 mask = GetChannelMask(spsm);
+	mask &= (fbmsk & 0xFF) ? (~0x1 & 0xf) : 0xf;
+	mask &= (fbmsk & 0xFF00) ? (~0x2 & 0xf) : 0xf;
+	mask &= (fbmsk & 0xFF0000) ? (~0x4 & 0xf) : 0xf;
+	mask &= (fbmsk & 0xFF000000) ? (~0x8 & 0xf) : 0xf;
+	return mask;
+}
+
 GSRendererType GSUtil::GetPreferredRenderer()
 {
 #if defined(__APPLE__)
diff --git a/pcsx2/GS/GSUtil.h b/pcsx2/GS/GSUtil.h
index 2da5ea1573..35b0f77019 100644
--- a/pcsx2/GS/GSUtil.h
+++ b/pcsx2/GS/GSUtil.h
@@ -34,6 +34,7 @@ public:
 	static bool HasCompatibleBits(u32 spsm, u32 dpsm);
 	static bool HasSameSwizzleBits(u32 spsm, u32 dpsm);
 	static u32 GetChannelMask(u32 spsm);
+	static u32 GetChannelMask(u32 spsm, u32 fbmsk);
 
 	static GSRendererType GetPreferredRenderer();
 };
diff --git a/pcsx2/GS/Renderers/Common/GSDirtyRect.h b/pcsx2/GS/Renderers/Common/GSDirtyRect.h
index e087495439..e1be80ff6b 100644
--- a/pcsx2/GS/Renderers/Common/GSDirtyRect.h
+++ b/pcsx2/GS/Renderers/Common/GSDirtyRect.h
@@ -19,6 +19,7 @@
 
 union RGBAMask
 {
+	u32 _u32;
 	struct
 	{
 		u32 r : 1;
@@ -26,7 +27,6 @@ union RGBAMask
 		u32 b : 1;
 		u32 a : 1;
 	} c;
-	u32 _u32;
 };
 
 class GSDirtyRect
diff --git a/pcsx2/GS/Renderers/HW/GSHwHack.cpp b/pcsx2/GS/Renderers/HW/GSHwHack.cpp
index fd008f7886..eb4a66835c 100644
--- a/pcsx2/GS/Renderers/HW/GSHwHack.cpp
+++ b/pcsx2/GS/Renderers/HW/GSHwHack.cpp
@@ -971,12 +971,67 @@ bool GSHwHack::OI_Battlefield2(GSRendererHW& r, GSTexture* rt, GSTexture* ds, GS
 
 bool GSHwHack::OI_HauntingGround(GSRendererHW& r, GSTexture* rt, GSTexture* ds, GSTextureCache::Source* t)
 {
-	// Haunting Ground clears two targets by doing a 256x448 direct colour write at 0x3000, covering a target at 0x3380.
-	// This currently isn't handled in our HLE clears, so we need to manually remove the other target.
+	// Haunting Ground clears two targets by doing a direct colour write at 0x3000, covering a target at 0x3380.
+	// To make matters worse, it's masked. This currently isn't handled in our HLE clears, so we need to manually
+	// remove the other target.
 	if (rt && !ds && !t && r.IsConstantDirectWriteMemClear())
 	{
 		GL_CACHE("GSHwHack::OI_HauntingGround()");
-		g_texture_cache->InvalidateVideoMemTargets(GSTextureCache::RenderTarget, RFRAME.Block(), RFRAME.FBW, RFRAME.PSM, r.m_r);
+
+		const u32 bp = RFBP;
+		const u32 bw = RFBW;
+		const u32 psm = RFPSM;
+		const u32 fbmsk = RFBMSK;
+		const GSVector4i rc = r.m_r;
+
+		for (int type = 0; type < 2; type++)
+		{
+			auto& list = g_texture_cache->m_dst[type];
+
+			for (auto i = list.begin(); i != list.end();)
+			{
+				GSTextureCache::Target* t = *i;
+				auto ei = i++;
+
+				// There's two cases we hit here - when we clear 3380 via 3000, and when we overlap 3000 by writing to 3380.
+				// The latter is actually only 256x224, which ends at 337F, but because the game's a pain in the ass, it
+				// shuffles 512x512, causing the target to expand. It'd actually be shuffling junk and wasting draw cycles,
+				// but when did that stop anyone? So, we can get away with just saying "if it's before, ignore".
+				if (t->m_TEX0.TBP0 <= bp)
+				{
+					// don't remove ourself..
+					continue;
+				}
+
+				// Has to intersect.
+				if (!t->Overlaps(bp, bw, psm, rc))
+					continue;
+
+				// Another annoying case. Sometimes it clears with RGB masked, only writing to A. We don't want to kill the
+				// target in this case, so we'll dirty A instead.
+				if (fbmsk != 0)
+				{
+					GL_CACHE("OI_HauntingGround(%x, %u, %s, %d,%d => %d,%d): Dirty target at %x %u %s %08X", bp, bw,
+						psm_str(psm), rc.x, rc.y, rc.z, rc.w, t->m_TEX0.TBP0, t->m_TEX0.TBW, psm_str(t->m_TEX0.PSM),
+						fbmsk);
+
+					g_texture_cache->AddDirtyRectTarget(t, rc, psm, bw, RGBAMask{GSUtil::GetChannelMask(psm, fbmsk)});
+				}
+				else
+				{
+					GL_CACHE("OI_HauntingGround(%x, %u, %s, %d,%d => %d,%d): Removing target at %x %u %s", bp, bw,
+						psm_str(psm), rc.x, rc.y, rc.z, rc.w, t->m_TEX0.TBP0, t->m_TEX0.TBW, psm_str(t->m_TEX0.PSM));
+
+					// Need to also remove any sources which reference this target.
+					g_texture_cache->InvalidateSourcesFromTarget(t);
+
+					list.erase(ei);
+					delete t;
+				}
+			}
+		}
+
+		g_texture_cache->InvalidateVideoMemType(GSTextureCache::DepthStencil, bp);
 	}
 
 	// Not skipping anything. This is just an invalidation hack.
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 066542d80a..0788f20363 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -2924,29 +2924,6 @@ void GSTextureCache::InvalidateVideoMemSubTarget(GSTextureCache::Target* rt)
 	}
 }
 
-void GSTextureCache::InvalidateVideoMemTargets(int type, u32 bp, u32 bw, u32 psm, const GSVector4i& r)
-{
-	auto& list = m_dst[type];
-
-	for (auto i = list.begin(); i != list.end();)
-	{
-		GSTextureCache::Target* t = *i;
-		auto ei = i++;
-
-		if (t->m_TEX0.TBP0 != bp && t->Overlaps(bp, bw, psm, r))
-		{
-			GL_CACHE("InvalidateVideoMemTargets(%x, %u, %s, %d,%d => %d,%d): Removing target at %x %u %s", bp, bw,
-				psm_str(psm), r.x, r.y, r.z, r.w, t->m_TEX0.TBP0, t->m_TEX0.TBW, psm_str(t->m_TEX0.PSM));
-
-			// Need to also remove any sources which reference this target.
-			InvalidateSourcesFromTarget(t);
-
-			list.erase(ei);
-			delete t;
-		}
-	}
-}
-
 void GSTextureCache::InvalidateSourcesFromTarget(const Target* t)
 {
 	for (auto it = m_src.m_surfaces.begin(); it != m_src.m_surfaces.end();)
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.h b/pcsx2/GS/Renderers/HW/GSTextureCache.h
index 5e8e9106cf..0fcfcf936f 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.h
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.h
@@ -22,6 +22,8 @@
 #include "GS/Renderers/Common/GSDirtyRect.h"
 #include <unordered_set>
 
+class GSHwHack;
+
 // Only for debugging. Reads back every target to local memory after drawing, effectively
 // disabling caching between draws.
 //#define DISABLE_HW_TEXTURE_CACHE
@@ -29,6 +31,8 @@
 class GSTextureCache
 {
 public:
+	friend GSHwHack;
+
 	enum
 	{
 		RenderTarget,
@@ -467,9 +471,6 @@ public:
 	void InvalidateVideoMem(const GSOffset& off, const GSVector4i& r, bool eewrite = false, bool target = true);
 	void InvalidateLocalMem(const GSOffset& off, const GSVector4i& r, bool full_flush = false);
 
-	/// Removes any targets overlapping the specified BP and rectangle.
-	void InvalidateVideoMemTargets(int type, u32 bp, u32 bw, u32 psm, const GSVector4i& r);
-
 	/// Removes any sources which point to the specified target.
 	void InvalidateSourcesFromTarget(const Target* t);