diff --git a/pcsx2/GS/GSClut.cpp b/pcsx2/GS/GSClut.cpp
index 72d9601c6c..53f7380765 100644
--- a/pcsx2/GS/GSClut.cpp
+++ b/pcsx2/GS/GSClut.cpp
@@ -155,8 +155,8 @@ void GSClut::Write(const GIFRegTEX0& TEX0, const GIFRegTEXCLUT& TEXCLUT)
 {
 	m_write.TEX0 = TEX0;
 	m_write.TEXCLUT = TEXCLUT;
-	m_write.dirty = false;
 	m_read.dirty = true;
+	m_write.dirty = false;
 
 	(this->*m_wc[TEX0.CSM][TEX0.CPSM][TEX0.PSM])(TEX0, TEXCLUT);
 }
diff --git a/pcsx2/GS/GSState.cpp b/pcsx2/GS/GSState.cpp
index f58efb6278..ed19e002a0 100644
--- a/pcsx2/GS/GSState.cpp
+++ b/pcsx2/GS/GSState.cpp
@@ -2546,8 +2546,13 @@ __forceinline void GSState::VertexKick(u32 skip)
 			__assume(0);
 	}
 
-	if (auto_flush && PRIM->TME && (m_context->FRAME.Block() == m_context->TEX0.TBP0))
-		FlushPrim();
+	if (m_context->FRAME.FBMSK != 0xFFFFFFFF)
+	{
+		m_mem.m_clut.Invalidate(m_context->FRAME.Block());
+
+		if (auto_flush && PRIM->TME && (m_context->FRAME.Block() == m_context->TEX0.TBP0))
+			FlushPrim();
+	}
 }
 
 void GSState::GetTextureMinMax(GSVector4i& r, const GIFRegTEX0& TEX0, const GIFRegCLAMP& CLAMP, bool linear)
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index 14e9075a7a..feee12db12 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -1009,7 +1009,6 @@ void GSRendererHW::SwSpriteRender()
 	}
 
 	m_tc->InvalidateVideoMem(dpo, m_r);
-	m_mem.m_clut.Invalidate();
 }
 
 bool GSRendererHW::CanUseSwSpriteRender()
@@ -2279,7 +2278,6 @@ bool GSRendererHW::OI_PointListPalette(GSTexture* rt, GSTexture* ds, GSTextureCa
 			m_mem.WritePixel32(x, y, c, FBP, FBW);
 		}
 		m_tc->InvalidateVideoMem(m_context->offset.fb, m_r);
-		m_mem.m_clut.Invalidate();
 		return false;
 	}
 	return true;
diff --git a/pcsx2/GS/Renderers/SW/GSRendererSW.cpp b/pcsx2/GS/Renderers/SW/GSRendererSW.cpp
index acf55e5c3d..e35812217a 100644
--- a/pcsx2/GS/Renderers/SW/GSRendererSW.cpp
+++ b/pcsx2/GS/Renderers/SW/GSRendererSW.cpp
@@ -578,8 +578,6 @@ void GSRendererSW::Queue(GSRingHeap::SharedPtr<GSRasterizerData>& item)
 	if (sd->global.sel.fwrite)
 	{
 		m_tc->InvalidatePages(sd->m_fb_pages, sd->m_fpsm);
-
-		m_mem.m_clut.Invalidate(m_context->FRAME.Block());
 	}
 
 	if (sd->global.sel.zwrite)