diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 763f40743c..21fb7a9ef6 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -5506,9 +5506,15 @@ GSTextureCache::Source* GSTextureCache::CreateSource(const GIFRegTEX0& TEX0, con
 			src->m_shared_texture = true;
 			src->m_32_bits_fmt = dst->m_32_bits_fmt;
 
-			// if the size doesn't match, we need to engage shader sampling.
-			if (new_size != dst_texture_size)
+			// kill source immediately if it's the RT/DS, because that'll get invalidated immediately
+			if (GSRendererHW::GetInstance()->IsTBPFrameOrZ(dst->m_TEX0.TBP0) || channel_shuffle)
 			{
+				GL_CACHE("TC: Source is RT or ZBUF, invalidating after draw.");
+				m_temporary_source = src;
+			}
+			else if (new_size != dst_texture_size)
+			{
+				// if the size doesn't match, we need to engage shader sampling.
 				GL_CACHE("TC: Sample reduced region directly from target: %dx%d -> %dx%d", dst_texture_size.x,
 					dst_texture_size.y, new_size.x, new_size.y);
 
@@ -5517,13 +5523,6 @@ GSTextureCache::Source* GSTextureCache::CreateSource(const GIFRegTEX0& TEX0, con
 				if (new_size.y != dst_texture_size.y)
 					src->m_region.SetY(region_rect.y, region_rect.w);
 			}
-
-			// kill source immediately if it's the RT/DS, because that'll get invalidated immediately
-			if (GSRendererHW::GetInstance()->IsTBPFrameOrZ(dst->m_TEX0.TBP0) || channel_shuffle)
-			{
-				GL_CACHE("TC: Source is RT or ZBUF, invalidating after draw.");
-				m_temporary_source = src;
-			}
 		}
 		else
 		{