diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index e02bc845a9..377b971377 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -1626,6 +1626,12 @@ void GSRendererHW::Draw()
 		}
 	}
 
+	if (m_src && m_src->m_shared_texture && m_src->m_texture != *m_src->m_from_target)
+	{
+		// Target texture changed, update reference.
+		m_src->m_texture = *m_src->m_from_target;
+	}
+
 	if (s_dump)
 	{
 		const u64 frame = g_perfmon.GetFrame();
@@ -2160,7 +2166,7 @@ void GSRendererHW::EmulateChannelShuffle(const GSTextureCache::Source* tex)
 			GL_INS("Gran Turismo RGB Channel");
 			m_conf.ps.channel = ChannelFetch_RGB;
 			m_context->TEX0.TFX = TFX_DECAL;
-			m_conf.rt = tex->m_from_target;
+			m_conf.rt = *tex->m_from_target;
 		}
 		else if (m_game.title == CRC::Tekken5)
 		{
@@ -2173,7 +2179,7 @@ void GSRendererHW::EmulateChannelShuffle(const GSTextureCache::Source* tex)
 				// 12 pages: 2 calls by channel, 3 channels, 1 blit
 				// Minus current draw call
 				m_skip = 12 * (3 + 3 + 1) - 1;
-				m_conf.rt = tex->m_from_target;
+				m_conf.rt = *tex->m_from_target;
 			}
 			else
 			{
@@ -2279,7 +2285,7 @@ void GSRendererHW::EmulateChannelShuffle(const GSTextureCache::Source* tex)
 	// Effect is really a channel shuffle effect so let's cheat a little
 	if (m_channel_shuffle)
 	{
-		m_conf.tex = tex->m_from_target;
+		m_conf.tex = *tex->m_from_target;
 		if (m_conf.tex)
 		{
 			if (m_conf.tex == m_conf.rt)
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index b450426996..b5e4538725 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -145,7 +145,7 @@ GSTextureCache::Source* GSTextureCache::LookupDepthSource(const GIFRegTEX0& TEX0
 		src->m_texture = dst->m_texture;
 		src->m_shared_texture = true;
 		src->m_target = true; // So renderer can check if a conversion is required
-		src->m_from_target = dst->m_texture; // avoid complex condition on the renderer
+		src->m_from_target = &dst->m_texture; // avoid complex condition on the renderer
 		src->m_from_target_TEX0 = dst->m_TEX0;
 		src->m_32_bits_fmt = dst->m_32_bits_fmt;
 		src->m_valid_rect = dst->m_valid;
@@ -1258,7 +1258,7 @@ GSTextureCache::Source* GSTextureCache::CreateSource(const GIFRegTEX0& TEX0, con
 		// Keep a trace of origin of the texture
 		src->m_texture = dTex;
 		src->m_target = true;
-		src->m_from_target = dst->m_texture;
+		src->m_from_target = &dst->m_texture;
 		src->m_from_target_TEX0 = dst->m_TEX0;
 		src->m_texture->SetScale(scale);
 		src->m_end_block = dst->m_end_block;
@@ -1287,7 +1287,7 @@ GSTextureCache::Source* GSTextureCache::CreateSource(const GIFRegTEX0& TEX0, con
 		// Keep a trace of origin of the texture
 		src->m_texture = dTex;
 		src->m_target = true;
-		src->m_from_target = dst->m_texture;
+		src->m_from_target = &dst->m_texture;
 		src->m_from_target_TEX0 = dst->m_TEX0;
 		src->m_end_block = dst->m_end_block;
 
@@ -1323,7 +1323,7 @@ GSTextureCache::Source* GSTextureCache::CreateSource(const GIFRegTEX0& TEX0, con
 
 		// Keep a trace of origin of the texture
 		src->m_target = true;
-		src->m_from_target = dst->m_texture;
+		src->m_from_target = &dst->m_texture;
 		src->m_from_target_TEX0 = dst->m_TEX0;
 		src->m_valid_rect = dst->m_valid;
 		src->m_end_block = dst->m_end_block;
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.h b/pcsx2/GS/Renderers/HW/GSTextureCache.h
index 16156191d5..5d671ed98e 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.h
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.h
@@ -164,7 +164,7 @@ public:
 		// Keep a trace of the target origin. There is no guarantee that pointer will
 		// still be valid on future. However it ought to be good when the source is created
 		// so it can be used to access un-converted data for the current draw call.
-		GSTexture* m_from_target;
+		GSTexture** m_from_target;
 		GIFRegTEX0 m_from_target_TEX0; // TEX0 of the target texture, if any, else equal to texture TEX0
 		GIFRegTEX0 m_layer_TEX0[7]; // Detect already loaded value
 		HashType m_layer_hash[7];