diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index d116d7ae57..25a7df769a 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -1742,9 +1742,12 @@ GSTextureCache::Source* GSTextureCache::LookupSource(const bool is_color, const
 			TEX0.TBP0, psm_s.pal > 0 ? TEX0.CBP : 0,
 			psm_str(TEX0.PSM));
 
-		// If it's from a target, we need to make sure the alpha information is up to date, especially in 16/24 bit formats where it can change draw to draw.
+		// If it's an old source made from target make sure it isn't a palette,
+		// alphas need to be used from the palette then.
+		// If it's from a target, we need to make sure the alpha information is up to date,
+		// especially in 16/24 bit formats where it can change draw to draw.
 		// Guard against merged targets which don't actually link.
-		if (src->m_target && src->m_from_target)
+		if (!src->m_palette && src->m_target && src->m_from_target)
 		{
 			src->m_valid_alpha_minmax = true;
 			if (src->m_target_direct)