diff --git a/pcsx2/GS/GSState.cpp b/pcsx2/GS/GSState.cpp
index ae93aae7fd..a776d83599 100644
--- a/pcsx2/GS/GSState.cpp
+++ b/pcsx2/GS/GSState.cpp
@@ -26,6 +26,7 @@
 #include <iomanip> // Dump Verticles
 
 int GSState::s_n = 0;
+int GSState::s_transfer_n = 0;
 
 static __fi bool IsAutoFlushEnabled()
 {
@@ -1520,6 +1521,7 @@ void GSState::FlushWrite()
 	m_tr.start += len;
 
 	g_perfmon.Put(GSPerfMon::Swizzle, len);
+	s_transfer_n++;
 }
 
 // This function decides if the context has changed in a way which warrants flushing the draw.
@@ -1788,6 +1790,7 @@ void GSState::Write(const u8* mem, int len)
 		m_tr.start = m_tr.end = m_tr.total;
 
 		g_perfmon.Put(GSPerfMon::Swizzle, len);
+		s_transfer_n++;
 	}
 	else
 	{
@@ -1854,6 +1857,7 @@ void GSState::Move()
 {
 	// ffxii uses this to move the top/bottom of the scrolling menus offscreen and then blends them back over the text to create a shading effect
 	// guitar hero copies the far end of the board to do a similar blend too
+	s_transfer_n++;
 
 	int sx = m_env.TRXPOS.SSAX;
 	int sy = m_env.TRXPOS.SSAY;
diff --git a/pcsx2/GS/GSState.h b/pcsx2/GS/GSState.h
index 8198380ce2..4ae9424d63 100644
--- a/pcsx2/GS/GSState.h
+++ b/pcsx2/GS/GSState.h
@@ -232,6 +232,7 @@ public:
 	std::vector<GSUploadQueue> m_draw_transfers;
 
 	static int s_n;
+	static int s_transfer_n;
 
 	static constexpr u32 STATE_VERSION = 8;
 
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index 8b0a301c2a..8092669e42 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -174,11 +174,22 @@ void GSRendererHW::VSync(u32 field, bool registers_written)
 	if (GSConfig.LoadTextureReplacements)
 		GSTextureReplacements::ProcessAsyncLoadedTextures();
 
-	m_tc->IncAge();
+	// Don't age the texture cache when no draws or EE writes have occurred.
+	// Xenosaga needs its targets kept around while it's loading, because it uses them for a fade transition.
+	if (m_last_draw_n == s_n && m_last_transfer_n == s_transfer_n)
+	{
+		GL_INS("No draws or transfers, not aging TC");
+	}
+	else
+	{
+		m_tc->IncAge();
+	}
+
+	m_last_draw_n = s_n + 1; // +1 for vsync
+	m_last_transfer_n = s_transfer_n;
 
 	GSRenderer::VSync(field, registers_written);
 
-
 	if (m_tc->GetHashCacheMemoryUsage() > 1024 * 1024 * 1024)
 	{
 		Host::AddKeyedFormattedOSDMessage("HashCacheOverflow", Host::OSD_ERROR_DURATION, "Hash cache has used %.2f MB of VRAM, disabling.",
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.h b/pcsx2/GS/Renderers/HW/GSRendererHW.h
index 0078a6fed0..c3e9a77b58 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.h
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.h
@@ -122,6 +122,10 @@ private:
 	std::unique_ptr<GSTextureCacheSW::Texture> m_sw_texture[7 + 1];
 	std::unique_ptr<GSVirtualAlignedClass<32>> m_sw_rasterizer;
 
+	// Tracking draw counters for idle frame detection.
+	int m_last_draw_n = 0;
+	int m_last_transfer_n = 0;
+
 public:
 	GSRendererHW();
 	virtual ~GSRendererHW() override;