diff --git a/pcsx2/GS/GSState.cpp b/pcsx2/GS/GSState.cpp
index 03c3fd3225..cc20d08daf 100644
--- a/pcsx2/GS/GSState.cpp
+++ b/pcsx2/GS/GSState.cpp
@@ -2001,8 +2001,11 @@ void GSState::Write(const u8* mem, int len)
 	if (GSConfig.PreloadFrameWithGSData)
 	{
 		// Store the transfer for preloading new RT's.
-		if(m_draw_transfers.size() == 0 || (m_draw_transfers.size() > 0 && blit.DBP != m_draw_transfers.front().DBP))
-			m_draw_transfers.push_front(blit);
+		if (m_draw_transfers.size() == 0 || (m_draw_transfers.size() > 0 && blit.DBP != m_draw_transfers.back().blit.DBP))
+		{
+			GSUploadQueue new_transfer = { blit, s_n };
+			m_draw_transfers.push_back(new_transfer);
+		}
 	}
 
 	GL_CACHE("Write! ...  => 0x%x W:%d F:%s (DIR %d%d), dPos(%d %d) size(%d %d)",
diff --git a/pcsx2/GS/GSState.h b/pcsx2/GS/GSState.h
index bfed894e4a..26150218da 100644
--- a/pcsx2/GS/GSState.h
+++ b/pcsx2/GS/GSState.h
@@ -206,6 +206,12 @@ protected:
 	bool IsCoverageAlpha();
 
 public:
+	struct GSUploadQueue
+	{
+		GIFRegBITBLTBUF blit;
+		int draw;
+	};
+
 	GIFPath m_path[4];
 	GIFRegPRIM* PRIM;
 	GSPrivRegSet* m_regs;
@@ -222,7 +228,7 @@ public:
 	bool m_mipmap;
 	u32 m_dirty_gs_regs;
 	int m_backed_up_ctx;
-	std::list<GIFRegBITBLTBUF> m_draw_transfers;
+	std::vector<GSUploadQueue> m_draw_transfers;
 	bool m_force_preload;
 
 	static int s_n;
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index 299ccb642e..08ab890981 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -207,9 +207,16 @@ void GSRendererHW::VSync(u32 field, bool registers_written)
 		m_force_preload = true;
 	}
 	else
+	{
 		m_force_preload = false;
-
-	m_draw_transfers.clear();
+		std::vector<GSState::GSUploadQueue>::iterator iter;
+		for (iter = m_draw_transfers.begin(); iter != m_draw_transfers.end(); ) {
+			if ((s_n - iter->draw) > 5)
+				iter = m_draw_transfers.erase(iter);
+			else
+				iter++;
+		}
+	}
 
 	if (GSConfig.LoadTextureReplacements)
 		GSTextureReplacements::ProcessAsyncLoadedTextures();
@@ -4003,10 +4010,12 @@ GSRendererHW::CLUTDrawTestResult GSRendererHW::PossibleCLUTDraw()
 		// If we have GPU CLUT enabled, don't do a CPU draw when it would result in a download.
 		if (GSConfig.UserHacks_GPUTargetCLUTMode != GSGPUTargetCLUTMode::Disabled)
 		{
-			for (auto upload : m_draw_transfers)
-			{
-				if (upload.DBP == m_context->TEX0.TBP0 && GSUtil::HasSharedBits(upload.DPSM, m_context->TEX0.PSM))
+			std::vector<GSState::GSUploadQueue>::iterator iter;
+			for (iter = m_draw_transfers.begin(); iter != m_draw_transfers.end(); ) {
+				if (iter->blit.DBP == m_context->TEX0.TBP0 && GSUtil::HasSharedBits(iter->blit.DPSM, m_context->TEX0.PSM))
 					return CLUTDrawTestResult::CLUTDrawOnCPU;
+
+				iter++;
 			}
 
 			GSTextureCache::Target* tgt = m_tc->GetExactTarget(m_context->TEX0.TBP0, m_context->TEX0.TBW, m_context->TEX0.PSM);
@@ -4030,10 +4039,12 @@ GSRendererHW::CLUTDrawTestResult GSRendererHW::PossibleCLUTDraw()
 		}
 		else
 		{
-			for (auto upload : m_draw_transfers)
-			{
-				if (upload.DBP == m_context->TEX0.TBP0 && GSUtil::HasSharedBits(upload.DPSM, m_context->TEX0.PSM))
+			std::vector<GSState::GSUploadQueue>::iterator iter;
+			for (iter = m_draw_transfers.begin(); iter != m_draw_transfers.end(); ) {
+				if (iter->blit.DBP == m_context->TEX0.TBP0 && GSUtil::HasSharedBits(iter->blit.DPSM, m_context->TEX0.PSM))
 					return CLUTDrawTestResult::CLUTDrawOnCPU;
+
+				iter++;
 			}
 		}
 
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 84ba7b86aa..05a58ed98c 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -706,16 +706,18 @@ GSTextureCache::Target* GSTextureCache::LookupTarget(const GIFRegTEX0& TEX0, con
 		{
 			if (!is_frame && !forced_preload)
 			{
-				for (auto transfer : static_cast<GSRendererHW*>(g_gs_renderer.get())->m_draw_transfers)
-				{
-					if (transfer.DBP == TEX0.TBP0 && GSUtil::HasSharedBits(transfer.DPSM, TEX0.PSM))
+				std::vector<GSState::GSUploadQueue>::iterator iter;
+				for (iter = static_cast<GSRendererHW*>(g_gs_renderer.get())->m_draw_transfers.begin(); iter != static_cast<GSRendererHW*>(g_gs_renderer.get())->m_draw_transfers.end(); ) {
+					if (iter->blit.DBP == TEX0.TBP0 && GSUtil::HasSharedBits(iter->blit.DPSM, TEX0.PSM))
 					{
+						iter = static_cast<GSRendererHW*>(g_gs_renderer.get())->m_draw_transfers.erase(iter);
 						GL_INS("Preloading the RT DATA");
 						const GSVector4i newrect = GSVector4i(0, 0, real_w, real_h);
 						AddDirtyRectTarget(dst, newrect, TEX0.PSM, TEX0.TBW);
 						dst->Update(true);
 						break;
 					}
+					iter++;
 				}
 			}
 			else
diff --git a/pcsx2/GS/Renderers/SW/GSRendererSW.cpp b/pcsx2/GS/Renderers/SW/GSRendererSW.cpp
index 3fdf961409..b4b48582ae 100644
--- a/pcsx2/GS/Renderers/SW/GSRendererSW.cpp
+++ b/pcsx2/GS/Renderers/SW/GSRendererSW.cpp
@@ -119,6 +119,7 @@ void GSRendererSW::VSync(u32 field, bool registers_written)
 
 	m_tc->IncAge();
 
+	m_draw_transfers.clear();
 	// if((m_perfmon.GetFrame() & 255) == 0) m_rl->PrintStats();
 }