diff --git a/pcsx2/GS/GSState.cpp b/pcsx2/GS/GSState.cpp
index 0c596c92f8..204ccd09b6 100644
--- a/pcsx2/GS/GSState.cpp
+++ b/pcsx2/GS/GSState.cpp
@@ -112,9 +112,6 @@ GSState::GSState()
 
 	GrowVertexBuffer();
 
-	m_draw_transfers.clear();
-	m_draw_transfers_double_buff.clear();
-
 	PRIM = &m_env.PRIM;
 	//CSR->rREV = 0x20;
 	m_env.PRMODECONT.AC = 1;
diff --git a/pcsx2/GS/GSState.h b/pcsx2/GS/GSState.h
index 089beda03a..82566394df 100644
--- a/pcsx2/GS/GSState.h
+++ b/pcsx2/GS/GSState.h
@@ -235,7 +235,6 @@ public:
 	u32 m_dirty_gs_regs = 0;
 	int m_backed_up_ctx = 0;
 	std::vector<GSUploadQueue> m_draw_transfers;
-	std::vector<GSUploadQueue> m_draw_transfers_double_buff;
 
 	static int s_n;
 	static int s_last_transfer_draw_n;
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index eb70fffc55..f0de09e58b 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -122,16 +122,11 @@ void GSRendererHW::VSync(u32 field, bool registers_written, bool idle_frame)
 			for (auto iter = m_draw_transfers.rbegin(); iter != m_draw_transfers.rend(); iter++)
 			{
 				if ((s_n - iter->draw) > 5)
-					break;
-				else // Keep the last 5 draws worth of transfers.
 				{
-					GSUploadQueue transfer = *iter;
-					m_draw_transfers_double_buff.push_back(transfer);
+					m_draw_transfers.erase(m_draw_transfers.begin(), std::next(iter).base());
+					break;
 				}
 			}
-			m_draw_transfers.clear();
-			// Flip EE queue.
-			m_draw_transfers.swap(m_draw_transfers_double_buff);
 		}
 	}
 	else if (!idle_frame)
@@ -143,17 +138,16 @@ void GSRendererHW::VSync(u32 field, bool registers_written, bool idle_frame)
 			for (auto iter = m_draw_transfers.rbegin(); iter != m_draw_transfers.rend(); iter++)
 			{
 				if ((s_n - iter->draw) > 5)
-					break;
-				else // Keep the last 5 draws worth of transfers.
 				{
-					GSUploadQueue transfer = *iter;
-					m_draw_transfers_double_buff.push_back(transfer);
+					m_draw_transfers.erase(m_draw_transfers.begin(), std::next(iter).base());
+					break;
 				}
 			}
 		}
-		m_draw_transfers.clear();
-		// Flip EE queue.
-		m_draw_transfers.swap(m_draw_transfers_double_buff);
+		else
+		{
+			m_draw_transfers.clear();
+		}
 	}
 
 	if (GSConfig.LoadTextureReplacements)
@@ -1094,11 +1088,11 @@ void GSRendererHW::InvalidateLocalMem(const GIFRegBITBLTBUF& BITBLTBUF, const GS
 	if (clut)
 		return; // FIXME
 
-	std::vector<GSState::GSUploadQueue>::iterator iter = GSRendererHW::GetInstance()->m_draw_transfers.end();
+	auto iter = m_draw_transfers.end();
 	bool skip = false;
 	// If the EE write overlaps the readback and was done since the last draw, there's no need to read it back.
 	// Dog's life does this.
-	while (iter != GSRendererHW::GetInstance()->m_draw_transfers.begin())
+	while (iter != m_draw_transfers.begin())
 	{
 		--iter;
 
@@ -4880,8 +4874,7 @@ __ri void GSRendererHW::DrawPrims(GSTextureCache::Target* rt, GSTextureCache::Ta
 // If the EE uploaded a new CLUT since the last draw, use that.
 bool GSRendererHW::HasEEUpload(GSVector4i r)
 {
-	std::vector<GSState::GSUploadQueue>::iterator iter;
-	for (iter = m_draw_transfers.begin(); iter != m_draw_transfers.end();)
+	for (auto iter = m_draw_transfers.begin(); iter != m_draw_transfers.end(); ++iter)
 	{
 		if (iter->draw == (s_n - 1) && iter->blit.DBP == m_cached_ctx.TEX0.TBP0 && GSUtil::HasSharedBits(iter->blit.DPSM, m_cached_ctx.TEX0.PSM))
 		{
@@ -4904,8 +4897,6 @@ bool GSRendererHW::HasEEUpload(GSVector4i r)
 			if (rect.rintersect(r).eq(r))
 				return true;
 		}
-
-		iter++;
 	}
 	return false;
 }
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 29bcdec641..d3108a4c03 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -1548,7 +1548,7 @@ GSTextureCache::Target* GSTextureCache::CreateTarget(GIFRegTEX0 TEX0, const GSVe
 						else
 							continue;
 					}
-					iter++;
+					++iter;
 				}
 
 				if (!eerect.rempty())