diff --git a/pcsx2/GS/Renderers/HW/GSHwHack.cpp b/pcsx2/GS/Renderers/HW/GSHwHack.cpp
index e4254d2302..f6fa33b13b 100644
--- a/pcsx2/GS/Renderers/HW/GSHwHack.cpp
+++ b/pcsx2/GS/Renderers/HW/GSHwHack.cpp
@@ -996,7 +996,7 @@ bool GSHwHack::OI_RozenMaidenGebetGarden(GSRendererHW& r, GSTexture* rt, GSTextu
 			TEX0.TBW = RCONTEXT->FRAME.FBW;
 			TEX0.PSM = RCONTEXT->FRAME.PSM;
 
-			if (GSTextureCache::Target* tmp_rt = g_texture_cache->LookupTarget(TEX0, r.GetTargetSize(), r.GetTextureScaleFactor(), GSTextureCache::RenderTarget, true))
+			if (GSTextureCache::Target* tmp_rt = g_texture_cache->LookupTarget(TEX0, r.GetTargetSize(), r.GetTextureScaleFactor(), GSTextureCache::RenderTarget))
 			{
 				GL_INS("OI_RozenMaidenGebetGarden FB clear");
 				g_gs_device->ClearRenderTarget(tmp_rt->m_texture, 0);
@@ -1014,7 +1014,7 @@ bool GSHwHack::OI_RozenMaidenGebetGarden(GSRendererHW& r, GSTexture* rt, GSTextu
 			TEX0.TBW = RCONTEXT->FRAME.FBW;
 			TEX0.PSM = RCONTEXT->ZBUF.PSM;
 
-			if (GSTextureCache::Target* tmp_ds = g_texture_cache->LookupTarget(TEX0, r.GetTargetSize(), r.GetTextureScaleFactor(), GSTextureCache::DepthStencil, true))
+			if (GSTextureCache::Target* tmp_ds = g_texture_cache->LookupTarget(TEX0, r.GetTargetSize(), r.GetTextureScaleFactor(), GSTextureCache::DepthStencil))
 			{
 				GL_INS("OI_RozenMaidenGebetGarden ZB clear");
 				g_gs_device->ClearDepth(tmp_ds->m_texture);
@@ -1047,7 +1047,7 @@ bool GSHwHack::OI_SonicUnleashed(GSRendererHW& r, GSTexture* rt, GSTexture* ds,
 
 	GL_INS("OI_SonicUnleashed replace draw by a copy");
 
-	GSTextureCache::Target* src = g_texture_cache->LookupTarget(Texture, GSVector2i(1, 1), r.GetTextureScaleFactor(), GSTextureCache::RenderTarget, true);
+	GSTextureCache::Target* src = g_texture_cache->LookupTarget(Texture, GSVector2i(1, 1), r.GetTextureScaleFactor(), GSTextureCache::RenderTarget);
 
 	const GSVector2i src_size(src->m_texture->GetSize());
 	GSVector2i rt_size(rt->GetSize());
@@ -1055,7 +1055,7 @@ bool GSHwHack::OI_SonicUnleashed(GSRendererHW& r, GSTexture* rt, GSTexture* ds,
 	// This is awful, but so is the CRC hack... it's a texture shuffle split horizontally instead of vertically.
 	if (rt_size.x < src_size.x || rt_size.y < src_size.y)
 	{
-		GSTextureCache::Target* rt_again = g_texture_cache->LookupTarget(Frame, src_size, src->m_scale, GSTextureCache::RenderTarget, true);
+		GSTextureCache::Target* rt_again = g_texture_cache->LookupTarget(Frame, src_size, src->m_scale, GSTextureCache::RenderTarget);
 		if (rt_again->m_unscaled_size.x < src->m_unscaled_size.x || rt_again->m_unscaled_size.y < src->m_unscaled_size.y)
 		{
 			rt_again->ResizeTexture(std::max(rt_again->m_unscaled_size.x, src->m_unscaled_size.x),
@@ -1137,7 +1137,7 @@ bool GSHwHack::GSC_Battlefield2(GSRendererHW& r, const GSFrameInfo& fi, int& ski
 			GIFRegTEX0 TEX0 = {};
 			TEX0.TBP0 = fi.FBP;
 			TEX0.TBW = 8;
-			GSTextureCache::Target* dst = g_texture_cache->LookupTarget(TEX0, r.GetTargetSize(), r.GetTextureScaleFactor(), GSTextureCache::DepthStencil, true);
+			GSTextureCache::Target* dst = g_texture_cache->LookupTarget(TEX0, r.GetTargetSize(), r.GetTextureScaleFactor(), GSTextureCache::DepthStencil);
 			if (dst)
 			{
 				g_gs_device->ClearDepth(dst->m_texture);
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index dfada31521..46fed44629 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -1806,7 +1806,9 @@ void GSRendererHW::Draw()
 		// (very close to 1024x1024, but apparently the GS rounds down..). So, catch that here, we don't want to
 		// create that target, because the clear isn't black, it'll hang around and never get invalidated.
 		const bool is_square = (t_size.y == t_size.x) && m_r.w >= 1023 && m_vertex.next == 2;
-		rt = g_texture_cache->LookupTarget(FRAME_TEX0, t_size, GetTextureScaleFactor(), GSTextureCache::RenderTarget, true, fm, false, force_preload, IsConstantDirectWriteMemClear(false) && is_square);
+		const bool is_clear = IsConstantDirectWriteMemClear(false) && is_square;
+		rt = g_texture_cache->LookupTarget(FRAME_TEX0, t_size, target_scale, GSTextureCache::RenderTarget, true,
+			fm, false, is_clear, force_preload);
 
 		// Draw skipped because it was a clear and there was no target.
 		if (!rt)
@@ -1826,7 +1828,8 @@ void GSRendererHW::Draw()
 		ZBUF_TEX0.TBW = context->FRAME.FBW;
 		ZBUF_TEX0.PSM = context->ZBUF.PSM;
 
-		ds = g_texture_cache->LookupTarget(ZBUF_TEX0, t_size, GetTextureScaleFactor(), GSTextureCache::DepthStencil, context->DepthWrite(), 0, false, force_preload);
+		ds = g_texture_cache->LookupTarget(ZBUF_TEX0, t_size, target_scale, GSTextureCache::DepthStencil,
+			context->DepthWrite(), 0, false, false, force_preload);
 	}
 
 	if (process_texture)
@@ -2029,7 +2032,6 @@ void GSRendererHW::Draw()
 	GSTextureCache::Target* old_ds = nullptr;
 	{
 		// We still need to make sure the dimensions of the targets match.
-		const float up_s = GetTextureScaleFactor();
 		const int new_w = std::max(t_size.x, std::max(rt ? rt->m_unscaled_size.x : 0, ds ? ds->m_unscaled_size.x : 0));
 		const int new_h = std::max(t_size.y, std::max(rt ? rt->m_unscaled_size.y : 0, ds ? ds->m_unscaled_size.y : 0));
 
@@ -2040,7 +2042,7 @@ void GSRendererHW::Draw()
 			const bool new_height = new_h > rt->GetUnscaledHeight();
 			const int old_height = rt->m_texture->GetHeight();
 
-			pxAssert(rt->GetScale() == up_s);
+			pxAssert(rt->GetScale() == target_scale);
 			rt->ResizeTexture(new_w, new_h);
 
 			if (!m_texture_shuffle && !m_channel_shuffle)
@@ -2080,7 +2082,7 @@ void GSRendererHW::Draw()
 			const bool new_height = new_h > ds->GetUnscaledHeight();
 			const int old_height = ds->m_texture->GetHeight();
 
-			pxAssert(ds->GetScale() == up_s);
+			pxAssert(ds->GetScale() == target_scale);
 			ds->ResizeTexture(new_w, new_h);
 
 			if (!m_texture_shuffle && !m_channel_shuffle)
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 860b30d0d8..4987340096 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -1070,7 +1070,8 @@ GSTextureCache::Target* GSTextureCache::FindTargetOverlap(u32 bp, u32 end_block,
 	return nullptr;
 }
 
-GSTextureCache::Target* GSTextureCache::LookupTarget(GIFRegTEX0 TEX0, const GSVector2i& size, float scale, int type, bool used, u32 fbmask, const bool is_frame, bool preload, bool is_clear)
+GSTextureCache::Target* GSTextureCache::LookupTarget(GIFRegTEX0 TEX0, const GSVector2i& size, float scale, int type,
+	bool used, u32 fbmask, bool is_frame, bool is_clear, bool preload)
 {
 	const GSLocalMemory::psm_t& psm_s = GSLocalMemory::m_psm[TEX0.PSM];
 	const u32 bp = TEX0.TBP0;
@@ -1386,10 +1387,9 @@ GSTextureCache::Target* GSTextureCache::LookupTarget(GIFRegTEX0 TEX0, const GSVe
 		}
 		dst->m_is_frame = is_frame;
 	}
-	if (used)
-	{
-		dst->m_used = true;
-	}
+
+	dst->m_used |= used;
+
 	if (is_frame)
 		dst->m_dirty_alpha = false;
 
@@ -2339,7 +2339,7 @@ bool GSTextureCache::Move(u32 SBP, u32 SBW, u32 SPSM, int sx, int sy, u32 DBP, u
 		new_TEX0.PSM = DPSM;
 
 		const GSVector2i target_size = GetTargetSize(DBP, DBW, DPSM, Common::AlignUpPow2(w, 64), h);
-		dst = LookupTarget(new_TEX0, target_size, src->m_scale, src->m_type, true);
+		dst = LookupTarget(new_TEX0, target_size, src->m_scale, src->m_type);
 		if (dst)
 		{
 			dst->UpdateValidity(GSVector4i(dx, dy, dx + w, dy + h));
@@ -3018,12 +3018,7 @@ GSTextureCache::Source* GSTextureCache::CreateSource(const GIFRegTEX0& TEX0, con
 			src->m_scale = dst->m_scale;
 			src->m_unscaled_size = dst->m_unscaled_size;
 			src->m_shared_texture = true;
-			src->m_target = true; // So renderer can check if a conversion is required
-			src->m_from_target = dst; // avoid complex condition on the renderer
-			src->m_from_target_TEX0 = dst->m_TEX0;
 			src->m_32_bits_fmt = dst->m_32_bits_fmt;
-			src->m_valid_rect = dst->m_valid;
-			src->m_end_block = dst->m_end_block;
 
 			// if the size doesn't match, we need to engage shader sampling.
 			if (new_size != dst_texture_size)
@@ -3577,7 +3572,7 @@ GSTextureCache::Target* GSTextureCache::CreateTarget(const GIFRegTEX0& TEX0, int
 
 	// TODO: This leaks if memory allocation fails. Use a unique_ptr so it gets freed, but these
 	// exceptions really need to get lost.
-	std::unique_ptr<Target> t = std::make_unique<Target>(TEX0, !GSConfig.UserHacks_DisableDepthSupport, type);
+	std::unique_ptr<Target> t = std::make_unique<Target>(TEX0, type);
 	t->m_unscaled_size = GSVector2i(w, h);
 	t->m_scale = scale;
 
@@ -4129,9 +4124,8 @@ bool GSTextureCache::Source::ClutMatch(const PaletteKey& palette_key)
 
 // GSTextureCache::Target
 
-GSTextureCache::Target::Target(const GIFRegTEX0& TEX0, const bool depth_supported, const int type)
+GSTextureCache::Target::Target(const GIFRegTEX0& TEX0, const int type)
 	: m_type(type)
-	, m_depth_supported(depth_supported)
 	, m_used(false)
 	, m_valid(GSVector4i::zero())
 {
@@ -4181,7 +4175,7 @@ void GSTextureCache::Target::Update(bool reset_age)
 		return;
 
 	// No handling please
-	if ((m_type == DepthStencil) && !m_depth_supported)
+	if (m_type == DepthStencil && GSConfig.UserHacks_DisableDepthSupport)
 	{
 		// do the most likely thing a direct write would do, clear it
 		GL_INS("ERROR: Update DepthStencil dummy");
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.h b/pcsx2/GS/Renderers/HW/GSTextureCache.h
index 031ca25d19..7fa3e19ed2 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.h
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.h
@@ -205,7 +205,6 @@ public:
 	{
 	public:
 		const int m_type = 0;
-		const bool m_depth_supported = false;
 		bool m_dirty_alpha = true;
 		bool m_is_frame = false;
 		bool m_used = false;
@@ -217,7 +216,7 @@ public:
 		int readbacks_since_draw = 0;
 
 	public:
-		Target(const GIFRegTEX0& TEX0, const bool depth_supported, const int type);
+		Target(const GIFRegTEX0& TEX0, const int type);
 		~Target();
 
 		void ResizeDrawn(const GSVector4i& rect);
@@ -437,7 +436,8 @@ public:
 	Source* LookupDepthSource(const GIFRegTEX0& TEX0, const GIFRegTEXA& TEXA, const GIFRegCLAMP& CLAMP, const GSVector4i& r, bool palette = false);
 
 	Target* FindTargetOverlap(u32 bp, u32 end_block, int type, int psm);
-	Target* LookupTarget(GIFRegTEX0 TEX0, const GSVector2i& size, float scale, int type, bool used, u32 fbmask = 0, const bool is_frame = false, bool preload = GSConfig.PreloadFrameWithGSData, bool is_clear = false);
+	Target* LookupTarget(GIFRegTEX0 TEX0, const GSVector2i& size, float scale, int type, bool used = true, u32 fbmask = 0,
+		bool is_frame = false, bool is_clear = false, bool preload = GSConfig.PreloadFrameWithGSData);
 	Target* LookupDisplayTarget(GIFRegTEX0 TEX0, const GSVector2i& size, float scale);
 
 	/// Looks up a target in the cache, and only returns it if the BP/BW/PSM match exactly.