diff --git a/pcsx2/GS/GS.cpp b/pcsx2/GS/GS.cpp
index fcefe2cec3..5a9f1b1807 100644
--- a/pcsx2/GS/GS.cpp
+++ b/pcsx2/GS/GS.cpp
@@ -264,7 +264,7 @@ bool GSreopen(bool recreate_device, bool recreate_renderer, const Pcsx2Config::G
 	else
 	{
 		// Make sure nothing is left over.
-		g_gs_renderer->PurgeTextureCache();
+		g_gs_renderer->PurgeTextureCache(true, true, true);
 		g_gs_renderer->PurgePool();
 	}
 
@@ -759,7 +759,7 @@ void GSUpdateConfig(const Pcsx2Config::GSOptions& new_config)
 	{
 		if (GSConfig.UserHacks_ReadTCOnClose)
 			g_gs_renderer->ReadbackTextureCache();
-		g_gs_renderer->PurgeTextureCache();
+		g_gs_renderer->PurgeTextureCache(true, true, true);
 		g_gs_renderer->PurgePool();
 	}
 
@@ -776,7 +776,7 @@ void GSUpdateConfig(const Pcsx2Config::GSOptions& new_config)
 	if (GSConfig.LoadTextureReplacements != old_config.LoadTextureReplacements ||
 		GSConfig.DumpReplaceableTextures != old_config.DumpReplaceableTextures)
 	{
-		g_gs_renderer->PurgeTextureCache();
+		g_gs_renderer->PurgeTextureCache(true, false, true);
 	}
 
 	if (GSConfig.OsdShowGPU != old_config.OsdShowGPU)
@@ -1118,7 +1118,7 @@ BEGIN_HOTKEY_LIST(g_gs_hotkeys){"Screenshot", TRANSLATE_NOOP("Hotkeys", "Graphic
 
 			MTGS::RunOnGSThread([new_level]() {
 				GSConfig.HWMipmap = new_level;
-				g_gs_renderer->PurgeTextureCache();
+				g_gs_renderer->PurgeTextureCache(true, false, true);
 				g_gs_renderer->PurgePool();
 			});
 		}},
@@ -1190,7 +1190,13 @@ BEGIN_HOTKEY_LIST(g_gs_hotkeys){"Screenshot", TRANSLATE_NOOP("Hotkeys", "Graphic
 				{
 					Host::AddKeyedOSDMessage("ReloadTextureReplacements",
 						TRANSLATE_STR("Hotkeys", "Reloading texture replacements..."), Host::OSD_INFO_DURATION);
-					MTGS::RunOnGSThread([]() { GSTextureReplacements::ReloadReplacementMap(); });
+					MTGS::RunOnGSThread([]() {
+						if (!g_gs_renderer)
+							return;
+
+						GSTextureReplacements::ReloadReplacementMap();
+						g_gs_renderer->PurgeTextureCache(true, false, true);
+					});
 				}
 			}
 		}},
diff --git a/pcsx2/GS/GSState.cpp b/pcsx2/GS/GSState.cpp
index 9b6c7c2d53..bf8ebb9288 100644
--- a/pcsx2/GS/GSState.cpp
+++ b/pcsx2/GS/GSState.cpp
@@ -2269,7 +2269,7 @@ void GSState::ReadLocalMemoryUnsync(u8* mem, int qwc, GIFRegBITBLTBUF BITBLTBUF,
 	}
 }
 
-void GSState::PurgeTextureCache()
+void GSState::PurgeTextureCache(bool sources, bool targets, bool hash_cache)
 {
 }
 
diff --git a/pcsx2/GS/GSState.h b/pcsx2/GS/GSState.h
index 9e3fd1de29..4a6f214c55 100644
--- a/pcsx2/GS/GSState.h
+++ b/pcsx2/GS/GSState.h
@@ -387,7 +387,7 @@ public:
 	bool TestDrawChanged();
 	void FlushWrite();
 	virtual void Draw() = 0;
-	virtual void PurgeTextureCache();
+	virtual void PurgeTextureCache(bool sources, bool targets, bool hash_cache);
 	virtual void ReadbackTextureCache();
 	virtual void InvalidateVideoMem(const GIFRegBITBLTBUF& BITBLTBUF, const GSVector4i& r) {}
 	virtual void InvalidateLocalMem(const GIFRegBITBLTBUF& BITBLTBUF, const GSVector4i& r, bool clut = false) {}
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index 9019fd7898..18500378dd 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -57,13 +57,13 @@ GSRendererHW::~GSRendererHW()
 
 void GSRendererHW::Destroy()
 {
-	g_texture_cache->RemoveAll();
+	g_texture_cache->RemoveAll(true, true, true);
 	GSRenderer::Destroy();
 }
 
-void GSRendererHW::PurgeTextureCache()
+void GSRendererHW::PurgeTextureCache(bool sources, bool targets, bool hash_cache)
 {
-	g_texture_cache->RemoveAll();
+	g_texture_cache->RemoveAll(sources, targets, hash_cache);
 }
 
 void GSRendererHW::ReadbackTextureCache()
@@ -92,7 +92,7 @@ void GSRendererHW::Reset(bool hardware_reset)
 	if (!hardware_reset)
 		g_texture_cache->ReadbackAll();
 
-	g_texture_cache->RemoveAll();
+	g_texture_cache->RemoveAll(true, true, true);
 
 	GSRenderer::Reset(hardware_reset);
 }
@@ -144,7 +144,7 @@ void GSRendererHW::VSync(u32 field, bool registers_written, bool idle_frame)
 			fmt::format(TRANSLATE_FS("GS", "Hash cache has used {:.2f} MB of VRAM, disabling."),
 				static_cast<float>(g_texture_cache->GetHashCacheMemoryUsage()) / 1048576.0f),
 			Host::OSD_ERROR_DURATION);
-		g_texture_cache->RemoveAll();
+		g_texture_cache->RemoveAll(true, false, true);
 		g_gs_device->PurgePool();
 		GSConfig.TexturePreloading = TexturePreloadingLevel::Partial;
 	}
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.h b/pcsx2/GS/Renderers/HW/GSRendererHW.h
index 83caef19a2..cc7c0db0a1 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.h
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.h
@@ -218,7 +218,7 @@ public:
 	void Move() override;
 	void Draw() override;
 
-	void PurgeTextureCache() override;
+	void PurgeTextureCache(bool sources, bool targets, bool hash_cache) override;
 	void ReadbackTextureCache() override;
 	GSTexture* LookupPaletteSource(u32 CBP, u32 CPSM, u32 CBW, GSVector2i& offset, float* scale, const GSVector2i& size) override;
 
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 3c44049036..9352d702ec 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -51,7 +51,7 @@ GSTextureCache::GSTextureCache()
 
 GSTextureCache::~GSTextureCache()
 {
-	RemoveAll();
+	RemoveAll(true, true, true);
 
 	s_hash_cache_purge_list = {};
 	_aligned_free(s_unswizzle_buffer);
@@ -66,32 +66,39 @@ void GSTextureCache::ReadbackAll()
 	}
 }
 
-void GSTextureCache::RemoveAll()
+void GSTextureCache::RemoveAll(bool sources, bool targets, bool hash_cache)
 {
-	m_src.RemoveAll();
-
-	for (int type = 0; type < 2; type++)
+	if (sources || targets)
 	{
-		for (auto t : m_dst[type])
-			delete t;
-
-		m_dst[type].clear();
+		m_src.RemoveAll();
+		m_palette_map.Clear();
+		m_source_memory_usage = 0;
 	}
 
-	for (auto it : m_hash_cache)
-		g_gs_device->Recycle(it.second.texture);
+	if (targets)
+	{
+		for (int type = 0; type < 2; type++)
+		{
+			for (auto t : m_dst[type])
+				delete t;
 
-	m_hash_cache.clear();
-	m_hash_cache_memory_usage = 0;
-	m_hash_cache_replacement_memory_usage = 0;
+			m_dst[type].clear();
+		}
 
-	m_palette_map.Clear();
-	m_target_heights.clear();
+		m_target_heights.clear();
+		m_surface_offset_cache.clear();
+		m_target_memory_usage = 0;
+	}
 
-	m_source_memory_usage = 0;
-	m_target_memory_usage = 0;
+	if (hash_cache)
+	{
+		for (auto it : m_hash_cache)
+			g_gs_device->Recycle(it.second.texture);
 
-	m_surface_offset_cache.clear();
+		m_hash_cache.clear();
+		m_hash_cache_memory_usage = 0;
+		m_hash_cache_replacement_memory_usage = 0;
+	}
 }
 
 bool GSTextureCache::FullRectDirty(Target* target, u32 rgba_mask)
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.h b/pcsx2/GS/Renderers/HW/GSTextureCache.h
index 054aaa7db0..42b00b4d8e 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.h
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.h
@@ -460,7 +460,7 @@ public:
 
 	void Read(Target* t, const GSVector4i& r);
 	void Read(Source* t, const GSVector4i& r);
-	void RemoveAll();
+	void RemoveAll(bool sources, bool targets, bool hash_cache);
 	void ReadbackAll();
 	void AddDirtyRectTarget(Target* target, GSVector4i rect, u32 psm, u32 bw, RGBAMask rgba, bool req_linear = false);
 	void ResizeTarget(Target* t, GSVector4i rect, u32 tbp, u32 psm, u32 tbw);