diff --git a/pcsx2/GS/Renderers/Common/GSDevice.cpp b/pcsx2/GS/Renderers/Common/GSDevice.cpp
index 84629b2256..cf23296996 100644
--- a/pcsx2/GS/Renderers/Common/GSDevice.cpp
+++ b/pcsx2/GS/Renderers/Common/GSDevice.cpp
@@ -241,6 +241,10 @@ void GSDevice::PurgePool()
 	m_pool.clear();
 }
 
+void GSDevice::ClearSamplerCache()
+{
+}
+
 GSTexture* GSDevice::CreateSparseRenderTarget(int w, int h, GSTexture::Format format, bool clear)
 {
 	return FetchSurface(HasColorSparse() ? GSTexture::Type::SparseRenderTarget : GSTexture::Type::RenderTarget, w, h, false, format, clear, true);
diff --git a/pcsx2/GS/Renderers/Common/GSDevice.h b/pcsx2/GS/Renderers/Common/GSDevice.h
index d788e097e5..49845720a5 100644
--- a/pcsx2/GS/Renderers/Common/GSDevice.h
+++ b/pcsx2/GS/Renderers/Common/GSDevice.h
@@ -656,6 +656,8 @@ public:
 	void AgePool();
 	void PurgePool();
 
+	virtual void ClearSamplerCache();
+
 	virtual void PrintMemoryUsage();
 
 	// Convert the GS blend equations to HW specific blend factors/ops
diff --git a/pcsx2/GS/Renderers/DX11/GSDevice11.h b/pcsx2/GS/Renderers/DX11/GSDevice11.h
index 185c3e5e7e..8a9287cfe8 100644
--- a/pcsx2/GS/Renderers/DX11/GSDevice11.h
+++ b/pcsx2/GS/Renderers/DX11/GSDevice11.h
@@ -297,6 +297,8 @@ public:
 
 	void RenderHW(GSHWDrawConfig& config) final;
 
+	void ClearSamplerCache() final;
+
 	ID3D11Device* operator->() { return m_dev.get(); }
 	operator ID3D11Device*() { return m_dev.get(); }
 	operator ID3D11DeviceContext*() { return m_ctx.get(); }
diff --git a/pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp b/pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp
index 66b9e70889..45b75bc06b 100644
--- a/pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp
+++ b/pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp
@@ -257,6 +257,11 @@ void GSDevice11::SetupPS(PSSelector sel, const GSHWDrawConfig::PSConstantBuffer*
 	PSSetShader(i->second.get(), m_ps_cb.get());
 }
 
+void GSDevice11::ClearSamplerCache()
+{
+	m_ps_ss.clear();
+}
+
 void GSDevice11::SetupOM(OMDepthStencilSelector dssel, OMBlendSelector bsel, u8 afix)
 {
 	auto i = std::as_const(m_om_dss).find(dssel.key);
diff --git a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
index 4b48a36eb5..cd24718eff 100644
--- a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
+++ b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp
@@ -1610,6 +1610,16 @@ void GSDeviceOGL::PSSetSamplerState(GLuint ss)
 	}
 }
 
+void GSDeviceOGL::ClearSamplerCache()
+{
+	glDeleteSamplers(std::size(m_ps_ss), m_ps_ss);
+
+	for (u32 key = 0; key < std::size(m_ps_ss); key++)
+	{
+		m_ps_ss[key] = CreateSampler(PSSamplerSelector(key));
+	}
+}
+
 void GSDeviceOGL::OMAttachRt(GSTextureOGL* rt)
 {
 	GLuint id = 0;
diff --git a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h
index c2c119248e..cf06c04151 100644
--- a/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h
+++ b/pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.h
@@ -371,6 +371,7 @@ public:
 	void PSSetShaderResource(int i, GSTexture* sr);
 	void PSSetShaderResources(GSTexture* sr0, GSTexture* sr1);
 	void PSSetSamplerState(GLuint ss);
+	void ClearSamplerCache() final;
 
 	void OMSetDepthStencilState(GSDepthStencilOGL* dss);
 	void OMSetBlendState(u8 blend_index = 0, u8 blend_factor = 0, bool is_blend_constant = false, bool accumulation_blend = false, bool blend_mix = false);
diff --git a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp
index 9e7bdd5c2d..63f0bbe8c2 100644
--- a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp
+++ b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp
@@ -990,6 +990,19 @@ VkSampler GSDeviceVK::GetSampler(GSHWDrawConfig::SamplerSelector ss)
 	return sampler;
 }
 
+void GSDeviceVK::ClearSamplerCache()
+{
+	for (const auto& it : m_samplers)
+		g_vulkan_context->DeferSamplerDestruction(it.second);
+	m_samplers.clear();
+	m_point_sampler = GetSampler(GSHWDrawConfig::SamplerSelector::Point());
+	m_linear_sampler = GetSampler(GSHWDrawConfig::SamplerSelector::Linear());
+	m_utility_sampler = m_point_sampler;
+
+	for (u32 i = 0; i < std::size(m_tfx_samplers); i++)
+		m_tfx_samplers[i] = GetSampler(m_tfx_sampler_sel[i]);
+}
+
 static void AddMacro(std::stringstream& ss, const char* name, const char* value)
 {
 	ss << "#define " << name << " " << value << "\n";
diff --git a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h
index 507419d4b8..24026007d4 100644
--- a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h
+++ b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h
@@ -167,6 +167,7 @@ private:
 	u16 ConvertBlendEnum(u16 generic) final;
 
 	VkSampler GetSampler(GSHWDrawConfig::SamplerSelector ss);
+	void ClearSamplerCache() final;
 
 	VkShaderModule GetTFXVertexShader(GSHWDrawConfig::VSSelector sel);
 	VkShaderModule GetTFXGeometryShader(GSHWDrawConfig::GSSelector sel);