diff --git a/pcsx2/GS/Renderers/DX11/GSDevice11.cpp b/pcsx2/GS/Renderers/DX11/GSDevice11.cpp
index 724f2d6c17..37bd2af213 100644
--- a/pcsx2/GS/Renderers/DX11/GSDevice11.cpp
+++ b/pcsx2/GS/Renderers/DX11/GSDevice11.cpp
@@ -58,6 +58,14 @@ GSDevice11::GSDevice11()
 	m_features.stencil_buffer = true;
 }
 
+GSDevice11::~GSDevice11()
+{
+	if (m_state.rt_view)
+		m_state.rt_view->Release();
+	if (m_state.dsv)
+		m_state.dsv->Release();
+}
+
 bool GSDevice11::Create(HostDisplay* display)
 {
 	if (!__super::Create(display))
@@ -1153,15 +1161,25 @@ void GSDevice11::OMSetRenderTargets(GSTexture* rt, GSTexture* ds, const GSVector
 	if (rt) rtv = *(GSTexture11*)rt;
 	if (ds) dsv = *(GSTexture11*)ds;
 
-	if (m_state.rt_view != rtv || m_state.dsv != dsv)
+	const bool changed = (m_state.rt_view != rtv || m_state.dsv != dsv);
+	if (m_state.rt_view != rtv)
 	{
+		if (m_state.rt_view)
+			m_state.rt_view->Release();
+		if (rtv)
+			rtv->AddRef();
 		m_state.rt_view = rtv;
-		m_state.rt_texture = static_cast<GSTexture11*>(rt);
-		m_state.dsv = dsv;
-		m_state.rt_ds = static_cast<GSTexture11*>(ds);
-
-		m_ctx->OMSetRenderTargets(1, &rtv, dsv);
 	}
+	if (m_state.dsv != dsv)
+	{
+		if (m_state.dsv)
+			m_state.dsv->Release();
+		if (dsv)
+			dsv->AddRef();
+		m_state.dsv = dsv;
+	}
+	if (changed)
+		m_ctx->OMSetRenderTargets(1, &rtv, dsv);
 
 	const GSVector2i size = rt ? rt->GetSize() : ds->GetSize();
 	if (m_state.viewport != size)
diff --git a/pcsx2/GS/Renderers/DX11/GSDevice11.h b/pcsx2/GS/Renderers/DX11/GSDevice11.h
index 051dcfab30..1033ac5604 100644
--- a/pcsx2/GS/Renderers/DX11/GSDevice11.h
+++ b/pcsx2/GS/Renderers/DX11/GSDevice11.h
@@ -154,8 +154,6 @@ private:
 		ID3D11BlendState* bs;
 		float bf;
 		ID3D11RenderTargetView* rt_view;
-		GSTexture11* rt_texture;
-		GSTexture11* rt_ds;
 		ID3D11DepthStencilView* dsv;
 	} m_state;
 
@@ -228,7 +226,7 @@ private:
 
 public:
 	GSDevice11();
-	virtual ~GSDevice11() {}
+	~GSDevice11() override;
 
 	__fi static GSDevice11* GetInstance() { return static_cast<GSDevice11*>(g_gs_device.get()); }
 	__fi ID3D11Device* GetD3DDevice() const { return m_dev.get(); }