From 83f307ec7ec77b2af13ed06d601e03a9d274bae8 Mon Sep 17 00:00:00 2001
From: Dentomologist <dentomologist@gmail.com>
Date: Fri, 4 Aug 2023 11:53:06 -0700
Subject: [PATCH] D3D12: Only use framebuffer integer descriptor if allocated

Verify that DXFramebuffer's integer RTV descriptor's cpu_handle has been
allocated before using it, and if it hasn't use the non-integer RTV
descriptor instead. This fixes a Dolphin crash in Twilight Princess, and
possibly other games (Issue 13312).

As an optimization to save space in the descriptor heap, DXFramebuffer's
integer descriptor is only initialized if the given abstract texture
format has different integer and non-integer RTV formats. This
previously wasn't accounted for by GetIntRTVDescriptorArray, which could
cause DX12::Gfx::BindFramebuffer to call OMSetRenderTargets with an
invalid descriptor which would lead to a crash.

Triggering the bug was fortunately rare because integer formats are only
used when blending is disabled and logic ops are enabled. Furthermore,
the standard integer abstract format is RGBA8 which has different
integer and non-integer RTV formats, causing the integer descriptor to
be initialized and avoiding the bug.

The crash started appearing in a2702c6 because it changed the
swapchain's abstract texture format from RGBA8 to RGB10_A2. Unlike
RGBA8, RGB10_A2 has the same integer and non-integer RTV formats and so
the bug can be triggered if the other requirements are met.
---
 .../Core/VideoBackends/D3D12/DX12Texture.cpp  | 21 +++++++++++++++++++
 Source/Core/VideoBackends/D3D12/DX12Texture.h |  5 +----
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/Source/Core/VideoBackends/D3D12/DX12Texture.cpp b/Source/Core/VideoBackends/D3D12/DX12Texture.cpp
index f20c37354b..e040d0483d 100644
--- a/Source/Core/VideoBackends/D3D12/DX12Texture.cpp
+++ b/Source/Core/VideoBackends/D3D12/DX12Texture.cpp
@@ -422,6 +422,23 @@ DXFramebuffer::~DXFramebuffer()
   }
 }
 
+const D3D12_CPU_DESCRIPTOR_HANDLE* DXFramebuffer::GetIntRTVDescriptorArray() const
+{
+  if (m_color_attachment == nullptr)
+    return nullptr;
+
+  const auto& handle = m_int_rtv_descriptor.cpu_handle;
+
+  // To save space in the descriptor heap, m_int_rtv_descriptor.cpu_handle.ptr is only allocated
+  // when the integer RTV format corresponding to the current abstract format differs from the
+  // non-integer RTV format. Only use the integer handle if it has been allocated.
+  if (handle.ptr != 0)
+    return &handle;
+
+  // The integer and non-integer RTV formats are the same, so use the non-integer descriptor.
+  return GetRTVDescriptorArray();
+}
+
 void DXFramebuffer::Unbind()
 {
   static const D3D12_DISCARD_REGION dr = {0, nullptr, 0, 1};
@@ -556,6 +573,10 @@ bool DXFramebuffer::CreateIRTVDescriptor()
   const bool multisampled = m_samples > 1;
   DXGI_FORMAT non_int_format = D3DCommon::GetRTVFormatForAbstractFormat(m_color_format, false);
   DXGI_FORMAT int_format = D3DCommon::GetRTVFormatForAbstractFormat(m_color_format, true);
+
+  // If the integer and non-integer RTV formats are the same for a given abstract format we can save
+  // space in the descriptor heap by only allocating the non-integer descriptor and using it for
+  // the integer RTV too.
   if (int_format != non_int_format)
   {
     if (!g_dx_context->GetRTVHeapManager().Allocate(&m_int_rtv_descriptor))
diff --git a/Source/Core/VideoBackends/D3D12/DX12Texture.h b/Source/Core/VideoBackends/D3D12/DX12Texture.h
index ea781fd899..2054b195b2 100644
--- a/Source/Core/VideoBackends/D3D12/DX12Texture.h
+++ b/Source/Core/VideoBackends/D3D12/DX12Texture.h
@@ -75,10 +75,7 @@ public:
   {
     return m_render_targets_raw.data();
   }
-  const D3D12_CPU_DESCRIPTOR_HANDLE* GetIntRTVDescriptorArray() const
-  {
-    return m_color_attachment ? &m_int_rtv_descriptor.cpu_handle : nullptr;
-  }
+  const D3D12_CPU_DESCRIPTOR_HANDLE* GetIntRTVDescriptorArray() const;
   const D3D12_CPU_DESCRIPTOR_HANDLE* GetDSVDescriptorArray() const
   {
     return m_depth_attachment ? &m_dsv_descriptor.cpu_handle : nullptr;