From 2b75f9638df3b12c1f93ac2e7b9cb4f87c6ddd84 Mon Sep 17 00:00:00 2001 From: Connor McLaughlin Date: Thu, 24 Nov 2022 19:42:40 +1000 Subject: [PATCH] GS/DX12/Vulkan: Handle out-of-VRAM better Instead of immediately failing the allocation, flush the texture pool, and execute/wait for the command buffer. This will clear out any textures which have already been freed, and the storage can be reused for. --- pcsx2/GS/Renderers/DX12/GSDevice12.cpp | 35 +++++++++++++++--------- pcsx2/GS/Renderers/DX12/GSDevice12.h | 2 +- pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp | 31 +++++++++++++-------- pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h | 2 +- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/pcsx2/GS/Renderers/DX12/GSDevice12.cpp b/pcsx2/GS/Renderers/DX12/GSDevice12.cpp index f788d400db..8c040fd6c5 100644 --- a/pcsx2/GS/Renderers/DX12/GSDevice12.cpp +++ b/pcsx2/GS/Renderers/DX12/GSDevice12.cpp @@ -324,7 +324,16 @@ GSTexture* GSDevice12::CreateSurface(GSTexture::Type type, int width, int height DXGI_FORMAT d3d_format, srv_format, rtv_format, dsv_format; LookupNativeFormat(format, &d3d_format, &srv_format, &rtv_format, &dsv_format); - return GSTexture12::Create(type, clamped_width, clamped_height, levels, format, d3d_format, srv_format, rtv_format, dsv_format).release(); + std::unique_ptr tex(GSTexture12::Create(type, clamped_width, clamped_height, levels, format, d3d_format, srv_format, rtv_format, dsv_format)); + if (!tex) + { + // We're probably out of vram, try flushing the command buffer to release pending textures. + PurgePool(); + ExecuteCommandListAndRestartRenderPass(true, "Couldn't allocate texture."); + tex = GSTexture12::Create(type, clamped_width, clamped_height, levels, format, d3d_format, srv_format, rtv_format, dsv_format); + } + + return tex.release(); } bool GSDevice12::DownloadTexture(GSTexture* src, const GSVector4i& rect, GSTexture::GSMap& out_map) @@ -869,7 +878,7 @@ void GSDevice12::IASetVertexBuffer(const void* vertex, size_t stride, size_t cou const u32 size = static_cast(stride) * static_cast(count); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) { - ExecuteCommandListAndRestartRenderPass("Uploading to vertex buffer"); + ExecuteCommandListAndRestartRenderPass(false, "Uploading to vertex buffer"); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) pxFailRel("Failed to reserve space for vertices"); } @@ -889,7 +898,7 @@ bool GSDevice12::IAMapVertexBuffer(void** vertex, size_t stride, size_t count) const u32 size = static_cast(stride) * static_cast(count); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) { - ExecuteCommandListAndRestartRenderPass("Mapping bytes to vertex buffer"); + ExecuteCommandListAndRestartRenderPass(false, "Mapping bytes to vertex buffer"); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) pxFailRel("Failed to reserve space for vertices"); } @@ -915,7 +924,7 @@ void GSDevice12::IASetIndexBuffer(const void* index, size_t count) const u32 size = sizeof(u32) * static_cast(count); if (!m_index_stream_buffer.ReserveMemory(size, sizeof(u32))) { - ExecuteCommandListAndRestartRenderPass("Uploading bytes to index buffer"); + ExecuteCommandListAndRestartRenderPass(false, "Uploading bytes to index buffer"); if (!m_index_stream_buffer.ReserveMemory(size, sizeof(u32))) pxFailRel("Failed to reserve space for vertices"); } @@ -1841,13 +1850,13 @@ void GSDevice12::ExecuteCommandList(bool wait_for_completion, const char* reason ExecuteCommandList(wait_for_completion); } -void GSDevice12::ExecuteCommandListAndRestartRenderPass(const char* reason) +void GSDevice12::ExecuteCommandListAndRestartRenderPass(bool wait_for_completion, const char* reason) { Console.Warning("Vulkan: Executing command buffer due to '%s'", reason); const bool was_in_render_pass = m_in_render_pass; EndRenderPass(); - g_d3d12_context->ExecuteCommandList(D3D12::Context::WaitType::None); + g_d3d12_context->ExecuteCommandList(GetWaitType(wait_for_completion, GSConfig.HWSpinCPUForReadbacks)); InvalidateCachedState(); if (was_in_render_pass) @@ -2002,7 +2011,7 @@ void GSDevice12::SetUtilityTexture(GSTexture* dtex, const D3D12::DescriptorHandl if (!GetTextureGroupDescriptors(&m_utility_texture_gpu, &handle, 1)) { - ExecuteCommandListAndRestartRenderPass("Ran out of utility texture descriptors"); + ExecuteCommandListAndRestartRenderPass(false, "Ran out of utility texture descriptors"); SetUtilityTexture(dtex, sampler); return; } @@ -2015,7 +2024,7 @@ void GSDevice12::SetUtilityTexture(GSTexture* dtex, const D3D12::DescriptorHandl if (!g_d3d12_context->GetSamplerAllocator().LookupSingle(&m_utility_sampler_gpu, sampler)) { - ExecuteCommandListAndRestartRenderPass("Ran out of utility sampler descriptors"); + ExecuteCommandListAndRestartRenderPass(false, "Ran out of utility sampler descriptors"); SetUtilityTexture(dtex, sampler); return; } @@ -2286,7 +2295,7 @@ bool GSDevice12::ApplyTFXState(bool already_execed) return false; } - ExecuteCommandListAndRestartRenderPass("Ran out of vertex uniform space"); + ExecuteCommandListAndRestartRenderPass(false, "Ran out of vertex uniform space"); return ApplyTFXState(true); } @@ -2307,7 +2316,7 @@ bool GSDevice12::ApplyTFXState(bool already_execed) return false; } - ExecuteCommandListAndRestartRenderPass("Ran out of pixel uniform space"); + ExecuteCommandListAndRestartRenderPass(false, "Ran out of pixel uniform space"); return ApplyTFXState(true); } @@ -2321,7 +2330,7 @@ bool GSDevice12::ApplyTFXState(bool already_execed) { if (!g_d3d12_context->GetSamplerAllocator().LookupGroup(&m_tfx_samplers_handle_gpu, m_tfx_samplers.data())) { - ExecuteCommandListAndRestartRenderPass("Ran out of sampler groups"); + ExecuteCommandListAndRestartRenderPass(false, "Ran out of sampler groups"); return ApplyTFXState(true); } @@ -2332,7 +2341,7 @@ bool GSDevice12::ApplyTFXState(bool already_execed) { if (!GetTextureGroupDescriptors(&m_tfx_textures_handle_gpu, m_tfx_textures.data(), 2)) { - ExecuteCommandListAndRestartRenderPass("Ran out of TFX texture descriptor groups"); + ExecuteCommandListAndRestartRenderPass(false, "Ran out of TFX texture descriptor groups"); return ApplyTFXState(true); } @@ -2343,7 +2352,7 @@ bool GSDevice12::ApplyTFXState(bool already_execed) { if (!GetTextureGroupDescriptors(&m_tfx_rt_textures_handle_gpu, m_tfx_textures.data() + 2, 2)) { - ExecuteCommandListAndRestartRenderPass("Ran out of TFX RT descriptor descriptor groups"); + ExecuteCommandListAndRestartRenderPass(false, "Ran out of TFX RT descriptor descriptor groups"); return ApplyTFXState(true); } diff --git a/pcsx2/GS/Renderers/DX12/GSDevice12.h b/pcsx2/GS/Renderers/DX12/GSDevice12.h index 78c23ed4ca..01e957c070 100644 --- a/pcsx2/GS/Renderers/DX12/GSDevice12.h +++ b/pcsx2/GS/Renderers/DX12/GSDevice12.h @@ -297,7 +297,7 @@ public: /// Ends any render pass, executes the command buffer, and invalidates cached state. void ExecuteCommandList(bool wait_for_completion); void ExecuteCommandList(bool wait_for_completion, const char* reason, ...); - void ExecuteCommandListAndRestartRenderPass(const char* reason); + void ExecuteCommandListAndRestartRenderPass(bool wait_for_completion, const char* reason); /// Set dirty flags on everything to force re-bind at next draw time. void InvalidateCachedState(); diff --git a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp index 3424d39dd4..50bb4f4552 100644 --- a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp +++ b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp @@ -406,7 +406,16 @@ GSTexture* GSDeviceVK::CreateSurface(GSTexture::Type type, int width, int height const u32 clamped_width = static_cast(std::clamp(1, width, g_vulkan_context->GetMaxImageDimension2D())); const u32 clamped_height = static_cast(std::clamp(1, height, g_vulkan_context->GetMaxImageDimension2D())); - return GSTextureVK::Create(type, clamped_width, clamped_height, levels, format, LookupNativeFormat(format)).release(); + std::unique_ptr tex(GSTextureVK::Create(type, clamped_width, clamped_height, levels, format, LookupNativeFormat(format))); + if (!tex) + { + // We're probably out of vram, try flushing the command buffer to release pending textures. + PurgePool(); + ExecuteCommandBufferAndRestartRenderPass(true, "Couldn't allocate texture."); + tex = GSTextureVK::Create(type, clamped_width, clamped_height, levels, format, LookupNativeFormat(format)); + } + + return tex.release(); } bool GSDeviceVK::DownloadTexture(GSTexture* src, const GSVector4i& rect, GSTexture::GSMap& out_map) @@ -928,7 +937,7 @@ void GSDeviceVK::IASetVertexBuffer(const void* vertex, size_t stride, size_t cou const u32 size = static_cast(stride) * static_cast(count); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) { - ExecuteCommandBufferAndRestartRenderPass("Uploading bytes to vertex buffer"); + ExecuteCommandBufferAndRestartRenderPass(false, "Uploading bytes to vertex buffer"); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) pxFailRel("Failed to reserve space for vertices"); } @@ -948,7 +957,7 @@ bool GSDeviceVK::IAMapVertexBuffer(void** vertex, size_t stride, size_t count) const u32 size = static_cast(stride) * static_cast(count); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) { - ExecuteCommandBufferAndRestartRenderPass("Mapping bytes to vertex buffer"); + ExecuteCommandBufferAndRestartRenderPass(false, "Mapping bytes to vertex buffer"); if (!m_vertex_stream_buffer.ReserveMemory(size, static_cast(stride))) pxFailRel("Failed to reserve space for vertices"); } @@ -974,7 +983,7 @@ void GSDeviceVK::IASetIndexBuffer(const void* index, size_t count) const u32 size = sizeof(u32) * static_cast(count); if (!m_index_stream_buffer.ReserveMemory(size, sizeof(u32))) { - ExecuteCommandBufferAndRestartRenderPass("Uploading bytes to index buffer"); + ExecuteCommandBufferAndRestartRenderPass(false, "Uploading bytes to index buffer"); if (!m_index_stream_buffer.ReserveMemory(size, sizeof(u32))) pxFailRel("Failed to reserve space for vertices"); } @@ -2353,14 +2362,14 @@ void GSDeviceVK::ExecuteCommandBuffer(bool wait_for_completion, const char* reas ExecuteCommandBuffer(wait_for_completion); } -void GSDeviceVK::ExecuteCommandBufferAndRestartRenderPass(const char* reason) +void GSDeviceVK::ExecuteCommandBufferAndRestartRenderPass(bool wait_for_completion, const char* reason) { Console.Warning("Vulkan: Executing command buffer due to '%s'", reason); const VkRenderPass render_pass = m_current_render_pass; const GSVector4i render_pass_area(m_current_render_pass_area); EndRenderPass(); - g_vulkan_context->ExecuteCommandBuffer(Vulkan::Context::WaitType::None); + g_vulkan_context->ExecuteCommandBuffer(GetWaitType(wait_for_completion, GSConfig.HWSpinCPUForReadbacks)); InvalidateCachedState(); if (render_pass != VK_NULL_HANDLE) @@ -2673,7 +2682,7 @@ bool GSDeviceVK::ApplyTFXState(bool already_execed) return false; } - ExecuteCommandBufferAndRestartRenderPass("Ran out of vertex uniform space"); + ExecuteCommandBufferAndRestartRenderPass(false, "Ran out of vertex uniform space"); return ApplyTFXState(true); } @@ -2694,7 +2703,7 @@ bool GSDeviceVK::ApplyTFXState(bool already_execed) return false; } - ExecuteCommandBufferAndRestartRenderPass("Ran out of pixel uniform space"); + ExecuteCommandBufferAndRestartRenderPass(false, "Ran out of pixel uniform space"); return ApplyTFXState(true); } @@ -2725,7 +2734,7 @@ bool GSDeviceVK::ApplyTFXState(bool already_execed) return false; } - ExecuteCommandBufferAndRestartRenderPass("Ran out of TFX texture descriptors"); + ExecuteCommandBufferAndRestartRenderPass(false, "Ran out of TFX texture descriptors"); return ApplyTFXState(true); } @@ -2749,7 +2758,7 @@ bool GSDeviceVK::ApplyTFXState(bool already_execed) return false; } - ExecuteCommandBufferAndRestartRenderPass("Ran out of TFX sampler descriptors"); + ExecuteCommandBufferAndRestartRenderPass(false, "Ran out of TFX sampler descriptors"); return ApplyTFXState(true); } @@ -2823,7 +2832,7 @@ bool GSDeviceVK::ApplyUtilityState(bool already_execed) return false; } - ExecuteCommandBufferAndRestartRenderPass("Ran out of utility descriptors"); + ExecuteCommandBufferAndRestartRenderPass(false, "Ran out of utility descriptors"); return ApplyTFXState(true); } diff --git a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h index fbca744888..4dc414c3e8 100644 --- a/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h +++ b/pcsx2/GS/Renderers/Vulkan/GSDeviceVK.h @@ -283,7 +283,7 @@ public: /// Ends any render pass, executes the command buffer, and invalidates cached state. void ExecuteCommandBuffer(bool wait_for_completion); void ExecuteCommandBuffer(bool wait_for_completion, const char* reason, ...); - void ExecuteCommandBufferAndRestartRenderPass(const char* reason); + void ExecuteCommandBufferAndRestartRenderPass(bool wait_for_completion, const char* reason); /// Set dirty flags on everything to force re-bind at next draw time. void InvalidateCachedState();