From c7352d9e10459a772f1846ea656b957814683c48 Mon Sep 17 00:00:00 2001 From: Stenzek Date: Sat, 11 Feb 2023 16:21:36 +1000 Subject: [PATCH] GS: Attempt to recreate device if GPU crashes --- common/D3D12/Context.cpp | 17 ++++- common/D3D12/Context.h | 2 +- common/Vulkan/Context.cpp | 42 ++++++++---- common/Vulkan/Context.h | 2 + pcsx2-gsrunner/Main.cpp | 15 +++-- pcsx2-qt/QtHost.cpp | 10 +-- pcsx2/Frontend/D3D11HostDisplay.cpp | 9 +-- pcsx2/Frontend/D3D11HostDisplay.h | 2 +- pcsx2/Frontend/D3D12HostDisplay.cpp | 18 +++-- pcsx2/Frontend/D3D12HostDisplay.h | 3 +- pcsx2/Frontend/ImGuiManager.cpp | 6 ++ pcsx2/Frontend/ImGuiManager.h | 3 + pcsx2/Frontend/MetalHostDisplay.h | 2 +- pcsx2/Frontend/MetalHostDisplay.mm | 8 +-- pcsx2/Frontend/OpenGLHostDisplay.cpp | 9 +-- pcsx2/Frontend/OpenGLHostDisplay.h | 2 +- pcsx2/Frontend/VulkanHostDisplay.cpp | 17 ++--- pcsx2/Frontend/VulkanHostDisplay.h | 2 +- pcsx2/GS/GS.cpp | 86 ++++++++++++++---------- pcsx2/GS/GS.h | 2 +- pcsx2/GS/Renderers/Common/GSRenderer.cpp | 42 +++++++++++- pcsx2/GS/Renderers/Common/GSRenderer.h | 1 + pcsx2/HostDisplay.h | 11 ++- tests/ctest/core/StubHost.cpp | 4 +- 24 files changed, 208 insertions(+), 107 deletions(-) diff --git a/common/D3D12/Context.cpp b/common/D3D12/Context.cpp index 5c34e77d48..a34a63e9c4 100644 --- a/common/D3D12/Context.cpp +++ b/common/D3D12/Context.cpp @@ -445,7 +445,7 @@ ID3D12GraphicsCommandList4* Context::GetInitCommandList() return res.command_lists[0].get(); } -void Context::ExecuteCommandList(WaitType wait_for_completion) +bool Context::ExecuteCommandList(WaitType wait_for_completion) { CommandListResources& res = m_command_lists[m_current_command_list]; HRESULT hr; @@ -463,12 +463,21 @@ void Context::ExecuteCommandList(WaitType wait_for_completion) if (res.init_command_list_used) { hr = res.command_lists[0]->Close(); - pxAssertRel(SUCCEEDED(hr), "Close init command list"); + if (FAILED(hr)) + { + Console.Error("Closing init command list failed with HRESULT %08X", hr); + return false; + } } // Close and queue command list. hr = res.command_lists[1]->Close(); - pxAssertRel(SUCCEEDED(hr), "Close command list"); + if (FAILED(hr)) + { + Console.Error("Closing main command list failed with HRESULT %08X", hr); + return false; + } + if (res.init_command_list_used) { const std::array execute_lists{res.command_lists[0].get(), res.command_lists[1].get()}; @@ -487,6 +496,8 @@ void Context::ExecuteCommandList(WaitType wait_for_completion) MoveToNextCommandList(); if (wait_for_completion != WaitType::None) WaitForFence(res.ready_fence_value, wait_for_completion == WaitType::Spin); + + return true; } void Context::InvalidateSamplerGroups() diff --git a/common/D3D12/Context.h b/common/D3D12/Context.h index d64be71dd7..1654840798 100644 --- a/common/D3D12/Context.h +++ b/common/D3D12/Context.h @@ -130,7 +130,7 @@ namespace D3D12 }; /// Executes the current command list. - void ExecuteCommandList(WaitType wait_for_completion); + bool ExecuteCommandList(WaitType wait_for_completion); /// Waits for a specific fence. void WaitForFence(u64 fence, bool spin); diff --git a/common/Vulkan/Context.cpp b/common/Vulkan/Context.cpp index 1f9af54021..a88290663a 100644 --- a/common/Vulkan/Context.cpp +++ b/common/Vulkan/Context.cpp @@ -1122,9 +1122,13 @@ namespace Vulkan void Context::WaitForCommandBufferCompletion(u32 index) { // Wait for this command buffer to be completed. - VkResult res = vkWaitForFences(m_device, 1, &m_frame_resources[index].fence, VK_TRUE, UINT64_MAX); + const VkResult res = vkWaitForFences(m_device, 1, &m_frame_resources[index].fence, VK_TRUE, UINT64_MAX); if (res != VK_SUCCESS) + { LOG_VULKAN_ERROR(res, "vkWaitForFences failed: "); + m_last_submit_failed.store(true, std::memory_order_release); + return; + } // Clean up any resources for command buffers between the last known completed buffer and this // now-completed command buffer. If we use >2 buffers, this may be more than one buffer. @@ -1266,11 +1270,12 @@ namespace Vulkan submit_info.pSignalSemaphores = &m_spin_resources[index].semaphore; } - VkResult res = vkQueueSubmit(m_graphics_queue, 1, &submit_info, resources.fence); + const VkResult res = vkQueueSubmit(m_graphics_queue, 1, &submit_info, resources.fence); if (res != VK_SUCCESS) { LOG_VULKAN_ERROR(res, "vkQueueSubmit failed: "); - pxFailRel("Failed to submit command buffer."); + m_last_submit_failed.store(true, std::memory_order_release); + return; } if (spin_cycles != 0) @@ -1286,14 +1291,14 @@ namespace Vulkan present_swap_chain->ReleaseCurrentImage(); - VkResult res = vkQueuePresentKHR(m_present_queue, &present_info); + const VkResult res = vkQueuePresentKHR(m_present_queue, &present_info); if (res != VK_SUCCESS) { // VK_ERROR_OUT_OF_DATE_KHR is not fatal, just means we need to recreate our swap chain. if (res != VK_ERROR_OUT_OF_DATE_KHR && res != VK_SUBOPTIMAL_KHR) LOG_VULKAN_ERROR(res, "vkQueuePresentKHR failed: "); - m_last_present_failed.store(true); + m_last_present_failed.store(true, std::memory_order_release); return; } @@ -1460,6 +1465,9 @@ namespace Vulkan void Context::ExecuteCommandBuffer(WaitType wait_for_completion) { + if (m_last_submit_failed.load(std::memory_order_acquire)) + return; + // If we're waiting for completion, don't bother waking the worker thread. const u32 current_frame = m_current_frame; SubmitCommandBuffer(); @@ -1481,9 +1489,12 @@ namespace Vulkan bool Context::CheckLastPresentFail() { - bool res = m_last_present_failed; - m_last_present_failed = false; - return res; + return m_last_present_failed.exchange(false, std::memory_order_acq_rel); + } + + bool Context::CheckLastSubmitFail() + { + return m_last_submit_failed.load(std::memory_order_acquire); } void Context::DeferBufferDestruction(VkBuffer object) @@ -1596,7 +1607,7 @@ namespace Vulkan VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT, DebugMessengerCallback, nullptr}; - VkResult res = + const VkResult res = vkCreateDebugUtilsMessengerEXT(m_instance, &messenger_info, nullptr, &m_debug_messenger_callback); if (res != VK_SUCCESS) { @@ -1688,7 +1699,7 @@ namespace Vulkan subpass_dependency_ptr}; VkRenderPass pass; - VkResult res = vkCreateRenderPass(m_device, &pass_info, nullptr, &pass); + const VkResult res = vkCreateRenderPass(m_device, &pass_info, nullptr, &pass); if (res != VK_SUCCESS) { LOG_VULKAN_ERROR(res, "vkCreateRenderPass failed: "); @@ -1894,9 +1905,14 @@ void main() SpinResources& resources = m_spin_resources[index]; if (!resources.in_progress) return; - VkResult res = vkWaitForFences(m_device, 1, &resources.fence, VK_TRUE, UINT64_MAX); + + const VkResult res = vkWaitForFences(m_device, 1, &resources.fence, VK_TRUE, UINT64_MAX); if (res != VK_SUCCESS) + { LOG_VULKAN_ERROR(res, "vkWaitForFences failed: "); + m_last_submit_failed.store(true, std::memory_order_release); + return; + } SpinCommandCompleted(index); } @@ -1906,7 +1922,7 @@ void main() resources.in_progress = false; const u32 timestamp_base = (index + NUM_COMMAND_BUFFERS) * 2; std::array timestamps; - VkResult res = vkGetQueryPoolResults(m_device, m_timestamp_query_pool, timestamp_base, static_cast(timestamps.size()), + const VkResult res = vkGetQueryPoolResults(m_device, m_timestamp_query_pool, timestamp_base, static_cast(timestamps.size()), sizeof(timestamps), timestamps.data(), sizeof(u64), VK_QUERY_RESULT_64_BIT); if (res == VK_SUCCESS) { @@ -2014,7 +2030,7 @@ void main() constexpr u64 MAX_MAX_DEVIATION = 100000; // 100µs for (int i = 0; i < 4; i++) // 4 tries to get under MAX_MAX_DEVIATION { - VkResult res = vkGetCalibratedTimestampsEXT(m_device, std::size(infos), infos, timestamps, &maxDeviation); + const VkResult res = vkGetCalibratedTimestampsEXT(m_device, std::size(infos), infos, timestamps, &maxDeviation); if (res != VK_SUCCESS) { LOG_VULKAN_ERROR(res, "vkGetCalibratedTimestampsEXT failed: "); diff --git a/common/Vulkan/Context.h b/common/Vulkan/Context.h index c9348e616a..828acbcaa9 100644 --- a/common/Vulkan/Context.h +++ b/common/Vulkan/Context.h @@ -209,6 +209,7 @@ namespace Vulkan // Was the last present submitted to the queue a failure? If so, we must recreate our swapchain. bool CheckLastPresentFail(); + bool CheckLastSubmitFail(); // Schedule a vulkan resource for destruction later on. This will occur when the command buffer // is next re-used, and the GPU has finished working with the specified resource. @@ -373,6 +374,7 @@ namespace Vulkan StreamBuffer m_texture_upload_buffer; + std::atomic_bool m_last_submit_failed{false}; std::atomic_bool m_last_present_failed{false}; std::atomic_bool m_present_done{true}; std::mutex m_present_mutex; diff --git a/pcsx2-gsrunner/Main.cpp b/pcsx2-gsrunner/Main.cpp index 1f5e2c2a75..7ad92e7486 100644 --- a/pcsx2-gsrunner/Main.cpp +++ b/pcsx2-gsrunner/Main.cpp @@ -280,7 +280,7 @@ void Host::ReleaseHostDisplay(bool clear_state) g_host_display.reset(); } -bool Host::BeginPresentFrame(bool frame_skip) +HostDisplay::PresentResult Host::BeginPresentFrame(bool frame_skip) { if (s_loop_number == 0) { @@ -291,12 +291,15 @@ bool Host::BeginPresentFrame(bool frame_skip) std::string dump_path(fmt::format("{}_frame{}.png", s_output_prefix, s_dump_frame_number)); GSQueueSnapshot(dump_path); } - if (g_host_display->BeginPresent(frame_skip)) - return true; - // don't render imgui - ImGuiManager::NewFrame(); - return false; + const HostDisplay::PresentResult result = g_host_display->BeginPresent(frame_skip); + if (result != HostDisplay::PresentResult::OK) + { + // don't render imgui + ImGuiManager::SkipFrame(); + } + + return result; } void Host::EndPresentFrame() diff --git a/pcsx2-qt/QtHost.cpp b/pcsx2-qt/QtHost.cpp index e697a988c2..51fbdb4055 100644 --- a/pcsx2-qt/QtHost.cpp +++ b/pcsx2-qt/QtHost.cpp @@ -967,17 +967,17 @@ void Host::ReleaseHostDisplay(bool clear_state) g_emu_thread->releaseHostDisplay(clear_state); } -bool Host::BeginPresentFrame(bool frame_skip) +HostDisplay::PresentResult Host::BeginPresentFrame(bool frame_skip) { - if (!g_host_display->BeginPresent(frame_skip)) + const HostDisplay::PresentResult result = g_host_display->BeginPresent(frame_skip); + if (result != HostDisplay::PresentResult::OK) { // if we're skipping a frame, we need to reset imgui's state, since // we won't be calling EndPresentFrame(). - ImGuiManager::NewFrame(); - return false; + ImGuiManager::SkipFrame(); } - return true; + return result; } void Host::EndPresentFrame() diff --git a/pcsx2/Frontend/D3D11HostDisplay.cpp b/pcsx2/Frontend/D3D11HostDisplay.cpp index 84b2ae6ced..af72cee10e 100644 --- a/pcsx2/Frontend/D3D11HostDisplay.cpp +++ b/pcsx2/Frontend/D3D11HostDisplay.cpp @@ -641,13 +641,10 @@ bool D3D11HostDisplay::UpdateImGuiFontTexture() return true; } -bool D3D11HostDisplay::BeginPresent(bool frame_skip) +HostDisplay::PresentResult D3D11HostDisplay::BeginPresent(bool frame_skip) { if (frame_skip || !m_swap_chain) - { - ImGui::EndFrame(); - return false; - } + return PresentResult::FrameSkipped; // When using vsync, the time here seems to include the time for the buffer to become available. // This blows our our GPU usage number considerably, so read the timestamp before the final blit @@ -664,7 +661,7 @@ bool D3D11HostDisplay::BeginPresent(bool frame_skip) const CD3D11_RECT scissor(0, 0, m_window_info.surface_width, m_window_info.surface_height); m_context->RSSetViewports(1, &vp); m_context->RSSetScissorRects(1, &scissor); - return true; + return PresentResult::OK; } void D3D11HostDisplay::EndPresent() diff --git a/pcsx2/Frontend/D3D11HostDisplay.h b/pcsx2/Frontend/D3D11HostDisplay.h index be65db4e94..4bfe1d2240 100644 --- a/pcsx2/Frontend/D3D11HostDisplay.h +++ b/pcsx2/Frontend/D3D11HostDisplay.h @@ -65,7 +65,7 @@ public: void SetVSync(VsyncMode mode) override; - bool BeginPresent(bool frame_skip) override; + PresentResult BeginPresent(bool frame_skip) override; void EndPresent() override; bool SetGPUTimingEnabled(bool enabled) override; diff --git a/pcsx2/Frontend/D3D12HostDisplay.cpp b/pcsx2/Frontend/D3D12HostDisplay.cpp index e40e55566a..7b5206a4c7 100644 --- a/pcsx2/Frontend/D3D12HostDisplay.cpp +++ b/pcsx2/Frontend/D3D12HostDisplay.cpp @@ -554,13 +554,13 @@ bool D3D12HostDisplay::UpdateImGuiFontTexture() return ImGui_ImplDX12_CreateFontsTexture(); } -bool D3D12HostDisplay::BeginPresent(bool frame_skip) +HostDisplay::PresentResult D3D12HostDisplay::BeginPresent(bool frame_skip) { + if (m_device_lost) + return HostDisplay::PresentResult::DeviceLost; + if (frame_skip || !m_swap_chain) - { - ImGui::EndFrame(); - return false; - } + return PresentResult::FrameSkipped; static constexpr std::array clear_color = {}; D3D12::Texture& swap_chain_buf = m_swap_chain_buffers[m_current_swap_chain_buffer]; @@ -574,7 +574,7 @@ bool D3D12HostDisplay::BeginPresent(bool frame_skip) const D3D12_RECT scissor{0, 0, static_cast(m_window_info.surface_width), static_cast(m_window_info.surface_height)}; cmdlist->RSSetViewports(1, &vp); cmdlist->RSSetScissorRects(1, &scissor); - return true; + return PresentResult::OK; } void D3D12HostDisplay::EndPresent() @@ -586,7 +586,11 @@ void D3D12HostDisplay::EndPresent() m_current_swap_chain_buffer = ((m_current_swap_chain_buffer + 1) % static_cast(m_swap_chain_buffers.size())); swap_chain_buf.TransitionToState(g_d3d12_context->GetCommandList(), D3D12_RESOURCE_STATE_PRESENT); - g_d3d12_context->ExecuteCommandList(D3D12::Context::WaitType::None); + if (!g_d3d12_context->ExecuteCommandList(D3D12::Context::WaitType::None)) + { + m_device_lost = true; + return; + } const bool vsync = static_cast(m_vsync_mode != VsyncMode::Off); if (!vsync && m_using_allow_tearing) diff --git a/pcsx2/Frontend/D3D12HostDisplay.h b/pcsx2/Frontend/D3D12HostDisplay.h index 696cece5c6..8487dd8de0 100644 --- a/pcsx2/Frontend/D3D12HostDisplay.h +++ b/pcsx2/Frontend/D3D12HostDisplay.h @@ -71,7 +71,7 @@ public: void SetVSync(VsyncMode mode) override; - bool BeginPresent(bool frame_skip) override; + PresentResult BeginPresent(bool frame_skip) override; void EndPresent() override; bool SetGPUTimingEnabled(bool enabled) override; @@ -97,4 +97,5 @@ protected: bool m_allow_tearing_supported = false; bool m_using_allow_tearing = false; + bool m_device_lost = false; }; diff --git a/pcsx2/Frontend/ImGuiManager.cpp b/pcsx2/Frontend/ImGuiManager.cpp index e721227c7c..155745ee69 100644 --- a/pcsx2/Frontend/ImGuiManager.cpp +++ b/pcsx2/Frontend/ImGuiManager.cpp @@ -229,6 +229,12 @@ void ImGuiManager::NewFrame() } } +void ImGuiManager::SkipFrame() +{ + ImGui::EndFrame(); + NewFrame(); +} + void ImGuiManager::SetStyle() { ImGuiStyle& style = ImGui::GetStyle(); diff --git a/pcsx2/Frontend/ImGuiManager.h b/pcsx2/Frontend/ImGuiManager.h index b75469c3d2..c8df60f283 100644 --- a/pcsx2/Frontend/ImGuiManager.h +++ b/pcsx2/Frontend/ImGuiManager.h @@ -40,6 +40,9 @@ namespace ImGuiManager /// Call at the beginning of the frame to set up ImGui state. void NewFrame(); + /// Call when skipping rendering a frame, to update internal state. + void SkipFrame(); + /// Renders any on-screen display elements. void RenderOSD(); diff --git a/pcsx2/Frontend/MetalHostDisplay.h b/pcsx2/Frontend/MetalHostDisplay.h index 436ca9a157..27e010675c 100644 --- a/pcsx2/Frontend/MetalHostDisplay.h +++ b/pcsx2/Frontend/MetalHostDisplay.h @@ -81,7 +81,7 @@ public: std::unique_ptr CreateTexture(u32 width, u32 height, const void* data, u32 data_stride, bool dynamic = false) override; void UpdateTexture(id texture, u32 x, u32 y, u32 width, u32 height, const void* data, u32 data_stride); void UpdateTexture(HostDisplayTexture* texture, u32 x, u32 y, u32 width, u32 height, const void* data, u32 data_stride) override; - bool BeginPresent(bool frame_skip) override; + PresentResult BeginPresent(bool frame_skip) override; void EndPresent() override; void SetVSync(VsyncMode mode) override; diff --git a/pcsx2/Frontend/MetalHostDisplay.mm b/pcsx2/Frontend/MetalHostDisplay.mm index 8e372cab9d..4bacd7a9d7 100644 --- a/pcsx2/Frontend/MetalHostDisplay.mm +++ b/pcsx2/Frontend/MetalHostDisplay.mm @@ -265,7 +265,7 @@ void MetalHostDisplay::UpdateTexture(HostDisplayTexture* texture, u32 x, u32 y, static bool s_capture_next = false; -bool MetalHostDisplay::BeginPresent(bool frame_skip) +HostDisplay::PresentResult MetalHostDisplay::BeginPresent(bool frame_skip) { @autoreleasepool { GSDeviceMTL* dev = static_cast(g_gs_device.get()); if (dev && m_capture_start_frame && dev->FrameNo() == m_capture_start_frame) @@ -273,7 +273,7 @@ bool MetalHostDisplay::BeginPresent(bool frame_skip) if (frame_skip || m_window_info.type == WindowInfo::Type::Surfaceless || !g_gs_device) { ImGui::EndFrame(); - return false; + return PresentResult::FrameSkipped; } id buf = dev->GetRenderCmdBuf(); m_current_drawable = MRCRetain([m_layer nextDrawable]); @@ -284,13 +284,13 @@ bool MetalHostDisplay::BeginPresent(bool frame_skip) [buf popDebugGroup]; dev->FlushEncoders(); ImGui::EndFrame(); - return false; + return PresentResult::FrameSkipped; } [m_pass_desc colorAttachments][0].texture = [m_current_drawable texture]; id enc = [buf renderCommandEncoderWithDescriptor:m_pass_desc]; [enc setLabel:@"Present"]; dev->m_current_render.encoder = MRCRetain(enc); - return true; + return PresentResult::OK; }} void MetalHostDisplay::EndPresent() diff --git a/pcsx2/Frontend/OpenGLHostDisplay.cpp b/pcsx2/Frontend/OpenGLHostDisplay.cpp index 373a85b9a0..4ce0a8c15b 100644 --- a/pcsx2/Frontend/OpenGLHostDisplay.cpp +++ b/pcsx2/Frontend/OpenGLHostDisplay.cpp @@ -335,13 +335,10 @@ bool OpenGLHostDisplay::UpdateImGuiFontTexture() return ImGui_ImplOpenGL3_CreateFontsTexture(); } -bool OpenGLHostDisplay::BeginPresent(bool frame_skip) +HostDisplay::PresentResult OpenGLHostDisplay::BeginPresent(bool frame_skip) { if (frame_skip || m_window_info.type == WindowInfo::Type::Surfaceless) - { - ImGui::EndFrame(); - return false; - } + return PresentResult::FrameSkipped; glDisable(GL_SCISSOR_TEST); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); @@ -350,7 +347,7 @@ bool OpenGLHostDisplay::BeginPresent(bool frame_skip) glClear(GL_COLOR_BUFFER_BIT); glViewport(0, 0, m_window_info.surface_width, m_window_info.surface_height); - return true; + return PresentResult::OK; } void OpenGLHostDisplay::EndPresent() diff --git a/pcsx2/Frontend/OpenGLHostDisplay.h b/pcsx2/Frontend/OpenGLHostDisplay.h index 20469fe3e9..07f5efb17f 100644 --- a/pcsx2/Frontend/OpenGLHostDisplay.h +++ b/pcsx2/Frontend/OpenGLHostDisplay.h @@ -58,7 +58,7 @@ public: void SetVSync(VsyncMode mode) override; - bool BeginPresent(bool frame_skip) override; + PresentResult BeginPresent(bool frame_skip) override; void EndPresent() override; bool SetGPUTimingEnabled(bool enabled) override; diff --git a/pcsx2/Frontend/VulkanHostDisplay.cpp b/pcsx2/Frontend/VulkanHostDisplay.cpp index 2d54066f17..e34200e255 100644 --- a/pcsx2/Frontend/VulkanHostDisplay.cpp +++ b/pcsx2/Frontend/VulkanHostDisplay.cpp @@ -339,17 +339,18 @@ bool VulkanHostDisplay::DoneCurrent() return true; } -bool VulkanHostDisplay::BeginPresent(bool frame_skip) +HostDisplay::PresentResult VulkanHostDisplay::BeginPresent(bool frame_skip) { if (frame_skip || !m_swap_chain) - { - ImGui::EndFrame(); - return false; - } + return PresentResult::FrameSkipped; // Previous frame needs to be presented before we can acquire the swap chain. g_vulkan_context->WaitForPresentComplete(); + // Check if the device was lost. + if (g_vulkan_context->CheckLastSubmitFail()) + return PresentResult::DeviceLost; + VkResult res = m_swap_chain->AcquireNextImage(); if (res != VK_SUCCESS) { @@ -367,7 +368,7 @@ bool VulkanHostDisplay::BeginPresent(bool frame_skip) { Console.Error("Failed to recreate surface after loss"); g_vulkan_context->ExecuteCommandBuffer(Vulkan::Context::WaitType::None); - return false; + return PresentResult::FrameSkipped; } res = m_swap_chain->AcquireNextImage(); @@ -380,7 +381,7 @@ bool VulkanHostDisplay::BeginPresent(bool frame_skip) // Still submit the command buffer, otherwise we'll end up with several frames waiting. LOG_VULKAN_ERROR(res, "vkAcquireNextImageKHR() failed: "); g_vulkan_context->ExecuteCommandBuffer(Vulkan::Context::WaitType::None); - return false; + return PresentResult::FrameSkipped; } } @@ -401,7 +402,7 @@ bool VulkanHostDisplay::BeginPresent(bool frame_skip) const VkRect2D scissor{{0, 0}, {static_cast(swap_chain_texture.GetWidth()), static_cast(swap_chain_texture.GetHeight())}}; vkCmdSetViewport(g_vulkan_context->GetCurrentCommandBuffer(), 0, 1, &vp); vkCmdSetScissor(g_vulkan_context->GetCurrentCommandBuffer(), 0, 1, &scissor); - return true; + return PresentResult::OK; } void VulkanHostDisplay::EndPresent() diff --git a/pcsx2/Frontend/VulkanHostDisplay.h b/pcsx2/Frontend/VulkanHostDisplay.h index 91d29c984b..607183d902 100644 --- a/pcsx2/Frontend/VulkanHostDisplay.h +++ b/pcsx2/Frontend/VulkanHostDisplay.h @@ -47,7 +47,7 @@ public: void SetVSync(VsyncMode mode) override; - bool BeginPresent(bool frame_skip) override; + PresentResult BeginPresent(bool frame_skip) override; void EndPresent() override; bool SetGPUTimingEnabled(bool enabled) override; diff --git a/pcsx2/GS/GS.cpp b/pcsx2/GS/GS.cpp index 7abec9daa0..3d1d48bd91 100644 --- a/pcsx2/GS/GS.cpp +++ b/pcsx2/GS/GS.cpp @@ -209,15 +209,15 @@ static bool DoGSOpen(GSRendererType renderer, u8* basemem) return false; } - try + if (!g_gs_device->Create()) { - if (!g_gs_device->Create()) - { - g_gs_device->Destroy(); - g_gs_device.reset(); - return false; - } + g_gs_device->Destroy(); + g_gs_device.reset(); + return false; + } + if (!g_gs_renderer) + { if (renderer == GSRendererType::Null) { g_gs_renderer = std::make_unique(); @@ -230,55 +230,67 @@ static bool DoGSOpen(GSRendererType renderer, u8* basemem) { g_gs_renderer = std::unique_ptr(MULTI_ISA_SELECT(makeGSRendererSW)(GSConfig.SWExtraThreads)); } + + g_gs_renderer->SetRegsMem(basemem); } - catch (std::exception& ex) + else { - Host::ReportFormattedErrorAsync("GS", "GS error: Exception caught in GSopen: %s", ex.what()); - g_gs_renderer.reset(); - g_gs_device->Destroy(); - g_gs_device.reset(); - return false; + Console.Warning("(DoGSOpen) Using existing renderer."); } GSConfig.OsdShowGPU = EmuConfig.GS.OsdShowGPU && g_host_display->SetGPUTimingEnabled(true); - g_gs_renderer->SetRegsMem(basemem); g_perfmon.Reset(); return true; } -bool GSreopen(bool recreate_display, const Pcsx2Config::GSOptions& old_config) +bool GSreopen(bool recreate_display, bool recreate_renderer, const Pcsx2Config::GSOptions& old_config) { Console.WriteLn("Reopening GS with %s display", recreate_display ? "new" : "existing"); - g_gs_renderer->Flush(GSState::GSFlushReason::GSREOPEN); + if (recreate_renderer) + g_gs_renderer->Flush(GSState::GSFlushReason::GSREOPEN); freezeData fd = {}; - if (g_gs_renderer->Freeze(&fd, true) != 0) + std::unique_ptr fd_data; + if (recreate_renderer) { - Console.Error("(GSreopen) Failed to get GS freeze size"); - return false; - } + if (g_gs_renderer->Freeze(&fd, true) != 0) + { + Console.Error("(GSreopen) Failed to get GS freeze size"); + return false; + } - std::unique_ptr fd_data = std::make_unique(fd.size); - fd.data = fd_data.get(); - if (g_gs_renderer->Freeze(&fd, false) != 0) + fd_data = std::make_unique(fd.size); + fd.data = fd_data.get(); + if (g_gs_renderer->Freeze(&fd, false) != 0) + { + Console.Error("(GSreopen) Failed to freeze GS"); + return false; + } + } + else { - Console.Error("(GSreopen) Failed to freeze GS"); - return false; + // Make sure nothing is left over. + g_gs_renderer->PurgePool(); + g_gs_renderer->PurgeTextureCache(); } if (recreate_display) { g_gs_device->ResetAPIState(); - if (Host::BeginPresentFrame(true)) + if (Host::BeginPresentFrame(false) == HostDisplay::PresentResult::OK) Host::EndPresentFrame(); } u8* basemem = g_gs_renderer->GetRegsMem(); const u32 gamecrc = g_gs_renderer->GetGameCRC(); - g_gs_renderer->Destroy(); - g_gs_renderer.reset(); + if (recreate_renderer) + { + g_gs_renderer->Destroy(); + g_gs_renderer.reset(); + } + g_gs_device->Destroy(); g_gs_device.reset(); @@ -327,13 +339,17 @@ bool GSreopen(bool recreate_display, const Pcsx2Config::GSOptions& old_config) } } - if (g_gs_renderer->Defrost(&fd) != 0) + if (recreate_renderer) { - Console.Error("(GSreopen) Failed to defrost"); - return false; + if (g_gs_renderer->Defrost(&fd) != 0) + { + Console.Error("(GSreopen) Failed to defrost"); + return false; + } + + g_gs_renderer->SetGameCRC(gamecrc); } - g_gs_renderer->SetGameCRC(gamecrc); return true; } @@ -698,7 +714,7 @@ void GSUpdateConfig(const Pcsx2Config::GSOptions& new_config) GSConfig.DisableShaderCache != old_config.DisableShaderCache || GSConfig.DisableThreadedPresentation != old_config.DisableThreadedPresentation ); - if (!GSreopen(do_full_restart, old_config)) + if (!GSreopen(do_full_restart, true, old_config)) pxFailRel("Failed to do full GS reopen"); return; } @@ -709,7 +725,7 @@ void GSUpdateConfig(const Pcsx2Config::GSOptions& new_config) GSConfig.SWExtraThreads != old_config.SWExtraThreads || GSConfig.SWExtraThreadsHeight != old_config.SWExtraThreadsHeight) { - if (!GSreopen(false, old_config)) + if (!GSreopen(false, true, old_config)) pxFailRel("Failed to do quick GS reopen"); return; @@ -787,7 +803,7 @@ void GSSwitchRenderer(GSRendererType new_renderer) const bool recreate_display = (!is_software_switch && existing_api != GetAPIForRenderer(new_renderer)); const Pcsx2Config::GSOptions old_config(GSConfig); GSConfig.Renderer = new_renderer; - if (!GSreopen(recreate_display, old_config)) + if (!GSreopen(recreate_display, true, old_config)) pxFailRel("Failed to reopen GS for renderer switch."); } diff --git a/pcsx2/GS/GS.h b/pcsx2/GS/GS.h index a82b67000c..a58850d3cd 100644 --- a/pcsx2/GS/GS.h +++ b/pcsx2/GS/GS.h @@ -53,7 +53,7 @@ s16 GSLookupBeforeDrawFunctionId(const std::string_view& name); int GSinit(); void GSshutdown(); bool GSopen(const Pcsx2Config::GSOptions& config, GSRendererType renderer, u8* basemem); -bool GSreopen(bool recreate_display, const Pcsx2Config::GSOptions& old_config); +bool GSreopen(bool recreate_display, bool recreate_renderer, const Pcsx2Config::GSOptions& old_config); void GSreset(bool hardware_reset); void GSclose(); void GSgifSoftReset(u32 mask); diff --git a/pcsx2/GS/Renderers/Common/GSRenderer.cpp b/pcsx2/GS/Renderers/Common/GSRenderer.cpp index a16b15fc62..a93af88802 100644 --- a/pcsx2/GS/Renderers/Common/GSRenderer.cpp +++ b/pcsx2/GS/Renderers/Common/GSRenderer.cpp @@ -50,6 +50,8 @@ std::unique_ptr g_gs_renderer; // we might be switching while the other thread reads it. static GSVector4 s_last_draw_rect; +// Last time we reset the renderer due to a GPU crash, if any. +static Common::Timer::Value s_last_gpu_reset_time; GSRenderer::GSRenderer() : m_shader_time_start(Common::Timer::GetCurrentValue()) @@ -602,6 +604,40 @@ void GSJoinSnapshotThreads() } } +bool GSRenderer::BeginPresentFrame(bool frame_skip) +{ + const HostDisplay::PresentResult result = Host::BeginPresentFrame(frame_skip); + if (result == HostDisplay::PresentResult::OK) + return true; + else if (result == HostDisplay::PresentResult::FrameSkipped) + return false; + + // If we're constantly crashing on something in particular, we don't want to end up in an + // endless reset loop.. that'd probably end up leaking memory and/or crashing us for other + // reasons. So just abort in such case. + const Common::Timer::Value current_time = Common::Timer::GetCurrentValue(); + if (s_last_gpu_reset_time != 0 && + Common::Timer::ConvertValueToSeconds(current_time - s_last_gpu_reset_time) < 15.0f) + { + pxFailRel("Host GPU lost too many times, device is probably completely wedged."); + } + s_last_gpu_reset_time = current_time; + + // Device lost, something went really bad. + // Let's just toss out everything, and try to hobble on. + if (!GSreopen(true, false, GSConfig)) + { + pxFailRel("Failed to recreate GS device after loss."); + return false; + } + + // First frame after reopening is definitely going to be trash, so skip it. + Host::AddIconOSDMessage("GSDeviceLost", ICON_FA_EXCLAMATION_TRIANGLE, + "Host GPU device encountered an error and was recovered. This may have broken rendering.", + Host::OSD_CRITICAL_ERROR_DURATION); + return false; +} + void GSRenderer::VSync(u32 field, bool registers_written) { Flush(GSFlushReason::VSYNC); @@ -647,7 +683,7 @@ void GSRenderer::VSync(u32 field, bool registers_written) if (skip_frame) { g_gs_device->ResetAPIState(); - if (Host::BeginPresentFrame(true)) + if (BeginPresentFrame(true)) Host::EndPresentFrame(); g_gs_device->RestoreAPIState(); PerformanceMetrics::Update(registers_written, fb_sprite_frame, true); @@ -694,7 +730,7 @@ void GSRenderer::VSync(u32 field, bool registers_written) } g_gs_device->ResetAPIState(); - if (Host::BeginPresentFrame(false)) + if (BeginPresentFrame(false)) { if (current && !blank_frame) { @@ -910,7 +946,7 @@ void GSRenderer::StopGSDump() void GSRenderer::PresentCurrentFrame() { g_gs_device->ResetAPIState(); - if (Host::BeginPresentFrame(false)) + if (BeginPresentFrame(false)) { GSTexture* current = g_gs_device->GetCurrent(); if (current) diff --git a/pcsx2/GS/Renderers/Common/GSRenderer.h b/pcsx2/GS/Renderers/Common/GSRenderer.h index 553dc3657f..b50e80a25e 100644 --- a/pcsx2/GS/Renderers/Common/GSRenderer.h +++ b/pcsx2/GS/Renderers/Common/GSRenderer.h @@ -23,6 +23,7 @@ class GSRenderer : public GSState { private: bool Merge(int field); + bool BeginPresentFrame(bool frame_skip); u64 m_shader_time_start = 0; diff --git a/pcsx2/HostDisplay.h b/pcsx2/HostDisplay.h index a3c6af85d1..e8befe59a4 100644 --- a/pcsx2/HostDisplay.h +++ b/pcsx2/HostDisplay.h @@ -60,6 +60,13 @@ public: RightOrBottom }; + enum class PresentResult + { + OK, + FrameSkipped, + DeviceLost + }; + struct AdapterAndModeList { std::vector adapter_names; @@ -137,7 +144,7 @@ public: /// Returns false if the window was completely occluded. If frame_skip is set, the frame won't be /// displayed, but the GPU command queue will still be flushed. - virtual bool BeginPresent(bool frame_skip) = 0; + virtual PresentResult BeginPresent(bool frame_skip) = 0; /// Presents the frame to the display, and renders OSD elements. virtual void EndPresent() = 0; @@ -184,7 +191,7 @@ namespace Host /// Returns false if the window was completely occluded. If frame_skip is set, the frame won't be /// displayed, but the GPU command queue will still be flushed. - bool BeginPresentFrame(bool frame_skip); + HostDisplay::PresentResult BeginPresentFrame(bool frame_skip); /// Presents the frame to the display, and renders OSD elements. void EndPresentFrame(); diff --git a/tests/ctest/core/StubHost.cpp b/tests/ctest/core/StubHost.cpp index db2edde932..74768ecad1 100644 --- a/tests/ctest/core/StubHost.cpp +++ b/tests/ctest/core/StubHost.cpp @@ -114,9 +114,9 @@ void Host::ReleaseHostDisplay(bool clear_state) { } -bool Host::BeginPresentFrame(bool frame_skip) +HostDisplay::PresentResult Host::BeginPresentFrame(bool frame_skip) { - return false; + return HostDisplay::PresentResult::FrameSkipped; } void Host::EndPresentFrame()