GS: Attempt to recreate device if GPU crashes

common/D3D12/Context.cpp

@@ -445,7 +445,7 @@ ID3D12GraphicsCommandList4* Context::GetInitCommandList()
 	return res.command_lists[0].get();
 }
 
-void Context::ExecuteCommandList(WaitType wait_for_completion)
+bool Context::ExecuteCommandList(WaitType wait_for_completion)
 {
 	CommandListResources& res = m_command_lists[m_current_command_list];
 	HRESULT hr;
@@ -463,12 +463,21 @@ void Context::ExecuteCommandList(WaitType wait_for_completion)
 	if (res.init_command_list_used)
 	{
 		hr = res.command_lists[0]->Close();
-		pxAssertRel(SUCCEEDED(hr), "Close init command list");
+		if (FAILED(hr))
+		{
+			Console.Error("Closing init command list failed with HRESULT %08X", hr);
+			return false;
+		}
 	}
 
 	// Close and queue command list.
 	hr = res.command_lists[1]->Close();
-	pxAssertRel(SUCCEEDED(hr), "Close command list");
+	if (FAILED(hr))
+	{
+		Console.Error("Closing main command list failed with HRESULT %08X", hr);
+		return false;
+	}
+
 	if (res.init_command_list_used)
 	{
 		const std::array<ID3D12CommandList*, 2> execute_lists{res.command_lists[0].get(), res.command_lists[1].get()};
@@ -487,6 +496,8 @@ void Context::ExecuteCommandList(WaitType wait_for_completion)
 	MoveToNextCommandList();
 	if (wait_for_completion != WaitType::None)
 		WaitForFence(res.ready_fence_value, wait_for_completion == WaitType::Spin);
+
+	return true;
 }
 
 void Context::InvalidateSamplerGroups()

common/D3D12/Context.h

@@ -130,7 +130,7 @@ namespace D3D12
 		};
 
 		/// Executes the current command list.
-		void ExecuteCommandList(WaitType wait_for_completion);
+		bool ExecuteCommandList(WaitType wait_for_completion);
 
 		/// Waits for a specific fence.
 		void WaitForFence(u64 fence, bool spin);

common/Vulkan/Context.cpp

@@ -1122,9 +1122,13 @@ namespace Vulkan
 	void Context::WaitForCommandBufferCompletion(u32 index)
 	{
 		// Wait for this command buffer to be completed.
-		VkResult res = vkWaitForFences(m_device, 1, &m_frame_resources[index].fence, VK_TRUE, UINT64_MAX);
+		const VkResult res = vkWaitForFences(m_device, 1, &m_frame_resources[index].fence, VK_TRUE, UINT64_MAX);
 		if (res != VK_SUCCESS)
+		{
 			LOG_VULKAN_ERROR(res, "vkWaitForFences failed: ");
+			m_last_submit_failed.store(true, std::memory_order_release);
+			return;
+		}
 
 		// Clean up any resources for command buffers between the last known completed buffer and this
 		// now-completed command buffer. If we use >2 buffers, this may be more than one buffer.
@@ -1266,11 +1270,12 @@ namespace Vulkan
 			submit_info.pSignalSemaphores = &m_spin_resources[index].semaphore;
 		}
 
-		VkResult res = vkQueueSubmit(m_graphics_queue, 1, &submit_info, resources.fence);
+		const VkResult res = vkQueueSubmit(m_graphics_queue, 1, &submit_info, resources.fence);
 		if (res != VK_SUCCESS)
 		{
 			LOG_VULKAN_ERROR(res, "vkQueueSubmit failed: ");
-			pxFailRel("Failed to submit command buffer.");
+			m_last_submit_failed.store(true, std::memory_order_release);
+			return;
 		}
 
 		if (spin_cycles != 0)
@@ -1286,14 +1291,14 @@ namespace Vulkan
 
 		present_swap_chain->ReleaseCurrentImage();
 
-		VkResult res = vkQueuePresentKHR(m_present_queue, &present_info);
+		const VkResult res = vkQueuePresentKHR(m_present_queue, &present_info);
 		if (res != VK_SUCCESS)
 		{
 			// VK_ERROR_OUT_OF_DATE_KHR is not fatal, just means we need to recreate our swap chain.
 			if (res != VK_ERROR_OUT_OF_DATE_KHR && res != VK_SUBOPTIMAL_KHR)
 				LOG_VULKAN_ERROR(res, "vkQueuePresentKHR failed: ");
 
-			m_last_present_failed.store(true);
+			m_last_present_failed.store(true, std::memory_order_release);
 			return;
 		}
 
@@ -1460,6 +1465,9 @@ namespace Vulkan
 
 	void Context::ExecuteCommandBuffer(WaitType wait_for_completion)
 	{
+		if (m_last_submit_failed.load(std::memory_order_acquire))
+			return;
+
 		// If we're waiting for completion, don't bother waking the worker thread.
 		const u32 current_frame = m_current_frame;
 		SubmitCommandBuffer();
@@ -1481,9 +1489,12 @@ namespace Vulkan
 
 	bool Context::CheckLastPresentFail()
 	{
-		bool res = m_last_present_failed;
+		return m_last_present_failed.exchange(false, std::memory_order_acq_rel);
-		m_last_present_failed = false;
+	}
-		return res;
+
+	bool Context::CheckLastSubmitFail()
+	{
+		return m_last_submit_failed.load(std::memory_order_acquire);
 	}
 
 	void Context::DeferBufferDestruction(VkBuffer object)
@@ -1596,7 +1607,7 @@ namespace Vulkan
 				VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT,
 			DebugMessengerCallback, nullptr};
 
-		VkResult res =
+		const VkResult res =
 			vkCreateDebugUtilsMessengerEXT(m_instance, &messenger_info, nullptr, &m_debug_messenger_callback);
 		if (res != VK_SUCCESS)
 		{
@@ -1688,7 +1699,7 @@ namespace Vulkan
 			subpass_dependency_ptr};
 
 		VkRenderPass pass;
-		VkResult res = vkCreateRenderPass(m_device, &pass_info, nullptr, &pass);
+		const VkResult res = vkCreateRenderPass(m_device, &pass_info, nullptr, &pass);
 		if (res != VK_SUCCESS)
 		{
 			LOG_VULKAN_ERROR(res, "vkCreateRenderPass failed: ");
@@ -1894,9 +1905,14 @@ void main()
 		SpinResources& resources = m_spin_resources[index];
 		if (!resources.in_progress)
 			return;
-		VkResult res = vkWaitForFences(m_device, 1, &resources.fence, VK_TRUE, UINT64_MAX);
+
+		const VkResult res = vkWaitForFences(m_device, 1, &resources.fence, VK_TRUE, UINT64_MAX);
 		if (res != VK_SUCCESS)
+		{
 			LOG_VULKAN_ERROR(res, "vkWaitForFences failed: ");
+			m_last_submit_failed.store(true, std::memory_order_release);
+			return;
+		}
 		SpinCommandCompleted(index);
 	}
 
@@ -1906,7 +1922,7 @@ void main()
 		resources.in_progress = false;
 		const u32 timestamp_base = (index + NUM_COMMAND_BUFFERS) * 2;
 		std::array<u64, 2> timestamps;
-		VkResult res = vkGetQueryPoolResults(m_device, m_timestamp_query_pool, timestamp_base, static_cast<u32>(timestamps.size()),
+		const VkResult res = vkGetQueryPoolResults(m_device, m_timestamp_query_pool, timestamp_base, static_cast<u32>(timestamps.size()),
 			sizeof(timestamps), timestamps.data(), sizeof(u64), VK_QUERY_RESULT_64_BIT);
 		if (res == VK_SUCCESS)
 		{
@@ -2014,7 +2030,7 @@ void main()
 		constexpr u64 MAX_MAX_DEVIATION = 100000; // 100µs
 		for (int i = 0; i < 4; i++) // 4 tries to get under MAX_MAX_DEVIATION
 		{
-			VkResult res = vkGetCalibratedTimestampsEXT(m_device, std::size(infos), infos, timestamps, &maxDeviation);
+			const VkResult res = vkGetCalibratedTimestampsEXT(m_device, std::size(infos), infos, timestamps, &maxDeviation);
 			if (res != VK_SUCCESS)
 			{
 				LOG_VULKAN_ERROR(res, "vkGetCalibratedTimestampsEXT failed: ");

common/Vulkan/Context.h

@@ -209,6 +209,7 @@ namespace Vulkan
 
 		// Was the last present submitted to the queue a failure? If so, we must recreate our swapchain.
 		bool CheckLastPresentFail();
+		bool CheckLastSubmitFail();
 
 		// Schedule a vulkan resource for destruction later on. This will occur when the command buffer
 		// is next re-used, and the GPU has finished working with the specified resource.
@@ -373,6 +374,7 @@ namespace Vulkan
 
 		StreamBuffer m_texture_upload_buffer;
 
+		std::atomic_bool m_last_submit_failed{false};
 		std::atomic_bool m_last_present_failed{false};
 		std::atomic_bool m_present_done{true};
 		std::mutex m_present_mutex;

pcsx2-gsrunner/Main.cpp

@@ -280,7 +280,7 @@ void Host::ReleaseHostDisplay(bool clear_state)
 	g_host_display.reset();
 }
 
-bool Host::BeginPresentFrame(bool frame_skip)
+HostDisplay::PresentResult Host::BeginPresentFrame(bool frame_skip)
 {
 	if (s_loop_number == 0)
 	{
@@ -291,12 +291,15 @@ bool Host::BeginPresentFrame(bool frame_skip)
 		std::string dump_path(fmt::format("{}_frame{}.png", s_output_prefix, s_dump_frame_number));
 		GSQueueSnapshot(dump_path);
 	}
-	if (g_host_display->BeginPresent(frame_skip))
-		return true;
 
-	// don't render imgui
+	const HostDisplay::PresentResult result = g_host_display->BeginPresent(frame_skip);
-	ImGuiManager::NewFrame();
+	if (result != HostDisplay::PresentResult::OK)
-	return false;
+	{
+		// don't render imgui
+		ImGuiManager::SkipFrame();
+	}
+
+	return result;
 }
 
 void Host::EndPresentFrame()

pcsx2-qt/QtHost.cpp

@@ -967,17 +967,17 @@ void Host::ReleaseHostDisplay(bool clear_state)
 	g_emu_thread->releaseHostDisplay(clear_state);
 }
 
-bool Host::BeginPresentFrame(bool frame_skip)
+HostDisplay::PresentResult Host::BeginPresentFrame(bool frame_skip)
 {
-	if (!g_host_display->BeginPresent(frame_skip))
+	const HostDisplay::PresentResult result = g_host_display->BeginPresent(frame_skip);
+	if (result != HostDisplay::PresentResult::OK)
 	{
 		// if we're skipping a frame, we need to reset imgui's state, since
 		// we won't be calling EndPresentFrame().
-		ImGuiManager::NewFrame();
+		ImGuiManager::SkipFrame();
-		return false;
 	}
 
-	return true;
+	return result;
 }
 
 void Host::EndPresentFrame()

pcsx2/Frontend/D3D11HostDisplay.cpp

@@ -641,13 +641,10 @@ bool D3D11HostDisplay::UpdateImGuiFontTexture()
 	return true;
 }
 
-bool D3D11HostDisplay::BeginPresent(bool frame_skip)
+HostDisplay::PresentResult D3D11HostDisplay::BeginPresent(bool frame_skip)
 {
 	if (frame_skip || !m_swap_chain)
-	{
+		return PresentResult::FrameSkipped;
-		ImGui::EndFrame();
-		return false;
-	}
 
 	// When using vsync, the time here seems to include the time for the buffer to become available.
 	// This blows our our GPU usage number considerably, so read the timestamp before the final blit
@@ -664,7 +661,7 @@ bool D3D11HostDisplay::BeginPresent(bool frame_skip)
 	const CD3D11_RECT scissor(0, 0, m_window_info.surface_width, m_window_info.surface_height);
 	m_context->RSSetViewports(1, &vp);
 	m_context->RSSetScissorRects(1, &scissor);
-	return true;
+	return PresentResult::OK;
 }
 
 void D3D11HostDisplay::EndPresent()

pcsx2/Frontend/D3D11HostDisplay.h

@@ -65,7 +65,7 @@ public:
 
 	void SetVSync(VsyncMode mode) override;
 
-	bool BeginPresent(bool frame_skip) override;
+	PresentResult BeginPresent(bool frame_skip) override;
 	void EndPresent() override;
 
 	bool SetGPUTimingEnabled(bool enabled) override;

pcsx2/Frontend/D3D12HostDisplay.cpp

@@ -554,13 +554,13 @@ bool D3D12HostDisplay::UpdateImGuiFontTexture()
 	return ImGui_ImplDX12_CreateFontsTexture();
 }
 
-bool D3D12HostDisplay::BeginPresent(bool frame_skip)
+HostDisplay::PresentResult D3D12HostDisplay::BeginPresent(bool frame_skip)
 {
+	if (m_device_lost)
+		return HostDisplay::PresentResult::DeviceLost;
+
 	if (frame_skip || !m_swap_chain)
-	{
+		return PresentResult::FrameSkipped;
-		ImGui::EndFrame();
-		return false;
-	}
 
 	static constexpr std::array<float, 4> clear_color = {};
 	D3D12::Texture& swap_chain_buf = m_swap_chain_buffers[m_current_swap_chain_buffer];
@@ -574,7 +574,7 @@ bool D3D12HostDisplay::BeginPresent(bool frame_skip)
 	const D3D12_RECT scissor{0, 0, static_cast<LONG>(m_window_info.surface_width), static_cast<LONG>(m_window_info.surface_height)};
 	cmdlist->RSSetViewports(1, &vp);
 	cmdlist->RSSetScissorRects(1, &scissor);
-	return true;
+	return PresentResult::OK;
 }
 
 void D3D12HostDisplay::EndPresent()
@@ -586,7 +586,11 @@ void D3D12HostDisplay::EndPresent()
 	m_current_swap_chain_buffer = ((m_current_swap_chain_buffer + 1) % static_cast<u32>(m_swap_chain_buffers.size()));
 
 	swap_chain_buf.TransitionToState(g_d3d12_context->GetCommandList(), D3D12_RESOURCE_STATE_PRESENT);
-	g_d3d12_context->ExecuteCommandList(D3D12::Context::WaitType::None);
+	if (!g_d3d12_context->ExecuteCommandList(D3D12::Context::WaitType::None))
+	{
+		m_device_lost = true;
+		return;
+	}
 
 	const bool vsync = static_cast<UINT>(m_vsync_mode != VsyncMode::Off);
 	if (!vsync && m_using_allow_tearing)

pcsx2/Frontend/D3D12HostDisplay.h

@@ -71,7 +71,7 @@ public:
 
 	void SetVSync(VsyncMode mode) override;
 
-	bool BeginPresent(bool frame_skip) override;
+	PresentResult BeginPresent(bool frame_skip) override;
 	void EndPresent() override;
 
 	bool SetGPUTimingEnabled(bool enabled) override;
@@ -97,4 +97,5 @@ protected:
 
 	bool m_allow_tearing_supported = false;
 	bool m_using_allow_tearing = false;
+	bool m_device_lost = false;
 };

pcsx2/Frontend/ImGuiManager.cpp

@@ -229,6 +229,12 @@ void ImGuiManager::NewFrame()
 	}
 }
 
+void ImGuiManager::SkipFrame()
+{
+	ImGui::EndFrame();
+	NewFrame();
+}
+
 void ImGuiManager::SetStyle()
 {
 	ImGuiStyle& style = ImGui::GetStyle();

pcsx2/Frontend/ImGuiManager.h

@@ -40,6 +40,9 @@ namespace ImGuiManager
 	/// Call at the beginning of the frame to set up ImGui state.
 	void NewFrame();
 
+	/// Call when skipping rendering a frame, to update internal state.
+	void SkipFrame();
+
 	/// Renders any on-screen display elements.
 	void RenderOSD();
 

pcsx2/Frontend/MetalHostDisplay.h

@@ -81,7 +81,7 @@ public:
 	std::unique_ptr<HostDisplayTexture> CreateTexture(u32 width, u32 height, const void* data, u32 data_stride, bool dynamic = false) override;
 	void UpdateTexture(id<MTLTexture> texture, u32 x, u32 y, u32 width, u32 height, const void* data, u32 data_stride);
 	void UpdateTexture(HostDisplayTexture* texture, u32 x, u32 y, u32 width, u32 height, const void* data, u32 data_stride) override;
-	bool BeginPresent(bool frame_skip) override;
+	PresentResult BeginPresent(bool frame_skip) override;
 	void EndPresent() override;
 	void SetVSync(VsyncMode mode) override;
 

pcsx2/Frontend/MetalHostDisplay.mm

@@ -265,7 +265,7 @@ void MetalHostDisplay::UpdateTexture(HostDisplayTexture* texture, u32 x, u32 y,
 
 static bool s_capture_next = false;
 
-bool MetalHostDisplay::BeginPresent(bool frame_skip)
+HostDisplay::PresentResult MetalHostDisplay::BeginPresent(bool frame_skip)
 { @autoreleasepool {
 	GSDeviceMTL* dev = static_cast<GSDeviceMTL*>(g_gs_device.get());
 	if (dev && m_capture_start_frame && dev->FrameNo() == m_capture_start_frame)
@@ -273,7 +273,7 @@ bool MetalHostDisplay::BeginPresent(bool frame_skip)
 	if (frame_skip || m_window_info.type == WindowInfo::Type::Surfaceless || !g_gs_device)
 	{
 		ImGui::EndFrame();
-		return false;
+		return PresentResult::FrameSkipped;
 	}
 	id<MTLCommandBuffer> buf = dev->GetRenderCmdBuf();
 	m_current_drawable = MRCRetain([m_layer nextDrawable]);
@@ -284,13 +284,13 @@ bool MetalHostDisplay::BeginPresent(bool frame_skip)
 		[buf popDebugGroup];
 		dev->FlushEncoders();
 		ImGui::EndFrame();
-		return false;
+		return PresentResult::FrameSkipped;
 	}
 	[m_pass_desc colorAttachments][0].texture = [m_current_drawable texture];
 	id<MTLRenderCommandEncoder> enc = [buf renderCommandEncoderWithDescriptor:m_pass_desc];
 	[enc setLabel:@"Present"];
 	dev->m_current_render.encoder = MRCRetain(enc);
-	return true;
+	return PresentResult::OK;
 }}
 
 void MetalHostDisplay::EndPresent()

pcsx2/Frontend/OpenGLHostDisplay.cpp

@@ -335,13 +335,10 @@ bool OpenGLHostDisplay::UpdateImGuiFontTexture()
 	return ImGui_ImplOpenGL3_CreateFontsTexture();
 }
 
-bool OpenGLHostDisplay::BeginPresent(bool frame_skip)
+HostDisplay::PresentResult OpenGLHostDisplay::BeginPresent(bool frame_skip)
 {
 	if (frame_skip || m_window_info.type == WindowInfo::Type::Surfaceless)
-	{
+		return PresentResult::FrameSkipped;
-		ImGui::EndFrame();
-		return false;
-	}
 
 	glDisable(GL_SCISSOR_TEST);
 	glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
@@ -350,7 +347,7 @@ bool OpenGLHostDisplay::BeginPresent(bool frame_skip)
 	glClear(GL_COLOR_BUFFER_BIT);
 	glViewport(0, 0, m_window_info.surface_width, m_window_info.surface_height);
 
-	return true;
+	return PresentResult::OK;
 }
 
 void OpenGLHostDisplay::EndPresent()

pcsx2/Frontend/OpenGLHostDisplay.h

@@ -58,7 +58,7 @@ public:
 
 	void SetVSync(VsyncMode mode) override;
 
-	bool BeginPresent(bool frame_skip) override;
+	PresentResult BeginPresent(bool frame_skip) override;
 	void EndPresent() override;
 
 	bool SetGPUTimingEnabled(bool enabled) override;

pcsx2/Frontend/VulkanHostDisplay.cpp

@@ -339,17 +339,18 @@ bool VulkanHostDisplay::DoneCurrent()
 	return true;
 }
 
-bool VulkanHostDisplay::BeginPresent(bool frame_skip)
+HostDisplay::PresentResult VulkanHostDisplay::BeginPresent(bool frame_skip)
 {
 	if (frame_skip || !m_swap_chain)
-	{
+		return PresentResult::FrameSkipped;
-		ImGui::EndFrame();
-		return false;
-	}
 
 	// Previous frame needs to be presented before we can acquire the swap chain.
 	g_vulkan_context->WaitForPresentComplete();
 
+	// Check if the device was lost.
+	if (g_vulkan_context->CheckLastSubmitFail())
+		return PresentResult::DeviceLost;
+
 	VkResult res = m_swap_chain->AcquireNextImage();
 	if (res != VK_SUCCESS)
 	{
@@ -367,7 +368,7 @@ bool VulkanHostDisplay::BeginPresent(bool frame_skip)
 			{
 				Console.Error("Failed to recreate surface after loss");
 				g_vulkan_context->ExecuteCommandBuffer(Vulkan::Context::WaitType::None);
-				return false;
+				return PresentResult::FrameSkipped;
 			}
 
 			res = m_swap_chain->AcquireNextImage();
@@ -380,7 +381,7 @@ bool VulkanHostDisplay::BeginPresent(bool frame_skip)
 			// Still submit the command buffer, otherwise we'll end up with several frames waiting.
 			LOG_VULKAN_ERROR(res, "vkAcquireNextImageKHR() failed: ");
 			g_vulkan_context->ExecuteCommandBuffer(Vulkan::Context::WaitType::None);
-			return false;
+			return PresentResult::FrameSkipped;
 		}
 	}
 
@@ -401,7 +402,7 @@ bool VulkanHostDisplay::BeginPresent(bool frame_skip)
 	const VkRect2D scissor{{0, 0}, {static_cast<u32>(swap_chain_texture.GetWidth()), static_cast<u32>(swap_chain_texture.GetHeight())}};
 	vkCmdSetViewport(g_vulkan_context->GetCurrentCommandBuffer(), 0, 1, &vp);
 	vkCmdSetScissor(g_vulkan_context->GetCurrentCommandBuffer(), 0, 1, &scissor);
-	return true;
+	return PresentResult::OK;
 }
 
 void VulkanHostDisplay::EndPresent()

pcsx2/Frontend/VulkanHostDisplay.h

@@ -47,7 +47,7 @@ public:
 
 	void SetVSync(VsyncMode mode) override;
 
-	bool BeginPresent(bool frame_skip) override;
+	PresentResult BeginPresent(bool frame_skip) override;
 	void EndPresent() override;
 
 	bool SetGPUTimingEnabled(bool enabled) override;

pcsx2/GS/GS.cpp

@@ -209,15 +209,15 @@ static bool DoGSOpen(GSRendererType renderer, u8* basemem)
 			return false;
 	}
 
-	try
+	if (!g_gs_device->Create())
 	{
-		if (!g_gs_device->Create())
+		g_gs_device->Destroy();
-		{
+		g_gs_device.reset();
-			g_gs_device->Destroy();
+		return false;
-			g_gs_device.reset();
+	}
-			return false;
-		}
 
+	if (!g_gs_renderer)
+	{
 		if (renderer == GSRendererType::Null)
 		{
 			g_gs_renderer = std::make_unique<GSRendererNull>();
@@ -230,55 +230,67 @@ static bool DoGSOpen(GSRendererType renderer, u8* basemem)
 		{
 			g_gs_renderer = std::unique_ptr<GSRenderer>(MULTI_ISA_SELECT(makeGSRendererSW)(GSConfig.SWExtraThreads));
 		}
+
+		g_gs_renderer->SetRegsMem(basemem);
 	}
-	catch (std::exception& ex)
+	else
 	{
-		Host::ReportFormattedErrorAsync("GS", "GS error: Exception caught in GSopen: %s", ex.what());
+		Console.Warning("(DoGSOpen) Using existing renderer.");
-		g_gs_renderer.reset();
-		g_gs_device->Destroy();
-		g_gs_device.reset();
-		return false;
 	}
 
 	GSConfig.OsdShowGPU = EmuConfig.GS.OsdShowGPU && g_host_display->SetGPUTimingEnabled(true);
 
-	g_gs_renderer->SetRegsMem(basemem);
 	g_perfmon.Reset();
 	return true;
 }
 
-bool GSreopen(bool recreate_display, const Pcsx2Config::GSOptions& old_config)
+bool GSreopen(bool recreate_display, bool recreate_renderer, const Pcsx2Config::GSOptions& old_config)
 {
 	Console.WriteLn("Reopening GS with %s display", recreate_display ? "new" : "existing");
 
-	g_gs_renderer->Flush(GSState::GSFlushReason::GSREOPEN);
+	if (recreate_renderer)
+		g_gs_renderer->Flush(GSState::GSFlushReason::GSREOPEN);
 
 	freezeData fd = {};
-	if (g_gs_renderer->Freeze(&fd, true) != 0)
+	std::unique_ptr<u8[]> fd_data;
+	if (recreate_renderer)
 	{
-		Console.Error("(GSreopen) Failed to get GS freeze size");
+		if (g_gs_renderer->Freeze(&fd, true) != 0)
-		return false;
+		{
-	}
+			Console.Error("(GSreopen) Failed to get GS freeze size");
+			return false;
+		}
 
-	std::unique_ptr<u8[]> fd_data = std::make_unique<u8[]>(fd.size);
+		fd_data = std::make_unique<u8[]>(fd.size);
-	fd.data = fd_data.get();
+		fd.data = fd_data.get();
-	if (g_gs_renderer->Freeze(&fd, false) != 0)
+		if (g_gs_renderer->Freeze(&fd, false) != 0)
+		{
+			Console.Error("(GSreopen) Failed to freeze GS");
+			return false;
+		}
+	}
+	else
 	{
-		Console.Error("(GSreopen) Failed to freeze GS");
+		// Make sure nothing is left over.
-		return false;
+		g_gs_renderer->PurgePool();
+		g_gs_renderer->PurgeTextureCache();
 	}
 
 	if (recreate_display)
 	{
 		g_gs_device->ResetAPIState();
-		if (Host::BeginPresentFrame(true))
+		if (Host::BeginPresentFrame(false) == HostDisplay::PresentResult::OK)
 			Host::EndPresentFrame();
 	}
 
 	u8* basemem = g_gs_renderer->GetRegsMem();
 	const u32 gamecrc = g_gs_renderer->GetGameCRC();
-	g_gs_renderer->Destroy();
+	if (recreate_renderer)
-	g_gs_renderer.reset();
+	{
+		g_gs_renderer->Destroy();
+		g_gs_renderer.reset();
+	}
+
 	g_gs_device->Destroy();
 	g_gs_device.reset();
 
@@ -327,13 +339,17 @@ bool GSreopen(bool recreate_display, const Pcsx2Config::GSOptions& old_config)
 		}
 	}
 
-	if (g_gs_renderer->Defrost(&fd) != 0)
+	if (recreate_renderer)
 	{
-		Console.Error("(GSreopen) Failed to defrost");
+		if (g_gs_renderer->Defrost(&fd) != 0)
-		return false;
+		{
+			Console.Error("(GSreopen) Failed to defrost");
+			return false;
+		}
+
+		g_gs_renderer->SetGameCRC(gamecrc);
 	}
 
-	g_gs_renderer->SetGameCRC(gamecrc);
 	return true;
 }
 
@@ -698,7 +714,7 @@ void GSUpdateConfig(const Pcsx2Config::GSOptions& new_config)
 			GSConfig.DisableShaderCache != old_config.DisableShaderCache ||
 			GSConfig.DisableThreadedPresentation != old_config.DisableThreadedPresentation
 		);
-		if (!GSreopen(do_full_restart, old_config))
+		if (!GSreopen(do_full_restart, true, old_config))
 			pxFailRel("Failed to do full GS reopen");
 		return;
 	}
@@ -709,7 +725,7 @@ void GSUpdateConfig(const Pcsx2Config::GSOptions& new_config)
 		GSConfig.SWExtraThreads != old_config.SWExtraThreads ||
 		GSConfig.SWExtraThreadsHeight != old_config.SWExtraThreadsHeight)
 	{
-		if (!GSreopen(false, old_config))
+		if (!GSreopen(false, true, old_config))
 			pxFailRel("Failed to do quick GS reopen");
 
 		return;
@@ -787,7 +803,7 @@ void GSSwitchRenderer(GSRendererType new_renderer)
 	const bool recreate_display = (!is_software_switch && existing_api != GetAPIForRenderer(new_renderer));
 	const Pcsx2Config::GSOptions old_config(GSConfig);
 	GSConfig.Renderer = new_renderer;
-	if (!GSreopen(recreate_display, old_config))
+	if (!GSreopen(recreate_display, true, old_config))
 		pxFailRel("Failed to reopen GS for renderer switch.");
 }
 

pcsx2/GS/GS.h

@@ -53,7 +53,7 @@ s16 GSLookupBeforeDrawFunctionId(const std::string_view& name);
 int GSinit();
 void GSshutdown();
 bool GSopen(const Pcsx2Config::GSOptions& config, GSRendererType renderer, u8* basemem);
-bool GSreopen(bool recreate_display, const Pcsx2Config::GSOptions& old_config);
+bool GSreopen(bool recreate_display, bool recreate_renderer, const Pcsx2Config::GSOptions& old_config);
 void GSreset(bool hardware_reset);
 void GSclose();
 void GSgifSoftReset(u32 mask);

pcsx2/GS/Renderers/Common/GSRenderer.cpp

@@ -50,6 +50,8 @@ std::unique_ptr<GSRenderer> g_gs_renderer;
 // we might be switching while the other thread reads it.
 static GSVector4 s_last_draw_rect;
 
+// Last time we reset the renderer due to a GPU crash, if any.
+static Common::Timer::Value s_last_gpu_reset_time;
 
 GSRenderer::GSRenderer()
 	: m_shader_time_start(Common::Timer::GetCurrentValue())
@@ -602,6 +604,40 @@ void GSJoinSnapshotThreads()
 	}
 }
 
+bool GSRenderer::BeginPresentFrame(bool frame_skip)
+{
+	const HostDisplay::PresentResult result = Host::BeginPresentFrame(frame_skip);
+	if (result == HostDisplay::PresentResult::OK)
+		return true;
+	else if (result == HostDisplay::PresentResult::FrameSkipped)
+		return false;
+
+	// If we're constantly crashing on something in particular, we don't want to end up in an
+	// endless reset loop.. that'd probably end up leaking memory and/or crashing us for other
+	// reasons. So just abort in such case.
+	const Common::Timer::Value current_time = Common::Timer::GetCurrentValue();
+	if (s_last_gpu_reset_time != 0 &&
+		Common::Timer::ConvertValueToSeconds(current_time - s_last_gpu_reset_time) < 15.0f)
+	{
+		pxFailRel("Host GPU lost too many times, device is probably completely wedged.");
+	}
+	s_last_gpu_reset_time = current_time;
+
+	// Device lost, something went really bad.
+	// Let's just toss out everything, and try to hobble on.
+	if (!GSreopen(true, false, GSConfig))
+	{
+		pxFailRel("Failed to recreate GS device after loss.");
+		return false;
+	}
+
+	// First frame after reopening is definitely going to be trash, so skip it.
+	Host::AddIconOSDMessage("GSDeviceLost", ICON_FA_EXCLAMATION_TRIANGLE,
+		"Host GPU device encountered an error and was recovered. This may have broken rendering.",
+		Host::OSD_CRITICAL_ERROR_DURATION);
+	return false;
+}
+
 void GSRenderer::VSync(u32 field, bool registers_written)
 {
 	Flush(GSFlushReason::VSYNC);
@@ -647,7 +683,7 @@ void GSRenderer::VSync(u32 field, bool registers_written)
 	if (skip_frame)
 	{
 		g_gs_device->ResetAPIState();
-		if (Host::BeginPresentFrame(true))
+		if (BeginPresentFrame(true))
 			Host::EndPresentFrame();
 		g_gs_device->RestoreAPIState();
 		PerformanceMetrics::Update(registers_written, fb_sprite_frame, true);
@@ -694,7 +730,7 @@ void GSRenderer::VSync(u32 field, bool registers_written)
 	}
 
 	g_gs_device->ResetAPIState();
-	if (Host::BeginPresentFrame(false))
+	if (BeginPresentFrame(false))
 	{
 		if (current && !blank_frame)
 		{
@@ -910,7 +946,7 @@ void GSRenderer::StopGSDump()
 void GSRenderer::PresentCurrentFrame()
 {
 	g_gs_device->ResetAPIState();
-	if (Host::BeginPresentFrame(false))
+	if (BeginPresentFrame(false))
 	{
 		GSTexture* current = g_gs_device->GetCurrent();
 		if (current)

pcsx2/GS/Renderers/Common/GSRenderer.h

@@ -23,6 +23,7 @@ class GSRenderer : public GSState
 {
 private:
 	bool Merge(int field);
+	bool BeginPresentFrame(bool frame_skip);
 
 	u64 m_shader_time_start = 0;
 

pcsx2/HostDisplay.h

@@ -60,6 +60,13 @@ public:
 		RightOrBottom
 	};
 
+	enum class PresentResult
+	{
+		OK,
+		FrameSkipped,
+		DeviceLost
+	};
+
 	struct AdapterAndModeList
 	{
 		std::vector<std::string> adapter_names;
@@ -137,7 +144,7 @@ public:
 
 	/// Returns false if the window was completely occluded. If frame_skip is set, the frame won't be
 	/// displayed, but the GPU command queue will still be flushed.
-	virtual bool BeginPresent(bool frame_skip) = 0;
+	virtual PresentResult BeginPresent(bool frame_skip) = 0;
 
 	/// Presents the frame to the display, and renders OSD elements.
 	virtual void EndPresent() = 0;
@@ -184,7 +191,7 @@ namespace Host
 
 	/// Returns false if the window was completely occluded. If frame_skip is set, the frame won't be
 	/// displayed, but the GPU command queue will still be flushed.
-	bool BeginPresentFrame(bool frame_skip);
+	HostDisplay::PresentResult BeginPresentFrame(bool frame_skip);
 
 	/// Presents the frame to the display, and renders OSD elements.
 	void EndPresentFrame();

tests/ctest/core/StubHost.cpp

@@ -114,9 +114,9 @@ void Host::ReleaseHostDisplay(bool clear_state)
 {
 }
 
-bool Host::BeginPresentFrame(bool frame_skip)
+HostDisplay::PresentResult Host::BeginPresentFrame(bool frame_skip)
 {
-	return false;
+	return HostDisplay::PresentResult::FrameSkipped;
 }
 
 void Host::EndPresentFrame()