vk: Improve video memory manager to attempt recovery in out of memory situations

rpcs3/Emu/RSX/Common/surface_store.h

@@ -660,49 +660,6 @@ namespace rsx
 				std::forward<Args>(extra_params)...);
 		}
 
-		bool check_memory_overload(u64 max_safe_memory) const
-		{
-			if (m_active_memory_used <= max_safe_memory) [[likely]]
-			{
-				return false;
-			}
-			else
-			{
-				rsx_log.warning("Surface cache is using too much memory! (%dM)", m_active_memory_used / 0x100000);
-				return true;
-			}
-		}
-
-		void handle_memory_overload(command_list_type cmd)
-		{
-			auto process_list_function = [&](std::unordered_map<u32, surface_storage_type>& data)
-			{
-				for (auto It = data.begin(); It != data.end();)
-				{
-					auto surface = Traits::get(It->second);
-					if (surface->dirty())
-					{
-						// Force memory barrier to release some resources
-						surface->memory_barrier(cmd, rsx::surface_access::read);
-					}
-					else if (!surface->test())
-					{
-						// Remove this
-						invalidate(It->second);
-						It = data.erase(It);
-					}
-					else
-					{
-						++It;
-					}
-				}
-			};
-
-			// Try and find old surfaces to remove
-			process_list_function(m_render_targets_storage);
-			process_list_function(m_depth_stencil_storage);
-		}
-
 	public:
 		/**
 		 * Update bound color and depth surface.
@@ -1122,5 +1079,52 @@ namespace rsx
 				}
 			}
 		}
+
+		bool check_memory_usage(u64 max_safe_memory) const
+		{
+			if (m_active_memory_used <= max_safe_memory) [[likely]]
+			{
+				return false;
+			}
+			else
+			{
+				rsx_log.warning("Surface cache is using too much memory! (%dM)", m_active_memory_used / 0x100000);
+				return true;
+			}
+		}
+
+		bool handle_memory_pressure(command_list_type cmd, problem_severity /*severity*/)
+		{
+			auto process_list_function = [&](std::unordered_map<u32, surface_storage_type>& data)
+			{
+				for (auto It = data.begin(); It != data.end();)
+				{
+					auto surface = Traits::get(It->second);
+					if (surface->dirty())
+					{
+						// Force memory barrier to release some resources
+						surface->memory_barrier(cmd, rsx::surface_access::read);
+					}
+					else if (!surface->test())
+					{
+						// Remove this
+						invalidate(It->second);
+						It = data.erase(It);
+					}
+					else
+					{
+						++It;
+					}
+				}
+			};
+
+			const auto old_usage = m_active_memory_used;
+
+			// Try and find old surfaces to remove
+			process_list_function(m_render_targets_storage);
+			process_list_function(m_depth_stencil_storage);
+
+			return (m_active_memory_used < old_usage);
+		}
 	};
 }

rpcs3/Emu/RSX/Common/texture_cache.h

@@ -1342,6 +1342,23 @@ namespace rsx
 			m_storage.purge_unreleased_sections();
 		}
 
+		bool handle_memory_pressure(problem_severity severity)
+		{
+			if (m_storage.m_unreleased_texture_objects)
+			{
+				m_storage.purge_unreleased_sections();
+				return true;
+			}
+
+			if (severity >= problem_severity::severe)
+			{
+				// Things are bad, previous check should have released 'unreleased' pool
+				return m_storage.purge_unlocked_sections();
+			}
+
+			return false;
+		}
+
 		image_view_type create_temporary_subresource(commandbuffer_type &cmd, deferred_subresource& desc)
 		{
 			if (!desc.do_not_cache) [[likely]]

rpcs3/Emu/RSX/Common/texture_cache_utils.h

@@ -682,6 +682,44 @@ namespace rsx
 			AUDIT(m_unreleased_texture_objects == 0);
 		}
 
+		bool purge_unlocked_sections()
+		{
+			// Reclaims all graphics memory consumed by unlocked textures
+			bool any_released = false;
+			for (auto it = m_in_use.begin(); it != m_in_use.end();)
+			{
+				auto* block = *it;
+
+				if (block->get_exists_count() > block->get_locked_count())
+				{
+					for (auto& tex : *block)
+					{
+						if (tex.get_context() == rsx::texture_upload_context::framebuffer_storage ||
+							tex.is_locked() ||
+							!tex.exists())
+						{
+							continue;
+						}
+
+						ASSERT(!tex.is_locked() && tex.exists());
+						tex.destroy();
+						any_released = true;
+					}
+				}
+
+				if (block->get_exists_count() == 0)
+				{
+					it = m_in_use.erase(it);
+				}
+				else
+				{
+					it++;
+				}
+			}
+
+			return any_released;
+		}
+
 
 		/**
 		 * Callbacks

rpcs3/Emu/RSX/GL/GLRenderTargets.h

@@ -358,9 +358,9 @@ struct gl_render_targets : public rsx::surface_store<gl_render_target_traits>
 	std::vector<GLuint> free_invalidated(gl::command_context& cmd)
 	{
 		// Do not allow more than 256M of RSX memory to be used by RTTs
-		if (check_memory_overload(256 * 0x100000))
+		if (check_memory_usage(256 * 0x100000))
 		{
-			handle_memory_overload(cmd);
+			handle_memory_pressure(cmd, rsx::problem_severity::moderate);
 		}
 
 		std::vector<GLuint> removed;

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -787,6 +787,26 @@ void VKGSRender::on_semaphore_acquire_wait()
 	}
 }
 
+bool VKGSRender::on_vram_exhausted(rsx::problem_severity severity)
+{
+	ASSERT(!vk::is_uninterruptible() && rsx::get_current_renderer()->is_current_thread());
+	bool released = m_texture_cache.handle_memory_pressure(severity);
+
+	if (severity <= rsx::problem_severity::moderate)
+	{
+		released |= m_rtts.handle_memory_pressure(*m_current_command_buffer, severity);
+		return released;
+	}
+
+	if (released && severity >= rsx::problem_severity::fatal)
+	{
+		// Imminent crash, full GPU sync is the least of our problems
+		flush_command_queue(true);
+	}
+
+	return released;
+}
+
 void VKGSRender::notify_tile_unbound(u32 tile)
 {
 	//TODO: Handle texture writeback

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -546,6 +546,9 @@ public:
 	// External callback in case we need to suddenly submit a commandlist unexpectedly, e.g in a violation handler
 	void emergency_query_cleanup(vk::command_buffer* commands);
 
+	// External callback to handle out of video memory problems
+	bool on_vram_exhausted(rsx::problem_severity severity);
+
 	// Conditional rendering
 	void begin_conditional_rendering(const std::vector<rsx::reports::occlusion_query_info*>& sources) override;
 	void end_conditional_rendering() override;

rpcs3/Emu/RSX/VK/VKHelpers.h

@@ -171,6 +171,8 @@ namespace vk
 	void vmm_notify_memory_allocated(void* handle, u32 memory_type, u64 memory_size);
 	void vmm_notify_memory_freed(void* handle);
 	void vmm_reset();
+	void vmm_check_memory_usage();
+	bool vmm_handle_memory_pressure(rsx::problem_severity severity);
 
 	/**
 	* Allocate enough space in upload_buffer and write all mipmap/layer data into the subbuffer.
@@ -323,6 +325,7 @@ namespace vk
 		virtual void unmap(mem_handle_t mem_handle) = 0;
 		virtual VkDeviceMemory get_vk_device_memory(mem_handle_t mem_handle) = 0;
 		virtual u64 get_vk_device_memory_offset(mem_handle_t mem_handle) = 0;
+		virtual f32 get_memory_usage() = 0;
 
 	protected:
 		VkDevice m_device;
@@ -337,6 +340,9 @@ namespace vk
 	public:
 		mem_allocator_vma(VkDevice dev, VkPhysicalDevice pdev) : mem_allocator_base(dev, pdev)
 		{
+			// Initialize stats pool
+			std::fill(stats.begin(), stats.end(), VmaBudget{});
+
 			VmaAllocatorCreateInfo allocatorInfo = {};
 			allocatorInfo.physicalDevice = pdev;
 			allocatorInfo.device = dev;
@@ -361,7 +367,26 @@ namespace vk
 			mem_req.size = block_sz;
 			mem_req.alignment = alignment;
 			create_info.memoryTypeBits = 1u << memory_type_index;
-			CHECK_RESULT(vmaAllocateMemory(m_allocator, &mem_req, &create_info, &vma_alloc, nullptr));
+
+			if (VkResult result = vmaAllocateMemory(m_allocator, &mem_req, &create_info, &vma_alloc, nullptr);
+				result != VK_SUCCESS)
+			{
+				if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY &&
+					vmm_handle_memory_pressure(rsx::problem_severity::fatal))
+				{
+					// If we just ran out of VRAM, attempt to release resources and try again
+					result = vmaAllocateMemory(m_allocator, &mem_req, &create_info, &vma_alloc, nullptr);
+				}
+
+				if (result != VK_SUCCESS)
+				{
+					die_with_error(HERE, result);
+				}
+				else
+				{
+					rsx_log.warning("Renderer ran out of video memory but successfully recovered.");
+				}
+			}
 
 			vmm_notify_memory_allocated(vma_alloc, memory_type_index, block_sz);
 			return vma_alloc;
@@ -405,8 +430,28 @@ namespace vk
 			return alloc_info.offset;
 		}
 
+		f32 get_memory_usage() override
+		{
+			vmaGetBudget(m_allocator, stats.data());
+
+			float max_usage = 0.f;
+			for (const auto& info : stats)
+			{
+				if (!info.budget)
+				{
+					break;
+				}
+
+				const float this_usage = (info.usage * 100.f) / info.budget;
+				max_usage = std::max(max_usage, this_usage);
+			}
+
+			return max_usage;
+		}
+
 	private:
 		VmaAllocator m_allocator;
+		std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> stats;
 	};
 
 	// Memory Allocator - built-in Vulkan device memory allocate/free
@@ -426,7 +471,26 @@ namespace vk
 			info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
 			info.allocationSize = block_sz;
 			info.memoryTypeIndex = memory_type_index;
-			CHECK_RESULT(vkAllocateMemory(m_device, &info, nullptr, &memory));
+
+			if (VkResult result = vkAllocateMemory(m_device, &info, nullptr, &memory);
+				result != VK_SUCCESS)
+			{
+				if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY &&
+					vmm_handle_memory_pressure(rsx::problem_severity::fatal))
+				{
+					// If we just ran out of VRAM, attempt to release resources and try again
+					result = vkAllocateMemory(m_device, &info, nullptr, &memory);
+				}
+
+				if (result != VK_SUCCESS)
+				{
+					die_with_error(HERE, result);
+				}
+				else
+				{
+					rsx_log.warning("Renderer ran out of video memory but successfully recovered.");
+				}
+			}
 
 			vmm_notify_memory_allocated(memory, memory_type_index, block_sz);
 			return memory;
@@ -460,6 +524,11 @@ namespace vk
 			return 0;
 		}
 
+		f32 get_memory_usage() override
+		{
+			return 0.f;
+		}
+
 	private:
 	};
 

rpcs3/Emu/RSX/VK/VKPresent.cpp

@@ -104,6 +104,9 @@ void VKGSRender::advance_queued_frames()
 	// Check all other frames for completion and clear resources
 	check_present_status();
 
+	// Run video memory balancer
+	vk::vmm_check_memory_usage();
+
 	// m_rtts storage is double buffered and should be safe to tag on frame boundary
 	m_rtts.free_invalidated(*m_current_command_buffer);
 

rpcs3/Emu/RSX/VK/VKRenderTargets.h

@@ -936,14 +936,14 @@ namespace rsx
 		void free_invalidated(vk::command_buffer& cmd)
 		{
 			// Do not allow more than 256M of RSX memory to be used by RTTs
-			if (check_memory_overload(256 * 0x100000))
+			if (check_memory_usage(256 * 0x100000))
 			{
 				if (!cmd.is_recording())
 				{
 					cmd.begin();
 				}
 
-				handle_memory_overload(cmd);
+				handle_memory_pressure(cmd, rsx::problem_severity::moderate);
 			}
 
 			const u64 last_finished_frame = vk::get_last_completed_frame_id();

rpcs3/Emu/RSX/VK/VKResourceManager.cpp

@@ -1,5 +1,6 @@
 #include "stdafx.h"
 #include "VKResourceManager.h"
+#include "VKGSRender.h"
 
 namespace vk
 {
@@ -75,4 +76,36 @@ namespace vk
 		g_vmm_memory_usage.clear();
 		g_vmm_allocations.clear();
 	}
+
+	bool vmm_handle_memory_pressure(rsx::problem_severity severity)
+	{
+		if (auto vkthr = dynamic_cast<VKGSRender*>(rsx::get_current_renderer()))
+		{
+			return vkthr->on_vram_exhausted(severity);
+		}
+
+		return false;
+	}
+
+	void vmm_check_memory_usage()
+	{
+		const auto vmm_load = get_current_mem_allocator()->get_memory_usage();
+		rsx::problem_severity load_severity = rsx::problem_severity::low;
+
+		if (vmm_load > 90.f)
+		{
+			rsx_log.warning("Video memory usage exceeding 90%. Will attempt to reclaim resources.");
+			load_severity = rsx::problem_severity::severe;
+		}
+		else if (vmm_load > 75.f)
+		{
+			rsx_log.notice("Video memory usage exceeding 75%. Will attempt to reclaim resources.");
+			load_severity = rsx::problem_severity::moderate;
+		}
+
+		if (load_severity >= rsx::problem_severity::moderate)
+		{
+			vmm_handle_memory_pressure(load_severity);
+		}
+	}
 }

rpcs3/Emu/RSX/rsx_utils.h

@@ -31,6 +31,14 @@ namespace rsx
 
 	extern atomic_t<u64> g_rsx_shared_tag;
 
+	enum class problem_severity : u8
+	{
+		low,
+		moderate,
+		severe,
+		fatal
+	};
+
 	//Base for resources with reference counting
 	class ref_counted
 	{