Performance enhancement - Vulkan memory allocator (#4635)

* Incorporates the vulkan memory allocator from the AMD GPUOpen project

rpcs3/CMakeLists.txt

@@ -309,6 +309,7 @@ ${LLVM_INCLUDE_DIRS}
 "${RPCS3_SRC_DIR}/../3rdparty/Optional"
 "${RPCS3_SRC_DIR}/../3rdparty/discord-rpc/include"
 "${RPCS3_SRC_DIR}/../3rdparty/xxHash"
+"${RPCS3_SRC_DIR}/../3rdparty/GPUOpen/include"
 )
 
 if(WIN32)

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -547,6 +547,19 @@ VKGSRender::VKGSRender() : GSRender()
 	vk::set_current_thread_ctx(m_thread_context);
 	vk::set_current_renderer(m_swapchain->get_device());
 
+	// Choose memory allocator (device memory)
+	if (g_cfg.video.disable_vulkan_mem_allocator) 
+	{
+		m_mem_allocator = std::make_shared<vk::mem_allocator_vk>(*m_device, m_device->gpu());
+	}
+	else 
+	{
+		m_mem_allocator = std::make_shared<vk::mem_allocator_vma>(*m_device, m_device->gpu());
+	}
+
+	
+	vk::set_current_mem_allocator(m_mem_allocator);
+
 	m_client_width = m_frame->client_width();
 	m_client_height = m_frame->client_height();
 	if (!m_swapchain->init(m_client_width, m_client_height))
@@ -775,6 +788,9 @@ VKGSRender::~VKGSRender()
 	m_secondary_command_buffer.destroy();
 	m_secondary_command_buffer_pool.destroy();
 
+	// Memory allocator (device memory)
+	m_mem_allocator->destroy();
+
 	//Device handles/contexts
 	m_swapchain->destroy();
 	m_thread_context.close();

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -281,6 +281,8 @@ public:
 	std::unique_ptr<vk::vertex_cache> m_vertex_cache;
 	std::unique_ptr<vk::shader_cache> m_shaders_cache;
 
+	std::shared_ptr<vk::mem_allocator_base> m_mem_allocator;
+
 private:
 	std::unique_ptr<VKProgramBuffer> m_prog_buffer;
 

rpcs3/Emu/RSX/VK/VKHelpers.cpp

@@ -1,12 +1,12 @@
 #include "stdafx.h"
 #include "VKHelpers.h"
-
 #include "Utilities/mutex.h"
 
 namespace vk
 {
 	context* g_current_vulkan_ctx = nullptr;
 	render_device g_current_renderer;
+	std::shared_ptr<vk::mem_allocator_base> g_mem_allocator = nullptr;
 
 	std::unique_ptr<image> g_null_texture;
 	std::unique_ptr<image_view> g_null_image_view;
@@ -188,6 +188,16 @@ namespace vk
 		g_null_sampler = nullptr;
 	}
 
+	void set_current_mem_allocator(std::shared_ptr<vk::mem_allocator_base> mem_allocator)
+	{
+		g_mem_allocator = mem_allocator;
+	}
+
+	std::shared_ptr<vk::mem_allocator_base> get_current_mem_allocator()
+	{
+		return g_mem_allocator;
+	}
+
 	void set_current_thread_ctx(const vk::context &ctx)
 	{
 		g_current_vulkan_ctx = (vk::context *)&ctx;

rpcs3/Emu/RSX/VK/VKHelpers.h

@@ -19,6 +19,8 @@
 #include "../Common/GLSLCommon.h"
 #include "../rsx_cache.h"
 
+#include "3rdparty/GPUOpen/include/vk_mem_alloc.h"
+
 #ifndef _WIN32
 #include <X11/Xutil.h>
 #endif
@@ -63,6 +65,7 @@ namespace vk
 	class command_buffer;
 	struct image;
 	struct vk_data_heap;
+	class mem_allocator_base;
 
 	vk::context *get_current_thread_ctx();
 	void set_current_thread_ctx(const vk::context &ctx);
@@ -70,6 +73,9 @@ namespace vk
 	vk::render_device *get_current_renderer();
 	void set_current_renderer(const vk::render_device &device);
 
+	void set_current_mem_allocator(std::shared_ptr<vk::mem_allocator_base> mem_allocator);
+	std::shared_ptr<vk::mem_allocator_base> get_current_mem_allocator();
+
 	//Compatibility workarounds
 	bool emulate_primitive_restart(rsx::primitive_type type);
 	bool sanitize_fp_values();
@@ -320,23 +326,190 @@ namespace vk
 		}
 	};
 
-	struct memory_block
-	{
-		VkMemoryAllocateInfo info = {};
-		VkDeviceMemory memory;
+	// Memory Allocator - base class
 
-		memory_block(VkDevice dev, u64 block_sz, uint32_t memory_type_index) : m_device(dev)
+	class mem_allocator_base
+	{
+	public:
+		using mem_handle_t = void *;
+
+		mem_allocator_base(VkDevice dev, VkPhysicalDevice pdev) : m_device(dev) {};
+		~mem_allocator_base() {};
+
+		virtual void destroy() = 0;
+
+		virtual mem_handle_t alloc(u64 block_sz, uint32_t memory_type_index) = 0;
+		virtual void free(mem_handle_t mem_handle) = 0;
+		virtual void *map(mem_handle_t mem_handle, u64 offset, u64 size) = 0;
+		virtual void unmap(mem_handle_t mem_handle) = 0;
+		virtual VkDeviceMemory get_vk_device_memory(mem_handle_t mem_handle) = 0;
+		virtual u64 get_vk_device_memory_offset(mem_handle_t mem_handle) = 0;
+
+	protected:
+		VkDevice m_device;
+	private:
+	};
+
+	// Memory Allocator - Vulkan Memory Allocator 
+	// https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator
+
+	class mem_allocator_vma : public mem_allocator_base
+	{
+	public:
+		mem_allocator_vma(VkDevice dev, VkPhysicalDevice pdev) : mem_allocator_base(dev, pdev)
 		{
+			VmaAllocatorCreateInfo allocatorInfo = {};
+			allocatorInfo.physicalDevice = pdev;
+			allocatorInfo.device = dev;
+
+			vmaCreateAllocator(&allocatorInfo, &m_allocator);
+		}
+
+		~mem_allocator_vma() {};
+
+		void destroy() override
+		{
+			vmaDestroyAllocator(m_allocator);
+		}
+
+		mem_handle_t alloc(u64 block_sz, uint32_t memory_type_index) override
+		{
+			VmaAllocation vma_alloc;
+			VkMemoryRequirements mem_req = {};
+			VmaAllocationCreateInfo create_info = {};
+
+			mem_req.memoryTypeBits = 1u << memory_type_index;
+			mem_req.size = block_sz;
+			create_info.memoryTypeBits = 1u << memory_type_index;
+			CHECK_RESULT(vmaAllocateMemory(m_allocator, &mem_req, &create_info, &vma_alloc, nullptr));
+			return vma_alloc;
+		}
+
+		void free(mem_handle_t mem_handle) override
+		{
+			vmaFreeMemory(m_allocator, static_cast<VmaAllocation>(mem_handle));
+		}
+
+		void *map(mem_handle_t mem_handle, u64 offset, u64 size) override
+		{
+			void *data = nullptr;
+
+			CHECK_RESULT(vmaMapMemory(m_allocator, static_cast<VmaAllocation>(mem_handle), &data));
+
+			// Add offset
+			data = static_cast<u8 *>(data) + offset;
+			return data;
+		}
+
+		void unmap(mem_handle_t mem_handle) override
+		{
+			vmaUnmapMemory(m_allocator, static_cast<VmaAllocation>(mem_handle));
+		}
+
+		VkDeviceMemory get_vk_device_memory(mem_handle_t mem_handle)
+		{
+			VmaAllocationInfo alloc_info;
+
+			vmaGetAllocationInfo(m_allocator, static_cast<VmaAllocation>(mem_handle), &alloc_info);
+			return alloc_info.deviceMemory;
+		}
+
+		u64 get_vk_device_memory_offset(mem_handle_t mem_handle)
+		{
+			VmaAllocationInfo alloc_info;
+
+			vmaGetAllocationInfo(m_allocator, static_cast<VmaAllocation>(mem_handle), &alloc_info);
+			return alloc_info.offset;
+		}
+
+	private:
+		VmaAllocator m_allocator;
+	};
+
+	// Memory Allocator - built-in Vulkan device memory allocate/free
+
+	class mem_allocator_vk : public mem_allocator_base
+	{
+	public:
+		mem_allocator_vk(VkDevice dev, VkPhysicalDevice pdev) : mem_allocator_base(dev, pdev) {};
+		~mem_allocator_vk() {};
+
+		void destroy() override {};
+
+		mem_handle_t alloc(u64 block_sz, uint32_t memory_type_index) override
+		{
+			VkDeviceMemory memory;
+			VkMemoryAllocateInfo info = {};
 			info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
 			info.allocationSize = block_sz;
 			info.memoryTypeIndex = memory_type_index;
 
 			CHECK_RESULT(vkAllocateMemory(m_device, &info, nullptr, &memory));
+			return memory;
+		}
+
+		void free(mem_handle_t mem_handle) override
+		{
+			vkFreeMemory(m_device, (VkDeviceMemory)mem_handle, nullptr);
+		}
+
+		void *map(mem_handle_t mem_handle, u64 offset, u64 size) override
+		{
+			void *data = nullptr;
+			CHECK_RESULT(vkMapMemory(m_device, (VkDeviceMemory)mem_handle, offset, std::max<u64>(size, 1u), 0, &data));
+			return data;
+		}
+
+		void unmap(mem_handle_t mem_handle) override
+		{
+			vkUnmapMemory(m_device, (VkDeviceMemory)mem_handle);
+		}
+
+		VkDeviceMemory get_vk_device_memory(mem_handle_t mem_handle) override
+		{
+			return (VkDeviceMemory)mem_handle;
+		}
+
+		u64 get_vk_device_memory_offset(mem_handle_t mem_handle)
+		{
+			return 0;
+		}
+
+	private:
+	};
+
+	struct memory_block
+	{
+
+		memory_block(VkDevice dev, u64 block_sz, uint32_t memory_type_index) : m_device(dev)
+		{
+			m_mem_allocator = get_current_mem_allocator();
+			m_mem_handle = m_mem_allocator->alloc(block_sz, memory_type_index);
 		}
 
 		~memory_block()
 		{
-			vkFreeMemory(m_device, memory, nullptr);
+			m_mem_allocator->free(m_mem_handle);
+		}
+
+		VkDeviceMemory get_vk_device_memory()
+		{
+			return m_mem_allocator->get_vk_device_memory(m_mem_handle);
+		}
+
+		u64 get_vk_device_memory_offset()
+		{
+			return m_mem_allocator->get_vk_device_memory_offset(m_mem_handle);
+		}
+
+		void *map(u64 offset, u64 size)
+		{
+			return m_mem_allocator->map(m_mem_handle, offset, size);
+		}
+
+		void unmap()
+		{
+			m_mem_allocator->unmap(m_mem_handle);
 		}
 
 		memory_block(const memory_block&) = delete;
@@ -344,6 +517,8 @@ namespace vk
 
 	private:
 		VkDevice m_device;
+		std::shared_ptr<vk::mem_allocator_base> m_mem_allocator;
+		mem_allocator_base::mem_handle_t m_mem_handle;
 	};
 
 	class memory_block_deprecated
@@ -466,7 +641,7 @@ namespace vk
 			}
 
 			memory = std::make_shared<vk::memory_block>(m_device, memory_req.size, memory_type_index);
-			CHECK_RESULT(vkBindImageMemory(m_device, value, memory->memory, 0));
+			CHECK_RESULT(vkBindImageMemory(m_device, value, memory->get_vk_device_memory(), memory->get_vk_device_memory_offset()));
 		}
 
 		// TODO: Ctor that uses a provided memory heap
@@ -591,7 +766,7 @@ namespace vk
 			}
 
 			memory.reset(new memory_block(m_device, memory_reqs.size, memory_type_index));
-			vkBindBufferMemory(dev, value, memory->memory, 0);
+			vkBindBufferMemory(dev, value, memory->get_vk_device_memory(), memory->get_vk_device_memory_offset());
 		}
 
 		~buffer()
@@ -601,14 +776,12 @@ namespace vk
 
 		void *map(u64 offset, u64 size)
 		{
-			void *data = nullptr;
-			CHECK_RESULT(vkMapMemory(m_device, memory->memory, offset, std::max<u64>(size, 1u), 0, &data));
-			return data;
+			return memory->map(offset, size);
 		}
 
 		void unmap()
 		{
-			vkUnmapMemory(m_device, memory->memory);
+			memory->unmap();
 		}
 
 		u32 size() const

rpcs3/Emu/RSX/VK/VKMemAlloc.cpp

@@ -0,0 +1,2 @@
+#define VMA_IMPLEMENTATION
+#include "3rdparty/GPUOpen/include/vk_mem_alloc.h"

rpcs3/Emu/RSX/VK/VKTextureCache.h

@@ -1143,8 +1143,7 @@ namespace vk
 			VkSubresourceLayout layout{};
 			vkGetImageSubresourceLayout(*m_device, image->value, &subresource, &layout);
 
-			void* mem = nullptr;
-			vkMapMemory(*m_device, image->memory->memory, 0, layout.rowPitch * height, 0, &mem);
+			void* mem = image->memory->map(0, layout.rowPitch * height);
 
 			u32 row_pitch = width * 4;
 			char *src = (char *)vm::base(address);
@@ -1163,7 +1162,7 @@ namespace vk
 				dst += layout.rowPitch;
 			}
 
-			vkUnmapMemory(*m_device, image->memory->memory);
+			image->memory->unmap();
 
 			auto result = image.get();
 			const u32 resource_memory = width * height * 4; //Rough approximate

rpcs3/Emu/System.h

@@ -365,6 +365,7 @@ struct cfg_root : cfg::node
 		cfg::_bool frame_skip_enabled{this, "Enable Frame Skip", false};
 		cfg::_bool force_cpu_blit_processing{this, "Force CPU Blit", false}; // Debugging option
 		cfg::_bool disable_on_disk_shader_cache{this, "Disable On-Disk Shader Cache", false};
+		cfg::_bool disable_vulkan_mem_allocator{ this, "Disable Vulkan Memory Allocator", false };
 		cfg::_bool full_rgb_range_output{this, "Use full RGB output range", true}; // Video out dynamic range
 		cfg::_int<1, 8> consequtive_frames_to_draw{this, "Consecutive Frames To Draw", 1};
 		cfg::_int<1, 8> consequtive_frames_to_skip{this, "Consecutive Frames To Skip", 1};

rpcs3/Json/tooltips.json

@@ -47,7 +47,8 @@
 		"logProg": "Dump game shaders to file. Only useful to developers.\nIf unsure, don't use this option.",
 		"disableOcclusionQueries": "Disables running occlusion queries. Minor to moderate performance boost.\nMight introduce issues with broken occlusion e.g missing geometry and extreme pop-in.",
 		"forceCpuBlitEmulation": "Forces emulation of all blit and image manipulation operations on the CPU.\nRequires 'Write Color Buffers' option to also be enabled in most cases to avoid missing graphics.\nSignificantly degrades performance but is more accurate in some cases.\nThis setting overrides the 'GPU texture scaling' option.",
-		"disableOnDiskShaderCache": "Disables the loading and saving of shaders from and to the shader cache in the data directory."
+		"disableOnDiskShaderCache": "Disables the loading and saving of shaders from and to the shader cache in the data directory.",
+		"disableVulkanMemAllocator": "Disables the custom Vulkan memory allocator and reverts to direct calls to VkAllocateMemory/VkFreeMemory."
 	},
 	"emulator": {
 		"gui": {

rpcs3/VKGSRender.vcxproj

@@ -46,6 +46,7 @@
     <ClCompile Include="Emu\RSX\VK\VKTexture.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKVertexBuffers.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKVertexProgram.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKMemAlloc.cpp" />
     <ClCompile Include="Emu\RSX\VK\VulkanAPI.cpp" />
   </ItemGroup>
   <ItemGroup>

rpcs3/VKGSRender.vcxproj.filters

@@ -75,5 +75,8 @@
     <ClCompile Include="Emu\RSX\VK\VKFormats.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="Emu\RSX\VK\VKMemAlloc.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

rpcs3/rpcs3qt/emu_settings.h

@@ -65,6 +65,7 @@ public:
 		MinimumScalableDimension,
 		ForceCPUBlitEmulation,
 		DisableOnDiskShaderCache,
+		DisableVulkanMemAllocator,
 
 		// Audio
 		AudioRenderer,
@@ -220,6 +221,7 @@ private:
 		{ DisableOcclusionQueries,  { "Video", "Disable ZCull Occlusion Queries" }},
 		{ ForceCPUBlitEmulation,    { "Video", "Force CPU Blit" }},
 		{ DisableOnDiskShaderCache, { "Video", "Disable On-Disk Shader Cache"}},
+		{ DisableVulkanMemAllocator, { "Video", "Disable Vulkan Memory Allocator" }},
 		{ AnisotropicFilterOverride,{ "Video", "Anisotropic Filter Override" }},
 		{ ResolutionScale,          { "Video", "Resolution Scale" }},
 		{ MinimumScalableDimension, { "Video", "Minimum Scalable Dimension" }},

rpcs3/rpcs3qt/settings_dialog.cpp

@@ -964,6 +964,9 @@ settings_dialog::settings_dialog(std::shared_ptr<gui_settings> guiSettings, std:
 	xemu_settings->EnhanceCheckBox(ui->disableOnDiskShaderCache, emu_settings::DisableOnDiskShaderCache);
 	SubscribeTooltip(ui->disableOnDiskShaderCache, json_debug["disableOnDiskShaderCache"].toString());
 
+	xemu_settings->EnhanceCheckBox(ui->disableVulkanMemAllocator, emu_settings::DisableVulkanMemAllocator);
+	SubscribeTooltip(ui->disableVulkanMemAllocator, json_debug["disableVulkanMemAllocator"].toString());
+
 	// Checkboxes: core debug options
 	xemu_settings->EnhanceCheckBox(ui->ppuDebug, emu_settings::PPUDebug);
 	SubscribeTooltip(ui->ppuDebug, json_debug["ppuDebug"].toString());

rpcs3/rpcs3qt/settings_dialog.ui

@@ -1699,6 +1699,13 @@
               </property>
              </widget>
             </item>
+            <item>
+               <widget class="QCheckBox" name="disableVulkanMemAllocator">
+                <property name="text">
+                <string>Disable Vulkan Memory Allocator</string>
+               </property>
+              </widget>
+            </item>             
            </layout>
           </widget>
          </item>