From 498eac6a32c45025a9e8c3a8c073e5700c9cba6f Mon Sep 17 00:00:00 2001 From: InoriRus Date: Mon, 4 Jul 2022 17:24:03 +0300 Subject: [PATCH] fix for AMD gpu (#21) --- appveyor.yml | 2 +- source/CMakeLists.txt | 2 +- .../Emulator/Graphics/GraphicContext.h | 26 +- .../Emulator/Graphics/GraphicsRender.h | 5 +- .../emulator/src/Graphics/GraphicsRender.cpp | 153 +++++---- source/emulator/src/Graphics/GraphicsRun.cpp | 4 +- source/emulator/src/Graphics/Utils.cpp | 28 +- source/emulator/src/Graphics/Window.cpp | 311 +++++++++++++----- 8 files changed, 345 insertions(+), 186 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4f06d8e..d679a9c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 0.1.7.build-{build} +version: 0.1.8.build-{build} image: Visual Studio 2019 environment: matrix: diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index e8d8708..8fcbf6d 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -82,7 +82,7 @@ if (KYTY_LINKER STREQUAL LD) set(KYTY_LD_OPTIONS "-Wl,--image-base=0x100000000000") endif() -project(Kyty${KYTY_PROJECT_NAME}${CMAKE_BUILD_TYPE}${KYTY_COMPILER} VERSION 0.1.7) +project(Kyty${KYTY_PROJECT_NAME}${CMAKE_BUILD_TYPE}${KYTY_COMPILER} VERSION 0.1.8) include(src_script.cmake) diff --git a/source/emulator/include/Emulator/Graphics/GraphicContext.h b/source/emulator/include/Emulator/Graphics/GraphicContext.h index abca6d0..fd7b070 100644 --- a/source/emulator/include/Emulator/Graphics/GraphicContext.h +++ b/source/emulator/include/Emulator/Graphics/GraphicContext.h @@ -38,23 +38,33 @@ struct VulkanCommandPool uint32_t buffers_count = 0; }; +struct VulkanQueueInfo +{ + Core::Mutex* mutex = nullptr; + uint32_t family = static_cast(-1); + uint32_t index = static_cast(-1); + VkQueue vk_queue = nullptr; +}; + struct GraphicContext { static constexpr int QUEUES_NUM = 11; static constexpr int QUEUE_GFX = 8; + static constexpr int QUEUE_GFX_NUM = 1; static constexpr int QUEUE_UTIL = 9; + static constexpr int QUEUE_UTIL_NUM = 1; static constexpr int QUEUE_PRESENT = 10; + static constexpr int QUEUE_PRESENT_NUM = 1; static constexpr int QUEUE_COMPUTE_START = 0; static constexpr int QUEUE_COMPUTE_NUM = 8; - uint32_t screen_width = 0; - uint32_t screen_height = 0; - VkInstance instance = nullptr; - VkDebugUtilsMessengerEXT debug_messenger = nullptr; - VkPhysicalDevice physical_device = nullptr; - VkDevice device = nullptr; - uint32_t queue_family_index = static_cast(-1); - VkQueue queue[QUEUES_NUM] = {}; + uint32_t screen_width = 0; + uint32_t screen_height = 0; + VkInstance instance = nullptr; + VkDebugUtilsMessengerEXT debug_messenger = nullptr; + VkPhysicalDevice physical_device = nullptr; + VkDevice device = nullptr; + VulkanQueueInfo queues[QUEUES_NUM]; }; struct VulkanMemory diff --git a/source/emulator/include/Emulator/Graphics/GraphicsRender.h b/source/emulator/include/Emulator/Graphics/GraphicsRender.h index 311889b..9688ac3 100644 --- a/source/emulator/include/Emulator/Graphics/GraphicsRender.h +++ b/source/emulator/include/Emulator/Graphics/GraphicsRender.h @@ -30,7 +30,7 @@ struct RenderColorInfo; class CommandBuffer { public: - CommandBuffer() { Allocate(); } + explicit CommandBuffer(int queue): m_queue(queue) { Allocate(); } virtual ~CommandBuffer() { Free(); } void SetParent(CommandProcessor* parent) { m_parent = parent; } @@ -55,9 +55,6 @@ public: VulkanCommandPool* GetPool() { return m_pool; } [[nodiscard]] bool IsExecute() const { return m_execute; } - void SetQueue(int queue) { m_queue = queue; } - - // void CommandProcessorWait(); private: VulkanCommandPool* m_pool = nullptr; diff --git a/source/emulator/src/Graphics/GraphicsRender.cpp b/source/emulator/src/Graphics/GraphicsRender.cpp index 18f40ac..fb9f5da 100644 --- a/source/emulator/src/Graphics/GraphicsRender.cpp +++ b/source/emulator/src/Graphics/GraphicsRender.cpp @@ -434,31 +434,28 @@ class CommandPool { public: CommandPool() = default; - ~CommandPool() + ~CommandPool() // NOLINT { - if (m_pool != nullptr) - { - // TODO(): check if destructor is called from std::_Exit() - // Delete(); - } + // TODO(): check if destructor is called from std::_Exit() + // DeleteAll(); } KYTY_CLASS_NO_COPY(CommandPool); - VulkanCommandPool* GetPool() + VulkanCommandPool* GetPool(int id) { - if (m_pool == nullptr) + if (m_pool[id] == nullptr) { - Create(); + Create(id); } - return m_pool; + return m_pool[id]; } private: - void Create(); - void Delete(); + void Create(int id); + void DeleteAll(); - VulkanCommandPool* m_pool = nullptr; + VulkanCommandPool* m_pool[GraphicContext::QUEUES_NUM] = {}; }; static RenderContext* g_render_ctx = nullptr; @@ -1190,60 +1187,61 @@ VulkanBuffer* GdsBuffer::GetBuffer(GraphicContext* ctx) return m_buffer; } -void CommandPool::Create() +void CommandPool::Create(int id) { auto* ctx = g_render_ctx->GetGraphicCtx(); - EXIT_IF(m_pool != nullptr); + EXIT_IF(id < 0 || id >= GraphicContext::QUEUES_NUM); + EXIT_IF(m_pool[id] != nullptr); - m_pool = new VulkanCommandPool; + m_pool[id] = new VulkanCommandPool; EXIT_IF(ctx == nullptr); EXIT_IF(ctx->device == nullptr); - EXIT_IF(ctx->queue_family_index == static_cast(-1)); - EXIT_IF(m_pool->pool != nullptr); - EXIT_IF(m_pool->buffers != nullptr); - EXIT_IF(m_pool->fences != nullptr); - EXIT_IF(m_pool->semaphores != nullptr); - EXIT_IF(m_pool->buffers_count != 0); + EXIT_IF(ctx->queues[id].family == static_cast(-1)); + EXIT_IF(m_pool[id]->pool != nullptr); + EXIT_IF(m_pool[id]->buffers != nullptr); + EXIT_IF(m_pool[id]->fences != nullptr); + EXIT_IF(m_pool[id]->semaphores != nullptr); + EXIT_IF(m_pool[id]->buffers_count != 0); VkCommandPoolCreateInfo pool_info {}; pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; pool_info.pNext = nullptr; - pool_info.queueFamilyIndex = ctx->queue_family_index; + pool_info.queueFamilyIndex = ctx->queues[id].family; pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - vkCreateCommandPool(ctx->device, &pool_info, nullptr, &m_pool->pool); + vkCreateCommandPool(ctx->device, &pool_info, nullptr, &m_pool[id]->pool); - EXIT_NOT_IMPLEMENTED(m_pool->pool == nullptr); + EXIT_NOT_IMPLEMENTED(m_pool[id]->pool == nullptr); - m_pool->buffers_count = 4; - m_pool->buffers = new VkCommandBuffer[m_pool->buffers_count]; - m_pool->fences = new VkFence[m_pool->buffers_count]; - m_pool->semaphores = new VkSemaphore[m_pool->buffers_count]; - m_pool->busy = new bool[m_pool->buffers_count]; + m_pool[id]->buffers_count = 4; + m_pool[id]->buffers = new VkCommandBuffer[m_pool[id]->buffers_count]; + m_pool[id]->fences = new VkFence[m_pool[id]->buffers_count]; + m_pool[id]->semaphores = new VkSemaphore[m_pool[id]->buffers_count]; + m_pool[id]->busy = new bool[m_pool[id]->buffers_count]; VkCommandBufferAllocateInfo alloc_info {}; alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; - alloc_info.commandPool = m_pool->pool; + alloc_info.commandPool = m_pool[id]->pool; alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - alloc_info.commandBufferCount = m_pool->buffers_count; + alloc_info.commandBufferCount = m_pool[id]->buffers_count; - if (vkAllocateCommandBuffers(ctx->device, &alloc_info, m_pool->buffers) != VK_SUCCESS) + if (vkAllocateCommandBuffers(ctx->device, &alloc_info, m_pool[id]->buffers) != VK_SUCCESS) { EXIT("Can't allocate command buffers"); } - for (uint32_t i = 0; i < m_pool->buffers_count; i++) + for (uint32_t i = 0; i < m_pool[id]->buffers_count; i++) { - m_pool->busy[i] = false; + m_pool[id]->busy[i] = false; VkFenceCreateInfo fence_info {}; fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; fence_info.pNext = nullptr; fence_info.flags = 0; - if (vkCreateFence(ctx->device, &fence_info, nullptr, &m_pool->fences[i]) != VK_SUCCESS) + if (vkCreateFence(ctx->device, &fence_info, nullptr, &m_pool[id]->fences[i]) != VK_SUCCESS) { EXIT("Can't create fence"); } @@ -1253,42 +1251,47 @@ void CommandPool::Create() semaphore_info.pNext = nullptr; semaphore_info.flags = 0; - if (vkCreateSemaphore(ctx->device, &semaphore_info, nullptr, &m_pool->semaphores[i]) != VK_SUCCESS) + if (vkCreateSemaphore(ctx->device, &semaphore_info, nullptr, &m_pool[id]->semaphores[i]) != VK_SUCCESS) { EXIT("Can't create semaphore"); } - EXIT_IF(m_pool->buffers[i] == nullptr); - EXIT_IF(m_pool->fences[i] == nullptr); - EXIT_IF(m_pool->semaphores[i] == nullptr); + EXIT_IF(m_pool[id]->buffers[i] == nullptr); + EXIT_IF(m_pool[id]->fences[i] == nullptr); + EXIT_IF(m_pool[id]->semaphores[i] == nullptr); } } -void CommandPool::Delete() +void CommandPool::DeleteAll() { auto* ctx = g_render_ctx->GetGraphicCtx(); - EXIT_IF(m_pool == nullptr); - EXIT_IF(ctx == nullptr); - EXIT_IF(ctx->device == nullptr); - - for (uint32_t i = 0; i < m_pool->buffers_count; i++) + for (auto& pool: m_pool) { - vkDestroySemaphore(ctx->device, m_pool->semaphores[i], nullptr); - vkDestroyFence(ctx->device, m_pool->fences[i], nullptr); + if (pool != nullptr) + { + EXIT_IF(ctx == nullptr); + EXIT_IF(ctx->device == nullptr); + + for (uint32_t i = 0; i < pool->buffers_count; i++) + { + vkDestroySemaphore(ctx->device, pool->semaphores[i], nullptr); + vkDestroyFence(ctx->device, pool->fences[i], nullptr); + } + + vkFreeCommandBuffers(ctx->device, pool->pool, pool->buffers_count, pool->buffers); + + vkDestroyCommandPool(ctx->device, pool->pool, nullptr); + + delete[] pool->semaphores; + delete[] pool->fences; + delete[] pool->buffers; + delete[] pool->busy; + + delete pool; + pool = nullptr; + } } - - vkFreeCommandBuffers(ctx->device, m_pool->pool, m_pool->buffers_count, m_pool->buffers); - - vkDestroyCommandPool(ctx->device, m_pool->pool, nullptr); - - delete[] m_pool->semaphores; - delete[] m_pool->fences; - delete[] m_pool->buffers; - delete[] m_pool->busy; - - delete m_pool; - m_pool = nullptr; } VulkanFramebuffer* FramebufferCache::CreateFramebuffer(RenderColorInfo* color, RenderDepthInfo* depth) @@ -5049,7 +5052,7 @@ void CommandBuffer::Allocate() { EXIT_IF(!IsInvalid()); - m_pool = g_command_pool.GetPool(); + m_pool = g_command_pool.GetPool(m_queue); Core::LockGuard lock(m_pool->mutex); @@ -5130,9 +5133,19 @@ void CommandBuffer::Execute() EXIT_IF(m_queue < 0 || m_queue >= GraphicContext::QUEUES_NUM); - auto* queue = g_render_ctx->GetGraphicCtx()->queue[m_queue]; + const auto& queue = g_render_ctx->GetGraphicCtx()->queues[m_queue]; - auto result = vkQueueSubmit(queue, 1, &submit_info, fence); + if (queue.mutex != nullptr) + { + queue.mutex->Lock(); + } + + auto result = vkQueueSubmit(queue.vk_queue, 1, &submit_info, fence); + + if (queue.mutex != nullptr) + { + queue.mutex->Unlock(); + } m_execute = true; @@ -5159,9 +5172,19 @@ void CommandBuffer::ExecuteWithSemaphore() EXIT_IF(m_queue < 0 || m_queue >= GraphicContext::QUEUES_NUM); - auto* queue = g_render_ctx->GetGraphicCtx()->queue[m_queue]; + const auto& queue = g_render_ctx->GetGraphicCtx()->queues[m_queue]; - auto result = vkQueueSubmit(queue, 1, &submit_info, fence); + if (queue.mutex != nullptr) + { + queue.mutex->Lock(); + } + + auto result = vkQueueSubmit(queue.vk_queue, 1, &submit_info, fence); + + if (queue.mutex != nullptr) + { + queue.mutex->Lock(); + } m_execute = true; diff --git a/source/emulator/src/Graphics/GraphicsRun.cpp b/source/emulator/src/Graphics/GraphicsRun.cpp index 20abc56..15c1f72 100644 --- a/source/emulator/src/Graphics/GraphicsRun.cpp +++ b/source/emulator/src/Graphics/GraphicsRun.cpp @@ -514,9 +514,9 @@ void CommandProcessor::BufferInit() { EXIT_IF(buf != nullptr); - buf = new CommandBuffer; + buf = new CommandBuffer(m_queue); buf->SetParent(this); - buf->SetQueue(m_queue); + // buf->SetQueue(m_queue); } m_current_buffer = 0; diff --git a/source/emulator/src/Graphics/Utils.cpp b/source/emulator/src/Graphics/Utils.cpp index 5719f15..8fe2aa1 100644 --- a/source/emulator/src/Graphics/Utils.cpp +++ b/source/emulator/src/Graphics/Utils.cpp @@ -333,8 +333,8 @@ void UtilFillImage(GraphicContext* ctx, VulkanImage* dst_image, const void* src_ std::memcpy(data, src_data, size); VulkanUnmapMemory(ctx, &staging_buffer.memory); - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_UTIL); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); @@ -360,8 +360,8 @@ void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t staging_buffer.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; VulkanCreateBuffer(ctx, size, &staging_buffer); - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_UTIL); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); @@ -381,8 +381,8 @@ void UtilFillBuffer(GraphicContext* ctx, void* dst_data, uint64_t size, uint32_t void UtilSetDepthLayoutOptimal(DepthStencilVulkanImage* image) { - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_UTIL); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); @@ -406,8 +406,8 @@ void UtilSetDepthLayoutOptimal(DepthStencilVulkanImage* image) void UtilSetImageLayoutOptimal(VulkanImage* image) { - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_UTIL); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); @@ -440,8 +440,8 @@ void UtilFillImage(GraphicContext* ctx, VulkanImage* image, const void* src_data std::memcpy(data, src_data, size); VulkanUnmapMemory(ctx, &staging_buffer.memory); - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_UTIL); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); @@ -459,8 +459,8 @@ void UtilFillImage(GraphicContext* ctx, const Vector& regions, V EXIT_IF(ctx == nullptr); EXIT_IF(dst_image == nullptr); - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_UTIL); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); @@ -478,8 +478,8 @@ void UtilCopyBuffer(VulkanBuffer* src_buffer, VulkanBuffer* dst_buffer, uint64_t EXIT_IF(dst_buffer == nullptr); EXIT_IF(dst_buffer->buffer == nullptr); - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_UTIL); + CommandBuffer buffer(GraphicContext::QUEUE_UTIL); + // buffer.SetQueue(GraphicContext::QUEUE_UTIL); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); diff --git a/source/emulator/src/Graphics/Window.cpp b/source/emulator/src/Graphics/Window.cpp index 6c96b41..c3f2ee4 100644 --- a/source/emulator/src/Graphics/Window.cpp +++ b/source/emulator/src/Graphics/Window.cpp @@ -1302,13 +1302,153 @@ static bool CheckFormat(VkPhysicalDevice device, VkFormat format, bool tile, VkF return false; } +struct QueueInfo +{ + uint32_t family = 0; + uint32_t index = 0; + bool graphics = false; + bool compute = false; + bool transfer = false; + bool present = false; +}; + +struct VulkanQueues +{ + uint32_t family_count = 0; + Vector family_used; + Vector available; + Vector graphics; + Vector compute; + Vector transfer; + Vector present; +}; + +static void VulkanDumpQueues(const VulkanQueues& qs) +{ + printf("Queues selected:\n"); + printf("\t family_count = %u\n", qs.family_count); + Core::StringList nums; + for (auto u: qs.family_used) + { + nums.Add(String::FromPrintf("%u", u)); + } + printf("\t family_used = [%s]\n", nums.Concat(U", ").C_Str()); + printf("\t graphics:\n"); + for (const auto& q: qs.graphics) + { + printf("\t\t family = %u, index = %u\n", q.family, q.index); + } + printf("\t compute:\n"); + for (const auto& q: qs.compute) + { + printf("\t\t family = %u, index = %u\n", q.family, q.index); + } + printf("\t transfer:\n"); + for (const auto& q: qs.transfer) + { + printf("\t\t family = %u, index = %u\n", q.family, q.index); + } + printf("\t present:\n"); + for (const auto& q: qs.present) + { + printf("\t\t family = %u, index = %u\n", q.family, q.index); + } +} + +static VulkanQueues VulkanFindQueues(VkPhysicalDevice device, VkSurfaceKHR surface, uint32_t graphics_num, uint32_t compute_num, + uint32_t transfer_num, uint32_t present_num) +{ + EXIT_IF(device == nullptr); + EXIT_IF(surface == nullptr); + + VulkanQueues qs; + + uint32_t queue_family_count = 0; + vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_family_count, nullptr); + Vector queue_families(queue_family_count); + vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_family_count, queue_families.GetData()); + + qs.family_count = queue_family_count; + + uint32_t family = 0; + for (auto& f: queue_families) + { + VkBool32 presentation_supported = VK_FALSE; + vkGetPhysicalDeviceSurfaceSupportKHR(device, family, surface, &presentation_supported); + + printf("\tqueue family: %s [count = %u], [present = %s]\n", string_VkQueueFlags(f.queueFlags).c_str(), f.queueCount, + (presentation_supported == VK_TRUE ? "true" : "false")); + + for (uint32_t i = 0; i < f.queueCount; i++) + { + QueueInfo info; + info.family = family; + info.index = i; + info.graphics = (f.queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0; + info.compute = (f.queueFlags & VK_QUEUE_COMPUTE_BIT) != 0; + info.transfer = (f.queueFlags & VK_QUEUE_TRANSFER_BIT) != 0; + info.present = (presentation_supported == VK_TRUE); + + qs.available.Add(info); + } + + qs.family_used.Add(0); + + family++; + } + + for (uint32_t i = 0; i < graphics_num; i++) + { + if (auto index = qs.available.Find(true, [](auto& q, auto& b) { return q.graphics == b; }); qs.available.IndexValid(index)) + { + qs.family_used[qs.available.At(index).family]++; + qs.graphics.Add(qs.available.At(index)); + qs.available.RemoveAt(index); + } + } + + for (uint32_t i = 0; i < compute_num; i++) + { + if (auto index = qs.available.Find(true, [](auto& q, auto& b) { return q.compute == b; }); qs.available.IndexValid(index)) + { + qs.family_used[qs.available.At(index).family]++; + qs.compute.Add(qs.available.At(index)); + qs.available.RemoveAt(index); + } + } + + for (uint32_t i = 0; i < transfer_num; i++) + { + if (auto index = qs.available.Find(true, [](auto& q, auto& b) { return q.transfer == b; }); qs.available.IndexValid(index)) + { + qs.family_used[qs.available.At(index).family]++; + qs.transfer.Add(qs.available.At(index)); + qs.available.RemoveAt(index); + } + } + + for (uint32_t i = 0; i < present_num; i++) + { + if (auto index = qs.available.Find(true, [](auto& q, auto& b) { return q.present == b; }); qs.available.IndexValid(index)) + { + qs.family_used[qs.available.At(index).family]++; + qs.present.Add(qs.available.At(index)); + qs.available.RemoveAt(index); + } + } + + return qs; +} + // NOLINTNEXTLINE(readability-function-cognitive-complexity) -static VkPhysicalDevice VulkanFindPhysicalDevice(VkInstance instance, VkSurfaceKHR surface, VkQueueFlags queue_family, uint32_t queue_count, - const Vector& device_extensions, - SurfaceCapabilities* r /*, uint32_t image_count*/) +static void VulkanFindPhysicalDevice(VkInstance instance, VkSurfaceKHR surface, const Vector& device_extensions, + SurfaceCapabilities* out_capabilities, VkPhysicalDevice* out_device, VulkanQueues* out_queues) { EXIT_IF(instance == nullptr); EXIT_IF(surface == nullptr); + EXIT_IF(out_capabilities == nullptr); + EXIT_IF(out_device == nullptr); + EXIT_IF(out_queues == nullptr); uint32_t devices_count = 0; vkEnumeratePhysicalDevices(instance, &devices_count, nullptr); @@ -1319,10 +1459,11 @@ static VkPhysicalDevice VulkanFindPhysicalDevice(VkInstance instance, VkSurfaceK vkEnumeratePhysicalDevices(instance, &devices_count, devices.GetData()); VkPhysicalDevice best_device = nullptr; + VulkanQueues best_queues; for (const auto& device: devices) { - bool skip_device = true; + bool skip_device = false; VkPhysicalDeviceProperties device_properties {}; VkPhysicalDeviceFeatures2 device_features2 {}; @@ -1339,35 +1480,21 @@ static VkPhysicalDevice VulkanFindPhysicalDevice(VkInstance instance, VkSurfaceK printf("Vulkan device: %s\n", device_properties.deviceName); - uint32_t queue_family_count = 0; - vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_family_count, nullptr); + auto qs = VulkanFindQueues(device, surface, GraphicContext::QUEUE_GFX_NUM, GraphicContext::QUEUE_COMPUTE_NUM, + GraphicContext::QUEUE_UTIL_NUM, GraphicContext::QUEUE_PRESENT_NUM); - Vector queue_families(queue_family_count); - vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_family_count, queue_families.GetData()); + VulkanDumpQueues(qs); - for (auto& f: queue_families) + if (qs.graphics.Size() != GraphicContext::QUEUE_GFX_NUM || + !(qs.compute.Size() >= 1 && qs.compute.Size() <= GraphicContext::QUEUE_COMPUTE_NUM) || + qs.transfer.Size() != GraphicContext::QUEUE_UTIL_NUM || qs.present.Size() != GraphicContext::QUEUE_PRESENT_NUM) { - printf("\tqueue family: %s [count = %u]\n", string_VkQueueFlags(f.queueFlags).c_str(), f.queueCount); - } - - Vector indexes; - queue_families.FindAll( - queue_family, [queue_count](auto f, auto bit) { return (f.queueFlags & bit) != 0 && f.queueCount >= queue_count; }, &indexes); - for (auto index: indexes) - { - VkBool32 presentation_supported = VK_FALSE; - vkGetPhysicalDeviceSurfaceSupportKHR(device, index, surface, &presentation_supported); - - if (presentation_supported == VK_TRUE) - { - skip_device = false; - break; - } + printf("Not enough queues\n"); + skip_device = true; } if (color_write_ext.colorWriteEnable != VK_TRUE) { - printf("colorWriteEnable is not supported\n"); skip_device = true; } @@ -1420,9 +1547,9 @@ static VkPhysicalDevice VulkanFindPhysicalDevice(VkInstance instance, VkSurfaceK if (!skip_device) { - VulkanGetSurfaceCapabilities(device, surface, r); + VulkanGetSurfaceCapabilities(device, surface, out_capabilities); - if ((r->capabilities.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) == 0) + if ((out_capabilities->capabilities.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) == 0) { printf("Surface cannot be destination of blit\n"); skip_device = true; @@ -1530,69 +1657,48 @@ static VkPhysicalDevice VulkanFindPhysicalDevice(VkInstance instance, VkSurfaceK continue; } - if (best_device == nullptr) - { - best_device = device; - } - - if (device_properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) + if (best_device == nullptr || device_properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) { best_device = device; + best_queues = qs; } } - return best_device; + *out_device = best_device; + *out_queues = best_queues; } static VkDevice VulkanCreateDevice(VkPhysicalDevice physical_device, VkSurfaceKHR surface, const VulkanExtensions* r, - VkQueueFlags queue_family, uint32_t queue_count, uint32_t* queue_family_index, - const Vector& device_extensions) + const VulkanQueues& queues, const Vector& device_extensions) { EXIT_IF(physical_device == nullptr); EXIT_IF(r == nullptr); - EXIT_IF(queue_family_index == nullptr); EXIT_IF(surface == nullptr); - Vector queue_priority(queue_count); - for (auto& f: queue_priority) + Vector queue_create_info(queues.family_count); + Vector> queue_priority(queues.family_count); + uint32_t queue_create_info_num = 0; + + for (uint32_t i = 0; i < queues.family_count; i++) { - f = 1.0f; - } - - uint32_t queue_family_count = 0; - vkGetPhysicalDeviceQueueFamilyProperties(physical_device, &queue_family_count, nullptr); - - Vector queue_families(queue_family_count); - vkGetPhysicalDeviceQueueFamilyProperties(physical_device, &queue_family_count, queue_families.GetData()); - - Vector indexes; - queue_families.FindAll( - queue_family, [queue_count](auto f, auto bit) { return (f.queueFlags & bit) != 0 && f.queueCount >= queue_count; }, &indexes); - - *queue_family_index = queue_families.Size() + 1; - - for (auto index: indexes) - { - VkBool32 presentation_supported = VK_FALSE; - vkGetPhysicalDeviceSurfaceSupportKHR(physical_device, index, surface, &presentation_supported); - - if (presentation_supported == VK_TRUE) + if (queues.family_used[i] != 0) { - *queue_family_index = index; - break; + for (uint32_t pi = 0; pi < queues.family_used[i]; pi++) + { + queue_priority[queue_create_info_num].Add(1.0f); + } + + queue_create_info[queue_create_info_num].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + queue_create_info[queue_create_info_num].pNext = nullptr; + queue_create_info[queue_create_info_num].flags = 0; + queue_create_info[queue_create_info_num].queueFamilyIndex = i; + queue_create_info[queue_create_info_num].queueCount = queues.family_used[i]; + queue_create_info[queue_create_info_num].pQueuePriorities = queue_priority[queue_create_info_num].GetDataConst(); + + queue_create_info_num++; } } - EXIT_IF(!queue_families.IndexValid(*queue_family_index)); - - VkDeviceQueueCreateInfo queue_create_info {}; - queue_create_info.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; - queue_create_info.pNext = nullptr; - queue_create_info.flags = 0; - queue_create_info.queueFamilyIndex = *queue_family_index; - queue_create_info.queueCount = queue_count; - queue_create_info.pQueuePriorities = queue_priority.GetDataConst(); - VkPhysicalDeviceFeatures device_features {}; device_features.fragmentStoresAndAtomics = VK_TRUE; device_features.samplerAnisotropy = VK_TRUE; @@ -1607,8 +1713,8 @@ static VkDevice VulkanCreateDevice(VkPhysicalDevice physical_device, VkSurfaceKH create_info.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; create_info.pNext = &color_write_ext; create_info.flags = 0; - create_info.pQueueCreateInfos = &queue_create_info; - create_info.queueCreateInfoCount = 1; + create_info.pQueueCreateInfos = queue_create_info.GetDataConst(); + create_info.queueCreateInfoCount = queue_create_info_num; create_info.enabledLayerCount = (r->enable_validation_layers ? r->required_layers.Size() : 0); create_info.ppEnabledLayerNames = (r->enable_validation_layers ? r->required_layers.GetDataConst() : nullptr); create_info.enabledExtensionCount = device_extensions.Size(); @@ -1902,17 +2008,36 @@ static VKAPI_ATTR VkResult VKAPI_CALL VulkanCreateDebugUtilsMessengerEXT(VkInsta return swapchain; } -static void VulkanCreateQueues(GraphicContext* ctx) +static void VulkanCreateQueues(GraphicContext* ctx, const VulkanQueues& queues) { EXIT_IF(ctx == nullptr); EXIT_IF(ctx->device == nullptr); - EXIT_IF(ctx->queue_family_index == static_cast(-1)); + EXIT_IF(queues.graphics.Size() != 1); + EXIT_IF(queues.transfer.Size() != 1); + EXIT_IF(queues.present.Size() != 1); + EXIT_IF(!(queues.compute.Size() >= 1 && queues.compute.Size() <= GraphicContext::QUEUE_COMPUTE_NUM)); - for (int i = 0; i < GraphicContext::QUEUES_NUM; i++) + auto get_queue = [ctx](int id, const QueueInfo& info, bool with_mutex = false) { - EXIT_IF(ctx->queue[i] != nullptr); - vkGetDeviceQueue(ctx->device, ctx->queue_family_index, i, &ctx->queue[i]); - EXIT_NOT_IMPLEMENTED(ctx->queue[i] == nullptr); + ctx->queues[id].family = info.family; + ctx->queues[id].index = info.index; + EXIT_IF(ctx->queues[id].vk_queue != nullptr); + vkGetDeviceQueue(ctx->device, ctx->queues[id].family, ctx->queues[id].index, &ctx->queues[id].vk_queue); + EXIT_NOT_IMPLEMENTED(ctx->queues[id].vk_queue == nullptr); + if (with_mutex) + { + ctx->queues[id].mutex = new Core::Mutex; + } + }; + + get_queue(GraphicContext::QUEUE_GFX, queues.graphics.At(0)); + get_queue(GraphicContext::QUEUE_UTIL, queues.transfer.At(0)); + get_queue(GraphicContext::QUEUE_PRESENT, queues.present.At(0)); + + for (int id = 0; id < GraphicContext::QUEUE_COMPUTE_NUM; id++) + { + bool with_mutex = (GraphicContext::QUEUE_COMPUTE_NUM == queues.compute.Size()); + get_queue(GraphicContext::QUEUE_COMPUTE_START + id, queues.compute.At(id % queues.compute.Size()), with_mutex); } } @@ -2063,9 +2188,11 @@ static void VulkanCreate(WindowContext* ctx) ctx->surface_capabilities = new SurfaceCapabilities {}; - ctx->graphic_ctx.physical_device = - VulkanFindPhysicalDevice(ctx->graphic_ctx.instance, ctx->surface, VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT, - GraphicContext::QUEUES_NUM, device_extensions, ctx->surface_capabilities); + VulkanQueues queues; + + VulkanFindPhysicalDevice(ctx->graphic_ctx.instance, ctx->surface, device_extensions, ctx->surface_capabilities, + &ctx->graphic_ctx.physical_device, &queues); + if (ctx->graphic_ctx.physical_device == nullptr) { EXIT("Could not find suitable device"); @@ -2079,15 +2206,13 @@ static void VulkanCreate(WindowContext* ctx) memcpy(ctx->device_name, device_properties.deviceName, sizeof(ctx->device_name)); memcpy(ctx->processor_name, Loader::GetSystemInfo().ProcessorName.C_Str(), sizeof(ctx->processor_name)); - ctx->graphic_ctx.device = - VulkanCreateDevice(ctx->graphic_ctx.physical_device, ctx->surface, &r, VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT, - GraphicContext::QUEUES_NUM, &ctx->graphic_ctx.queue_family_index, device_extensions); + ctx->graphic_ctx.device = VulkanCreateDevice(ctx->graphic_ctx.physical_device, ctx->surface, &r, queues, device_extensions); if (ctx->graphic_ctx.device == nullptr) { EXIT("Could not create device"); } - VulkanCreateQueues(&ctx->graphic_ctx); + VulkanCreateQueues(&ctx->graphic_ctx, queues); ctx->swapchain = VulkanCreateSwapchain(&ctx->graphic_ctx, 2); } @@ -2238,8 +2363,8 @@ void WindowDrawBuffer(VideoOutVulkanImage* image) EXIT_IF(blt_src_image == nullptr); EXIT_IF(blt_dst_image == nullptr); - CommandBuffer buffer; - buffer.SetQueue(GraphicContext::QUEUE_PRESENT); + CommandBuffer buffer(GraphicContext::QUEUE_PRESENT); + // buffer.SetQueue(GraphicContext::QUEUE_PRESENT); EXIT_NOT_IMPLEMENTED(buffer.IsInvalid()); @@ -2280,7 +2405,11 @@ void WindowDrawBuffer(VideoOutVulkanImage* image) present.waitSemaphoreCount = 1; present.pResults = nullptr; - result = vkQueuePresentKHR(g_window_ctx->graphic_ctx.queue[GraphicContext::QUEUE_PRESENT], &present); + const auto& queue = g_window_ctx->graphic_ctx.queues[GraphicContext::QUEUE_PRESENT]; + + EXIT_IF(queue.mutex != nullptr); + + result = vkQueuePresentKHR(queue.vk_queue, &present); EXIT_NOT_IMPLEMENTED(result != VK_SUCCESS); WindowUpdateTitle();