vulkan: fix various validation errors

oit: skip to new frame even if previous frame was never presented
libretro: quad pipeline used for overlay must use subpass 2 in per-pixel
overlay: delay destruction of VMUs images
libretro: WaitIdle on queue instead of wait_sync_index

core/rend/vulkan/oit/oit_drawer.cpp

@@ -693,6 +693,12 @@ void OITTextureDrawer::EndFrame()
 
 vk::CommandBuffer OITScreenDrawer::NewFrame()
 {
+	if (frameRendered)
+	{
+		// in case the previous image was never presented
+		frameRendered = false;
+		NewImage();
+	}
 	vk::CommandBuffer commandBuffer = commandPool->Allocate();
 	commandBuffer.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit));
 

core/rend/vulkan/overlay.cpp

@@ -43,7 +43,7 @@ std::unique_ptr<Texture> VulkanOverlay::createTexture(vk::CommandBuffer commandB
 	return texture;
 }
 
-void VulkanOverlay::Prepare(vk::CommandBuffer cmdBuffer, bool vmu, bool crosshair)
+void VulkanOverlay::Prepare(vk::CommandBuffer cmdBuffer, bool vmu, bool crosshair, TextureCache& textureCache)
 {
 	if (vmu)
 	{
@@ -52,12 +52,18 @@ void VulkanOverlay::Prepare(vk::CommandBuffer cmdBuffer, bool vmu, bool crosshai
 			std::unique_ptr<Texture>& texture = vmuTextures[i];
 			if (!vmu_lcd_status[i])
 			{
-				texture.reset();
+				if (texture)
+				{
+					textureCache.DestroyLater(texture.get());
+					texture.reset();
+				}
 				continue;
 			}
 			if (texture != nullptr && !vmu_lcd_changed[i])
 				continue;
 
+			if (texture)
+				textureCache.DestroyLater(texture.get());
 			texture = createTexture(cmdBuffer, 48, 32, (u8*)vmu_lcd_data[i]);
 			vmu_lcd_changed[i] = false;
 		}
@@ -69,7 +75,7 @@ void VulkanOverlay::Prepare(vk::CommandBuffer cmdBuffer, bool vmu, bool crosshai
 	}
 }
 
-vk::CommandBuffer VulkanOverlay::Prepare(vk::CommandPool commandPool, bool vmu, bool crosshair)
+vk::CommandBuffer VulkanOverlay::Prepare(vk::CommandPool commandPool, bool vmu, bool crosshair, TextureCache& textureCache)
 {
 	VulkanContext *context = VulkanContext::Instance();
 	commandBuffers.resize(context->GetSwapChainSize());
@@ -78,7 +84,7 @@ vk::CommandBuffer VulkanOverlay::Prepare(vk::CommandPool commandPool, bool vmu,
 			.front());
 	vk::CommandBuffer cmdBuffer = *commandBuffers[context->GetCurrentImageIndex()];
 	cmdBuffer.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit));
-	Prepare(cmdBuffer, vmu, crosshair);
+	Prepare(cmdBuffer, vmu, crosshair, textureCache);
 	cmdBuffer.end();
 
 	return cmdBuffer;

core/rend/vulkan/overlay.h

@@ -26,6 +26,7 @@
 #include <vector>
 
 class Texture;
+class TextureCache;
 
 class VulkanOverlay
 {
@@ -52,8 +53,8 @@ public:
 		xhairDrawer.reset();
 	}
 
-	vk::CommandBuffer Prepare(vk::CommandPool commandPool, bool vmu, bool crosshair);
-	void Prepare(vk::CommandBuffer commandBuffer, bool vmu, bool crosshair);
+	vk::CommandBuffer Prepare(vk::CommandPool commandPool, bool vmu, bool crosshair, TextureCache& textureCache);
+	void Prepare(vk::CommandBuffer commandBuffer, bool vmu, bool crosshair, TextureCache& textureCache);
 	void Draw(vk::CommandBuffer commandBuffer, vk::Extent2D viewport, float scaling, bool vmu, bool crosshair);
 
 private:

core/rend/vulkan/quad.cpp

@@ -109,13 +109,14 @@ void QuadPipeline::CreatePipeline()
 	  &pipelineColorBlendStateCreateInfo,         // pColorBlendState
 	  &pipelineDynamicStateCreateInfo,            // pDynamicState
 	  *pipelineLayout,                            // layout
-	  renderPass                                  // renderPass
+	  renderPass,                                 // renderPass
+	  subpass                                     // subpass
 	);
 
 	pipeline = GetContext()->GetDevice().createGraphicsPipelineUnique(GetContext()->GetPipelineCache(), graphicsPipelineCreateInfo);
 }
 
-void QuadPipeline::Init(ShaderManager *shaderManager, vk::RenderPass renderPass)
+void QuadPipeline::Init(ShaderManager *shaderManager, vk::RenderPass renderPass, int subpass)
 {
 	this->shaderManager = shaderManager;
 	if (!pipelineLayout)
@@ -156,6 +157,7 @@ void QuadPipeline::Init(ShaderManager *shaderManager, vk::RenderPass renderPass)
 	if (this->renderPass != renderPass)
 	{
 		this->renderPass = renderPass;
+		this->subpass = subpass;
 		pipeline.reset();
 	}
 }

core/rend/vulkan/quad.h

@@ -72,8 +72,8 @@ class QuadPipeline
 public:
 	QuadPipeline(bool ignoreTexAlpha, bool rotate = false)
 		: rotate(rotate), ignoreTexAlpha(ignoreTexAlpha) {}
-	void Init(ShaderManager *shaderManager, vk::RenderPass renderPass);
-	void Init(const QuadPipeline& other) { Init(other.shaderManager, other.renderPass); }
+	void Init(ShaderManager *shaderManager, vk::RenderPass renderPass, int subpass);
+	void Init(const QuadPipeline& other) { Init(other.shaderManager, other.renderPass, other.subpass); }
 	void Term() {
 		pipeline.reset();
 		linearSampler.reset();
@@ -97,6 +97,7 @@ private:
 	void CreatePipeline();
 
 	vk::RenderPass renderPass;
+	int subpass = 0;
 	vk::UniquePipeline pipeline;
 	vk::UniqueSampler linearSampler;
 	vk::UniqueSampler nearestSampler;

core/rend/vulkan/vk_context_lr.h

@@ -46,8 +46,8 @@ public:
 	vk::DescriptorPool GetDescriptorPool() const { return *descriptorPool; }
 	u32 GetSwapChainSize() const { u32 m = retro_render_if->get_sync_index_mask(retro_render_if->handle); u32 n = 1; while (m >>= 1) n++; return n; }
 	int GetCurrentImageIndex() const { return retro_render_if->get_sync_index(retro_render_if->handle); }
-	// FIXME that's not quite correct
-	void WaitIdle() const { retro_render_if->wait_sync_index(retro_render_if->handle); }
+
+	void WaitIdle() const { queue.waitIdle(); }
 	void SubmitCommandBuffers(u32 bufferCount, vk::CommandBuffer *buffers, vk::Fence fence) {
 		retro_render_if->lock_queue(retro_render_if->handle);
 		queue.submit(vk::SubmitInfo(0, nullptr, nullptr, bufferCount, buffers, 0, nullptr), fence);

core/rend/vulkan/vulkan_context.cpp

@@ -656,10 +656,10 @@ void VulkanContext::CreateSwapChain()
 	    	renderCompleteSemaphores.push_back(device->createSemaphoreUnique(vk::SemaphoreCreateInfo()));
 	    	imageAcquiredSemaphores.push_back(device->createSemaphoreUnique(vk::SemaphoreCreateInfo()));
 	    }
-	    quadPipeline->Init(shaderManager.get(), *renderPass);
-	    quadPipelineWithAlpha->Init(shaderManager.get(), *renderPass);
+	    quadPipeline->Init(shaderManager.get(), *renderPass, 0);
+	    quadPipelineWithAlpha->Init(shaderManager.get(), *renderPass, 0);
 	    quadDrawer->Init(quadPipeline.get());
-	    quadRotatePipeline->Init(shaderManager.get(), *renderPass);
+	    quadRotatePipeline->Init(shaderManager.get(), *renderPass, 0);
 	    quadRotateDrawer->Init(quadRotatePipeline.get());
 	    overlay->Init(quadPipelineWithAlpha.get());
 
@@ -739,6 +739,7 @@ bool VulkanContext::init()
 #error "Unknown Vulkan platform"
 #endif
 	overlay = std::unique_ptr<VulkanOverlay>(new VulkanOverlay());
+	textureCache = std::unique_ptr<TextureCache>(new TextureCache());
 
 	return InitDevice();
 }
@@ -758,6 +759,7 @@ void VulkanContext::NewFrame()
 	device->resetCommandPool(*commandPools[currentImage], vk::CommandPoolResetFlagBits::eReleaseResources);
 	vk::CommandBuffer commandBuffer = *commandBuffers[currentImage];
 	commandBuffer.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit));
+	textureCache->SetCurrentIndex(GetCurrentImageIndex());
 	verify(!rendering);
 	rendering = true;
 }
@@ -884,7 +886,7 @@ std::string VulkanContext::getDriverVersion()
 
 vk::CommandBuffer VulkanContext::PrepareOverlay(bool vmu, bool crosshair)
 {
-	return overlay->Prepare(*commandPools[GetCurrentImageIndex()], vmu, crosshair);
+	return overlay->Prepare(*commandPools[GetCurrentImageIndex()], vmu, crosshair, *textureCache);
 }
 
  void VulkanContext::DrawOverlay(float scaling, bool vmu, bool crosshair)
@@ -1147,6 +1149,18 @@ void VulkanContext::SetWindowSize(u32 width, u32 height)
 	}
 }
 
+VulkanContext::VulkanContext()
+{
+	verify(contextInstance == nullptr);
+	contextInstance = this;
+}
+
+VulkanContext::~VulkanContext()
+{
+	verify(contextInstance == this);
+	contextInstance = nullptr;
+}
+
 void ImGui_ImplVulkan_RenderDrawData(ImDrawData *draw_data)
 {
 	VulkanContext *context = VulkanContext::Instance();

core/rend/vulkan/vulkan_context.h

@@ -40,14 +40,15 @@ public:
 #include "wsi/context.h"
 
 struct ImDrawData;
+struct TextureCache;
 void ImGui_ImplVulkan_RenderDrawData(ImDrawData *draw_data);
 static vk::Format findDepthFormat(vk::PhysicalDevice physicalDevice);
 
 class VulkanContext : public GraphicsContext
 {
 public:
-	VulkanContext() { verify(contextInstance == nullptr); contextInstance = this; }
-	~VulkanContext() { verify(contextInstance == this); contextInstance = nullptr; }
+	VulkanContext();
+	~VulkanContext();
 
 	bool init();
 	void term() override;
@@ -210,6 +211,8 @@ private:
 	vk::Extent2D lastFrameExtent;
 
 	std::unique_ptr<VulkanOverlay> overlay;
+	// only used to delay the destruction of overlay textures
+	std::unique_ptr<TextureCache> textureCache;
 
 #ifdef VK_DEBUG
 #ifndef __ANDROID__

core/rend/vulkan/vulkan_renderer.h

@@ -67,7 +67,7 @@ protected:
 #endif
 #ifdef LIBRETRO
 		quadPipeline = std::unique_ptr<QuadPipeline>(new QuadPipeline(false, false));
-		quadPipeline->Init(&shaderManager, renderPass);
+		quadPipeline->Init(&shaderManager, renderPass, subpass);
 		overlay = std::unique_ptr<VulkanOverlay>(new VulkanOverlay());
 		overlay->Init(quadPipeline.get());
 #endif
@@ -147,7 +147,7 @@ public:
 		{
 #ifdef LIBRETRO
 			if (!ctx->rend.isRTT)
-				overlay->Prepare(texCommandBuffer, true, true);
+				overlay->Prepare(texCommandBuffer, true, true, textureCache);
 #endif
 			CheckFogTexture();
 			CheckPaletteTexture();