HostDisplay: Clamp scissor rect for cursor

Scissor with x/y < 0 is invalid.

To you know who you are, stop copying these changes and putting your
name on it, or respect the copyright declared in the files. You're
violating both copyright as well as the license by not attributing.

src/common/d3d12/util.cpp

@@ -14,6 +14,52 @@ Log_SetChannel(D3D12);
 
 namespace D3D12 {
 
+void ResourceBarrier(ID3D12GraphicsCommandList* cmdlist, ID3D12Resource* resource, D3D12_RESOURCE_STATES from_state,
+                     D3D12_RESOURCE_STATES to_state)
+{
+  const D3D12_RESOURCE_BARRIER barrier = {D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
+                                          D3D12_RESOURCE_BARRIER_FLAG_NONE,
+                                          {{resource, D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES, from_state, to_state}}};
+  cmdlist->ResourceBarrier(1, &barrier);
+}
+
+void SetViewport(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height, float min_depth /*= 0.0f*/,
+                 float max_depth /*= 1.0f*/)
+{
+  const D3D12_VIEWPORT vp{static_cast<float>(x),
+                          static_cast<float>(y),
+                          static_cast<float>(width),
+                          static_cast<float>(height),
+                          min_depth,
+                          max_depth};
+  cmdlist->RSSetViewports(1, &vp);
+}
+
+void SetScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height)
+{
+  const D3D12_RECT r{x, y, x + width, y + height};
+  cmdlist->RSSetScissorRects(1, &r);
+}
+
+void SetViewportAndScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height,
+                           float min_depth /*= 0.0f*/, float max_depth /*= 1.0f*/)
+{
+  SetViewport(cmdlist, x, y, width, height, min_depth, max_depth);
+  SetScissor(cmdlist, x, y, width, height);
+}
+
+void SetViewportAndClampScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height,
+                                float min_depth /*= 0.0f*/, float max_depth /*= 1.0f*/)
+{
+  SetViewport(cmdlist, x, y, width, height, min_depth, max_depth);
+
+  const int cx = std::max(x, 0);
+  const int cy = std::max(y, 0);
+  const int cwidth = width - (cx - x);
+  const int cheight = height - (cy - y);
+  SetScissor(cmdlist, cx, cy, cwidth, cheight);
+}
+
 u32 GetTexelSize(DXGI_FORMAT format)
 {
   switch (format)

src/common/d3d12/util.h

@@ -12,39 +12,19 @@ namespace D3D12 {
 
 class ShaderCache;
 
-static inline void ResourceBarrier(ID3D12GraphicsCommandList* cmdlist, ID3D12Resource* resource,
-                                   D3D12_RESOURCE_STATES from_state, D3D12_RESOURCE_STATES to_state)
-{
-  const D3D12_RESOURCE_BARRIER barrier = {D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
-                                          D3D12_RESOURCE_BARRIER_FLAG_NONE,
-                                          {{resource, D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES, from_state, to_state}}};
-  cmdlist->ResourceBarrier(1, &barrier);
-}
+void ResourceBarrier(ID3D12GraphicsCommandList* cmdlist, ID3D12Resource* resource, D3D12_RESOURCE_STATES from_state,
+                     D3D12_RESOURCE_STATES to_state);
 
-static inline void SetViewport(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height,
-                               float min_depth = 0.0f, float max_depth = 1.0f)
-{
-  const D3D12_VIEWPORT vp{static_cast<float>(x),
-                          static_cast<float>(y),
-                          static_cast<float>(width),
-                          static_cast<float>(height),
-                          min_depth,
-                          max_depth};
-  cmdlist->RSSetViewports(1, &vp);
-}
+void SetViewport(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height, float min_depth = 0.0f,
+                 float max_depth = 1.0f);
 
-static inline void SetScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height)
-{
-  const D3D12_RECT r{x, y, x + width, y + height};
-  cmdlist->RSSetScissorRects(1, &r);
-}
+void SetScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height);
 
-static inline void SetViewportAndScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height,
-                                         float min_depth = 0.0f, float max_depth = 1.0f)
-{
-  SetViewport(cmdlist, x, y, width, height, min_depth, max_depth);
-  SetScissor(cmdlist, x, y, width, height);
-}
+void SetViewportAndScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height,
+                           float min_depth = 0.0f, float max_depth = 1.0f);
+
+void SetViewportAndClampScissor(ID3D12GraphicsCommandList* cmdlist, int x, int y, int width, int height,
+                                float min_depth = 0.0f, float max_depth = 1.0f);
 
 u32 GetTexelSize(DXGI_FORMAT format);
 

src/common/vulkan/util.cpp

@@ -172,6 +172,25 @@ void Vulkan::Util::SetViewportAndScissor(VkCommandBuffer command_buffer, int x,
   vkCmdSetScissor(command_buffer, 0, 1, &scissor);
 }
 
+void Vulkan::Util::SetViewportAndClampScissor(VkCommandBuffer command_buffer, int x, int y, int width, int height,
+                                              float min_depth /*= 0.0f*/, float max_depth /*= 1.0f*/)
+{
+  const VkViewport vp{static_cast<float>(x),
+                      static_cast<float>(y),
+                      static_cast<float>(width),
+                      static_cast<float>(height),
+                      min_depth,
+                      max_depth};
+  vkCmdSetViewport(command_buffer, 0, 1, &vp);
+
+  const int cx = std::max(x, 0);
+  const int cy = std::max(y, 0);
+  const int cwidth = width - (cx - x);
+  const int cheight = height - (cy - y);
+  const VkRect2D scissor{{cx, cy}, {static_cast<u32>(cwidth), static_cast<u32>(cheight)}};
+  vkCmdSetScissor(command_buffer, 0, 1, &scissor);
+}
+
 void Vulkan::Util::SafeDestroyFramebuffer(VkFramebuffer& fb)
 {
   if (fb != VK_NULL_HANDLE)

src/common/vulkan/util.h

@@ -52,6 +52,8 @@ void SetScissor(VkCommandBuffer command_buffer, int x, int y, int width, int hei
 // Combines viewport and scissor updates
 void SetViewportAndScissor(VkCommandBuffer command_buffer, int x, int y, int width, int height, float min_depth = 0.0f,
                            float max_depth = 1.0f);
+void SetViewportAndClampScissor(VkCommandBuffer command_buffer, int x, int y, int width, int height,
+                                float min_depth = 0.0f, float max_depth = 1.0f);
 
 // Wrapper for creating an barrier on a buffer
 void BufferMemoryBarrier(VkCommandBuffer command_buffer, VkBuffer buffer, VkAccessFlags src_access_mask,

src/frontend-common/d3d12_host_display.cpp

@@ -727,7 +727,7 @@ void D3D12HostDisplay::RenderDisplay(ID3D12GraphicsCommandList* cmdlist, s32 lef
   cmdlist->SetGraphicsRootDescriptorTable(1, texture->GetSRVDescriptor());
   cmdlist->SetGraphicsRootDescriptorTable(2, linear_filter ? m_linear_sampler : m_point_sampler);
 
-  D3D12::SetViewportAndScissor(cmdlist, left, top, width, height);
+  D3D12::SetViewportAndClampScissor(cmdlist, left, top, width, height);
 
   cmdlist->IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
   cmdlist->DrawInstanced(3, 1, 0, 0);
@@ -753,7 +753,7 @@ void D3D12HostDisplay::RenderSoftwareCursor(ID3D12GraphicsCommandList* cmdlist,
   cmdlist->SetGraphicsRootDescriptorTable(1, static_cast<D3D12::Texture*>(texture_handle)->GetSRVDescriptor());
   cmdlist->SetGraphicsRootDescriptorTable(2, m_linear_sampler);
 
-  D3D12::SetViewportAndScissor(cmdlist, left, top, width, height);
+  D3D12::SetViewportAndClampScissor(cmdlist, left, top, width, height);
 
   cmdlist->IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
   cmdlist->DrawInstanced(3, 1, 0, 0);

src/frontend-common/vulkan_host_display.cpp

@@ -851,7 +851,7 @@ void VulkanHostDisplay::RenderDisplay(s32 left, s32 top, s32 width, s32 height,
   vkCmdBindPipeline(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_display_pipeline);
   vkCmdPushConstants(cmdbuffer, m_pipeline_layout, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(pc), &pc);
   vkCmdBindDescriptorSets(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipeline_layout, 0, 1, &ds, 0, nullptr);
-  Vulkan::Util::SetViewportAndScissor(cmdbuffer, left, top, width, height);
+  Vulkan::Util::SetViewportAndClampScissor(cmdbuffer, left, top, width, height);
   vkCmdDraw(cmdbuffer, 3, 1, 0, 0);
 }
 
@@ -895,7 +895,7 @@ void VulkanHostDisplay::RenderSoftwareCursor(s32 left, s32 top, s32 width, s32 h
   vkCmdBindPipeline(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_cursor_pipeline);
   vkCmdPushConstants(cmdbuffer, m_pipeline_layout, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(pc), &pc);
   vkCmdBindDescriptorSets(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipeline_layout, 0, 1, &ds, 0, nullptr);
-  Vulkan::Util::SetViewportAndScissor(cmdbuffer, left, top, width, height);
+  Vulkan::Util::SetViewportAndClampScissor(cmdbuffer, left, top, width, height);
   vkCmdDraw(cmdbuffer, 3, 1, 0, 0);
 }