libretro: Re-query hardware render interface after AV system info change

I suspect the frontend is supposed to call context_reset/destroy here,
but it's not for whatever reason, and this works around it.

src/duckstation-libretro/libretro_d3d11_host_display.cpp

@@ -71,6 +71,34 @@ void LibretroD3D11HostDisplay::ResizeRenderWindow(s32 new_window_width, s32 new_
   m_window_info.surface_height = static_cast<u32>(new_window_height);
 }
 
+bool LibretroD3D11HostDisplay::ChangeRenderWindow(const WindowInfo& new_wi)
+{
+  // Check that the device hasn't changed.
+  retro_hw_render_interface* ri = nullptr;
+  if (!g_retro_environment_callback(RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE, &ri))
+  {
+    Log_ErrorPrint("Failed to get HW render interface");
+    return false;
+  }
+  else if (ri->interface_type != RETRO_HW_RENDER_INTERFACE_D3D11 ||
+           ri->interface_version != RETRO_HW_RENDER_INTERFACE_D3D11_VERSION)
+  {
+    Log_ErrorPrintf("Unexpected HW interface - type %u version %u", static_cast<unsigned>(ri->interface_type),
+                    static_cast<unsigned>(ri->interface_version));
+    return false;
+  }
+
+  const retro_hw_render_interface_d3d11* d3d11_ri = reinterpret_cast<const retro_hw_render_interface_d3d11*>(ri);
+  if (d3d11_ri->device != m_device.Get() || d3d11_ri->context != m_context.Get())
+  {
+    Log_ErrorPrintf("D3D device/context changed outside our control");
+    return false;
+  }
+
+  m_window_info = new_wi;
+  return true;
+}
+
 bool LibretroD3D11HostDisplay::Render()
 {
   const u32 resolution_scale = g_libretro_host_interface.GetResolutionScale();

src/duckstation-libretro/libretro_d3d11_host_display.h

@@ -14,6 +14,7 @@ public:
   bool CreateRenderDevice(const WindowInfo& wi, std::string_view adapter_name, bool debug_device) override;
 
   void ResizeRenderWindow(s32 new_window_width, s32 new_window_height) override;
+  bool ChangeRenderWindow(const WindowInfo& new_wi) override;
 
   void SetVSync(bool enabled) override;
 

src/duckstation-libretro/libretro_host_interface.cpp

@@ -1145,6 +1145,16 @@ void LibretroHostInterface::HardwareRendererContextReset()
 
 void LibretroHostInterface::SwitchToHardwareRenderer()
 {
+  struct retro_system_av_info avi;
+  g_libretro_host_interface.GetSystemAVInfo(&avi, true);
+
+  WindowInfo wi;
+  wi.type = WindowInfo::Type::Libretro;
+  wi.display_connection = &g_libretro_host_interface.m_hw_render_callback;
+  wi.surface_width = avi.geometry.base_width;
+  wi.surface_height = avi.geometry.base_height;
+  wi.surface_scale = 1.0f;
+
   // use the existing device if we just resized the window
   std::optional<GPURenderer> renderer;
   std::unique_ptr<HostDisplay> display = std::move(m_hw_render_display);
@@ -1152,10 +1162,15 @@ void LibretroHostInterface::SwitchToHardwareRenderer()
   {
     Log_InfoPrintf("Using existing hardware display");
     renderer = RenderAPIToRenderer(display->GetRenderAPI());
-    if (!display->CreateResources())
+    if (!display->ChangeRenderWindow(wi) || !display->CreateResources())
-      Panic("Failed to recreate resources after reinit");
+    {
+      Log_ErrorPrintf("Failed to recreate resources after reinit");
+      display->DestroyRenderDevice();
+      display.reset();
     }
-  else
+  }
+
+  if (!display)
   {
     renderer = RetroHwContextToRenderer(m_hw_render_callback.context_type);
     if (!renderer.has_value())
@@ -1184,16 +1199,6 @@ void LibretroHostInterface::SwitchToHardwareRenderer()
         Log_ErrorPrintf("Unhandled renderer '%s'", Settings::GetRendererName(renderer.value()));
         return;
     }
-
-    struct retro_system_av_info avi;
-    g_libretro_host_interface.GetSystemAVInfo(&avi, true);
-
-    WindowInfo wi;
-    wi.type = WindowInfo::Type::Libretro;
-    wi.display_connection = &g_libretro_host_interface.m_hw_render_callback;
-    wi.surface_width = avi.geometry.base_width;
-    wi.surface_height = avi.geometry.base_height;
-    wi.surface_scale = 1.0f;
     if (!display || !display->CreateRenderDevice(wi, {}, g_settings.gpu_use_debug_device) ||
         !display->InitializeRenderDevice(GetShaderCacheBasePath(), g_settings.gpu_use_debug_device))
     {

src/duckstation-libretro/libretro_opengl_host_display.cpp

@@ -135,6 +135,12 @@ void LibretroOpenGLHostDisplay::ResizeRenderWindow(s32 new_window_width, s32 new
   m_window_info.surface_height = static_cast<u32>(new_window_height);
 }
 
+bool LibretroOpenGLHostDisplay::ChangeRenderWindow(const WindowInfo& new_wi)
+{
+  m_window_info = new_wi;
+  return true;
+}
+
 bool LibretroOpenGLHostDisplay::Render()
 {
   const GLuint fbo = static_cast<GLuint>(

src/duckstation-libretro/libretro_opengl_host_display.h

@@ -21,6 +21,7 @@ public:
   void DestroyRenderDevice() override;
 
   void ResizeRenderWindow(s32 new_window_width, s32 new_window_height) override;
+  bool ChangeRenderWindow(const WindowInfo& new_wi) override;
 
   void SetVSync(bool enabled) override;
 

src/duckstation-libretro/libretro_vulkan_host_display.cpp

@@ -107,11 +107,8 @@ bool LibretroVulkanHostDisplay::CreateRenderDevice(const WindowInfo& wi, std::st
     return false;
   }
 
-  // Keeping the pointer instead of memcpying can cause crashes, e.g. fullscreen switches.
-  std::memcpy(&m_ri, reinterpret_cast<const retro_hw_render_interface_vulkan*>(ri),
-              sizeof(retro_hw_render_interface_vulkan));
-
   // TODO: Grab queue? it should be the same
+  m_ri = reinterpret_cast<retro_hw_render_interface_vulkan*>(ri);
   return true;
 }
 
@@ -150,6 +147,33 @@ void LibretroVulkanHostDisplay::ResizeRenderWindow(s32 new_window_width, s32 new
   m_window_info.surface_height = static_cast<u32>(new_window_height);
 }
 
+bool LibretroVulkanHostDisplay::ChangeRenderWindow(const WindowInfo& new_wi)
+{
+  // re-query hardware render interface - in vulkan, things get recreated without us being notified
+  retro_hw_render_interface* ri = nullptr;
+  if (!g_retro_environment_callback(RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE, &ri))
+  {
+    Log_ErrorPrint("Failed to get HW render interface");
+    return false;
+  }
+  else if (ri->interface_type != RETRO_HW_RENDER_INTERFACE_VULKAN ||
+           ri->interface_version != RETRO_HW_RENDER_INTERFACE_VULKAN_VERSION)
+  {
+    Log_ErrorPrintf("Unexpected HW interface - type %u version %u", static_cast<unsigned>(ri->interface_type),
+                    static_cast<unsigned>(ri->interface_version));
+    return false;
+  }
+
+  retro_hw_render_interface_vulkan* vri = reinterpret_cast<retro_hw_render_interface_vulkan*>(ri);
+  if (vri != m_ri)
+  {
+    Log_WarningPrintf("HW render interface pointer changed without us being notified, this might cause issues?");
+    m_ri = vri;
+  }
+
+  return true;
+}
+
 bool LibretroVulkanHostDisplay::Render()
 {
   const u32 resolution_scale = g_libretro_host_interface.GetResolutionScale();
@@ -186,13 +210,13 @@ bool LibretroVulkanHostDisplay::Render()
   vkCmdEndRenderPass(cmdbuffer);
   m_frame_texture.TransitionToLayout(cmdbuffer, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
   m_frame_view.image_layout = m_frame_texture.GetLayout();
-  m_ri.set_image(m_ri.handle, &m_frame_view, 0, nullptr, VK_QUEUE_FAMILY_IGNORED);
+  m_ri->set_image(m_ri->handle, &m_frame_view, 0, nullptr, VK_QUEUE_FAMILY_IGNORED);
 
   // TODO: We can't use this because it doesn't support passing fences...
   // m_ri.set_command_buffers(m_ri.handle, 1, &cmdbuffer);
-  m_ri.lock_queue(m_ri.handle);
+  m_ri->lock_queue(m_ri->handle);
   g_vulkan_context->SubmitCommandBuffer();
-  m_ri.unlock_queue(m_ri.handle);
+  m_ri->unlock_queue(m_ri->handle);
   g_vulkan_context->MoveToNextCommandBuffer();
 
   g_retro_video_refresh_callback(RETRO_HW_FRAME_BUFFER_VALID, display_width, display_height, 0);

src/duckstation-libretro/libretro_vulkan_host_display.h

@@ -18,6 +18,7 @@ public:
   void DestroyRenderDevice() override;
 
   void ResizeRenderWindow(s32 new_window_width, s32 new_window_height) override;
+  bool ChangeRenderWindow(const WindowInfo& new_wi) override;
 
   void SetVSync(bool enabled) override;
 
@@ -33,7 +34,7 @@ private:
 
   bool CheckFramebufferSize(u32 width, u32 height);
 
-  retro_hw_render_interface_vulkan m_ri;
+  retro_hw_render_interface_vulkan* m_ri = nullptr;
 
   Vulkan::Texture m_frame_texture;
   retro_vulkan_image m_frame_view = {};