Qt: Fix selecting fullscreen mode breaking other backend borderless

src/core/host_display.h

@@ -66,6 +66,7 @@ public:
   virtual void DestroyRenderDevice() = 0;
   virtual void DestroyRenderSurface() = 0;
   virtual bool ChangeRenderWindow(const WindowInfo& wi) = 0;
+  virtual bool SupportsFullscreen() const = 0;
   virtual bool IsFullscreen() = 0;
   virtual bool SetFullscreen(bool fullscreen, u32 width, u32 height, float refresh_rate) = 0;
   virtual bool CreateResources() = 0;

src/duckstation-libretro/libretro_host_display.cpp

@@ -156,6 +156,11 @@ void LibretroHostDisplay::ResizeRenderWindow(s32 new_window_width, s32 new_windo
   m_window_info.surface_height = new_window_height;
 }
 
+bool LibretroHostDisplay::SupportsFullscreen() const
+{
+  return false;
+}
+
 bool LibretroHostDisplay::IsFullscreen()
 {
   return false;

src/duckstation-libretro/libretro_host_display.h

@@ -23,6 +23,7 @@ public:
 
   bool ChangeRenderWindow(const WindowInfo& wi) override;
   void ResizeRenderWindow(s32 new_window_width, s32 new_window_height) override;
+  bool SupportsFullscreen() const override;
   bool IsFullscreen() override;
   bool SetFullscreen(bool fullscreen, u32 width, u32 height, float refresh_rate) override;
   void DestroyRenderSurface() override;

src/duckstation-qt/mainwindow.cpp

@@ -87,8 +87,15 @@ QtDisplayWidget* MainWindow::createDisplay(QThread* worker_thread, const QString
   Assert(!m_host_display && !m_display_widget);
   Assert(!fullscreen || !render_to_main);
 
+  m_host_display = m_host_interface->createHostDisplay();
+  if (!m_host_display)
+  {
+    reportError(tr("Failed to create host display."));
+    return nullptr;
+  }
+
   const std::string fullscreen_mode = m_host_interface->GetStringSettingValue("GPU", "FullscreenMode", "");
-  const bool is_exclusive_fullscreen = (fullscreen && !fullscreen_mode.empty());
+  const bool is_exclusive_fullscreen = (fullscreen && !fullscreen_mode.empty() && m_host_display->SupportsFullscreen());
 
   m_display_widget = new QtDisplayWidget((!fullscreen && render_to_main) ? m_ui.mainContainer : nullptr);
   m_display_widget->setWindowTitle(windowTitle());
@@ -122,14 +129,17 @@ QtDisplayWidget* MainWindow::createDisplay(QThread* worker_thread, const QString
   {
     reportError(QStringLiteral("Failed to get window info from widget"));
     destroyDisplayWidget();
+    delete m_host_display;
+    m_host_display = nullptr;
     return nullptr;
   }
 
-  m_host_display = m_host_interface->createHostDisplay();
-  if (!m_host_display || !m_host_display->CreateRenderDevice(wi.value(), adapter_name.toStdString(), use_debug_device))
+  if (!m_host_display->CreateRenderDevice(wi.value(), adapter_name.toStdString(), use_debug_device))
   {
     reportError(tr("Failed to create host display device context."));
     destroyDisplayWidget();
+    delete m_host_display;
+    m_host_display = nullptr;
     return nullptr;
   }
 
@@ -145,7 +155,7 @@ QtDisplayWidget* MainWindow::updateDisplay(QThread* worker_thread, bool fullscre
   const bool is_fullscreen = m_display_widget->isFullScreen();
   const bool is_rendering_to_main = (!is_fullscreen && m_display_widget->parent());
   const std::string fullscreen_mode = m_host_interface->GetStringSettingValue("GPU", "FullscreenMode", "");
-  const bool is_exclusive_fullscreen = (fullscreen && !fullscreen_mode.empty());
+  const bool is_exclusive_fullscreen = (fullscreen && !fullscreen_mode.empty() && m_host_display->SupportsFullscreen());
   if (fullscreen == is_fullscreen && is_rendering_to_main == render_to_main)
     return m_display_widget;
 

src/frontend-common/d3d11_host_display.cpp

@@ -476,6 +476,15 @@ void D3D11HostDisplay::ResizeRenderWindow(s32 new_window_width, s32 new_window_h
 #endif
 }
 
+bool D3D11HostDisplay::SupportsFullscreen() const
+{
+#ifndef LIBRETRO
+  return true;
+#else
+  return false;
+#endif
+}
+
 bool D3D11HostDisplay::IsFullscreen()
 {
 #ifndef LIBRETRO

src/frontend-common/d3d11_host_display.h

@@ -44,6 +44,7 @@ public:
 
   virtual bool ChangeRenderWindow(const WindowInfo& new_wi) override;
   virtual void ResizeRenderWindow(s32 new_window_width, s32 new_window_height) override;
+  virtual bool SupportsFullscreen() const override;
   virtual bool IsFullscreen() override;
   virtual bool SetFullscreen(bool fullscreen, u32 width, u32 height, float refresh_rate) override;
   virtual void DestroyRenderSurface() override;

src/frontend-common/opengl_host_display.cpp

@@ -315,6 +315,11 @@ void OpenGLHostDisplay::ResizeRenderWindow(s32 new_window_width, s32 new_window_
 #endif
 }
 
+bool OpenGLHostDisplay::SupportsFullscreen() const
+{
+  return false;
+}
+
 bool OpenGLHostDisplay::IsFullscreen()
 {
   return false;

src/frontend-common/opengl_host_display.h

@@ -44,6 +44,7 @@ public:
 
   virtual bool ChangeRenderWindow(const WindowInfo& new_wi) override;
   virtual void ResizeRenderWindow(s32 new_window_width, s32 new_window_height) override;
+  virtual bool SupportsFullscreen() const override;
   virtual bool IsFullscreen() override;
   virtual bool SetFullscreen(bool fullscreen, u32 width, u32 height, float refresh_rate) override;
   virtual void DestroyRenderSurface() override;

src/frontend-common/vulkan_host_display.cpp

@@ -174,6 +174,11 @@ void VulkanHostDisplay::ResizeRenderWindow(s32 new_window_width, s32 new_window_
 #endif
 }
 
+bool VulkanHostDisplay::SupportsFullscreen() const
+{
+  return false;
+}
+
 bool VulkanHostDisplay::IsFullscreen()
 {
   return false;

src/frontend-common/vulkan_host_display.h

@@ -41,6 +41,7 @@ public:
 
   virtual bool ChangeRenderWindow(const WindowInfo& new_wi) override;
   virtual void ResizeRenderWindow(s32 new_window_width, s32 new_window_height) override;
+  virtual bool SupportsFullscreen() const override;
   virtual bool IsFullscreen() override;
   virtual bool SetFullscreen(bool fullscreen, u32 width, u32 height, float refresh_rate) override;
   virtual void DestroyRenderSurface() override;