diff --git a/Source/Android/jni/MainAndroid.cpp b/Source/Android/jni/MainAndroid.cpp
index 4cd975fae8..24fabfb72a 100644
--- a/Source/Android/jni/MainAndroid.cpp
+++ b/Source/Android/jni/MainAndroid.cpp
@@ -113,10 +113,6 @@ void Host_RequestRenderWindowSize(int width, int height)
 {
 }
 
-void Host_RequestFullscreen(bool enable_fullscreen)
-{
-}
-
 void Host_SetStartupDebuggingParameters()
 {
 }
diff --git a/Source/Core/Core/Analytics.cpp b/Source/Core/Core/Analytics.cpp
index 65d298af0d..4d2dcd8e41 100644
--- a/Source/Core/Core/Analytics.cpp
+++ b/Source/Core/Core/Analytics.cpp
@@ -211,8 +211,6 @@ void DolphinAnalytics::MakePerGameBuilder()
   builder.AddData("cfg-gfx-realxfb", g_Config.RealXFBEnabled());
   builder.AddData("cfg-gfx-virtualxfb", g_Config.VirtualXFBEnabled());
   builder.AddData("cfg-gfx-vsync", g_Config.bVSync);
-  builder.AddData("cfg-gfx-fullscreen", g_Config.bFullscreen);
-  builder.AddData("cfg-gfx-exclusive-mode", g_Config.bExclusiveMode);
   builder.AddData("cfg-gfx-aspect-ratio", g_Config.iAspectRatio);
   builder.AddData("cfg-gfx-efb-access", g_Config.bEFBAccessEnable);
   builder.AddData("cfg-gfx-efb-scale", g_Config.iEFBScale);
diff --git a/Source/Core/Core/Host.h b/Source/Core/Core/Host.h
index 2f517c4c8e..349d65f7a6 100644
--- a/Source/Core/Core/Host.h
+++ b/Source/Core/Core/Host.h
@@ -31,7 +31,6 @@ void Host_Message(int Id);
 void Host_NotifyMapLoaded();
 void Host_RefreshDSPDebuggerWindow();
 void Host_RequestRenderWindowSize(int width, int height);
-void Host_RequestFullscreen(bool enable_fullscreen);
 void Host_SetStartupDebuggingParameters();
 void Host_SetWiiMoteConnectionState(int _State);
 void Host_UpdateDisasmDialog();
diff --git a/Source/Core/DolphinQt2/Host.cpp b/Source/Core/DolphinQt2/Host.cpp
index 906853bc3c..0797f4f39e 100644
--- a/Source/Core/DolphinQt2/Host.cpp
+++ b/Source/Core/DolphinQt2/Host.cpp
@@ -100,9 +100,6 @@ void Host_YieldToUI()
 void Host_UpdateMainFrame()
 {
 }
-void Host_RequestFullscreen(bool enable)
-{
-}
 void Host_RequestRenderWindowSize(int w, int h)
 {
 }
diff --git a/Source/Core/DolphinWX/Frame.cpp b/Source/Core/DolphinWX/Frame.cpp
index 129cd16108..290b1a8e10 100644
--- a/Source/Core/DolphinWX/Frame.cpp
+++ b/Source/Core/DolphinWX/Frame.cpp
@@ -202,11 +202,6 @@ bool CRenderFrame::ShowFullScreen(bool show, long style)
   {
     // OpenGL requires the pop-up style to activate exclusive mode.
     SetWindowStyle((GetWindowStyle() & ~wxDEFAULT_FRAME_STYLE) | wxPOPUP_WINDOW);
-
-    // Some backends don't support exclusive fullscreen, so we
-    // can't tell exactly when exclusive mode is activated.
-    if (!g_Config.backend_info.bSupportsExclusiveFullscreen)
-      OSD::AddMessage("Entered exclusive fullscreen.");
   }
 #endif
 
@@ -525,6 +520,8 @@ void CFrame::OnActive(wxActivateEvent& event)
     {
       if (SConfig::GetInstance().bRenderToMain)
         m_RenderParent->SetFocus();
+      else if (RendererIsFullscreen() && g_ActiveConfig.ExclusiveFullscreenEnabled())
+        DoExclusiveFullscreen(true);  // Regain exclusive mode
 
       if (SConfig::GetInstance().m_PauseOnFocusLost && Core::GetState() == Core::CORE_PAUSE)
         DoPause();
@@ -701,21 +698,6 @@ void CFrame::OnHostMessage(wxCommandEvent& event)
   }
   break;
 
-  case IDM_FULLSCREEN_REQUEST:
-  {
-    bool enable_fullscreen = event.GetInt() == 0 ? false : true;
-    ToggleDisplayMode(enable_fullscreen);
-    if (m_RenderFrame != nullptr)
-      m_RenderFrame->ShowFullScreen(enable_fullscreen);
-
-    // If the stop dialog initiated this fullscreen switch then we need
-    // to pause the emulator after we've completed the switch.
-    // TODO: Allow the renderer to switch fullscreen modes while paused.
-    if (m_confirmStop)
-      Core::SetState(Core::CORE_PAUSE);
-  }
-  break;
-
   case WM_USER_CREATE:
     if (SConfig::GetInstance().bHideCursor)
       m_RenderParent->SetCursor(wxCURSOR_BLANK);
@@ -1131,31 +1113,12 @@ void CFrame::OnMouse(wxMouseEvent& event)
 
 void CFrame::DoFullscreen(bool enable_fullscreen)
 {
-  if (g_Config.bExclusiveMode && Core::GetState() == Core::CORE_PAUSE)
-  {
-    // A responsive renderer is required for exclusive fullscreen, but the
-    // renderer can only respond in the running state. Therefore we ignore
-    // fullscreen switches if we are in exclusive fullscreen, but the
-    // renderer is not running.
-    // TODO: Allow the renderer to switch fullscreen modes while paused.
-    return;
-  }
-
   ToggleDisplayMode(enable_fullscreen);
 
-  if (enable_fullscreen)
-  {
-    m_RenderFrame->ShowFullScreen(true, wxFULLSCREEN_ALL);
-  }
-  else if (!g_Config.bExclusiveMode)
-  {
-    // Exiting exclusive fullscreen should be done from a Renderer callback.
-    // Therefore we don't exit fullscreen from here if we are in exclusive mode.
-    m_RenderFrame->ShowFullScreen(false, wxFULLSCREEN_ALL);
-  }
-
   if (SConfig::GetInstance().bRenderToMain)
   {
+    m_RenderFrame->ShowFullScreen(enable_fullscreen, wxFULLSCREEN_ALL);
+
     if (enable_fullscreen)
     {
       // Save the current mode before going to fullscreen
@@ -1197,12 +1160,32 @@ void CFrame::DoFullscreen(bool enable_fullscreen)
       }
     }
   }
+  else if (g_ActiveConfig.ExclusiveFullscreenEnabled())
+  {
+    if (!enable_fullscreen)
+      DoExclusiveFullscreen(false);
+
+    m_RenderFrame->ShowFullScreen(enable_fullscreen, wxFULLSCREEN_ALL);
+    m_RenderFrame->Raise();
+
+    if (enable_fullscreen)
+      DoExclusiveFullscreen(true);
+  }
   else
   {
+    m_RenderFrame->ShowFullScreen(enable_fullscreen, wxFULLSCREEN_ALL);
     m_RenderFrame->Raise();
   }
+}
 
-  g_Config.bFullscreen = enable_fullscreen;
+void CFrame::DoExclusiveFullscreen(bool enable_fullscreen)
+{
+  if (!g_renderer || g_renderer->IsFullscreen() == enable_fullscreen)
+    return;
+
+  bool was_unpaused = Core::PauseAndLock(true);
+  g_renderer->SetFullscreen(enable_fullscreen);
+  Core::PauseAndLock(false, was_unpaused);
 }
 
 const CGameListCtrl* CFrame::GetGameListCtrl() const
diff --git a/Source/Core/DolphinWX/Frame.h b/Source/Core/DolphinWX/Frame.h
index c621043aff..dc31fed7cd 100644
--- a/Source/Core/DolphinWX/Frame.h
+++ b/Source/Core/DolphinWX/Frame.h
@@ -105,7 +105,8 @@ public:
   void OnRenderParentMove(wxMoveEvent& event);
   bool RendererHasFocus();
   bool RendererIsFullscreen();
-  void DoFullscreen(bool bF);
+  void DoFullscreen(bool enable_fullscreen);
+  void DoExclusiveFullscreen(bool enable_fullscreen);
   void ToggleDisplayMode(bool bFullscreen);
   static void ConnectWiimote(int wm_idx, bool connect);
   void UpdateTitle(const std::string& str);
diff --git a/Source/Core/DolphinWX/FrameTools.cpp b/Source/Core/DolphinWX/FrameTools.cpp
index 58f8c34afb..63416f6464 100644
--- a/Source/Core/DolphinWX/FrameTools.cpp
+++ b/Source/Core/DolphinWX/FrameTools.cpp
@@ -787,12 +787,6 @@ void CFrame::DoStop()
       // waiting on inputs
       bool should_pause = !NetPlayDialog::GetNetPlayClient();
 
-      // If exclusive fullscreen is not enabled then we can pause the emulation
-      // before we've exited fullscreen. If not then we need to exit fullscreen first.
-      should_pause =
-          should_pause && (!RendererIsFullscreen() || !g_Config.ExclusiveFullscreenEnabled() ||
-                           SConfig::GetInstance().bRenderToMain);
-
       if (should_pause)
       {
         Core::SetState(Core::CORE_PAUSE);
diff --git a/Source/Core/DolphinWX/Globals.h b/Source/Core/DolphinWX/Globals.h
index cd36f68c1a..3ab8c809cd 100644
--- a/Source/Core/DolphinWX/Globals.h
+++ b/Source/Core/DolphinWX/Globals.h
@@ -292,7 +292,6 @@ enum
   IDM_WINDOW_SIZE_REQUEST,
   IDM_STOPPED,
   IDM_HOST_MESSAGE,
-  IDM_FULLSCREEN_REQUEST,
 
   // Used for Host_ConnectWiimote()
   IDM_FORCE_CONNECT_WIIMOTE1,
diff --git a/Source/Core/DolphinWX/Main.cpp b/Source/Core/DolphinWX/Main.cpp
index 444b270d39..f87de73a0c 100644
--- a/Source/Core/DolphinWX/Main.cpp
+++ b/Source/Core/DolphinWX/Main.cpp
@@ -452,13 +452,6 @@ void Host_RequestRenderWindowSize(int width, int height)
   main_frame->GetEventHandler()->AddPendingEvent(event);
 }
 
-void Host_RequestFullscreen(bool enable_fullscreen)
-{
-  wxCommandEvent event(wxEVT_HOST_COMMAND, IDM_FULLSCREEN_REQUEST);
-  event.SetInt(enable_fullscreen ? 1 : 0);
-  main_frame->GetEventHandler()->AddPendingEvent(event);
-}
-
 void Host_SetStartupDebuggingParameters()
 {
   SConfig& StartUp = SConfig::GetInstance();
diff --git a/Source/Core/DolphinWX/MainNoGUI.cpp b/Source/Core/DolphinWX/MainNoGUI.cpp
index 6c8a53b88d..b0c6e9a321 100644
--- a/Source/Core/DolphinWX/MainNoGUI.cpp
+++ b/Source/Core/DolphinWX/MainNoGUI.cpp
@@ -113,10 +113,6 @@ void Host_RequestRenderWindowSize(int width, int height)
 {
 }
 
-void Host_RequestFullscreen(bool enable_fullscreen)
-{
-}
-
 void Host_SetStartupDebuggingParameters()
 {
   SConfig& StartUp = SConfig::GetInstance();
diff --git a/Source/Core/VideoBackends/D3D/D3DBase.cpp b/Source/Core/VideoBackends/D3D/D3DBase.cpp
index 55f3e524bd..3edd936efa 100644
--- a/Source/Core/VideoBackends/D3D/D3DBase.cpp
+++ b/Source/Core/VideoBackends/D3D/D3DBase.cpp
@@ -8,6 +8,7 @@
 #include "Common/Logging/Log.h"
 #include "Common/MsgHandler.h"
 #include "Common/StringUtil.h"
+#include "Core/ConfigManager.h"
 #include "VideoBackends/D3D/D3DBase.h"
 #include "VideoBackends/D3D/D3DState.h"
 #include "VideoBackends/D3D/D3DTexture.h"
@@ -315,7 +316,8 @@ HRESULT Create(HWND wnd)
   swap_chain_desc.OutputWindow = wnd;
   swap_chain_desc.SampleDesc.Count = 1;
   swap_chain_desc.SampleDesc.Quality = 0;
-  swap_chain_desc.Windowed = !g_Config.bFullscreen;
+  swap_chain_desc.Windowed =
+      !SConfig::GetInstance().bFullscreen || g_ActiveConfig.bBorderlessFullscreen;
 
   DXGI_OUTPUT_DESC out_desc = {};
   output->GetDesc(&out_desc);
@@ -610,17 +612,11 @@ HRESULT SetFullscreenState(bool enable_fullscreen)
   return swapchain->SetFullscreenState(enable_fullscreen, nullptr);
 }
 
-HRESULT GetFullscreenState(bool* fullscreen_state)
+bool GetFullscreenState()
 {
-  if (fullscreen_state == nullptr)
-  {
-    return E_POINTER;
-  }
-
-  BOOL state;
-  HRESULT hr = swapchain->GetFullscreenState(&state, nullptr);
-  *fullscreen_state = !!state;
-  return hr;
+  BOOL state = FALSE;
+  swapchain->GetFullscreenState(&state, nullptr);
+  return !!state;
 }
 
 }  // namespace D3D
diff --git a/Source/Core/VideoBackends/D3D/D3DBase.h b/Source/Core/VideoBackends/D3D/D3DBase.h
index 24c6848434..668671fee6 100644
--- a/Source/Core/VideoBackends/D3D/D3DBase.h
+++ b/Source/Core/VideoBackends/D3D/D3DBase.h
@@ -75,7 +75,7 @@ bool BGRATexturesSupported();
 unsigned int GetMaxTextureSize();
 
 HRESULT SetFullscreenState(bool enable_fullscreen);
-HRESULT GetFullscreenState(bool* fullscreen_state);
+bool GetFullscreenState();
 
 // This function will assign a name to the given resource.
 // The DirectX debug layer will make it easier to identify resources that way,
diff --git a/Source/Core/VideoBackends/D3D/Render.cpp b/Source/Core/VideoBackends/D3D/Render.cpp
index bf0b487276..7845c84662 100644
--- a/Source/Core/VideoBackends/D3D/Render.cpp
+++ b/Source/Core/VideoBackends/D3D/Render.cpp
@@ -851,8 +851,6 @@ void Renderer::SwapImpl(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight,
   SetWindowSize(fbStride, fbHeight);
 
   const bool windowResized = CheckForResize();
-  const bool fullscreen = g_ActiveConfig.bFullscreen && !g_ActiveConfig.bBorderlessFullscreen &&
-                          !SConfig::GetInstance().bRenderToMain;
 
   bool xfbchanged = s_last_xfb_mode != g_ActiveConfig.bUseRealXFB;
 
@@ -869,33 +867,9 @@ void Renderer::SwapImpl(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight,
   // Flip/present backbuffer to frontbuffer here
   D3D::Present();
 
-  // Check exclusive fullscreen state
-  bool exclusive_mode, fullscreen_changed = false;
-  if (SUCCEEDED(D3D::GetFullscreenState(&exclusive_mode)))
-  {
-    if (fullscreen && !exclusive_mode)
-    {
-      if (g_Config.bExclusiveMode)
-        OSD::AddMessage("Lost exclusive fullscreen.");
-
-      // Exclusive fullscreen is enabled in the configuration, but we're
-      // not in exclusive mode. Either exclusive fullscreen was turned on
-      // or the render frame lost focus. When the render frame is in focus
-      // we can apply exclusive mode.
-      fullscreen_changed = Host_RendererHasFocus();
-
-      g_Config.bExclusiveMode = false;
-    }
-    else if (!fullscreen && exclusive_mode)
-    {
-      // Exclusive fullscreen is disabled, but we're still in exclusive mode.
-      fullscreen_changed = true;
-    }
-  }
-
   // Resize the back buffers NOW to avoid flickering
   if (CalculateTargetSize(s_backbuffer_width, s_backbuffer_height) || xfbchanged || windowResized ||
-      fullscreen_changed || s_last_efb_scale != g_ActiveConfig.iEFBScale ||
+      s_last_efb_scale != g_ActiveConfig.iEFBScale ||
       s_last_multisamples != g_ActiveConfig.iMultisamples ||
       s_last_stereo_mode != (g_ActiveConfig.iStereoMode > 0))
   {
@@ -903,23 +877,8 @@ void Renderer::SwapImpl(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight,
     s_last_multisamples = g_ActiveConfig.iMultisamples;
     PixelShaderCache::InvalidateMSAAShaders();
 
-    if (windowResized || fullscreen_changed)
+    if (windowResized)
     {
-      // Apply fullscreen state
-      if (fullscreen_changed)
-      {
-        g_Config.bExclusiveMode = fullscreen;
-
-        if (fullscreen)
-          OSD::AddMessage("Entered exclusive fullscreen.");
-
-        D3D::SetFullscreenState(fullscreen);
-
-        // If fullscreen is disabled we can safely notify the UI to exit fullscreen.
-        if (!g_ActiveConfig.bFullscreen)
-          Host_RequestFullscreen(false);
-      }
-
       // TODO: Aren't we still holding a reference to the back buffer right now?
       D3D::Reset();
       SAFE_RELEASE(s_screenshot_texture);
@@ -1291,4 +1250,14 @@ void Renderer::BlitScreen(TargetRectangle src, TargetRectangle dst, D3DTexture2D
   }
 }
 
+void Renderer::SetFullscreen(bool enable_fullscreen)
+{
+  D3D::SetFullscreenState(enable_fullscreen);
+}
+
+bool Renderer::IsFullscreen() const
+{
+  return D3D::GetFullscreenState();
+}
+
 }  // namespace DX11
diff --git a/Source/Core/VideoBackends/D3D/Render.h b/Source/Core/VideoBackends/D3D/Render.h
index 66f546be6c..0507b76c80 100644
--- a/Source/Core/VideoBackends/D3D/Render.h
+++ b/Source/Core/VideoBackends/D3D/Render.h
@@ -25,6 +25,8 @@ public:
   void SetSamplerState(int stage, int texindex, bool custom_tex) override;
   void SetInterlacingMode() override;
   void SetViewport() override;
+  void SetFullscreen(bool enable_fullscreen) override;
+  bool IsFullscreen() const override;
 
   // TODO: Fix confusing names (see ResetAPIState and RestoreAPIState)
   void ApplyState(bool bUseDstAlpha) override;
diff --git a/Source/Core/VideoBackends/D3D12/D3DBase.cpp b/Source/Core/VideoBackends/D3D12/D3DBase.cpp
index 5405ec1f2e..9f1da4042e 100644
--- a/Source/Core/VideoBackends/D3D12/D3DBase.cpp
+++ b/Source/Core/VideoBackends/D3D12/D3DBase.cpp
@@ -871,12 +871,11 @@ HRESULT SetFullscreenState(bool enable_fullscreen)
   return S_OK;
 }
 
-HRESULT GetFullscreenState(bool* fullscreen_state)
+bool GetFullscreenState()
 {
   // Fullscreen exclusive intentionally not supported in DX12 backend. No performance
   // difference between it and windowed full-screen due to usage of a FLIP swap chain.
-  *fullscreen_state = false;
-  return S_OK;
+  return false;
 }
 
 }  // namespace D3D
diff --git a/Source/Core/VideoBackends/D3D12/D3DBase.h b/Source/Core/VideoBackends/D3D12/D3DBase.h
index a23bad1feb..89cd02738d 100644
--- a/Source/Core/VideoBackends/D3D12/D3DBase.h
+++ b/Source/Core/VideoBackends/D3D12/D3DBase.h
@@ -124,7 +124,7 @@ const std::string VertexShaderVersionString();
 unsigned int GetMaxTextureSize();
 
 HRESULT SetFullscreenState(bool enable_fullscreen);
-HRESULT GetFullscreenState(bool* fullscreen_state);
+bool GetFullscreenState();
 
 // This function will assign a name to the given resource.
 // The DirectX debug layer will make it easier to identify resources that way,
diff --git a/Source/Core/VideoBackends/D3D12/Render.cpp b/Source/Core/VideoBackends/D3D12/Render.cpp
index 8b3c36651b..f37357aef4 100644
--- a/Source/Core/VideoBackends/D3D12/Render.cpp
+++ b/Source/Core/VideoBackends/D3D12/Render.cpp
@@ -803,8 +803,6 @@ void Renderer::SwapImpl(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height
   SetWindowSize(fb_stride, fb_height);
 
   const bool window_resized = CheckForResize();
-  const bool fullscreen = g_ActiveConfig.bFullscreen && !g_ActiveConfig.bBorderlessFullscreen &&
-                          !SConfig::GetInstance().bRenderToMain;
 
   bool xfb_changed = s_last_xfb_mode != g_ActiveConfig.bUseRealXFB;
 
diff --git a/Source/Core/VideoCommon/RenderBase.h b/Source/Core/VideoCommon/RenderBase.h
index 94f48d0b9b..3d37b716f9 100644
--- a/Source/Core/VideoCommon/RenderBase.h
+++ b/Source/Core/VideoCommon/RenderBase.h
@@ -72,6 +72,8 @@ public:
   virtual void SetSamplerState(int stage, int texindex, bool custom_tex) {}
   virtual void SetInterlacingMode() {}
   virtual void SetViewport() {}
+  virtual void SetFullscreen(bool enable_fullscreen) {}
+  virtual bool IsFullscreen() const { return false; }
   virtual void ApplyState(bool bUseDstAlpha) {}
   virtual void RestoreState() {}
   virtual void ResetAPIState() {}
diff --git a/Source/Core/VideoCommon/VideoConfig.cpp b/Source/Core/VideoCommon/VideoConfig.cpp
index 0e79b4b180..115dbf8162 100644
--- a/Source/Core/VideoCommon/VideoConfig.cpp
+++ b/Source/Core/VideoCommon/VideoConfig.cpp
@@ -31,10 +31,6 @@ VideoConfig::VideoConfig()
 {
   bRunning = false;
 
-  // Exclusive fullscreen flags
-  bFullscreen = false;
-  bExclusiveMode = false;
-
   // Needed for the first frame, I think
   fAspectRatioHackW = 1;
   fAspectRatioHackH = 1;
diff --git a/Source/Core/VideoCommon/VideoConfig.h b/Source/Core/VideoCommon/VideoConfig.h
index b866ce6343..225c12d590 100644
--- a/Source/Core/VideoCommon/VideoConfig.h
+++ b/Source/Core/VideoCommon/VideoConfig.h
@@ -63,8 +63,6 @@ struct VideoConfig final
 
   // General
   bool bVSync;
-  bool bFullscreen;
-  bool bExclusiveMode;
   bool bRunning;
   bool bWidescreenHack;
   int iAspectRatio;