diff --git a/Source/Core/Common/WorkQueueThread.h b/Source/Core/Common/WorkQueueThread.h
index c0a437c168..9f553d0c89 100644
--- a/Source/Core/Common/WorkQueueThread.h
+++ b/Source/Core/Common/WorkQueueThread.h
@@ -27,6 +27,7 @@ public:
   {
     Shutdown();
     m_shutdown.Clear();
+    m_cancelled.Clear();
     m_function = std::move(function);
     m_thread = std::thread(&WorkQueueThread::ThreadLoop, this);
   }
@@ -34,6 +35,7 @@ public:
   template <typename... Args>
   void EmplaceItem(Args&&... args)
   {
+    if (!m_cancelled.IsSet())
     {
       std::lock_guard lg(m_lock);
       m_items.emplace(std::forward<Args>(args)...);
@@ -41,6 +43,24 @@ public:
     m_wakeup.Set();
   }
 
+  void Clear()
+  {
+    {
+      std::lock_guard lg(m_lock);
+      m_items = std::queue<T>();
+    }
+    m_wakeup.Set();
+  }
+
+  void Cancel()
+  {
+    m_cancelled.Set();
+    Clear();
+    Shutdown();
+  }
+
+  bool IsCancelled() const { return m_cancelled.IsSet(); }
+
 private:
   void Shutdown()
   {
@@ -81,6 +101,7 @@ private:
   std::thread m_thread;
   Common::Event m_wakeup;
   Common::Flag m_shutdown;
+  Common::Flag m_cancelled;
   std::mutex m_lock;
   std::queue<T> m_items;
 };
diff --git a/Source/Core/DolphinQt/CheatsManager.cpp b/Source/Core/DolphinQt/CheatsManager.cpp
index edb9537c86..3c0dc72e76 100644
--- a/Source/Core/DolphinQt/CheatsManager.cpp
+++ b/Source/Core/DolphinQt/CheatsManager.cpp
@@ -33,7 +33,6 @@
 
 #include "DolphinQt/Config/ARCodeWidget.h"
 #include "DolphinQt/Config/GeckoCodeWidget.h"
-#include "DolphinQt/GameList/GameListModel.h"
 #include "DolphinQt/Settings.h"
 
 constexpr u32 MAX_RESULTS = 50;
@@ -152,7 +151,8 @@ static bool Compare(T mem_value, T value, CompareType op)
   }
 }
 
-CheatsManager::CheatsManager(QWidget* parent) : QDialog(parent)
+CheatsManager::CheatsManager(const GameListModel& game_list_model, QWidget* parent)
+    : QDialog(parent), m_game_list_model(game_list_model)
 {
   setWindowTitle(tr("Cheats Manager"));
   setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint);
@@ -175,11 +175,9 @@ void CheatsManager::OnStateChanged(Core::State state)
   if (state != Core::State::Running && state != Core::State::Paused)
     return;
 
-  auto* model = Settings::Instance().GetGameListModel();
-
-  for (int i = 0; i < model->rowCount(QModelIndex()); i++)
+  for (int i = 0; i < m_game_list_model.rowCount(QModelIndex()); i++)
   {
-    auto file = model->GetGameFile(i);
+    auto file = m_game_list_model.GetGameFile(i);
 
     if (file->GetGameID() == SConfig::GetInstance().GetGameID())
     {
diff --git a/Source/Core/DolphinQt/CheatsManager.h b/Source/Core/DolphinQt/CheatsManager.h
index 7d0d1297f6..f1e0c1ec86 100644
--- a/Source/Core/DolphinQt/CheatsManager.h
+++ b/Source/Core/DolphinQt/CheatsManager.h
@@ -11,6 +11,7 @@
 #include <QDialog>
 
 #include "Common/CommonTypes.h"
+#include "DolphinQt/GameList/GameListModel.h"
 
 class ARCodeWidget;
 class QComboBox;
@@ -39,7 +40,7 @@ class CheatsManager : public QDialog
 {
   Q_OBJECT
 public:
-  explicit CheatsManager(QWidget* parent = nullptr);
+  explicit CheatsManager(const GameListModel& game_list_model, QWidget* parent = nullptr);
   ~CheatsManager();
 
 private:
@@ -61,6 +62,7 @@ private:
   void OnMatchContextMenu();
   void OnWatchItemChanged(QTableWidgetItem* item);
 
+  const GameListModel& m_game_list_model;
   std::vector<Result> m_results;
   std::vector<Result> m_watch;
   std::shared_ptr<const UICommon::GameFile> m_game_file;
diff --git a/Source/Core/DolphinQt/GameList/GameList.cpp b/Source/Core/DolphinQt/GameList/GameList.cpp
index 11cbadb2d3..e3056b4ef4 100644
--- a/Source/Core/DolphinQt/GameList/GameList.cpp
+++ b/Source/Core/DolphinQt/GameList/GameList.cpp
@@ -41,7 +41,6 @@
 
 #include "DolphinQt/Config/PropertiesDialog.h"
 #include "DolphinQt/ConvertDialog.h"
-#include "DolphinQt/GameList/GameListModel.h"
 #include "DolphinQt/GameList/GridProxyModel.h"
 #include "DolphinQt/GameList/ListProxyModel.h"
 #include "DolphinQt/MenuBar.h"
@@ -54,27 +53,26 @@
 
 #include "UICommon/GameFile.h"
 
-GameList::GameList(QWidget* parent) : QStackedWidget(parent)
+GameList::GameList(QWidget* parent) : QStackedWidget(parent), m_model(this)
 {
-  m_model = Settings::Instance().GetGameListModel();
   m_list_proxy = new ListProxyModel(this);
   m_list_proxy->setSortCaseSensitivity(Qt::CaseInsensitive);
   m_list_proxy->setSortRole(GameListModel::SORT_ROLE);
-  m_list_proxy->setSourceModel(m_model);
+  m_list_proxy->setSourceModel(&m_model);
   m_grid_proxy = new GridProxyModel(this);
-  m_grid_proxy->setSourceModel(m_model);
+  m_grid_proxy->setSourceModel(&m_model);
 
   MakeListView();
   MakeGridView();
   MakeEmptyView();
 
   if (Settings::GetQSettings().contains(QStringLiteral("gridview/scale")))
-    m_model->SetScale(Settings::GetQSettings().value(QStringLiteral("gridview/scale")).toFloat());
+    m_model.SetScale(Settings::GetQSettings().value(QStringLiteral("gridview/scale")).toFloat());
 
   connect(m_list, &QTableView::doubleClicked, this, &GameList::GameSelected);
   connect(m_grid, &QListView::doubleClicked, this, &GameList::GameSelected);
-  connect(m_model, &QAbstractItemModel::rowsInserted, this, &GameList::ConsiderViewChange);
-  connect(m_model, &QAbstractItemModel::rowsRemoved, this, &GameList::ConsiderViewChange);
+  connect(&m_model, &QAbstractItemModel::rowsInserted, this, &GameList::ConsiderViewChange);
+  connect(&m_model, &QAbstractItemModel::rowsRemoved, this, &GameList::ConsiderViewChange);
 
   addWidget(m_list);
   addWidget(m_grid);
@@ -94,7 +92,7 @@ GameList::GameList(QWidget* parent) : QStackedWidget(parent)
 
 void GameList::PurgeCache()
 {
-  m_model->PurgeCache();
+  m_model.PurgeCache();
 }
 
 void GameList::MakeListView()
@@ -176,7 +174,7 @@ GameList::~GameList()
 {
   Settings::GetQSettings().setValue(QStringLiteral("tableheader/state"),
                                     m_list->horizontalHeader()->saveState());
-  Settings::GetQSettings().setValue(QStringLiteral("gridview/scale"), m_model->GetScale());
+  Settings::GetQSettings().setValue(QStringLiteral("gridview/scale"), m_model.GetScale());
 }
 
 void GameList::UpdateColumnVisibility()
@@ -205,9 +203,13 @@ void GameList::UpdateColumnVisibility()
 
 void GameList::MakeEmptyView()
 {
+  const QString refreshing_msg = tr("Refreshing...");
+  const QString empty_msg = tr("Dolphin could not find any GameCube/Wii ISOs or WADs.\n"
+                               "Double-click here to set a games directory...");
+
   m_empty = new QLabel(this);
-  m_empty->setText(tr("Dolphin could not find any GameCube/Wii ISOs or WADs.\n"
-                      "Double-click here to set a games directory..."));
+  m_empty->setText(refreshing_msg);
+  m_empty->setEnabled(false);
   m_empty->setAlignment(Qt::AlignHCenter | Qt::AlignVCenter);
 
   auto event_filter = new DoubleClickEventFilter{m_empty};
@@ -218,6 +220,21 @@ void GameList::MakeEmptyView()
     if (!dir.isEmpty())
       Settings::Instance().AddPath(dir);
   });
+
+  QSizePolicy size_policy{m_empty->sizePolicy()};
+  size_policy.setRetainSizeWhenHidden(true);
+  m_empty->setSizePolicy(size_policy);
+
+  connect(&Settings::Instance(), &Settings::GameListRefreshRequested, this,
+          [this, refreshing_msg = refreshing_msg] {
+            m_empty->setText(refreshing_msg);
+            m_empty->setEnabled(false);
+          });
+  connect(&Settings::Instance(), &Settings::GameListRefreshCompleted, this,
+          [this, empty_msg = empty_msg] {
+            m_empty->setText(empty_msg);
+            m_empty->setEnabled(true);
+          });
 }
 
 void GameList::resizeEvent(QResizeEvent* event)
@@ -368,21 +385,19 @@ void GameList::ShowContextMenu(const QPoint&)
 
     menu->addSeparator();
 
-    auto* model = Settings::Instance().GetGameListModel();
-
     auto* tags_menu = menu->addMenu(tr("Tags"));
 
     auto path = game->GetFilePath();
-    auto game_tags = model->GetGameTags(path);
+    auto game_tags = m_model.GetGameTags(path);
 
-    for (const auto& tag : model->GetAllTags())
+    for (const auto& tag : m_model.GetAllTags())
     {
       auto* tag_action = tags_menu->addAction(tag);
 
       tag_action->setCheckable(true);
       tag_action->setChecked(game_tags.contains(tag));
 
-      connect(tag_action, &QAction::toggled, [path, tag, model](bool checked) {
+      connect(tag_action, &QAction::toggled, [path, tag, model = &m_model](bool checked) {
         if (!checked)
           model->RemoveGameTag(path, tag);
         else
@@ -635,7 +650,7 @@ void GameList::DeleteFile()
 
         if (deletion_successful)
         {
-          m_model->RemoveGame(game->GetFilePath());
+          m_model.RemoveGame(game->GetFilePath());
         }
         else
         {
@@ -686,7 +701,7 @@ std::shared_ptr<const UICommon::GameFile> GameList::GetSelectedGame() const
   if (sel_model->hasSelection())
   {
     QModelIndex model_index = proxy->mapToSource(sel_model->selectedIndexes()[0]);
-    return m_model->GetGameFile(model_index.row());
+    return m_model.GetGameFile(model_index.row());
   }
   return {};
 }
@@ -714,7 +729,7 @@ QList<std::shared_ptr<const UICommon::GameFile>> GameList::GetSelectedGames() co
     for (const auto& index : index_list)
     {
       QModelIndex model_index = proxy->mapToSource(index);
-      selected_list.push_back(m_model->GetGameFile(model_index.row()));
+      selected_list.push_back(m_model.GetGameFile(model_index.row()));
     }
   }
   return selected_list;
@@ -728,18 +743,18 @@ bool GameList::HasMultipleSelected() const
 
 std::shared_ptr<const UICommon::GameFile> GameList::FindGame(const std::string& path) const
 {
-  return m_model->FindGame(path);
+  return m_model.FindGame(path);
 }
 
 std::shared_ptr<const UICommon::GameFile>
 GameList::FindSecondDisc(const UICommon::GameFile& game) const
 {
-  return m_model->FindSecondDisc(game);
+  return m_model.FindSecondDisc(game);
 }
 
 std::string GameList::GetNetPlayName(const UICommon::GameFile& game) const
 {
-  return m_model->GetNetPlayName(game);
+  return m_model.GetNetPlayName(game);
 }
 
 void GameList::SetViewColumn(int col, bool view)
@@ -756,7 +771,7 @@ void GameList::SetPreferredView(bool list)
 
 void GameList::ConsiderViewChange()
 {
-  if (m_model->rowCount(QModelIndex()) > 0)
+  if (m_model.rowCount(QModelIndex()) > 0)
   {
     if (m_prefer_list)
       setCurrentWidget(m_list);
@@ -905,7 +920,7 @@ void GameList::NewTag()
   if (tag.isEmpty())
     return;
 
-  Settings::Instance().GetGameListModel()->NewTag(tag);
+  m_model.NewTag(tag);
 }
 
 void GameList::DeleteTag()
@@ -915,12 +930,12 @@ void GameList::DeleteTag()
   if (tag.isEmpty())
     return;
 
-  Settings::Instance().GetGameListModel()->DeleteTag(tag);
+  m_model.DeleteTag(tag);
 }
 
 void GameList::SetSearchTerm(const QString& term)
 {
-  m_model->SetSearchTerm(term);
+  m_model.SetSearchTerm(term);
 
   m_list_proxy->invalidate();
   m_grid_proxy->invalidate();
@@ -930,7 +945,7 @@ void GameList::SetSearchTerm(const QString& term)
 
 void GameList::ZoomIn()
 {
-  m_model->SetScale(m_model->GetScale() + 0.1);
+  m_model.SetScale(m_model.GetScale() + 0.1);
 
   m_list_proxy->invalidate();
   m_grid_proxy->invalidate();
@@ -940,10 +955,10 @@ void GameList::ZoomIn()
 
 void GameList::ZoomOut()
 {
-  if (m_model->GetScale() <= 0.1)
+  if (m_model.GetScale() <= 0.1)
     return;
 
-  m_model->SetScale(m_model->GetScale() - 0.1);
+  m_model.SetScale(m_model.GetScale() - 0.1);
 
   m_list_proxy->invalidate();
   m_grid_proxy->invalidate();
@@ -955,7 +970,7 @@ void GameList::UpdateFont()
 {
   QFont f;
 
-  f.setPointSizeF(m_model->GetScale() * f.pointSize());
+  f.setPointSizeF(m_model.GetScale() * f.pointSize());
 
   m_grid->setFont(f);
 }
diff --git a/Source/Core/DolphinQt/GameList/GameList.h b/Source/Core/DolphinQt/GameList/GameList.h
index ec92ea08e0..8ed4c87ec8 100644
--- a/Source/Core/DolphinQt/GameList/GameList.h
+++ b/Source/Core/DolphinQt/GameList/GameList.h
@@ -8,7 +8,8 @@
 
 #include <QStackedWidget>
 
-class GameListModel;
+#include "DolphinQt/GameList/GameListModel.h"
+
 class QLabel;
 class QListView;
 class QSortFilterProxyModel;
@@ -46,6 +47,8 @@ public:
 
   void PurgeCache();
 
+  const GameListModel& GetGameListModel() const { return m_model; }
+
 signals:
   void GameSelected();
   void NetPlayHost(const UICommon::GameFile& game);
@@ -85,7 +88,7 @@ private:
   void ConsiderViewChange();
   void UpdateFont();
 
-  GameListModel* m_model;
+  GameListModel m_model;
   QSortFilterProxyModel* m_list_proxy;
   QSortFilterProxyModel* m_grid_proxy;
 
diff --git a/Source/Core/DolphinQt/GameList/GameTracker.cpp b/Source/Core/DolphinQt/GameList/GameTracker.cpp
index 3055ad3170..a641640d6d 100644
--- a/Source/Core/DolphinQt/GameList/GameTracker.cpp
+++ b/Source/Core/DolphinQt/GameList/GameTracker.cpp
@@ -4,6 +4,7 @@
 
 #include "DolphinQt/GameList/GameTracker.h"
 
+#include <QApplication>
 #include <QDir>
 #include <QDirIterator>
 #include <QFile>
@@ -15,7 +16,6 @@
 #include "DiscIO/DirectoryBlob.h"
 
 #include "DolphinQt/QtUtils/QueueOnObject.h"
-#include "DolphinQt/QtUtils/RunOnObject.h"
 
 #include "DolphinQt/Settings.h"
 
@@ -35,6 +35,10 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
   qRegisterMetaType<std::shared_ptr<const UICommon::GameFile>>();
   qRegisterMetaType<std::string>();
 
+  connect(qApp, &QApplication::aboutToQuit, this, [this] {
+    m_processing_halted = true;
+    m_load_thread.Cancel();
+  });
   connect(this, &QFileSystemWatcher::directoryChanged, this, &GameTracker::UpdateDirectory);
   connect(this, &QFileSystemWatcher::fileChanged, this, &GameTracker::UpdateFile);
   connect(&Settings::Instance(), &Settings::AutoRefreshToggled, this, [] {
@@ -74,27 +78,24 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
       break;
     case CommandType::UpdateMetadata:
       m_cache.UpdateAdditionalMetadata(
-          [this](const std::shared_ptr<const UICommon::GameFile>& game) {
-            emit GameUpdated(game);
-          });
+          [this](const std::shared_ptr<const UICommon::GameFile>& game) { emit GameUpdated(game); },
+          m_processing_halted);
       QueueOnObject(this, [] { Settings::Instance().NotifyMetadataRefreshComplete(); });
       break;
+    case CommandType::ResumeProcessing:
+      m_processing_halted = false;
+      break;
     case CommandType::PurgeCache:
       m_cache.Clear(UICommon::GameFileCache::DeleteOnDisk::Yes);
       break;
     case CommandType::BeginRefresh:
-      if (m_busy_count++ == 0)
-      {
-        for (auto& file : m_tracked_files.keys())
-          emit GameRemoved(file.toStdString());
-        m_tracked_files.clear();
-      }
+      QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListStarted(); });
+      for (auto& file : m_tracked_files.keys())
+        emit GameRemoved(file.toStdString());
+      m_tracked_files.clear();
       break;
     case CommandType::EndRefresh:
-      if (--m_busy_count == 0)
-      {
-        QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListComplete(); });
-      }
+      QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListComplete(); });
       break;
     }
   });
@@ -134,6 +135,8 @@ void GameTracker::StartInternal()
 
   m_started = true;
 
+  QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListStarted(); });
+
   std::vector<std::string> paths;
   paths.reserve(m_tracked_files.size());
   for (const QString& path : m_tracked_files.keys())
@@ -149,18 +152,20 @@ void GameTracker::StartInternal()
 
   m_initial_games_emitted_event.Wait();
 
-  bool cache_updated = m_cache.Update(paths, emit_game_loaded, emit_game_removed);
-  cache_updated |= m_cache.UpdateAdditionalMetadata(emit_game_updated);
+  bool cache_updated =
+      m_cache.Update(paths, emit_game_loaded, emit_game_removed, m_processing_halted);
+  cache_updated |= m_cache.UpdateAdditionalMetadata(emit_game_updated, m_processing_halted);
   if (cache_updated)
     m_cache.Save();
 
   QueueOnObject(this, [] { Settings::Instance().NotifyMetadataRefreshComplete(); });
+  QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListComplete(); });
 }
 
 bool GameTracker::AddPath(const QString& dir)
 {
   if (Settings::Instance().IsAutoRefreshEnabled())
-    RunOnObject(this, [this, dir] { return addPath(dir); });
+    QueueOnObject(this, [this, dir] { return addPath(dir); });
 
   m_tracked_paths.push_back(dir);
 
@@ -170,7 +175,7 @@ bool GameTracker::AddPath(const QString& dir)
 bool GameTracker::RemovePath(const QString& dir)
 {
   if (Settings::Instance().IsAutoRefreshEnabled())
-    RunOnObject(this, [this, dir] { return removePath(dir); });
+    QueueOnObject(this, [this, dir] { return removePath(dir); });
 
   const auto index = m_tracked_paths.indexOf(dir);
 
@@ -194,6 +199,16 @@ void GameTracker::RemoveDirectory(const QString& dir)
 
 void GameTracker::RefreshAll()
 {
+  m_processing_halted = true;
+  m_load_thread.Clear();
+  m_load_thread.EmplaceItem(Command{CommandType::ResumeProcessing, {}});
+
+  if (m_needs_purge)
+  {
+    m_load_thread.EmplaceItem(Command{CommandType::PurgeCache, {}});
+    m_needs_purge = false;
+  }
+
   m_load_thread.EmplaceItem(Command{CommandType::BeginRefresh});
 
   for (const QString& dir : Settings::Instance().GetPaths())
@@ -255,7 +270,7 @@ void GameTracker::RemoveDirectoryInternal(const QString& dir)
 void GameTracker::UpdateDirectoryInternal(const QString& dir)
 {
   auto it = GetIterator(dir);
-  while (it->hasNext())
+  while (it->hasNext() && !m_processing_halted)
   {
     QString path = QFileInfo(it->next()).canonicalFilePath();
 
@@ -275,6 +290,9 @@ void GameTracker::UpdateDirectoryInternal(const QString& dir)
 
   for (const auto& missing : FindMissingFiles(dir))
   {
+    if (m_processing_halted)
+      break;
+
     auto& tracked_file = m_tracked_files[missing];
 
     tracked_file.remove(dir);
@@ -345,6 +363,6 @@ void GameTracker::LoadGame(const QString& path)
 
 void GameTracker::PurgeCache()
 {
-  m_load_thread.EmplaceItem(Command{CommandType::PurgeCache, {}});
-  RefreshAll();
+  m_needs_purge = true;
+  Settings::Instance().RefreshGameList();
 }
diff --git a/Source/Core/DolphinQt/GameList/GameTracker.h b/Source/Core/DolphinQt/GameList/GameTracker.h
index 93720614b3..135ee43fd0 100644
--- a/Source/Core/DolphinQt/GameList/GameTracker.h
+++ b/Source/Core/DolphinQt/GameList/GameTracker.h
@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
 #include <string>
 
@@ -72,6 +73,7 @@ private:
     UpdateDirectory,
     UpdateFile,
     UpdateMetadata,
+    ResumeProcessing,
     PurgeCache,
     BeginRefresh,
     EndRefresh,
@@ -92,8 +94,8 @@ private:
   Common::Event m_initial_games_emitted_event;
   bool m_initial_games_emitted = false;
   bool m_started = false;
-  // Count of currently running refresh jobs
-  u32 m_busy_count = 0;
+  bool m_needs_purge = false;
+  std::atomic_bool m_processing_halted = false;
 };
 
 Q_DECLARE_METATYPE(std::shared_ptr<const UICommon::GameFile>)
diff --git a/Source/Core/DolphinQt/MainWindow.cpp b/Source/Core/DolphinQt/MainWindow.cpp
index 069af901fb..54deeb47c6 100644
--- a/Source/Core/DolphinQt/MainWindow.cpp
+++ b/Source/Core/DolphinQt/MainWindow.cpp
@@ -399,7 +399,7 @@ void MainWindow::CreateComponents()
   m_watch_widget = new WatchWidget(this);
   m_breakpoint_widget = new BreakpointWidget(this);
   m_code_widget = new CodeWidget(this);
-  m_cheats_manager = new CheatsManager(this);
+  m_cheats_manager = new CheatsManager(m_game_list->GetGameListModel(), this);
 
   const auto request_watch = [this](QString name, u32 addr) {
     m_watch_widget->AddWatch(name, addr);
@@ -1276,8 +1276,9 @@ void MainWindow::BootWiiSystemMenu()
 
 void MainWindow::NetPlayInit()
 {
-  m_netplay_setup_dialog = new NetPlaySetupDialog(this);
-  m_netplay_dialog = new NetPlayDialog;
+  const auto& game_list_model = m_game_list->GetGameListModel();
+  m_netplay_setup_dialog = new NetPlaySetupDialog(game_list_model, this);
+  m_netplay_dialog = new NetPlayDialog(game_list_model);
 #ifdef USE_DISCORD_PRESENCE
   m_netplay_discord = new DiscordHandler(this);
 #endif
diff --git a/Source/Core/DolphinQt/MenuBar.cpp b/Source/Core/DolphinQt/MenuBar.cpp
index 5c9f91544b..6b6617cff8 100644
--- a/Source/Core/DolphinQt/MenuBar.cpp
+++ b/Source/Core/DolphinQt/MenuBar.cpp
@@ -506,7 +506,13 @@ void MenuBar::AddViewMenu()
   AddShowRegionsMenu(view_menu);
 
   view_menu->addSeparator();
-  view_menu->addAction(tr("Purge Game List Cache"), this, &MenuBar::PurgeGameListCache);
+  QAction* const purge_action =
+      view_menu->addAction(tr("Purge Game List Cache"), this, &MenuBar::PurgeGameListCache);
+  purge_action->setEnabled(false);
+  connect(&Settings::Instance(), &Settings::GameListRefreshRequested, purge_action,
+          [purge_action] { purge_action->setEnabled(false); });
+  connect(&Settings::Instance(), &Settings::GameListRefreshStarted, purge_action,
+          [purge_action] { purge_action->setEnabled(true); });
   view_menu->addSeparator();
   view_menu->addAction(tr("Search"), this, &MenuBar::ShowSearch, QKeySequence::Find);
 }
diff --git a/Source/Core/DolphinQt/NetPlay/GameListDialog.cpp b/Source/Core/DolphinQt/NetPlay/GameListDialog.cpp
index d0723d3766..1117cbe4c6 100644
--- a/Source/Core/DolphinQt/NetPlay/GameListDialog.cpp
+++ b/Source/Core/DolphinQt/NetPlay/GameListDialog.cpp
@@ -10,11 +10,10 @@
 #include <QListWidget>
 #include <QVBoxLayout>
 
-#include "DolphinQt/GameList/GameListModel.h"
-#include "DolphinQt/Settings.h"
 #include "UICommon/GameFile.h"
 
-GameListDialog::GameListDialog(QWidget* parent) : QDialog(parent)
+GameListDialog::GameListDialog(const GameListModel& game_list_model, QWidget* parent)
+    : QDialog(parent), m_game_list_model(game_list_model)
 {
   setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint);
   setWindowTitle(tr("Select a game"));
@@ -47,16 +46,14 @@ void GameListDialog::ConnectWidgets()
 
 void GameListDialog::PopulateGameList()
 {
-  auto* game_list_model = Settings::Instance().GetGameListModel();
-
   m_game_list->clear();
 
-  for (int i = 0; i < game_list_model->rowCount(QModelIndex()); i++)
+  for (int i = 0; i < m_game_list_model.rowCount(QModelIndex()); i++)
   {
-    std::shared_ptr<const UICommon::GameFile> game = game_list_model->GetGameFile(i);
+    std::shared_ptr<const UICommon::GameFile> game = m_game_list_model.GetGameFile(i);
 
     auto* item =
-        new QListWidgetItem(QString::fromStdString(game_list_model->GetNetPlayName(*game)));
+        new QListWidgetItem(QString::fromStdString(m_game_list_model.GetNetPlayName(*game)));
     item->setData(Qt::UserRole, QVariant::fromValue(std::move(game)));
     m_game_list->addItem(item);
   }
diff --git a/Source/Core/DolphinQt/NetPlay/GameListDialog.h b/Source/Core/DolphinQt/NetPlay/GameListDialog.h
index 78e5f90ff3..11e50e05aa 100644
--- a/Source/Core/DolphinQt/NetPlay/GameListDialog.h
+++ b/Source/Core/DolphinQt/NetPlay/GameListDialog.h
@@ -6,7 +6,8 @@
 
 #include <QDialog>
 
-class GameListModel;
+#include "DolphinQt/GameList/GameListModel.h"
+
 class QVBoxLayout;
 class QListWidget;
 class QDialogButtonBox;
@@ -20,7 +21,7 @@ class GameListDialog : public QDialog
 {
   Q_OBJECT
 public:
-  explicit GameListDialog(QWidget* parent);
+  explicit GameListDialog(const GameListModel& game_list_model, QWidget* parent);
 
   int exec() override;
   const UICommon::GameFile& GetSelectedGame() const;
@@ -30,6 +31,7 @@ private:
   void ConnectWidgets();
   void PopulateGameList();
 
+  const GameListModel& m_game_list_model;
   QVBoxLayout* m_main_layout;
   QListWidget* m_game_list;
   QDialogButtonBox* m_button_box;
diff --git a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp
index dd3be6cc65..39b989d141 100644
--- a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp
+++ b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp
@@ -40,7 +40,6 @@
 #include "Core/NetPlayServer.h"
 #include "Core/SyncIdentifier.h"
 
-#include "DolphinQt/GameList/GameListModel.h"
 #include "DolphinQt/NetPlay/ChunkedProgressDialog.h"
 #include "DolphinQt/NetPlay/GameListDialog.h"
 #include "DolphinQt/NetPlay/MD5Dialog.h"
@@ -60,8 +59,8 @@
 #include "VideoCommon/RenderBase.h"
 #include "VideoCommon/VideoConfig.h"
 
-NetPlayDialog::NetPlayDialog(QWidget* parent)
-    : QDialog(parent), m_game_list_model(Settings::Instance().GetGameListModel())
+NetPlayDialog::NetPlayDialog(const GameListModel& game_list_model, QWidget* parent)
+    : QDialog(parent), m_game_list_model(game_list_model)
 {
   setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint);
 
@@ -158,7 +157,7 @@ void NetPlayDialog::CreateMainLayout()
     Settings::Instance().GetNetPlayServer()->ComputeMD5(m_current_game_identifier);
   });
   m_md5_menu->addAction(tr("Other game..."), this, [this] {
-    GameListDialog gld(this);
+    GameListDialog gld(m_game_list_model, this);
 
     if (gld.exec() != QDialog::Accepted)
       return;
@@ -322,13 +321,13 @@ void NetPlayDialog::ConnectWidgets()
   connect(m_quit_button, &QPushButton::clicked, this, &NetPlayDialog::reject);
 
   connect(m_game_button, &QPushButton::clicked, [this] {
-    GameListDialog gld(this);
+    GameListDialog gld(m_game_list_model, this);
     if (gld.exec() == QDialog::Accepted)
     {
       Settings& settings = Settings::Instance();
 
       const UICommon::GameFile& game = gld.GetSelectedGame();
-      const std::string netplay_name = settings.GetGameListModel()->GetNetPlayName(game);
+      const std::string netplay_name = m_game_list_model.GetNetPlayName(game);
 
       settings.GetNetPlayServer()->ChangeGame(game.GetSyncIdentifier(), netplay_name);
       Settings::GetQSettings().setValue(QStringLiteral("netplay/hostgame"),
@@ -1048,9 +1047,9 @@ NetPlayDialog::FindGameFile(const NetPlay::SyncIdentifier& sync_identifier,
 
   std::optional<std::shared_ptr<const UICommon::GameFile>> game_file =
       RunOnObject(this, [this, &sync_identifier, found] {
-        for (int i = 0; i < m_game_list_model->rowCount(QModelIndex()); i++)
+        for (int i = 0; i < m_game_list_model.rowCount(QModelIndex()); i++)
         {
-          auto game_file = m_game_list_model->GetGameFile(i);
+          auto game_file = m_game_list_model.GetGameFile(i);
           *found = std::min(*found, game_file->CompareSyncIdentifier(sync_identifier));
           if (*found == NetPlay::SyncIdentifierComparison::SameGame)
             return game_file;
diff --git a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h
index 7459be4f16..bd32dd61f4 100644
--- a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h
+++ b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h
@@ -9,11 +9,11 @@
 
 #include "Common/Lazy.h"
 #include "Core/NetPlayClient.h"
+#include "DolphinQt/GameList/GameListModel.h"
 #include "VideoCommon/OnScreenDisplay.h"
 
 class ChunkedProgressDialog;
 class MD5Dialog;
-class GameListModel;
 class PadMappingDialog;
 class QCheckBox;
 class QComboBox;
@@ -31,7 +31,7 @@ class NetPlayDialog : public QDialog, public NetPlay::NetPlayUI
 {
   Q_OBJECT
 public:
-  explicit NetPlayDialog(QWidget* parent = nullptr);
+  explicit NetPlayDialog(const GameListModel& game_list_model, QWidget* parent = nullptr);
   ~NetPlayDialog();
 
   void show(std::string nickname, bool use_traversal);
@@ -151,7 +151,7 @@ private:
   std::string m_current_game_name;
   Common::Lazy<std::string> m_external_ip_address;
   std::string m_nickname;
-  GameListModel* m_game_list_model = nullptr;
+  const GameListModel& m_game_list_model;
   bool m_use_traversal = false;
   bool m_is_copy_button_retry = false;
   bool m_got_stop_request = true;
diff --git a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp
index adcfa3faf3..521ec48745 100644
--- a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp
+++ b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp
@@ -21,7 +21,6 @@
 #include "Core/Config/NetplaySettings.h"
 #include "Core/NetPlayProto.h"
 
-#include "DolphinQt/GameList/GameListModel.h"
 #include "DolphinQt/QtUtils/ModalMessageBox.h"
 #include "DolphinQt/QtUtils/UTF8CodePointCountValidator.h"
 #include "DolphinQt/Settings.h"
@@ -29,8 +28,8 @@
 #include "UICommon/GameFile.h"
 #include "UICommon/NetPlayIndex.h"
 
-NetPlaySetupDialog::NetPlaySetupDialog(QWidget* parent)
-    : QDialog(parent), m_game_list_model(Settings::Instance().GetGameListModel())
+NetPlaySetupDialog::NetPlaySetupDialog(const GameListModel& game_list_model, QWidget* parent)
+    : QDialog(parent), m_game_list_model(game_list_model)
 {
   setWindowTitle(tr("NetPlay Setup"));
   setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint);
@@ -359,12 +358,12 @@ void NetPlaySetupDialog::PopulateGameList()
   QSignalBlocker blocker(m_host_games);
 
   m_host_games->clear();
-  for (int i = 0; i < m_game_list_model->rowCount(QModelIndex()); i++)
+  for (int i = 0; i < m_game_list_model.rowCount(QModelIndex()); i++)
   {
-    std::shared_ptr<const UICommon::GameFile> game = m_game_list_model->GetGameFile(i);
+    std::shared_ptr<const UICommon::GameFile> game = m_game_list_model.GetGameFile(i);
 
     auto* item =
-        new QListWidgetItem(QString::fromStdString(m_game_list_model->GetNetPlayName(*game)));
+        new QListWidgetItem(QString::fromStdString(m_game_list_model.GetNetPlayName(*game)));
     item->setData(Qt::UserRole, QVariant::fromValue(std::move(game)));
     m_host_games->addItem(item);
   }
diff --git a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h
index a2ff67264d..b3247df03b 100644
--- a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h
+++ b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h
@@ -6,7 +6,8 @@
 
 #include <QDialog>
 
-class GameListModel;
+#include "DolphinQt/GameList/GameListModel.h"
+
 class QCheckBox;
 class QComboBox;
 class QDialogButtonBox;
@@ -27,7 +28,7 @@ class NetPlaySetupDialog : public QDialog
 {
   Q_OBJECT
 public:
-  explicit NetPlaySetupDialog(QWidget* parent);
+  explicit NetPlaySetupDialog(const GameListModel& game_list_model, QWidget* parent);
 
   void accept() override;
   void show();
@@ -79,5 +80,5 @@ private:
   QCheckBox* m_host_upnp;
 #endif
 
-  GameListModel* m_game_list_model;
+  const GameListModel& m_game_list_model;
 };
diff --git a/Source/Core/DolphinQt/Settings.cpp b/Source/Core/DolphinQt/Settings.cpp
index 6cb3435bf5..c7c156bfd5 100644
--- a/Source/Core/DolphinQt/Settings.cpp
+++ b/Source/Core/DolphinQt/Settings.cpp
@@ -24,7 +24,6 @@
 #include "Core/NetPlayClient.h"
 #include "Core/NetPlayServer.h"
 
-#include "DolphinQt/GameList/GameListModel.h"
 #include "DolphinQt/QtUtils/QueueOnObject.h"
 
 #include "InputCommon/ControllerInterface/ControllerInterface.h"
@@ -148,6 +147,11 @@ void Settings::RefreshGameList()
   emit GameListRefreshRequested();
 }
 
+void Settings::NotifyRefreshGameListStarted()
+{
+  emit GameListRefreshStarted();
+}
+
 void Settings::NotifyRefreshGameListComplete()
 {
   emit GameListRefreshCompleted();
@@ -296,12 +300,6 @@ void Settings::SetLogConfigVisible(bool visible)
   }
 }
 
-GameListModel* Settings::GetGameListModel() const
-{
-  static GameListModel* model = new GameListModel;
-  return model;
-}
-
 std::shared_ptr<NetPlay::NetPlayClient> Settings::GetNetPlayClient()
 {
   return m_client;
diff --git a/Source/Core/DolphinQt/Settings.h b/Source/Core/DolphinQt/Settings.h
index eb7f7ab4ae..2e493feddd 100644
--- a/Source/Core/DolphinQt/Settings.h
+++ b/Source/Core/DolphinQt/Settings.h
@@ -26,7 +26,6 @@ class NetPlayClient;
 class NetPlayServer;
 }  // namespace NetPlay
 
-class GameListModel;
 class InputConfig;
 
 // UI settings to be stored in the config directory.
@@ -73,6 +72,7 @@ public:
   QString GetDefaultGame() const;
   void SetDefaultGame(QString path);
   void RefreshGameList();
+  void NotifyRefreshGameListStarted();
   void NotifyRefreshGameListComplete();
   void RefreshMetadata();
   void NotifyMetadataRefreshComplete();
@@ -143,8 +143,6 @@ public:
   bool IsAnalyticsEnabled() const;
   void SetAnalyticsEnabled(bool enabled);
 
-  // Other
-  GameListModel* GetGameListModel() const;
 signals:
   void ConfigChanged();
   void EmulationStateChanged(Core::State new_state);
@@ -153,6 +151,7 @@ signals:
   void PathRemoved(const QString&);
   void DefaultGameChanged(const QString&);
   void GameListRefreshRequested();
+  void GameListRefreshStarted();
   void GameListRefreshCompleted();
   void TitleDBReloadRequested();
   void MetadataRefreshRequested();
diff --git a/Source/Core/DolphinQt/Settings/InterfacePane.cpp b/Source/Core/DolphinQt/Settings/InterfacePane.cpp
index 145d532213..cd0596f319 100644
--- a/Source/Core/DolphinQt/Settings/InterfacePane.cpp
+++ b/Source/Core/DolphinQt/Settings/InterfacePane.cpp
@@ -23,7 +23,6 @@
 #include "Core/Config/UISettings.h"
 #include "Core/ConfigManager.h"
 
-#include "DolphinQt/GameList/GameListModel.h"
 #include "DolphinQt/QtUtils/ModalMessageBox.h"
 #include "DolphinQt/Settings.h"
 
diff --git a/Source/Core/DolphinQt/ToolBar.cpp b/Source/Core/DolphinQt/ToolBar.cpp
index bdcbc0a4bb..e36bcd4f72 100644
--- a/Source/Core/DolphinQt/ToolBar.cpp
+++ b/Source/Core/DolphinQt/ToolBar.cpp
@@ -46,7 +46,9 @@ ToolBar::ToolBar(QWidget* parent) : QToolBar(parent)
   connect(&Settings::Instance(), &Settings::WidgetLockChanged, this,
           [this](bool locked) { setMovable(!locked); });
 
-  connect(&Settings::Instance(), &Settings::GameListRefreshCompleted, this,
+  connect(&Settings::Instance(), &Settings::GameListRefreshRequested, this,
+          [this] { m_refresh_action->setEnabled(false); });
+  connect(&Settings::Instance(), &Settings::GameListRefreshStarted, this,
           [this] { m_refresh_action->setEnabled(true); });
 
   OnEmulationStateChanged(Core::GetState());
@@ -112,10 +114,8 @@ void ToolBar::MakeActions()
   m_set_pc_action = addAction(tr("Set PC"), this, &ToolBar::SetPCPressed);
 
   m_open_action = addAction(tr("Open"), this, &ToolBar::OpenPressed);
-  m_refresh_action = addAction(tr("Refresh"), [this] {
-    m_refresh_action->setEnabled(false);
-    emit RefreshPressed();
-  });
+  m_refresh_action = addAction(tr("Refresh"), [this] { emit RefreshPressed(); });
+  m_refresh_action->setEnabled(false);
 
   addSeparator();
 
diff --git a/Source/Core/UICommon/GameFileCache.cpp b/Source/Core/UICommon/GameFileCache.cpp
index 35d4fe3571..38dd09f8a8 100644
--- a/Source/Core/UICommon/GameFileCache.cpp
+++ b/Source/Core/UICommon/GameFileCache.cpp
@@ -90,7 +90,8 @@ std::shared_ptr<const GameFile> GameFileCache::AddOrGet(const std::string& path,
 bool GameFileCache::Update(
     const std::vector<std::string>& all_game_paths,
     std::function<void(const std::shared_ptr<const GameFile>&)> game_added_to_cache,
-    std::function<void(const std::string&)> game_removed_from_cache)
+    std::function<void(const std::string&)> game_removed_from_cache,
+    const std::atomic_bool& processing_halted)
 {
   // Copy game paths into a set, except ones that match DiscIO::ShouldHideFromGameList.
   // TODO: Prevent DoFileSearch from looking inside /files/ directories of DirectoryBlobs at all?
@@ -113,6 +114,9 @@ bool GameFileCache::Update(
     auto end = m_cached_files.end();
     while (it != end)
     {
+      if (processing_halted)
+        break;
+
       if (game_paths.erase((*it)->GetFilePath()))
       {
         ++it;
@@ -134,6 +138,9 @@ bool GameFileCache::Update(
   // aren't in m_cached_files, so we simply add all of them to m_cached_files.
   for (const std::string& path : game_paths)
   {
+    if (processing_halted)
+      break;
+
     auto file = std::make_shared<GameFile>(path);
     if (file->IsValid())
     {
@@ -149,12 +156,16 @@ bool GameFileCache::Update(
 }
 
 bool GameFileCache::UpdateAdditionalMetadata(
-    std::function<void(const std::shared_ptr<const GameFile>&)> game_updated)
+    std::function<void(const std::shared_ptr<const GameFile>&)> game_updated,
+    const std::atomic_bool& processing_halted)
 {
   bool cache_changed = false;
 
   for (std::shared_ptr<GameFile>& file : m_cached_files)
   {
+    if (processing_halted)
+      break;
+
     const bool updated = UpdateAdditionalMetadata(&file);
     cache_changed |= updated;
     if (game_updated && updated)
diff --git a/Source/Core/UICommon/GameFileCache.h b/Source/Core/UICommon/GameFileCache.h
index 2e09c6da16..2c3fd39492 100644
--- a/Source/Core/UICommon/GameFileCache.h
+++ b/Source/Core/UICommon/GameFileCache.h
@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <cstddef>
 #include <functional>
 #include <memory>
@@ -44,9 +45,11 @@ public:
   // These functions return true if the call modified the cache.
   bool Update(const std::vector<std::string>& all_game_paths,
               std::function<void(const std::shared_ptr<const GameFile>&)> game_added_to_cache = {},
-              std::function<void(const std::string&)> game_removed_from_cache = {});
+              std::function<void(const std::string&)> game_removed_from_cache = {},
+              const std::atomic_bool& processing_halted = false);
   bool UpdateAdditionalMetadata(
-      std::function<void(const std::shared_ptr<const GameFile>&)> game_updated = {});
+      std::function<void(const std::shared_ptr<const GameFile>&)> game_updated = {},
+      const std::atomic_bool& processing_halted = false);
 
   bool Load();
   bool Save();