diff --git a/Source/Core/DolphinQt/GameList/GameTracker.cpp b/Source/Core/DolphinQt/GameList/GameTracker.cpp
index 59006d7ee4..4caf643dd5 100644
--- a/Source/Core/DolphinQt/GameList/GameTracker.cpp
+++ b/Source/Core/DolphinQt/GameList/GameTracker.cpp
@@ -34,7 +34,7 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
 
   connect(this, &QFileSystemWatcher::directoryChanged, this, &GameTracker::UpdateDirectory);
   connect(this, &QFileSystemWatcher::fileChanged, this, &GameTracker::UpdateFile);
-  connect(&Settings::Instance(), &Settings::AutoRefreshToggled, this, [] {
+  connect(&Settings::Instance(), &Settings::AutoRefreshToggled, [] {
     const auto paths = Settings::Instance().GetPaths();
 
     for (const auto& path : paths)
@@ -44,7 +44,7 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
     }
   });
 
-  connect(&Settings::Instance(), &Settings::MetadataRefreshRequested, this, [this] {
+  connect(&Settings::Instance(), &Settings::MetadataRefreshRequested, [this] {
     m_load_thread.EmplaceItem(Command{CommandType::UpdateMetadata, {}});
   });
 
@@ -80,9 +80,18 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
       m_cache.Clear(UICommon::GameFileCache::DeleteOnDisk::Yes);
       break;
     case CommandType::BeginRefresh:
-      for (auto& file : m_tracked_files.keys())
-        emit GameRemoved(file.toStdString());
-      m_tracked_files.clear();
+      if (m_busy_count++ == 0)
+      {
+        for (auto& file : m_tracked_files.keys())
+          emit GameRemoved(file.toStdString());
+        m_tracked_files.clear();
+      }
+      break;
+    case CommandType::EndRefresh:
+      if (--m_busy_count == 0)
+      {
+        QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListComplete(); });
+      }
       break;
     }
   });
@@ -189,6 +198,8 @@ void GameTracker::RefreshAll()
     m_load_thread.EmplaceItem(Command{CommandType::RemoveDirectory, dir});
     m_load_thread.EmplaceItem(Command{CommandType::AddDirectory, dir});
   }
+
+  m_load_thread.EmplaceItem(Command{CommandType::EndRefresh});
 }
 
 void GameTracker::UpdateDirectory(const QString& dir)
diff --git a/Source/Core/DolphinQt/GameList/GameTracker.h b/Source/Core/DolphinQt/GameList/GameTracker.h
index c411854a33..93720614b3 100644
--- a/Source/Core/DolphinQt/GameList/GameTracker.h
+++ b/Source/Core/DolphinQt/GameList/GameTracker.h
@@ -74,6 +74,7 @@ private:
     UpdateMetadata,
     PurgeCache,
     BeginRefresh,
+    EndRefresh,
   };
 
   struct Command
@@ -91,6 +92,8 @@ private:
   Common::Event m_initial_games_emitted_event;
   bool m_initial_games_emitted = false;
   bool m_started = false;
+  // Count of currently running refresh jobs
+  u32 m_busy_count = 0;
 };
 
 Q_DECLARE_METATYPE(std::shared_ptr<const UICommon::GameFile>)
diff --git a/Source/Core/DolphinQt/Settings.cpp b/Source/Core/DolphinQt/Settings.cpp
index 99322db7fe..8dc68cd742 100644
--- a/Source/Core/DolphinQt/Settings.cpp
+++ b/Source/Core/DolphinQt/Settings.cpp
@@ -141,6 +141,11 @@ void Settings::RefreshGameList()
   emit GameListRefreshRequested();
 }
 
+void Settings::NotifyRefreshGameListComplete()
+{
+  emit GameListRefreshCompleted();
+}
+
 void Settings::RefreshMetadata()
 {
   emit MetadataRefreshRequested();
diff --git a/Source/Core/DolphinQt/Settings.h b/Source/Core/DolphinQt/Settings.h
index f16ce2a474..7997ca0ec3 100644
--- a/Source/Core/DolphinQt/Settings.h
+++ b/Source/Core/DolphinQt/Settings.h
@@ -75,6 +75,7 @@ public:
   QString GetDefaultGame() const;
   void SetDefaultGame(QString path);
   void RefreshGameList();
+  void NotifyRefreshGameListComplete();
   void RefreshMetadata();
   void NotifyMetadataRefreshComplete();
   void ReloadTitleDB();
@@ -150,6 +151,7 @@ signals:
   void PathRemoved(const QString&);
   void DefaultGameChanged(const QString&);
   void GameListRefreshRequested();
+  void GameListRefreshCompleted();
   void TitleDBReloadRequested();
   void MetadataRefreshRequested();
   void MetadataRefreshCompleted();
diff --git a/Source/Core/DolphinQt/ToolBar.cpp b/Source/Core/DolphinQt/ToolBar.cpp
index d6ba3e9c96..80ce2d755c 100644
--- a/Source/Core/DolphinQt/ToolBar.cpp
+++ b/Source/Core/DolphinQt/ToolBar.cpp
@@ -32,10 +32,10 @@ ToolBar::ToolBar(QWidget* parent) : QToolBar(parent)
   connect(&Settings::Instance(), &Settings::ThemeChanged, this, &ToolBar::UpdateIcons);
   UpdateIcons();
 
-  connect(&Settings::Instance(), &Settings::EmulationStateChanged, this,
+  connect(&Settings::Instance(), &Settings::EmulationStateChanged,
           [this](Core::State state) { OnEmulationStateChanged(state); });
 
-  connect(Host::GetInstance(), &Host::UpdateDisasmDialog, this,
+  connect(Host::GetInstance(), &Host::UpdateDisasmDialog,
           [this] { OnEmulationStateChanged(Core::GetState()); });
 
   connect(&Settings::Instance(), &Settings::DebugModeToggled, this, &ToolBar::OnDebugModeToggled);
@@ -43,9 +43,12 @@ ToolBar::ToolBar(QWidget* parent) : QToolBar(parent)
   connect(&Settings::Instance(), &Settings::ToolBarVisibilityChanged, this, &ToolBar::setVisible);
   connect(this, &ToolBar::visibilityChanged, &Settings::Instance(), &Settings::SetToolBarVisible);
 
-  connect(&Settings::Instance(), &Settings::WidgetLockChanged, this,
+  connect(&Settings::Instance(), &Settings::WidgetLockChanged,
           [this](bool locked) { setMovable(!locked); });
 
+  connect(&Settings::Instance(), &Settings::GameListRefreshCompleted,
+          [this] { m_refresh_action->setEnabled(true); });
+
   OnEmulationStateChanged(Core::GetState());
   OnDebugModeToggled(Settings::Instance().IsDebugModeEnabled());
 }
@@ -109,7 +112,10 @@ void ToolBar::MakeActions()
   m_set_pc_action = addAction(tr("Set PC"), this, &ToolBar::SetPCPressed);
 
   m_open_action = addAction(tr("Open"), this, &ToolBar::OpenPressed);
-  m_refresh_action = addAction(tr("Refresh"), this, &ToolBar::RefreshPressed);
+  m_refresh_action = addAction(tr("Refresh"), [this] {
+    m_refresh_action->setEnabled(false);
+    emit RefreshPressed();
+  });
 
   addSeparator();