DolphinWX: defer gamelist scanning and switch to single-file cache.

Source/Core/Common/ChunkFile.h

@@ -296,125 +296,3 @@ private:
     *ptr += size;
   }
 };
-
-// NOTE: this class is only used in DolphinWX/ISOFile.cpp for caching loaded
-// ISO data. It will be removed when DolphinWX is, so please don't use it.
-class CChunkFileReader
-{
-public:
-  // Load file template
-  template <class T>
-  static bool Load(const std::string& _rFilename, u32 _Revision, T& _class)
-  {
-    INFO_LOG(COMMON, "ChunkReader: Loading %s", _rFilename.c_str());
-
-    if (!File::Exists(_rFilename))
-      return false;
-
-    // Check file size
-    const u64 fileSize = File::GetSize(_rFilename);
-    static const u64 headerSize = sizeof(SChunkHeader);
-    if (fileSize < headerSize)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: File too small");
-      return false;
-    }
-
-    File::IOFile pFile(_rFilename, "rb");
-    if (!pFile)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Can't open file for reading");
-      return false;
-    }
-
-    // read the header
-    SChunkHeader header;
-    if (!pFile.ReadArray(&header, 1))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Bad header size");
-      return false;
-    }
-
-    // Check revision
-    if (header.Revision != _Revision)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Wrong file revision, got %d expected %d", header.Revision,
-                _Revision);
-      return false;
-    }
-
-    // get size
-    const u32 sz = (u32)(fileSize - headerSize);
-    if (header.ExpectedSize != sz)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Bad file size, got %d expected %d", sz, header.ExpectedSize);
-      return false;
-    }
-
-    // read the state
-    std::vector<u8> buffer(sz);
-    if (!pFile.ReadArray(&buffer[0], sz))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Error reading file");
-      return false;
-    }
-
-    u8* ptr = &buffer[0];
-    PointerWrap p(&ptr, PointerWrap::MODE_READ);
-    _class.DoState(p);
-
-    INFO_LOG(COMMON, "ChunkReader: Done loading %s", _rFilename.c_str());
-    return true;
-  }
-
-  // Save file template
-  template <class T>
-  static bool Save(const std::string& _rFilename, u32 _Revision, T& _class)
-  {
-    INFO_LOG(COMMON, "ChunkReader: Writing %s", _rFilename.c_str());
-    File::IOFile pFile(_rFilename, "wb");
-    if (!pFile)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Error opening file for write");
-      return false;
-    }
-
-    // Get data
-    u8* ptr = nullptr;
-    PointerWrap p(&ptr, PointerWrap::MODE_MEASURE);
-    _class.DoState(p);
-    size_t const sz = (size_t)ptr;
-    std::vector<u8> buffer(sz);
-    ptr = &buffer[0];
-    p.SetMode(PointerWrap::MODE_WRITE);
-    _class.DoState(p);
-
-    // Create header
-    SChunkHeader header;
-    header.Revision = _Revision;
-    header.ExpectedSize = (u32)sz;
-
-    // Write to file
-    if (!pFile.WriteArray(&header, 1))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Failed writing header");
-      return false;
-    }
-
-    if (!pFile.WriteArray(&buffer[0], sz))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Failed writing data");
-      return false;
-    }
-
-    INFO_LOG(COMMON, "ChunkReader: Done writing %s", _rFilename.c_str());
-    return true;
-  }
-
-private:
-  struct SChunkHeader
-  {
-    u32 Revision;
-    u32 ExpectedSize;
-  };
-};

Source/Core/DolphinWX/Config/ConfigMain.cpp

@@ -24,21 +24,21 @@
 #include "DolphinWX/GameListCtrl.h"
 #include "DolphinWX/WxUtils.h"
 
-// Sent by child panes to signify that the game list should
-// be updated when this modal dialog closes.
 wxDEFINE_EVENT(wxDOLPHIN_CFG_REFRESH_LIST, wxCommandEvent);
+wxDEFINE_EVENT(wxDOLPHIN_CFG_RESCAN_LIST, wxCommandEvent);
 
 CConfigMain::CConfigMain(wxWindow* parent, wxWindowID id, const wxString& title,
                          const wxPoint& position, const wxSize& size, long style)
     : wxDialog(parent, id, title, position, size, style)
 {
-  // Control refreshing of the ISOs list
+  // Control refreshing of the GameListCtrl
-  m_refresh_game_list_on_close = false;
+  m_event_on_close = wxEVT_NULL;
 
   Bind(wxEVT_CLOSE_WINDOW, &CConfigMain::OnClose, this);
   Bind(wxEVT_BUTTON, &CConfigMain::OnCloseButton, this, wxID_CLOSE);
   Bind(wxEVT_SHOW, &CConfigMain::OnShow, this);
   Bind(wxDOLPHIN_CFG_REFRESH_LIST, &CConfigMain::OnSetRefreshGameListOnClose, this);
+  Bind(wxDOLPHIN_CFG_RESCAN_LIST, &CConfigMain::OnSetRescanGameListOnClose, this);
 
   wxDialog::SetExtraStyle(GetExtraStyle() & ~wxWS_EX_BLOCK_EVENTS);
 
@@ -115,8 +115,8 @@ void CConfigMain::OnClose(wxCloseEvent& WXUNUSED(event))
 
   SConfig::GetInstance().SaveSettings();
 
-  if (m_refresh_game_list_on_close)
+  if (m_event_on_close != wxEVT_NULL)
-    AddPendingEvent(wxCommandEvent{DOLPHIN_EVT_RELOAD_GAMELIST});
+    AddPendingEvent(wxCommandEvent{m_event_on_close});
 }
 
 void CConfigMain::OnShow(wxShowEvent& event)
@@ -132,5 +132,12 @@ void CConfigMain::OnCloseButton(wxCommandEvent& WXUNUSED(event))
 
 void CConfigMain::OnSetRefreshGameListOnClose(wxCommandEvent& WXUNUSED(event))
 {
-  m_refresh_game_list_on_close = true;
+  // Don't override a rescan
+  if (m_event_on_close == wxEVT_NULL)
+    m_event_on_close = DOLPHIN_EVT_REFRESH_GAMELIST;
+}
+
+void CConfigMain::OnSetRescanGameListOnClose(wxCommandEvent& WXUNUSED(event))
+{
+  m_event_on_close = DOLPHIN_EVT_RESCAN_GAMELIST;
 }

Source/Core/DolphinWX/Config/ConfigMain.h

@@ -10,7 +10,10 @@
 class wxNotebook;
 class wxPanel;
 
+// Fast refresh - can be fulfilled from cache
 wxDECLARE_EVENT(wxDOLPHIN_CFG_REFRESH_LIST, wxCommandEvent);
+// Rescan and refresh - modifies cache
+wxDECLARE_EVENT(wxDOLPHIN_CFG_RESCAN_LIST, wxCommandEvent);
 
 class CConfigMain : public wxDialog
 {
@@ -41,8 +44,8 @@ private:
   void OnCloseButton(wxCommandEvent& event);
   void OnShow(wxShowEvent& event);
   void OnSetRefreshGameListOnClose(wxCommandEvent& event);
+  void OnSetRescanGameListOnClose(wxCommandEvent& event);
 
   wxNotebook* Notebook;
-
+  wxEventType m_event_on_close;
-  bool m_refresh_game_list_on_close;
 };

Source/Core/DolphinWX/Config/PathConfigPane.cpp

@@ -163,7 +163,7 @@ void PathConfigPane::OnRecursiveISOCheckBoxChanged(wxCommandEvent& event)
 {
   SConfig::GetInstance().m_RecursiveISOFolder = m_recursive_iso_paths_checkbox->IsChecked();
 
-  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_REFRESH_LIST));
+  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_RESCAN_LIST));
 }
 
 void PathConfigPane::OnAddISOPath(wxCommandEvent& event)
@@ -179,7 +179,7 @@ void PathConfigPane::OnAddISOPath(wxCommandEvent& event)
     }
     else
     {
-      AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_REFRESH_LIST));
+      AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_RESCAN_LIST));
       m_iso_paths_listbox->Append(dialog.GetPath());
     }
   }
@@ -189,7 +189,7 @@ void PathConfigPane::OnAddISOPath(wxCommandEvent& event)
 
 void PathConfigPane::OnRemoveISOPath(wxCommandEvent& event)
 {
-  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_REFRESH_LIST));
+  AddPendingEvent(wxCommandEvent(wxDOLPHIN_CFG_RESCAN_LIST));
   m_iso_paths_listbox->Delete(m_iso_paths_listbox->GetSelection());
 
 // This seems to not be activated on Windows when it should be. wxw bug?

Source/Core/DolphinWX/Frame.cpp

@@ -351,7 +351,8 @@ CFrame::CFrame(wxFrame* parent, wxWindowID id, const wxString& title, wxRect geo
   // This panel is the parent for rendering and it holds the gamelistctrl
   m_panel = new wxPanel(this, IDM_MPANEL, wxDefaultPosition, wxDefaultSize, 0);
 
-  m_game_list_ctrl = new GameListCtrl(m_panel, wxID_ANY, wxDefaultPosition, wxDefaultSize,
+  m_game_list_ctrl =
+      new GameListCtrl(m_batch_mode, m_panel, wxID_ANY, wxDefaultPosition, wxDefaultSize,
                        wxLC_REPORT | wxSUNKEN_BORDER | wxLC_ALIGN_LEFT);
   m_game_list_ctrl->Bind(wxEVT_LIST_ITEM_ACTIVATED, &CFrame::OnGameListCtrlItemActivated, this);
 
@@ -490,7 +491,8 @@ void CFrame::BindEvents()
   BindMenuBarEvents();
 
   Bind(DOLPHIN_EVT_RELOAD_THEME_BITMAPS, &CFrame::OnReloadThemeBitmaps, this);
-  Bind(DOLPHIN_EVT_RELOAD_GAMELIST, &CFrame::OnReloadGameList, this);
+  Bind(DOLPHIN_EVT_REFRESH_GAMELIST, &CFrame::OnRefreshGameList, this);
+  Bind(DOLPHIN_EVT_RESCAN_GAMELIST, &CFrame::OnRescanGameList, this);
   Bind(DOLPHIN_EVT_UPDATE_LOAD_WII_MENU_ITEM, &CFrame::OnUpdateLoadWiiMenuItem, this);
   Bind(DOLPHIN_EVT_BOOT_SOFTWARE, &CFrame::OnPlay, this);
   Bind(DOLPHIN_EVT_STOP_SOFTWARE, &CFrame::OnStop, this);
@@ -909,7 +911,7 @@ void CFrame::OnGameListCtrlItemActivated(wxListEvent& WXUNUSED(event))
     GetMenuBar()->FindItem(IDM_LIST_WORLD)->Check(true);
     GetMenuBar()->FindItem(IDM_LIST_UNKNOWN)->Check(true);
 
-    UpdateGameList();
+    GameListRefresh();
   }
   else if (!m_game_list_ctrl->GetISO(0))
   {

Source/Core/DolphinWX/Frame.h

@@ -98,7 +98,8 @@ public:
 
   void DoStop();
   void UpdateGUI();
-  void UpdateGameList();
+  void GameListRefresh();
+  void GameListRescan();
   void ToggleLogWindow(bool bShow);
   void ToggleLogConfigWindow(bool bShow);
   void StatusBarMessage(const char* Text, ...);
@@ -267,7 +268,8 @@ private:
   void OnHelp(wxCommandEvent& event);
 
   void OnReloadThemeBitmaps(wxCommandEvent& event);
-  void OnReloadGameList(wxCommandEvent& event);
+  void OnRefreshGameList(wxCommandEvent& event);
+  void OnRescanGameList(wxCommandEvent& event);
 
   void OnUpdateInterpreterMenuItem(wxUpdateUIEvent& event);
 

Source/Core/DolphinWX/FrameTools.cpp

@@ -774,7 +774,7 @@ void CFrame::OnBootDrive(wxCommandEvent& event)
 
 void CFrame::OnRefresh(wxCommandEvent& WXUNUSED(event))
 {
-  UpdateGameList();
+  GameListRescan();
 }
 
 void CFrame::OnScreenshot(wxCommandEvent& WXUNUSED(event))
@@ -1097,12 +1097,17 @@ void CFrame::OnReloadThemeBitmaps(wxCommandEvent& WXUNUSED(event))
   reload_event.SetEventObject(this);
   wxPostEvent(GetToolBar(), reload_event);
 
-  UpdateGameList();
+  GameListRefresh();
 }
 
-void CFrame::OnReloadGameList(wxCommandEvent& WXUNUSED(event))
+void CFrame::OnRefreshGameList(wxCommandEvent& WXUNUSED(event))
 {
-  UpdateGameList();
+  GameListRefresh();
+}
+
+void CFrame::OnRescanGameList(wxCommandEvent& WXUNUSED(event))
+{
+  GameListRescan();
 }
 
 void CFrame::OnUpdateInterpreterMenuItem(wxUpdateUIEvent& event)
@@ -1605,9 +1610,16 @@ void CFrame::UpdateGUI()
   }
 }
 
-void CFrame::UpdateGameList()
+void CFrame::GameListRefresh()
 {
-  wxCommandEvent event{DOLPHIN_EVT_RELOAD_GAMELIST, GetId()};
+  wxCommandEvent event{DOLPHIN_EVT_REFRESH_GAMELIST, GetId()};
+  event.SetEventObject(this);
+  wxPostEvent(m_game_list_ctrl, event);
+}
+
+void CFrame::GameListRescan()
+{
+  wxCommandEvent event{DOLPHIN_EVT_RESCAN_GAMELIST, GetId()};
   event.SetEventObject(this);
   wxPostEvent(m_game_list_ctrl, event);
 }
@@ -1674,17 +1686,19 @@ void CFrame::GameListChanged(wxCommandEvent& event)
     SConfig::GetInstance().m_ListDrives = event.IsChecked();
     break;
   case IDM_PURGE_GAME_LIST_CACHE:
-    std::vector<std::string> rFilenames =
+    std::vector<std::string> filenames =
         Common::DoFileSearch({".cache"}, {File::GetUserPath(D_CACHE_IDX)});
 
-    for (const std::string& rFilename : rFilenames)
+    for (const std::string& filename : filenames)
     {
-      File::Delete(rFilename);
+      File::Delete(filename);
     }
-    break;
+    // Do rescan after cache has been cleared
+    GameListRescan();
+    return;
   }
 
-  UpdateGameList();
+  GameListRefresh();
 }
 
 // Enable and disable the toolbar
@@ -1743,6 +1757,6 @@ void CFrame::OnChangeColumnsVisible(wxCommandEvent& event)
   default:
     return;
   }
-  UpdateGameList();
+  GameListRefresh();
   SConfig::GetInstance().SaveSettings();
 }

Source/Core/DolphinWX/GameListCtrl.cpp

@@ -44,6 +44,7 @@
 #include "Common/MathUtil.h"
 #include "Common/StringUtil.h"
 #include "Common/SysConf.h"
+#include "Common/Thread.h"
 #include "Core/Boot/Boot.h"
 #include "Core/ConfigManager.h"
 #include "Core/Core.h"
@@ -155,36 +156,6 @@ static int CompareGameListItems(const GameListItem* iso1, const GameListItem* is
   return 0;
 }
 
-static std::vector<std::string> GetFileSearchExtensions()
-{
-  std::vector<std::string> extensions;
-
-  if (SConfig::GetInstance().m_ListGC)
-  {
-    extensions.push_back(".gcm");
-    extensions.push_back(".tgc");
-  }
-
-  if (SConfig::GetInstance().m_ListWii || SConfig::GetInstance().m_ListGC)
-  {
-    extensions.push_back(".iso");
-    extensions.push_back(".ciso");
-    extensions.push_back(".gcz");
-    extensions.push_back(".wbfs");
-  }
-
-  if (SConfig::GetInstance().m_ListWad)
-    extensions.push_back(".wad");
-
-  if (SConfig::GetInstance().m_ListElfDol)
-  {
-    extensions.push_back(".dol");
-    extensions.push_back(".elf");
-  }
-
-  return extensions;
-}
-
 static bool ShouldDisplayGameListItem(const GameListItem& item)
 {
   const bool show_platform = [&item] {
@@ -240,7 +211,8 @@ static bool ShouldDisplayGameListItem(const GameListItem& item)
   }
 }
 
-wxDEFINE_EVENT(DOLPHIN_EVT_RELOAD_GAMELIST, wxCommandEvent);
+wxDEFINE_EVENT(DOLPHIN_EVT_REFRESH_GAMELIST, wxCommandEvent);
+wxDEFINE_EVENT(DOLPHIN_EVT_RESCAN_GAMELIST, wxCommandEvent);
 
 struct GameListCtrl::ColumnInfo
 {
@@ -250,8 +222,8 @@ struct GameListCtrl::ColumnInfo
   bool& visible;
 };
 
-GameListCtrl::GameListCtrl(wxWindow* parent, const wxWindowID id, const wxPoint& pos,
+GameListCtrl::GameListCtrl(bool disable_scanning, wxWindow* parent, const wxWindowID id,
-                             const wxSize& size, long style)
+                           const wxPoint& pos, const wxSize& size, long style)
     : wxListCtrl(parent, id, pos, size, style), m_tooltip(nullptr),
       m_columns({// {COLUMN, {default_width (without platform padding), resizability, visibility}}
                  {COLUMN_PLATFORM, 32 + 1 /* icon padding */, false,
@@ -286,9 +258,57 @@ GameListCtrl::GameListCtrl(wxWindow* parent, const wxWindowID id, const wxPoint&
   Bind(wxEVT_MENU, &GameListCtrl::OnChangeDisc, this, IDM_LIST_CHANGE_DISC);
   Bind(wxEVT_MENU, &GameListCtrl::OnNetPlayHost, this, IDM_START_NETPLAY);
 
-  Bind(DOLPHIN_EVT_RELOAD_GAMELIST, &GameListCtrl::OnReloadGameList, this);
+  Bind(DOLPHIN_EVT_REFRESH_GAMELIST, &GameListCtrl::OnRefreshGameList, this);
+  Bind(DOLPHIN_EVT_RESCAN_GAMELIST, &GameListCtrl::OnRescanGameList, this);
 
   wxTheApp->Bind(DOLPHIN_EVT_LOCAL_INI_CHANGED, &GameListCtrl::OnLocalIniModified, this);
+
+  if (!disable_scanning)
+  {
+    m_scan_thread = std::thread([&] {
+      Common::SetCurrentThreadName("gamelist scanner");
+
+      if (SyncCacheFile(false))
+      {
+        // Account for changes outside the cache (just wii banners atm; could scan Ini here)
+        // Note this is on the initial load path. Further improvements could be made if updates
+        // from scan->ui threads described small changelists instead of updating the entire list
+        // at once.
+        bool cache_modified = false;
+        for (auto& file : m_cached_files)
+        {
+          if (file->ReloadBannerIfNeeded())
+            cache_modified = true;
+        }
+
+        QueueEvent(new wxCommandEvent(DOLPHIN_EVT_REFRESH_GAMELIST));
+
+        if (cache_modified)
+          SyncCacheFile(true);
+      }
+      else
+      {
+        RescanList();
+      }
+
+      m_scan_trigger.Wait();
+      while (!m_scan_exiting.IsSet())
+      {
+        RescanList();
+        m_scan_trigger.Wait();
+      }
+    });
+  }
+}
+
+GameListCtrl::~GameListCtrl()
+{
+  if (m_scan_thread.joinable())
+  {
+    m_scan_exiting.Set();
+    m_scan_trigger.Set();
+    m_scan_thread.join();
+  }
 }
 
 template <typename T>
@@ -345,8 +365,8 @@ void GameListCtrl::InitBitmaps()
 
   auto& platform_indexes = m_image_indexes.platform;
   platform_indexes.resize(static_cast<size_t>(DiscIO::Platform::NUMBER_OF_PLATFORMS));
-  InitBitmap(img_list, &platform_indexes, this, platform_bmp_size,
+  InitBitmap(img_list, &platform_indexes, this, platform_bmp_size, DiscIO::Platform::GAMECUBE_DISC,
-             DiscIO::Platform::GAMECUBE_DISC, "Platform_Gamecube");
+             "Platform_Gamecube");
   InitBitmap(img_list, &platform_indexes, this, platform_bmp_size, DiscIO::Platform::WII_DISC,
              "Platform_Wii");
   InitBitmap(img_list, &platform_indexes, this, platform_bmp_size, DiscIO::Platform::WII_WAD,
@@ -388,24 +408,42 @@ void GameListCtrl::BrowseForDirectory()
     {
       SConfig::GetInstance().m_ISOFolder.push_back(sPath);
       SConfig::GetInstance().SaveSettings();
-      ReloadList();
+      m_scan_trigger.Set();
     }
   }
 }
 
-void GameListCtrl::ReloadList()
+void GameListCtrl::RefreshList()
 {
   int scrollPos = wxWindow::GetScrollPos(wxVERTICAL);
   // Don't let the user refresh it while a game is running
   if (Core::GetState() != Core::State::Uninitialized)
     return;
 
-  ScanForISOs();
+  m_shown_files.clear();
+  for (auto& item : m_cached_files)
+  {
+    if (ShouldDisplayGameListItem(*item))
+      m_shown_files.push_back(item);
+  }
+
+  // Drives are not cached. Not sure if this is required, but better to err on the
+  // side of caution if cross-platform issues could come into play.
+  if (SConfig::GetInstance().m_ListDrives)
+  {
+    const Core::TitleDatabase title_database;
+    for (const auto& drive : cdio_get_devices())
+    {
+      auto file = std::make_shared<GameListItem>(drive, title_database);
+      if (file->IsValid())
+        m_shown_files.push_back(file);
+    }
+  }
 
   Freeze();
   ClearAll();
 
-  if (m_ISOFiles.size() != 0)
+  if (!m_shown_files.empty())
   {
     // Don't load bitmaps unless there are games to list
     InitBitmaps();
@@ -430,14 +468,13 @@ void GameListCtrl::ReloadList()
 #endif
     // set initial sizes for columns
     SetColumnWidth(COLUMN_DUMMY, 0);
-
     for (const auto& c : m_columns)
     {
       SetColumnWidth(c.id, c.visible ? FromDIP(c.default_width + platform_padding) : 0);
     }
 
     // add all items
-    for (int i = 0; i < (int)m_ISOFiles.size(); i++)
+    for (int i = 0; i < (int)m_shown_files.size(); i++)
       InsertItemInReportView(i);
     SetColors();
 
@@ -511,7 +548,7 @@ static wxString NiceSizeFormat(u64 size)
   return wxString::Format("%.2f %s", size / unit_size, unit_symbols[unit]);
 }
 
-// Update the column content of the item at _Index
+// Update the column content of the item at index
 void GameListCtrl::UpdateItemAtColumn(long index, int column)
 {
   const auto& iso_file = *GetISO(GetItemData(index));
@@ -631,96 +668,148 @@ void GameListCtrl::SetColors()
   }
 }
 
-void GameListCtrl::ScanForISOs()
+void GameListCtrl::DoState(PointerWrap* p, u32 size)
 {
-  m_ISOFiles.clear();
+  struct
+  {
+    u32 Revision;
+    u32 ExpectedSize;
+  } header = {CACHE_REVISION, size};
+  p->Do(header);
+  if (p->GetMode() == PointerWrap::MODE_READ)
+  {
+    if (header.Revision != CACHE_REVISION || header.ExpectedSize != size)
+    {
+      p->SetMode(PointerWrap::MODE_MEASURE);
+      return;
+    }
+  }
+  p->DoEachElement(m_cached_files, [](PointerWrap& state, std::shared_ptr<GameListItem>& elem) {
+    if (state.GetMode() == PointerWrap::MODE_READ)
+    {
+      elem = std::make_shared<GameListItem>();
+    }
+    elem->DoState(state);
+  });
+}
 
-  const Core::TitleDatabase title_database;
+bool GameListCtrl::SyncCacheFile(bool write)
-  auto rFilenames =
+{
-      Common::DoFileSearch(GetFileSearchExtensions(), SConfig::GetInstance().m_ISOFolder,
+  std::string filename(File::GetUserPath(D_CACHE_IDX) + "wx_gamelist.cache");
+  const char* open_mode = write ? "wb" : "rb";
+  File::IOFile f(filename, open_mode);
+  if (!f)
+    return false;
+  bool success = false;
+  if (write)
+  {
+    // Measure the size of the buffer.
+    u8* ptr = nullptr;
+    PointerWrap p(&ptr, PointerWrap::MODE_MEASURE);
+    DoState(&p);
+    const size_t buffer_size = reinterpret_cast<size_t>(ptr);
+
+    // Then actually do the write.
+    std::vector<u8> buffer(buffer_size);
+    ptr = &buffer[0];
+    p.SetMode(PointerWrap::MODE_WRITE);
+    DoState(&p, buffer_size);
+    if (f.WriteBytes(buffer.data(), buffer.size()))
+      success = true;
+  }
+  else
+  {
+    std::vector<u8> buffer(f.GetSize());
+    if (buffer.size() && f.ReadBytes(buffer.data(), buffer.size()))
+    {
+      u8* ptr = buffer.data();
+      PointerWrap p(&ptr, PointerWrap::MODE_READ);
+      DoState(&p, buffer.size());
+      if (p.GetMode() == PointerWrap::MODE_READ)
+        success = true;
+    }
+  }
+  if (!success)
+  {
+    // If some file operation failed, try to delete the probably-corrupted cache
+    f.Close();
+    File::Delete(filename);
+  }
+  return success;
+}
+
+void GameListCtrl::RescanList()
+{
+  const std::vector<std::string> search_extensions = {".gcm",  ".tgc", ".iso", ".ciso", ".gcz",
+                                                      ".wbfs", ".wad", ".dol", ".elf"};
+  // TODO This could process paths iteratively as they are found
+  auto search_results = Common::DoFileSearch(search_extensions, SConfig::GetInstance().m_ISOFolder,
                                              SConfig::GetInstance().m_RecursiveISOFolder);
 
-  if (rFilenames.size() > 0)
+  // TODO rethink some good algorithms to use here
+  std::vector<std::string> cached_paths;
+  for (const auto& file : m_cached_files)
+    cached_paths.emplace_back(file->GetFileName());
+  std::sort(cached_paths.begin(), cached_paths.end());
+
+  std::list<std::string> removed_paths;
+  std::set_difference(cached_paths.cbegin(), cached_paths.cend(), search_results.cbegin(),
+                      search_results.cend(), std::back_inserter(removed_paths));
+
+  std::vector<std::string> new_paths;
+  std::set_difference(search_results.cbegin(), search_results.cend(), cached_paths.cbegin(),
+                      cached_paths.cend(), std::back_inserter(new_paths));
+
+  const Core::TitleDatabase title_database;
+  // TODO we could store all paths and modification times to judge if file needs to be rescanned.
+  // If we cached paths that turned out to be invalid, this would save failing on them each time
+  // refresh is done.
+  // However if people e.g. set dolphin to recursively scan the root of their drive(s), then we
+  // would cache way too much data. Maybe just use an upper bound of invalid paths to cache?
+  // For now, only scan new_paths. This could cause false negatives (file actively being written),
+  // but otherwise
+  // should be fine.
+  for (const auto& path : removed_paths)
   {
-    wxProgressDialog dialog(
+    auto it = std::find_if(
-        _("Scanning for ISOs"), _("Scanning..."), (int)rFilenames.size() - 1, this,
+        m_cached_files.cbegin(), m_cached_files.cend(),
-        wxPD_APP_MODAL | wxPD_AUTO_HIDE | wxPD_CAN_ABORT | wxPD_ELAPSED_TIME | wxPD_ESTIMATED_TIME |
+        [&path](const std::shared_ptr<GameListItem>& file) { return file->GetFileName() == path; });
-            wxPD_REMAINING_TIME | wxPD_SMOOTH  // - makes updates as small as possible (down to 1px)
+    if (it != m_cached_files.cend())
-        );
+      m_cached_files.erase(it);
-
-    for (u32 i = 0; i < rFilenames.size(); i++)
-    {
-      std::string FileName;
-      SplitPath(rFilenames[i], nullptr, &FileName, nullptr);
-
-      // Update with the progress (i) and the message
-      dialog.Update(i, wxString::Format(_("Scanning %s"), StrToWxStr(FileName)));
-      if (dialog.WasCancelled())
-        break;
-
-      auto iso_file = std::make_unique<GameListItem>(rFilenames[i], title_database);
-
-      if (iso_file->IsValid() && ShouldDisplayGameListItem(*iso_file))
-      {
-        m_ISOFiles.push_back(std::move(iso_file));
-      }
   }
+  for (const auto& path : new_paths)
+  {
+    auto file = std::make_shared<GameListItem>(path, title_database);
+    if (file->IsValid())
+      m_cached_files.push_back(std::move(file));
   }
 
-  if (SConfig::GetInstance().m_ListDrives)
+  for (auto& file : m_cached_files)
-  {
+    file->ReloadINI();
-    const std::vector<std::string> drives = cdio_get_devices();
 
-    for (const auto& drive : drives)
+  // Post UI event to update the displayed list
-    {
+  QueueEvent(new wxCommandEvent(DOLPHIN_EVT_REFRESH_GAMELIST));
-      auto gli = std::make_unique<GameListItem>(drive, title_database);
 
-      if (gli->IsValid())
+  SyncCacheFile(true);
-        m_ISOFiles.push_back(std::move(gli));
-    }
 }
 
-  std::sort(m_ISOFiles.begin(), m_ISOFiles.end());
+void GameListCtrl::OnRefreshGameList(wxCommandEvent& WXUNUSED(event))
+{
+  RefreshList();
 }
 
-void GameListCtrl::OnReloadGameList(wxCommandEvent& WXUNUSED(event))
+void GameListCtrl::OnRescanGameList(wxCommandEvent& WXUNUSED(event))
 {
-  ReloadList();
+  m_scan_trigger.Set();
 }
 
 void GameListCtrl::OnLocalIniModified(wxCommandEvent& ev)
 {
   ev.Skip();
-  std::string game_id = WxStrToStr(ev.GetString());
+  // We need show any changes to the ini which could impact our columns. Currently only the
-  // NOTE: The same game may occur multiple times if there are multiple
+  // EmuState/Issues settings can do that. We also need to persist the changes to the cache - so
-  //   physical copies in the search paths.
+  // just trigger a rescan which will sync the cache and then display the new values.
-  for (std::size_t i = 0; i < m_ISOFiles.size(); ++i)
+  m_scan_trigger.Set();
-  {
-    if (m_ISOFiles[i]->GetGameID() != game_id)
-      continue;
-    m_ISOFiles[i]->ReloadINI();
-
-    // The indexes in m_ISOFiles and the list do not line up.
-    // We need to find the corresponding item in the list (if it exists)
-    long item_id = 0;
-    for (; item_id < GetItemCount(); ++item_id)
-    {
-      if (i == static_cast<std::size_t>(GetItemData(item_id)))
-        break;
-    }
-    // If the item is not currently being displayed then we're done.
-    if (item_id == GetItemCount())
-      continue;
-
-    // Update all the columns
-    for (int j = FIRST_COLUMN_WITH_CONTENT; j < NUMBER_OF_COLUMN; ++j)
-    {
-      // NOTE: Banner is not modified by the INI and updating it will
-      //  duplicate it in memory which is not wanted.
-      if (j != COLUMN_BANNER && GetColumnWidth(j) != 0)
-        UpdateItemAtColumn(item_id, j);
-    }
-  }
 }
 
 void GameListCtrl::OnColBeginDrag(wxListEvent& event)
@@ -733,8 +822,8 @@ void GameListCtrl::OnColBeginDrag(wxListEvent& event)
 
 const GameListItem* GameListCtrl::GetISO(size_t index) const
 {
-  if (index < m_ISOFiles.size())
+  if (index < m_shown_files.size())
-    return m_ISOFiles[index].get();
+    return m_shown_files[index].get();
 
   return nullptr;
 }
@@ -790,7 +879,7 @@ void GameListCtrl::OnKeyPress(wxListEvent& event)
   static int lastKey = 0, sLoop = 0;
   int Loop = 0;
 
-  for (int i = 0; i < (int)m_ISOFiles.size(); i++)
+  for (int i = 0; i < (int)m_shown_files.size(); i++)
   {
     // Easy way to get game string
     wxListItem bleh;
@@ -806,7 +895,7 @@ void GameListCtrl::OnKeyPress(wxListEvent& event)
       if (lastKey == event.GetKeyCode() && Loop < sLoop)
       {
         Loop++;
-        if (i + 1 == (int)m_ISOFiles.size())
+        if (i + 1 == (int)m_shown_files.size())
           i = -1;
         continue;
       }
@@ -827,7 +916,7 @@ void GameListCtrl::OnKeyPress(wxListEvent& event)
 
     // If we get past the last game in the list,
     // we'll have to go back to the first one.
-    if (i + 1 == (int)m_ISOFiles.size() && sLoop > 0 && Loop > 0)
+    if (i + 1 == (int)m_shown_files.size() && sLoop > 0 && Loop > 0)
       i = -1;
   }
 
@@ -1044,7 +1133,7 @@ void GameListCtrl::OnRightClick(wxMouseEvent& event)
 
 const GameListItem* GameListCtrl::GetSelectedISO() const
 {
-  if (m_ISOFiles.empty())
+  if (m_shown_files.empty())
     return nullptr;
 
   if (GetSelectedItemCount() == 0)
@@ -1143,7 +1232,7 @@ void GameListCtrl::OnDeleteISO(wxCommandEvent& WXUNUSED(event))
   {
     for (const GameListItem* iso : GetAllSelectedISOs())
       File::Delete(iso->GetFileName());
-    ReloadList();
+    m_scan_trigger.Set();
   }
 }
 
@@ -1315,7 +1404,7 @@ void GameListCtrl::CompressSelection(bool _compress)
   if (!all_good)
     WxUtils::ShowErrorDialog(_("Dolphin was unable to complete the requested action."));
 
-  ReloadList();
+  m_scan_trigger.Set();
 }
 
 bool GameListCtrl::CompressCB(const std::string& text, float percent, void* arg)
@@ -1388,7 +1477,7 @@ void GameListCtrl::OnCompressISO(wxCommandEvent& WXUNUSED(event))
   if (!all_good)
     WxUtils::ShowErrorDialog(_("Dolphin was unable to complete the requested action."));
 
-  ReloadList();
+  m_scan_trigger.Set();
 }
 
 void GameListCtrl::OnChangeDisc(wxCommandEvent& WXUNUSED(event))

Source/Core/DolphinWX/GameListCtrl.h

@@ -12,6 +12,8 @@
 #include <wx/listctrl.h>
 #include <wx/tipwin.h>
 
+#include "Common/ChunkFile.h"
+#include "Common/Event.h"
 #include "DolphinWX/ISOFile.h"
 
 class wxEmuStateTip : public wxTipWindow
@@ -31,19 +33,19 @@ public:
   }
 };
 
-wxDECLARE_EVENT(DOLPHIN_EVT_RELOAD_GAMELIST, wxCommandEvent);
+wxDECLARE_EVENT(DOLPHIN_EVT_REFRESH_GAMELIST, wxCommandEvent);
+wxDECLARE_EVENT(DOLPHIN_EVT_RESCAN_GAMELIST, wxCommandEvent);
 
 class GameListCtrl : public wxListCtrl
 {
 public:
-  GameListCtrl(wxWindow* parent, const wxWindowID id, const wxPoint& pos, const wxSize& size,
+  GameListCtrl(bool disable_scanning, wxWindow* parent, const wxWindowID id, const wxPoint& pos,
-                long style);
+               const wxSize& size, long style);
-  ~GameListCtrl() = default;
+  ~GameListCtrl();
 
   void BrowseForDirectory();
   const GameListItem* GetISO(size_t index) const;
   const GameListItem* GetSelectedISO() const;
-  std::vector<const GameListItem*> GetAllSelectedISOs() const;
 
   static bool IsHidingItems();
 
@@ -70,16 +72,19 @@ public:
 private:
   struct ColumnInfo;
 
-  void ReloadList();
-
   void InitBitmaps();
   void UpdateItemAtColumn(long index, int column);
   void InsertItemInReportView(long index);
   void SetColors();
-  void ScanForISOs();
+  void RefreshList();
+  void RescanList();
+  void DoState(PointerWrap* p, u32 size = 0);
+  bool SyncCacheFile(bool write);
+  std::vector<const GameListItem*> GetAllSelectedISOs() const;
 
   // events
-  void OnReloadGameList(wxCommandEvent& event);
+  void OnRefreshGameList(wxCommandEvent& event);
+  void OnRescanGameList(wxCommandEvent& event);
   void OnLeftClick(wxMouseEvent& event);
   void OnRightClick(wxMouseEvent& event);
   void OnMouseMotion(wxMouseEvent& event);
@@ -109,14 +114,23 @@ private:
   static bool MultiCompressCB(const std::string& text, float percent, void* arg);
   static bool WiiCompressWarning();
 
-  std::vector<std::unique_ptr<GameListItem>> m_ISOFiles;
+  struct
-  struct {
+  {
     std::vector<int> flag;
     std::vector<int> platform;
     std::vector<int> utility_banner;
     std::vector<int> emu_state;
   } m_image_indexes;
 
+  // Actual backing GameListItems are maintained in a background thread and cached to file
+  static constexpr u32 CACHE_REVISION = 0;
+  std::list<std::shared_ptr<GameListItem>> m_cached_files;
+  std::thread m_scan_thread;
+  Common::Event m_scan_trigger;
+  Common::Flag m_scan_exiting;
+  // UI thread's view into the cache
+  std::vector<std::shared_ptr<GameListItem>> m_shown_files;
+
   int m_last_column;
   int m_last_sort;
   wxSize m_lastpos;

Source/Core/DolphinWX/ISOFile.cpp

@@ -38,8 +38,6 @@
 #include "DolphinWX/ISOFile.h"
 #include "DolphinWX/WxUtils.h"
 
-static const u32 CACHE_REVISION = 0x129;  // Last changed in PR 5102
-
 static std::string GetLanguageString(DiscIO::Language language,
                                      std::map<DiscIO::Language, std::string> strings)
 {
@@ -69,26 +67,8 @@ GameListItem::GameListItem(const std::string& _rFileName, const Core::TitleDatab
       m_Revision(0), m_Valid(false), m_ImageWidth(0), m_ImageHeight(0), m_disc_number(0),
       m_has_custom_name(false)
 {
-  if (LoadFromCache())
-  {
-    m_Valid = true;
-
-    // Wii banners can only be read if there is a savefile,
-    // so sometimes caches don't contain banners. Let's check
-    // if a banner has become available after the cache was made.
-    if (m_pImage.empty())
-    {
-      std::vector<u32> buffer =
-          DiscIO::Volume::GetWiiBanner(&m_ImageWidth, &m_ImageHeight, m_title_id);
-      ReadVolumeBanner(buffer, m_ImageWidth, m_ImageHeight);
-      if (!m_pImage.empty())
-        SaveToCache();
-    }
-  }
-  else
   {
     std::unique_ptr<DiscIO::Volume> volume(DiscIO::CreateVolumeFromFilename(_rFileName));
-
     if (volume != nullptr)
     {
       m_Platform = volume->GetVolumeType();
@@ -116,7 +96,6 @@ GameListItem::GameListItem(const std::string& _rFileName, const Core::TitleDatab
       ReadVolumeBanner(buffer, m_ImageWidth, m_ImageHeight);
 
       m_Valid = true;
-      SaveToCache();
     }
   }
 
@@ -138,7 +117,6 @@ GameListItem::GameListItem(const std::string& _rFileName, const Core::TitleDatab
     m_FileSize = File::GetSize(_rFileName);
     m_Platform = DiscIO::Platform::ELF_DOL;
     m_blob_type = DiscIO::BlobType::DIRECTORY;
-  }
 
     std::string path, name;
     SplitPath(m_FileName, &path, &name, nullptr);
@@ -146,29 +124,21 @@ GameListItem::GameListItem(const std::string& _rFileName, const Core::TitleDatab
     // A bit like the Homebrew Channel icon, except there can be multiple files
     // in a folder with their own icons. Useful for those who don't want to have
     // a Homebrew Channel-style folder structure.
-  if (ReadPNGBanner(path + name + ".png"))
+    if (SetWxBannerFromPngFile(path + name + ".png"))
       return;
 
     // Homebrew Channel icon. Typical for DOLs and ELFs,
     // but can be also used with volumes.
-  if (ReadPNGBanner(path + "icon.png"))
+    if (SetWxBannerFromPngFile(path + "icon.png"))
       return;
-
+  }
+  else
+  {
     // Volume banner. Typical for everything that isn't a DOL or ELF.
-  if (!m_pImage.empty())
+    SetWxBannerFromRaw();
-  {
-    // Need to make explicit copy as wxImage uses reference counting for copies combined with only
-    // taking a pointer, not the content, when given a buffer to its constructor.
-    m_image.Create(m_ImageWidth, m_ImageHeight, false);
-    std::memcpy(m_image.GetData(), m_pImage.data(), m_pImage.size());
-    return;
   }
 }
 
-GameListItem::~GameListItem()
-{
-}
-
 bool GameListItem::IsValid() const
 {
   if (!m_Valid)
@@ -198,37 +168,35 @@ void GameListItem::ReloadINI()
   }
 }
 
-bool GameListItem::LoadFromCache()
-{
-  return CChunkFileReader::Load<GameListItem>(CreateCacheFilename(), CACHE_REVISION, *this);
-}
-
-void GameListItem::SaveToCache()
-{
-  if (!File::IsDirectory(File::GetUserPath(D_CACHE_IDX)))
-    File::CreateDir(File::GetUserPath(D_CACHE_IDX));
-
-  CChunkFileReader::Save<GameListItem>(CreateCacheFilename(), CACHE_REVISION, *this);
-}
-
 void GameListItem::DoState(PointerWrap& p)
 {
+  p.Do(m_FileName);
   p.Do(m_names);
   p.Do(m_descriptions);
   p.Do(m_company);
   p.Do(m_game_id);
   p.Do(m_title_id);
+  p.Do(m_issues);
+  p.Do(m_emu_state);
   p.Do(m_FileSize);
   p.Do(m_VolumeSize);
   p.Do(m_region);
   p.Do(m_Country);
+  p.Do(m_Platform);
   p.Do(m_blob_type);
+  p.Do(m_Revision);
   p.Do(m_pImage);
+  p.Do(m_Valid);
   p.Do(m_ImageWidth);
   p.Do(m_ImageHeight);
-  p.Do(m_Platform);
   p.Do(m_disc_number);
-  p.Do(m_Revision);
+  p.Do(m_custom_name_titles_txt);
+  p.Do(m_custom_name);
+  p.Do(m_has_custom_name);
+  if (p.GetMode() == PointerWrap::MODE_READ)
+  {
+    SetWxBannerFromRaw();
+  }
 }
 
 bool GameListItem::IsElfOrDol() const
@@ -241,27 +209,6 @@ bool GameListItem::IsElfOrDol() const
   return name_end == ".elf" || name_end == ".dol";
 }
 
-std::string GameListItem::CreateCacheFilename() const
-{
-  std::string Filename, LegalPathname, extension;
-  SplitPath(m_FileName, &LegalPathname, &Filename, &extension);
-
-  if (Filename.empty())
-    return Filename;  // Disc Drive
-
-  // Filename.extension_HashOfFolderPath_Size.cache
-  // Append hash to prevent ISO name-clashing in different folders.
-  Filename.append(
-      StringFromFormat("%s_%x_%" PRIx64 ".cache", extension.c_str(),
-                       HashFletcher((const u8*)LegalPathname.c_str(), LegalPathname.size()),
-                       File::GetSize(m_FileName)));
-
-  std::string fullname(File::GetUserPath(D_CACHE_IDX));
-  fullname += Filename;
-  return fullname;
-}
-
-// Outputs to m_pImage
 void GameListItem::ReadVolumeBanner(const std::vector<u32>& buffer, int width, int height)
 {
   m_pImage.resize(width * height * 3);
@@ -273,8 +220,7 @@ void GameListItem::ReadVolumeBanner(const std::vector<u32>& buffer, int width, i
   }
 }
 
-// Outputs to m_Bitmap
+bool GameListItem::SetWxBannerFromPngFile(const std::string& path)
-bool GameListItem::ReadPNGBanner(const std::string& path)
 {
   if (!File::Exists(path))
     return false;
@@ -287,6 +233,37 @@ bool GameListItem::ReadPNGBanner(const std::string& path)
   return true;
 }
 
+void GameListItem::SetWxBannerFromRaw()
+{
+  // Need to make explicit copy as wxImage uses reference counting for copies combined with only
+  // taking a pointer, not the content, when given a buffer to its constructor.
+  if (!m_pImage.empty())
+  {
+    m_image.Create(m_ImageWidth, m_ImageHeight, false);
+    std::memcpy(m_image.GetData(), m_pImage.data(), m_pImage.size());
+  }
+}
+
+bool GameListItem::ReloadBannerIfNeeded()
+{
+  // Wii banners can only be read if there is a savefile,
+  // so sometimes caches don't contain banners. Let's check
+  // if a banner has become available after the cache was made.
+  if ((m_Platform == DiscIO::Platform::WII_DISC || m_Platform == DiscIO::Platform::WII_WAD) &&
+      m_pImage.empty())
+  {
+    std::vector<u32> buffer =
+        DiscIO::Volume::GetWiiBanner(&m_ImageWidth, &m_ImageHeight, m_title_id);
+    if (buffer.size())
+    {
+      ReadVolumeBanner(buffer, m_ImageWidth, m_ImageHeight);
+      SetWxBannerFromRaw();
+      return true;
+    }
+  }
+  return false;
+}
+
 std::string GameListItem::GetDescription(DiscIO::Language language) const
 {
   return GetLanguageString(language, m_descriptions);

Source/Core/DolphinWX/ISOFile.h

@@ -32,8 +32,9 @@ class PointerWrap;
 class GameListItem
 {
 public:
+  GameListItem() = default;
   GameListItem(const std::string& file_name, const Core::TitleDatabase& title_database);
-  ~GameListItem();
+  ~GameListItem() = default;
 
   // Reload settings after INI changes
   void ReloadINI();
@@ -65,8 +66,20 @@ public:
   //   to display it
   const wxImage& GetBannerImage() const { return m_image; }
   void DoState(PointerWrap& p);
+  bool ReloadBannerIfNeeded();
 
 private:
+  bool IsElfOrDol() const;
+  // Outputs to m_pImage
+  void ReadVolumeBanner(const std::vector<u32>& buffer, int width, int height);
+  // Outputs to m_image
+  bool SetWxBannerFromPngFile(const std::string& path);
+  void SetWxBannerFromRaw();
+
+  // IMPORTANT: All data members must be save/restored in DoState.
+  // If anything is changed, make sure DoState handles it properly and
+  // GameListCtrl::CACHE_REVISION is incremented.
+
   std::string m_FileName;
 
   std::map<DiscIO::Language, std::string> m_names;
@@ -97,15 +110,4 @@ private:
   std::string m_custom_name_titles_txt;  // Custom title from titles.txt
   std::string m_custom_name;             // Custom title from INI or titles.txt
   bool m_has_custom_name;
-
-  bool LoadFromCache();
-  void SaveToCache();
-
-  bool IsElfOrDol() const;
-  std::string CreateCacheFilename() const;
-
-  // Outputs to m_pImage
-  void ReadVolumeBanner(const std::vector<u32>& buffer, int width, int height);
-  // Outputs to m_Bitmap
-  bool ReadPNGBanner(const std::string& path);
 };

Source/Core/DolphinWX/Main.cpp

@@ -208,9 +208,6 @@ void DolphinApp::MacOpenFile(const wxString& fileName)
 
 void DolphinApp::AfterInit()
 {
-  if (!m_batch_mode)
-    main_frame->UpdateGameList();
-
 #if defined(USE_ANALYTICS) && USE_ANALYTICS
   if (!SConfig::GetInstance().m_analytics_permission_asked)
   {