diff --git a/Source/Core/DolphinQt2/CMakeLists.txt b/Source/Core/DolphinQt2/CMakeLists.txt
index 04726b8434..6b42dea3bc 100644
--- a/Source/Core/DolphinQt2/CMakeLists.txt
+++ b/Source/Core/DolphinQt2/CMakeLists.txt
@@ -59,6 +59,7 @@ set(SRCS
   Config/Mapping/WiimoteEmuMotionControl.cpp
   Config/PropertiesDialog.cpp
   Config/SettingsWindow.cpp
+  GameList/GameFileCache.cpp
   GameList/GameFile.cpp
   GameList/GameList.cpp
   GameList/GameListModel.cpp
diff --git a/Source/Core/DolphinQt2/DolphinQt2.vcxproj b/Source/Core/DolphinQt2/DolphinQt2.vcxproj
index b9c97c4ccc..d160584d4f 100644
--- a/Source/Core/DolphinQt2/DolphinQt2.vcxproj
+++ b/Source/Core/DolphinQt2/DolphinQt2.vcxproj
@@ -188,6 +188,7 @@
     <ClCompile Include="Config\PropertiesDialog.cpp" />
     <ClCompile Include="Config\SettingsWindow.cpp" />
     <ClCompile Include="GameList\GameFile.cpp" />
+    <ClCompile Include="GameList\GameFileCache.cpp" />
     <ClCompile Include="GameList\GameList.cpp" />
     <ClCompile Include="GameList\GameListModel.cpp" />
     <ClCompile Include="GameList\GameTracker.cpp" />
@@ -236,6 +237,7 @@
     <ClInclude Include="Config\Mapping\WiimoteEmuExtension.h" />
     <ClInclude Include="Config\Mapping\WiimoteEmuGeneral.h" />
     <ClInclude Include="Config\Mapping\WiimoteEmuMotionControl.h" />
+    <ClInclude Include="GameList\GameFileCache.h" />
     <ClInclude Include="QtUtils\BlockUserInputFilter.h" />
     <ClInclude Include="QtUtils\ElidedButton.h" />
     <ClInclude Include="QtUtils\ListTabWidget.h" />
diff --git a/Source/Core/DolphinQt2/GameList/GameFile.cpp b/Source/Core/DolphinQt2/GameList/GameFile.cpp
index c22b5e0ecf..035028c9f9 100644
--- a/Source/Core/DolphinQt2/GameList/GameFile.cpp
+++ b/Source/Core/DolphinQt2/GameList/GameFile.cpp
@@ -2,8 +2,6 @@
 // Licensed under GPLv2+
 // Refer to the license.txt file included.
 
-#include <QCryptographicHash>
-#include <QDataStream>
 #include <QDir>
 #include <QFileInfo>
 #include <QImage>
@@ -26,9 +24,6 @@
 #include "DolphinQt2/Resources.h"
 #include "DolphinQt2/Settings.h"
 
-static const int CACHE_VERSION = 13;  // Last changed in PR #3261
-static const int DATASTREAM_VERSION = QDataStream::Qt_5_5;
-
 QList<DiscIO::Language> GameFile::GetAvailableLanguages() const
 {
   return m_long_names.keys();
@@ -43,6 +38,11 @@ ConvertLanguageMap(const std::map<DiscIO::Language, std::string>& map)
   return result;
 }
 
+GameFile::GameFile()
+{
+  m_valid = false;
+}
+
 GameFile::GameFile(const QString& path) : m_path(path)
 {
   m_valid = false;
@@ -50,16 +50,13 @@ GameFile::GameFile(const QString& path) : m_path(path)
   if (!LoadFileInfo(path))
     return;
 
-  if (!TryLoadCache())
+  if (TryLoadVolume())
   {
-    if (TryLoadVolume())
-    {
-      LoadState();
-    }
-    else if (!TryLoadElfDol())
-    {
-      return;
-    }
+    LoadState();
+  }
+  else if (!TryLoadElfDol())
+  {
+    return;
   }
 
   m_valid = true;
@@ -76,16 +73,6 @@ bool GameFile::IsValid() const
   return true;
 }
 
-QString GameFile::GetCacheFileName() const
-{
-  QString folder = QString::fromStdString(File::GetUserPath(D_CACHE_IDX));
-  // Append a hash of the full path to prevent name clashes between
-  // files with the same names in different folders.
-  QString hash =
-      QString::fromUtf8(QCryptographicHash::hash(m_path.toUtf8(), QCryptographicHash::Md5).toHex());
-  return folder + GetFileName() + hash;
-}
-
 void GameFile::ReadBanner(const DiscIO::Volume& volume)
 {
   int width, height;
@@ -129,27 +116,6 @@ bool GameFile::IsElfOrDol()
   return extension == QStringLiteral("elf") || extension == QStringLiteral("dol");
 }
 
-bool GameFile::TryLoadCache()
-{
-  QFile cache(GetCacheFileName());
-  if (!cache.exists())
-    return false;
-  if (!cache.open(QIODevice::ReadOnly))
-    return false;
-  if (QFileInfo(cache).lastModified() < m_last_modified)
-    return false;
-
-  QDataStream in(&cache);
-  in.setVersion(DATASTREAM_VERSION);
-
-  int cache_version;
-  in >> cache_version;
-  if (cache_version != CACHE_VERSION)
-    return false;
-
-  return false;
-}
-
 bool GameFile::TryLoadVolume()
 {
   QSharedPointer<DiscIO::Volume> volume(
@@ -179,7 +145,6 @@ bool GameFile::TryLoadVolume()
 
   ReadBanner(*volume);
 
-  SaveCache();
   return true;
 }
 
@@ -199,11 +164,6 @@ bool GameFile::TryLoadElfDol()
   return true;
 }
 
-void GameFile::SaveCache()
-{
-  // TODO
-}
-
 QString GameFile::GetFileName() const
 {
   return QFileInfo(m_path).fileName();
@@ -427,3 +387,96 @@ QString FormatSize(qint64 size)
   }
   return QStringLiteral("%1 %2").arg(QString::number(num, 'f', 1)).arg(unit);
 }
+
+template <typename T, typename U = std::enable_if_t<std::is_enum<T>::value>>
+QDataStream& operator<<(QDataStream& out, const T& enum_value)
+{
+  out << static_cast<std::underlying_type_t<T>>(enum_value);
+  return out;
+}
+
+template <typename T, typename U = std::enable_if_t<std::is_enum<T>::value>>
+QDataStream& operator>>(QDataStream& in, T& enum_value)
+{
+  std::underlying_type_t<T> tmp;
+  in >> tmp;
+  enum_value = static_cast<T>(tmp);
+  return in;
+}
+
+// Some C++ implementations define uint64_t as an 'unsigned long', but QDataStream only has built-in
+// overloads for quint64, which is an 'unsigned long long' on Unix
+QDataStream& operator<<(QDataStream& out, const unsigned long& integer)
+{
+  out << static_cast<quint64>(integer);
+  return out;
+}
+QDataStream& operator>>(QDataStream& in, unsigned long& integer)
+{
+  quint64 tmp;
+  in >> tmp;
+  integer = static_cast<unsigned long>(tmp);
+  return in;
+}
+
+QDataStream& operator<<(QDataStream& out, const GameFile& file)
+{
+  out << file.m_last_modified;
+  out << file.m_path;
+  out << file.m_title_id;
+  out << file.m_game_id;
+  out << file.m_maker_id;
+  out << file.m_maker;
+  out << file.m_long_makers;
+  out << file.m_short_makers;
+  out << file.m_internal_name;
+  out << file.m_long_names;
+  out << file.m_short_names;
+  out << file.m_platform;
+  out << file.m_region;
+  out << file.m_country;
+  out << file.m_blob_type;
+  out << file.m_size;
+  out << file.m_raw_size;
+  out << file.m_descriptions;
+  out << file.m_revision;
+  out << file.m_disc_number;
+  out << file.m_issues;
+  out << file.m_rating;
+  out << file.m_apploader_date;
+  out << file.m_banner;
+
+  return out;
+}
+
+QDataStream& operator>>(QDataStream& in, GameFile& file)
+{
+  in >> file.m_last_modified;
+  in >> file.m_path;
+  in >> file.m_title_id;
+  in >> file.m_game_id;
+  in >> file.m_maker_id;
+  in >> file.m_maker;
+  in >> file.m_long_makers;
+  in >> file.m_short_makers;
+  in >> file.m_internal_name;
+  in >> file.m_long_names;
+  in >> file.m_short_names;
+  in >> file.m_platform;
+  in >> file.m_region;
+  in >> file.m_country;
+  in >> file.m_blob_type;
+  in >> file.m_size;
+  in >> file.m_raw_size;
+  in >> file.m_descriptions;
+  in >> file.m_revision;
+  in >> file.m_disc_number;
+  in >> file.m_issues;
+  in >> file.m_rating;
+  in >> file.m_apploader_date;
+  in >> file.m_banner;
+
+  file.m_valid = true;
+
+  return in;
+}
diff --git a/Source/Core/DolphinQt2/GameList/GameFile.h b/Source/Core/DolphinQt2/GameList/GameFile.h
index cc36d52f64..c5791ed7cc 100644
--- a/Source/Core/DolphinQt2/GameList/GameFile.h
+++ b/Source/Core/DolphinQt2/GameList/GameFile.h
@@ -5,6 +5,7 @@
 #pragma once
 
 #include <QDateTime>
+#include <QFile>
 #include <QMap>
 #include <QPixmap>
 #include <QString>
@@ -21,10 +22,10 @@ enum class Platform;
 class Volume;
 }
 
-// TODO cache
 class GameFile final
 {
 public:
+  GameFile();
   explicit GameFile(const QString& path);
 
   bool IsValid() const;
@@ -34,6 +35,7 @@ public:
   QString GetFileExtension() const;
   QString GetFileFolder() const;
   qint64 GetFileSize() const { return m_size; }
+  QDateTime GetLastModified() const { return m_last_modified; }
   // The rest will not.
   QString GetGameID() const { return m_game_id; }
   QString GetMakerID() const { return m_maker_id; }
@@ -73,18 +75,18 @@ public:
   bool Uninstall();
   bool ExportWiiSave();
 
+  friend QDataStream& operator<<(QDataStream& out, const GameFile& file);
+  friend QDataStream& operator>>(QDataStream& in, GameFile& file);
+
 private:
   QString GetBannerString(const QMap<DiscIO::Language, QString>& m) const;
 
-  QString GetCacheFileName() const;
   void ReadBanner(const DiscIO::Volume& volume);
   bool LoadFileInfo(const QString& path);
   void LoadState();
   bool IsElfOrDol();
   bool TryLoadElfDol();
-  bool TryLoadCache();
   bool TryLoadVolume();
-  void SaveCache();
 
   bool m_valid;
   QString m_path;
@@ -115,3 +117,9 @@ private:
 };
 
 QString FormatSize(qint64 size);
+
+QDataStream& operator<<(QDataStream& out, const GameFile& file);
+QDataStream& operator>>(QDataStream& in, GameFile& file);
+
+QDataStream& operator<<(QDataStream& out, const unsigned long& file);
+QDataStream& operator>>(QDataStream& in, unsigned long& file);
diff --git a/Source/Core/DolphinQt2/GameList/GameFileCache.cpp b/Source/Core/DolphinQt2/GameList/GameFileCache.cpp
new file mode 100644
index 0000000000..40e1928649
--- /dev/null
+++ b/Source/Core/DolphinQt2/GameList/GameFileCache.cpp
@@ -0,0 +1,87 @@
+// Copyright 2017 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#include "GameFileCache.h"
+
+#include <QDataStream>
+#include <QDir>
+#include <QFileInfo>
+
+#include "Common/FileUtil.h"
+#include "Core/ConfigManager.h"
+#include "DolphinQt2/Settings.h"
+
+static const int CACHE_VERSION = 1;  // Last changed in PR #5927
+static const int DATASTREAM_VERSION = QDataStream::Qt_5_0;
+
+GameFileCache::GameFileCache()
+    : m_path(QString::fromStdString(File::GetUserPath(D_CACHE_IDX) + "qt_gamefile.cache"))
+{
+}
+
+bool GameFileCache::IsCached(const QString& path)
+{
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  return m_gamefiles.contains(path);
+}
+
+GameFile GameFileCache::GetFile(const QString& path)
+{
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  return m_gamefiles[path];
+}
+
+void GameFileCache::Load()
+{
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  QFile file(m_path);
+
+  if (!file.open(QIODevice::ReadOnly))
+    return;
+
+  QDataStream stream(&file);
+  stream.setVersion(DATASTREAM_VERSION);
+
+  qint32 cache_version;
+  stream >> cache_version;
+
+  // If the cache file is using an older version, ignore it and create it from scratch
+  if (cache_version != CACHE_VERSION)
+    return;
+
+  stream >> m_gamefiles;
+}
+
+void GameFileCache::Save()
+{
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  QFile file(m_path);
+
+  if (!file.open(QIODevice::WriteOnly))
+    return;
+
+  QDataStream stream(&file);
+  stream.setVersion(DATASTREAM_VERSION);
+
+  stream << static_cast<qint32>(CACHE_VERSION);
+  stream << m_gamefiles;
+}
+
+void GameFileCache::Update(const GameFile& gamefile)
+{
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  m_gamefiles[gamefile.GetFilePath()] = gamefile;
+}
+
+QList<QString> GameFileCache::GetCached()
+{
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  return m_gamefiles.keys();
+}
diff --git a/Source/Core/DolphinQt2/GameList/GameFileCache.h b/Source/Core/DolphinQt2/GameList/GameFileCache.h
new file mode 100644
index 0000000000..b1a505c0f2
--- /dev/null
+++ b/Source/Core/DolphinQt2/GameList/GameFileCache.h
@@ -0,0 +1,30 @@
+// Copyright 2017 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <QFile>
+
+#include <mutex>
+
+#include "DolphinQt2/GameList/GameFile.h"
+
+class GameFileCache
+{
+public:
+  explicit GameFileCache();
+
+  void Update(const GameFile& gamefile);
+  void Save();
+  void Load();
+  bool IsCached(const QString& path);
+  GameFile GetFile(const QString& path);
+  QList<QString> GetCached();
+
+private:
+  QString m_path;
+
+  QMap<QString, GameFile> m_gamefiles;
+  std::mutex m_mutex;
+};
diff --git a/Source/Core/DolphinQt2/GameList/GameTracker.cpp b/Source/Core/DolphinQt2/GameList/GameTracker.cpp
index f1558f153d..85539f4d33 100644
--- a/Source/Core/DolphinQt2/GameList/GameTracker.cpp
+++ b/Source/Core/DolphinQt2/GameList/GameTracker.cpp
@@ -7,7 +7,9 @@
 #include <QFile>
 
 #include "DiscIO/DirectoryBlob.h"
+#include "DolphinQt2/GameList/GameFileCache.h"
 #include "DolphinQt2/GameList/GameTracker.h"
+#include "DolphinQt2/QtUtils/QueueOnObject.h"
 #include "DolphinQt2/Settings.h"
 
 static const QStringList game_filters{
@@ -21,6 +23,8 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
   connect(this, &QFileSystemWatcher::directoryChanged, this, &GameTracker::UpdateDirectory);
   connect(this, &QFileSystemWatcher::fileChanged, this, &GameTracker::UpdateFile);
 
+  cache.Load();
+
   m_load_thread.Reset([this](const QString& path) { LoadGame(path); });
 }
 
@@ -128,8 +132,23 @@ void GameTracker::LoadGame(const QString& path)
 {
   if (!DiscIO::ShouldHideFromGameList(path.toStdString()))
   {
+    if (cache.IsCached(path))
+    {
+      const QDateTime last_modified = QFileInfo(path).lastModified();
+      auto cached_file = cache.GetFile(path);
+      if (cached_file.GetLastModified() >= last_modified)
+      {
+        emit GameLoaded(QSharedPointer<GameFile>::create(cached_file));
+        return;
+      }
+    }
+
     auto game = QSharedPointer<GameFile>::create(path);
     if (game->IsValid())
+    {
       emit GameLoaded(game);
+      cache.Update(*game);
+      cache.Save();
+    }
   }
 }
diff --git a/Source/Core/DolphinQt2/GameList/GameTracker.h b/Source/Core/DolphinQt2/GameList/GameTracker.h
index 550a6548ee..b063b7dcbe 100644
--- a/Source/Core/DolphinQt2/GameList/GameTracker.h
+++ b/Source/Core/DolphinQt2/GameList/GameTracker.h
@@ -12,6 +12,7 @@
 
 #include "Common/WorkQueueThread.h"
 #include "DolphinQt2/GameList/GameFile.h"
+#include "DolphinQt2/GameList/GameFileCache.h"
 
 // Watches directories and loads GameFiles in a separate thread.
 // To use this, just add directories using AddDirectory, and listen for the
@@ -39,6 +40,7 @@ private:
   // game path -> directories that track it
   QMap<QString, QSet<QString>> m_tracked_files;
   Common::WorkQueueThread<QString> m_load_thread;
+  GameFileCache cache;
 };
 
 Q_DECLARE_METATYPE(QSharedPointer<GameFile>)