diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h b/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h
index 6eec276620..dba514f4ce 100644
--- a/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h
+++ b/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h
@@ -20,14 +20,7 @@ struct PixelShaderData;
 class CustomAssetLibrary
 {
 public:
-  // TODO: this should be std::chrono::system_clock::time_point to
-  // support any type of loader where the time isn't from the filesystem
-  // but there's no way to convert filesystem times to system times
-  // without 'clock_cast', once our builders catch up
-  // to support 'clock_cast' we should update this
-  // For now, it's fine as a filesystem library is all that is
-  // available
-  using TimeType = std::filesystem::file_time_type;
+  using TimeType = std::chrono::system_clock::time_point;
 
   // The AssetID is a unique identifier for a particular asset
   using AssetID = std::string;
diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp
index 3d92c7fb82..7a1f84ca08 100644
--- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp
+++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp
@@ -17,6 +17,20 @@ namespace VideoCommon
 {
 namespace
 {
+std::chrono::system_clock::time_point FileTimeToSysTime(std::filesystem::file_time_type file_time)
+{
+#ifdef _WIN32
+  return std::chrono::clock_cast<std::chrono::system_clock>(file_time);
+#else
+  // Note: all compilers should switch to chrono::clock_cast
+  // once it is available for use
+  const auto system_time_now = std::chrono::system_clock::now();
+  const auto file_time_now = decltype(file_time)::clock::now();
+  return std::chrono::time_point_cast<std::chrono::system_clock::duration>(
+      file_time - file_time_now + system_time_now);
+#endif
+}
+
 std::size_t GetAssetSize(const CustomTextureData& data)
 {
   std::size_t total = 0;
@@ -42,8 +56,9 @@ DirectFilesystemAssetLibrary::GetLastAssetWriteTime(const AssetID& asset_id) con
       const auto tp = std::filesystem::last_write_time(value, ec);
       if (ec)
         continue;
-      if (tp > max_entry)
-        max_entry = tp;
+      auto tp_sys = FileTimeToSysTime(tp);
+      if (tp_sys > max_entry)
+        max_entry = tp_sys;
     }
     return max_entry;
   }
@@ -235,7 +250,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass
     if (!LoadMips(asset_path, data))
       return {};
 
-    return LoadInfo{GetAssetSize(*data), last_loaded_time};
+    return LoadInfo{GetAssetSize(*data), FileTimeToSysTime(last_loaded_time)};
   }
   else if (ext == ".png")
   {
@@ -252,7 +267,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass
     if (!LoadMips(asset_path, data))
       return {};
 
-    return LoadInfo{GetAssetSize(*data), last_loaded_time};
+    return LoadInfo{GetAssetSize(*data), FileTimeToSysTime(last_loaded_time)};
   }
 
   ERROR_LOG_FMT(VIDEO, "Asset '{}' error - extension '{}' unknown!", asset_id, ext);