diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index 2615200591..d8e8a02a5f 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -20,6 +20,7 @@ target_sources(common PRIVATE
FastJmp.cpp
GL/Context.cpp
GL/Program.cpp
+ GL/ShaderCache.cpp
GL/StreamBuffer.cpp
FileSystem.cpp
IniInterface.cpp
@@ -78,6 +79,7 @@ target_sources(common PRIVATE
General.h
GL/Context.h
GL/Program.h
+ GL/ShaderCache.h
GL/StreamBuffer.h
HashCombine.h
MemcpyFast.h
diff --git a/common/GL/ShaderCache.cpp b/common/GL/ShaderCache.cpp
new file mode 100644
index 0000000000..eda9578789
--- /dev/null
+++ b/common/GL/ShaderCache.cpp
@@ -0,0 +1,402 @@
+/* PCSX2 - PS2 Emulator for PCs
+ * Copyright (C) 2002-2021 PCSX2 Dev Team
+ *
+ * PCSX2 is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU Lesser General Public License as published by the Free Software Found-
+ * ation, either version 3 of the License, or (at your option) any later version.
+ *
+ * PCSX2 is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with PCSX2.
+ * If not, see .
+ */
+
+#include "common/GL/ShaderCache.h"
+#include "common/FileSystem.h"
+#include "common/Console.h"
+#include "common/MD5Digest.h"
+#include "common/StringUtil.h"
+
+namespace GL
+{
+#pragma pack(push, 1)
+ struct CacheIndexEntry
+ {
+ u64 vertex_source_hash_low;
+ u64 vertex_source_hash_high;
+ u32 vertex_source_length;
+ u64 geometry_source_hash_low;
+ u64 geometry_source_hash_high;
+ u32 geometry_source_length;
+ u64 fragment_source_hash_low;
+ u64 fragment_source_hash_high;
+ u32 fragment_source_length;
+ u32 file_offset;
+ u32 blob_size;
+ u32 blob_format;
+ };
+#pragma pack(pop)
+
+ ShaderCache::ShaderCache() = default;
+
+ ShaderCache::~ShaderCache()
+ {
+ Close();
+ }
+
+ bool ShaderCache::CacheIndexKey::operator==(const CacheIndexKey& key) const
+ {
+ return (
+ vertex_source_hash_low == key.vertex_source_hash_low && vertex_source_hash_high == key.vertex_source_hash_high &&
+ vertex_source_length == key.vertex_source_length && geometry_source_hash_low == key.geometry_source_hash_low &&
+ geometry_source_hash_high == key.geometry_source_hash_high &&
+ geometry_source_length == key.geometry_source_length && fragment_source_hash_low == key.fragment_source_hash_low &&
+ fragment_source_hash_high == key.fragment_source_hash_high && fragment_source_length == key.fragment_source_length);
+ }
+
+ bool ShaderCache::CacheIndexKey::operator!=(const CacheIndexKey& key) const
+ {
+ return (
+ vertex_source_hash_low != key.vertex_source_hash_low || vertex_source_hash_high != key.vertex_source_hash_high ||
+ vertex_source_length != key.vertex_source_length || geometry_source_hash_low != key.geometry_source_hash_low ||
+ geometry_source_hash_high != key.geometry_source_hash_high ||
+ geometry_source_length != key.geometry_source_length || fragment_source_hash_low != key.fragment_source_hash_low ||
+ fragment_source_hash_high != key.fragment_source_hash_high || fragment_source_length != key.fragment_source_length);
+ }
+
+ bool ShaderCache::Open(bool is_gles, std::string_view base_path, u32 version)
+ {
+ m_base_path = base_path;
+ m_version = version;
+ m_program_binary_supported = is_gles || GLAD_GL_ARB_get_program_binary;
+ if (m_program_binary_supported)
+ {
+ // check that there's at least one format and the extension isn't being "faked"
+ GLint num_formats = 0;
+ glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &num_formats);
+ Console.WriteLn("%u program binary formats supported by driver", num_formats);
+ m_program_binary_supported = (num_formats > 0);
+ }
+
+ if (!m_program_binary_supported)
+ {
+ Console.Warning("Your GL driver does not support program binaries. Hopefully it has a built-in cache.");
+ return true;
+ }
+
+ if (!base_path.empty())
+ {
+ const std::string index_filename = GetIndexFileName();
+ const std::string blob_filename = GetBlobFileName();
+
+ if (ReadExisting(index_filename, blob_filename))
+ return true;
+
+ return CreateNew(index_filename, blob_filename);
+ }
+
+ return true;
+ }
+
+ bool ShaderCache::CreateNew(const std::string& index_filename, const std::string& blob_filename)
+ {
+ if (FileSystem::FileExists(index_filename.c_str()))
+ {
+ Console.Warning("Removing existing index file '%s'", index_filename.c_str());
+ FileSystem::DeleteFilePath(index_filename.c_str());
+ }
+ if (FileSystem::FileExists(blob_filename.c_str()))
+ {
+ Console.Warning("Removing existing blob file '%s'", blob_filename.c_str());
+ FileSystem::DeleteFilePath(blob_filename.c_str());
+ }
+
+ m_index_file = FileSystem::OpenCFile(index_filename.c_str(), "wb");
+ if (!m_index_file)
+ {
+ Console.Error("Failed to open index file '%s' for writing", index_filename.c_str());
+ return false;
+ }
+
+ const u32 index_version = FILE_VERSION;
+ if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1 ||
+ std::fwrite(&m_version, sizeof(m_version), 1, m_index_file) != 1)
+ {
+ Console.Error("Failed to write version to index file '%s'", index_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ FileSystem::DeleteFilePath(index_filename.c_str());
+ return false;
+ }
+
+ m_blob_file = FileSystem::OpenCFile(blob_filename.c_str(), "w+b");
+ if (!m_blob_file)
+ {
+ Console.Error("Failed to open blob file '%s' for writing", blob_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ FileSystem::DeleteFilePath(index_filename.c_str());
+ return false;
+ }
+
+ return true;
+ }
+
+ bool ShaderCache::ReadExisting(const std::string& index_filename, const std::string& blob_filename)
+ {
+ m_index_file = FileSystem::OpenCFile(index_filename.c_str(), "r+b");
+ if (!m_index_file)
+ {
+ // special case here: when there's a sharing violation (i.e. two instances running),
+ // we don't want to blow away the cache. so just continue without a cache.
+ if (errno == EACCES)
+ {
+ Console.WriteLn("Failed to open shader cache index with EACCES, are you running two instances?");
+ return true;
+ }
+
+ return false;
+ }
+
+ u32 file_version = 0;
+ u32 data_version = 0;
+ if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION ||
+ std::fread(&data_version, sizeof(data_version), 1, m_index_file) != 1 || data_version != m_version)
+ {
+ Console.Error("Bad file/data version in '%s'", index_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ return false;
+ }
+
+ m_blob_file = FileSystem::OpenCFile(blob_filename.c_str(), "a+b");
+ if (!m_blob_file)
+ {
+ Console.Error("Blob file '%s' is missing", blob_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ return false;
+ }
+
+ std::fseek(m_blob_file, 0, SEEK_END);
+ const u32 blob_file_size = static_cast(std::ftell(m_blob_file));
+
+ for (;;)
+ {
+ CacheIndexEntry entry;
+ if (std::fread(&entry, sizeof(entry), 1, m_index_file) != 1 ||
+ (entry.file_offset + entry.blob_size) > blob_file_size)
+ {
+ if (std::feof(m_index_file))
+ break;
+
+ Console.Error("Failed to read entry from '%s', corrupt file?", index_filename.c_str());
+ m_index.clear();
+ std::fclose(m_blob_file);
+ m_blob_file = nullptr;
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ return false;
+ }
+
+ const CacheIndexKey key{
+ entry.vertex_source_hash_low, entry.vertex_source_hash_high, entry.vertex_source_length,
+ entry.geometry_source_hash_low, entry.geometry_source_hash_high, entry.geometry_source_length,
+ entry.fragment_source_hash_low, entry.fragment_source_hash_high, entry.fragment_source_length};
+ const CacheIndexData data{entry.file_offset, entry.blob_size, entry.blob_format};
+ m_index.emplace(key, data);
+ }
+
+ Console.WriteLn("Read %zu entries from '%s'", m_index.size(), index_filename.c_str());
+ return true;
+ }
+
+ void ShaderCache::Close()
+ {
+ m_index.clear();
+ if (m_index_file)
+ std::fclose(m_index_file);
+ if (m_blob_file)
+ std::fclose(m_blob_file);
+ }
+
+ bool ShaderCache::Recreate()
+ {
+ Close();
+
+ const std::string index_filename = GetIndexFileName();
+ const std::string blob_filename = GetBlobFileName();
+
+ return CreateNew(index_filename, blob_filename);
+ }
+
+ ShaderCache::CacheIndexKey ShaderCache::GetCacheKey(const std::string_view& vertex_shader,
+ const std::string_view& geometry_shader,
+ const std::string_view& fragment_shader)
+ {
+ union ShaderHash
+ {
+ struct
+ {
+ u64 low;
+ u64 high;
+ };
+ u8 bytes[16];
+ };
+
+ ShaderHash vertex_hash = {};
+ ShaderHash geometry_hash = {};
+ ShaderHash fragment_hash = {};
+
+ MD5Digest digest;
+ if (!vertex_shader.empty())
+ {
+ digest.Update(vertex_shader.data(), static_cast(vertex_shader.length()));
+ digest.Final(vertex_hash.bytes);
+ }
+
+ if (!geometry_shader.empty())
+ {
+ digest.Reset();
+ digest.Update(geometry_shader.data(), static_cast(geometry_shader.length()));
+ digest.Final(geometry_hash.bytes);
+ }
+
+ if (!fragment_shader.empty())
+ {
+ digest.Reset();
+ digest.Update(fragment_shader.data(), static_cast(fragment_shader.length()));
+ digest.Final(fragment_hash.bytes);
+ }
+
+ return CacheIndexKey{vertex_hash.low, vertex_hash.high, static_cast(vertex_shader.length()),
+ geometry_hash.low, geometry_hash.high, static_cast(geometry_shader.length()),
+ fragment_hash.low, fragment_hash.high, static_cast(fragment_shader.length())};
+ }
+
+ std::string ShaderCache::GetIndexFileName() const
+ {
+ return StringUtil::StdStringFromFormat("%s/gl_programs.idx", m_base_path.c_str());
+ }
+
+ std::string ShaderCache::GetBlobFileName() const
+ {
+ return StringUtil::StdStringFromFormat("%s/gl_programs.bin", m_base_path.c_str());
+ }
+
+ std::optional ShaderCache::GetProgram(const std::string_view vertex_shader,
+ const std::string_view geometry_shader,
+ const std::string_view fragment_shader, const PreLinkCallback& callback)
+ {
+ if (!m_program_binary_supported || !m_blob_file)
+ return CompileProgram(vertex_shader, geometry_shader, fragment_shader, callback, false);
+
+ const auto key = GetCacheKey(vertex_shader, geometry_shader, fragment_shader);
+ auto iter = m_index.find(key);
+ if (iter == m_index.end())
+ return CompileAndAddProgram(key, vertex_shader, geometry_shader, fragment_shader, callback);
+
+ std::vector data(iter->second.blob_size);
+ if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 ||
+ std::fread(data.data(), 1, iter->second.blob_size, m_blob_file) != iter->second.blob_size)
+ {
+ Console.Error("Read blob from file failed");
+ return {};
+ }
+
+ Program prog;
+ if (prog.CreateFromBinary(data.data(), static_cast(data.size()), iter->second.blob_format))
+ return std::optional(std::move(prog));
+
+ Console.Warning(
+ "Failed to create program from binary, this may be due to a driver or GPU Change. Recreating cache.");
+ if (!Recreate())
+ return CompileProgram(vertex_shader, geometry_shader, fragment_shader, callback, false);
+ else
+ return CompileAndAddProgram(key, vertex_shader, geometry_shader, fragment_shader, callback);
+ }
+
+ bool ShaderCache::GetProgram(Program* out_program, const std::string_view vertex_shader,
+ const std::string_view geometry_shader, const std::string_view fragment_shader,
+ const PreLinkCallback& callback /* = */)
+ {
+ auto prog = GetProgram(vertex_shader, geometry_shader, fragment_shader, callback);
+ if (!prog)
+ return false;
+
+ *out_program = std::move(*prog);
+ return true;
+ }
+
+ std::optional ShaderCache::CompileProgram(const std::string_view& vertex_shader,
+ const std::string_view& geometry_shader,
+ const std::string_view& fragment_shader,
+ const PreLinkCallback& callback, bool set_retrievable)
+ {
+ Program prog;
+ if (!prog.Compile(vertex_shader, geometry_shader, fragment_shader))
+ return std::nullopt;
+
+ if (callback)
+ callback(prog);
+
+ if (set_retrievable)
+ prog.SetBinaryRetrievableHint();
+
+ if (!prog.Link())
+ return std::nullopt;
+
+ return std::optional(std::move(prog));
+ }
+
+ std::optional ShaderCache::CompileAndAddProgram(const CacheIndexKey& key,
+ const std::string_view& vertex_shader,
+ const std::string_view& geometry_shader,
+ const std::string_view& fragment_shader,
+ const PreLinkCallback& callback)
+ {
+ std::optional prog = CompileProgram(vertex_shader, geometry_shader, fragment_shader, callback, true);
+ if (!prog)
+ return std::nullopt;
+
+ std::vector prog_data;
+ u32 prog_format = 0;
+ if (!prog->GetBinary(&prog_data, &prog_format))
+ return std::nullopt;
+
+ if (!m_blob_file || std::fseek(m_blob_file, 0, SEEK_END) != 0)
+ return prog;
+
+ CacheIndexData data;
+ data.file_offset = static_cast(std::ftell(m_blob_file));
+ data.blob_size = static_cast(prog_data.size());
+ data.blob_format = prog_format;
+
+ CacheIndexEntry entry = {};
+ entry.vertex_source_hash_low = key.vertex_source_hash_low;
+ entry.vertex_source_hash_high = key.vertex_source_hash_high;
+ entry.vertex_source_length = key.vertex_source_length;
+ entry.geometry_source_hash_low = key.geometry_source_hash_low;
+ entry.geometry_source_hash_high = key.geometry_source_hash_high;
+ entry.geometry_source_length = key.geometry_source_length;
+ entry.fragment_source_hash_low = key.fragment_source_hash_low;
+ entry.fragment_source_hash_high = key.fragment_source_hash_high;
+ entry.fragment_source_length = key.fragment_source_length;
+ entry.file_offset = data.file_offset;
+ entry.blob_size = data.blob_size;
+ entry.blob_format = data.blob_format;
+
+ if (std::fwrite(prog_data.data(), 1, entry.blob_size, m_blob_file) != entry.blob_size ||
+ std::fflush(m_blob_file) != 0 || std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 ||
+ std::fflush(m_index_file) != 0)
+ {
+ Console.Error("Failed to write shader blob to file");
+ return prog;
+ }
+
+ m_index.emplace(key, data);
+ return prog;
+ }
+} // namespace GL
diff --git a/common/GL/ShaderCache.h b/common/GL/ShaderCache.h
new file mode 100644
index 0000000000..20a04841c5
--- /dev/null
+++ b/common/GL/ShaderCache.h
@@ -0,0 +1,112 @@
+/* PCSX2 - PS2 Emulator for PCs
+ * Copyright (C) 2002-2021 PCSX2 Dev Team
+ *
+ * PCSX2 is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU Lesser General Public License as published by the Free Software Found-
+ * ation, either version 3 of the License, or (at your option) any later version.
+ *
+ * PCSX2 is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with PCSX2.
+ * If not, see .
+ */
+
+#pragma once
+#include "common/Pcsx2Defs.h"
+#include "common/HashCombine.h"
+#include "common/GL/Program.h"
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace GL
+{
+ class ShaderCache
+ {
+ public:
+ using PreLinkCallback = std::function;
+
+ ShaderCache();
+ ~ShaderCache();
+
+ bool Open(bool is_gles, std::string_view base_path, u32 version);
+
+ std::optional GetProgram(const std::string_view vertex_shader, const std::string_view geometry_shader,
+ const std::string_view fragment_shader, const PreLinkCallback& callback = {});
+ bool GetProgram(Program* out_program, const std::string_view vertex_shader, const std::string_view geometry_shader,
+ const std::string_view fragment_shader, const PreLinkCallback& callback = {});
+
+ private:
+ static constexpr u32 FILE_VERSION = 1;
+
+ struct CacheIndexKey
+ {
+ u64 vertex_source_hash_low;
+ u64 vertex_source_hash_high;
+ u32 vertex_source_length;
+ u64 geometry_source_hash_low;
+ u64 geometry_source_hash_high;
+ u32 geometry_source_length;
+ u64 fragment_source_hash_low;
+ u64 fragment_source_hash_high;
+ u32 fragment_source_length;
+
+ bool operator==(const CacheIndexKey& key) const;
+ bool operator!=(const CacheIndexKey& key) const;
+ };
+
+ struct CacheIndexEntryHasher
+ {
+ std::size_t operator()(const CacheIndexKey& e) const noexcept
+ {
+ std::size_t h = 0;
+ HashCombine(h,
+ e.vertex_source_hash_low, e.vertex_source_hash_high, e.vertex_source_length,
+ e.geometry_source_hash_low, e.geometry_source_hash_high, e.geometry_source_length,
+ e.fragment_source_hash_low, e.fragment_source_hash_high, e.fragment_source_length);
+ return h;
+ }
+ };
+
+ struct CacheIndexData
+ {
+ u32 file_offset;
+ u32 blob_size;
+ u32 blob_format;
+ };
+
+ using CacheIndex = std::unordered_map;
+
+ static CacheIndexKey GetCacheKey(const std::string_view& vertex_shader, const std::string_view& geometry_shader,
+ const std::string_view& fragment_shader);
+
+ std::string GetIndexFileName() const;
+ std::string GetBlobFileName() const;
+
+ bool CreateNew(const std::string& index_filename, const std::string& blob_filename);
+ bool ReadExisting(const std::string& index_filename, const std::string& blob_filename);
+ void Close();
+ bool Recreate();
+
+ std::optional CompileProgram(const std::string_view& vertex_shader, const std::string_view& geometry_shader,
+ const std::string_view& fragment_shader, const PreLinkCallback& callback,
+ bool set_retrievable);
+ std::optional CompileAndAddProgram(const CacheIndexKey& key, const std::string_view& vertex_shader,
+ const std::string_view& geometry_shader,
+ const std::string_view& fragment_shader, const PreLinkCallback& callback);
+
+ std::string m_base_path;
+ std::FILE* m_index_file = nullptr;
+ std::FILE* m_blob_file = nullptr;
+
+ CacheIndex m_index;
+ u32 m_version = 0;
+ bool m_program_binary_supported = false;
+ };
+} // namespace GL
diff --git a/common/common.vcxproj b/common/common.vcxproj
index 1f8e37ce41..34d6b6e48d 100644
--- a/common/common.vcxproj
+++ b/common/common.vcxproj
@@ -52,6 +52,7 @@
+
@@ -113,6 +114,7 @@
+
diff --git a/common/common.vcxproj.filters b/common/common.vcxproj.filters
index 107ab30cde..f2b229c59e 100644
--- a/common/common.vcxproj.filters
+++ b/common/common.vcxproj.filters
@@ -148,6 +148,9 @@
Source Files\GL
+
+ Source Files\GL
+
@@ -345,6 +348,9 @@
Header Files\GL
+
+ Header Files\GL
+