GS: Support creating/replaying dumps in zstandard format

pcsx2-qt/MainWindow.cpp

@@ -46,7 +46,7 @@
 #include "SettingWidgetBinder.h"
 
 static constexpr char DISC_IMAGE_FILTER[] =
-	QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.chd *.cso *.gz *.elf *.irx *.m3u *.gs *.gs.xz);;"
+	QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.chd *.cso *.gz *.elf *.irx *.m3u *.gs *.gs.xz *.gs.zst);;"
 									"Single-Track Raw Images (*.bin *.iso);;"
 									"Cue Sheets (*.cue);;"
 									"MAME CHD Images (*.chd);;"
@@ -55,7 +55,7 @@ static constexpr char DISC_IMAGE_FILTER[] =
 									"ELF Executables (*.elf);;"
 									"IRX Executables (*.irx);;"
 									"Playlists (*.m3u);;"
-									"GS Dumps (*.gs *.gs.xz)");
+									"GS Dumps (*.gs *.gs.xz *.gs.zst)");
 
 const char* MainWindow::DEFAULT_THEME_NAME = "darkfusion";
 

pcsx2-qt/Settings/GraphicsSettingsWidget.ui

@@ -1194,7 +1194,12 @@
             </item>
             <item>
              <property name="text">
-              <string>LZMA (XZ)</string>
+              <string>LZMA (xz)</string>
+             </property>
+            </item>
+			<item>
+             <property name="text">
+              <string>Zstandard (zst)</string>
              </property>
             </item>
            </widget>

pcsx2/CMakeLists.txt

@@ -1604,6 +1604,7 @@ target_link_libraries(PCSX2_FLAGS INTERFACE
 	PkgConfig::SAMPLERATE
 	PNG::PNG
 	LibLZMA::LibLZMA
+	Zstd::Zstd
 	${LIBC_LIBRARIES}
 )
 

pcsx2/Config.h

@@ -195,7 +195,8 @@ enum class TexturePreloadingLevel : u8
 enum class GSDumpCompressionMethod : u8
 {
 	Uncompressed,
-	LZMA
+	LZMA,
+	Zstandard,
 };
 
 // Template function for casting enumerations to their underlying type

pcsx2/GS/GS.cpp

@@ -1276,7 +1276,8 @@ void GSApp::Init()
 	m_gs_tv_shaders.push_back(GSSetting(4, "Wave filter", ""));
 
 	m_gs_dump_compression.push_back(GSSetting(static_cast<u32>(GSDumpCompressionMethod::Uncompressed), "Uncompressed", ""));
-	m_gs_dump_compression.push_back(GSSetting(static_cast<u32>(GSDumpCompressionMethod::LZMA), "LZMA (XZ)", ""));
+	m_gs_dump_compression.push_back(GSSetting(static_cast<u32>(GSDumpCompressionMethod::LZMA), "LZMA (xz)", ""));
+	m_gs_dump_compression.push_back(GSSetting(static_cast<u32>(GSDumpCompressionMethod::Zstandard), "Zstandard (zst)", ""));
 
 	// clang-format off
 	// Avoid to clutter the ini file with useless options

pcsx2/GS/GSDump.cpp

@@ -226,3 +226,92 @@ void GSDumpXz::Compress(lzma_action action, lzma_ret expected_status)
 
 	} while (m_strm.avail_out == 0);
 }
+
+//////////////////////////////////////////////////////////////////////
+// GSDumpZstd implementation
+//////////////////////////////////////////////////////////////////////
+
+GSDumpZst::GSDumpZst(const std::string& fn, const std::string& serial, u32 crc,
+	u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
+	const freezeData& fd, const GSPrivRegSet* regs)
+	: GSDumpBase(fn + ".gs.zst")
+{
+	m_strm = ZSTD_createCStream();
+
+	// Compression level 6 provides a good balance between speed and ratio.
+	ZSTD_CCtx_setParameter(m_strm, ZSTD_c_compressionLevel, 6);
+
+	m_in_buff.reserve(_1mb);
+	m_out_buff.resize(_1mb);
+
+	AddHeader(serial, crc, screenshot_width, screenshot_height, screenshot_pixels, fd, regs);
+}
+
+GSDumpZst::~GSDumpZst()
+{
+	// Finish the stream
+	Compress(ZSTD_e_end);
+
+	ZSTD_freeCStream(m_strm);
+}
+
+void GSDumpZst::AppendRawData(const void* data, size_t size)
+{
+	size_t old_size = m_in_buff.size();
+	m_in_buff.resize(old_size + size);
+	memcpy(&m_in_buff[old_size], data, size);
+	MayFlush();
+}
+
+void GSDumpZst::AppendRawData(u8 c)
+{
+	m_in_buff.push_back(c);
+	MayFlush();
+}
+
+void GSDumpZst::MayFlush()
+{
+	if (m_in_buff.size() >= _1mb)
+		Compress(ZSTD_e_continue);
+}
+
+void GSDumpZst::Compress(ZSTD_EndDirective action)
+{
+	if (m_in_buff.empty())
+		return;
+
+	ZSTD_inBuffer inbuf = {m_in_buff.data(), m_in_buff.size(), 0};
+
+	for (;;)
+	{
+		ZSTD_outBuffer outbuf = {m_out_buff.data(), m_out_buff.size(), 0};
+
+		const size_t remaining = ZSTD_compressStream2(m_strm, &outbuf, &inbuf, action);
+		if (ZSTD_isError(remaining))
+		{
+			fprintf(stderr, "GSDumpZstd: Error %s\n", ZSTD_getErrorName(remaining));
+			return;
+		}
+
+		if (outbuf.pos > 0)
+		{
+			Write(m_out_buff.data(), outbuf.pos);
+			outbuf.pos = 0;
+		}
+
+		if (action == ZSTD_e_end)
+		{
+			// break when compression output has finished
+			if (remaining == 0)
+				break;
+		}
+		else
+		{
+			// break when all input data is consumed
+			if (inbuf.pos == inbuf.size)
+				break;
+		}
+	}
+
+	m_in_buff.clear();
+}

pcsx2/GS/GSDump.h

@@ -19,6 +19,7 @@
 #include "GSRegs.h"
 #include "Renderers/SW/GSVertexSW.h"
 #include <lzma.h>
+#include <zstd.h>
 
 /*
 
@@ -110,3 +111,22 @@ public:
 		const freezeData& fd, const GSPrivRegSet* regs);
 	virtual ~GSDumpXz();
 };
+
+class GSDumpZst final : public GSDumpBase
+{
+	ZSTD_CStream* m_strm;
+
+	std::vector<u8> m_in_buff;
+	std::vector<u8> m_out_buff;
+
+	void MayFlush();
+	void Compress(ZSTD_EndDirective action);
+	void AppendRawData(const void* data, size_t size);
+	void AppendRawData(u8 c);
+
+public:
+	GSDumpZst(const std::string& fn, const std::string& serial, u32 crc,
+		u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
+		const freezeData& fd, const GSPrivRegSet* regs);
+	virtual ~GSDumpZst();
+};

pcsx2/GS/GSLzma.cpp

@@ -66,6 +66,8 @@ std::unique_ptr<GSDumpFile> GSDumpFile::OpenGSDump(const char* filename, const c
 
 	if (StringUtil::EndsWithNoCase(filename, ".xz"))
 		return std::make_unique<GSDumpLzma>(fp, nullptr);
+	else if (StringUtil::EndsWithNoCase(filename, ".zst"))
+		return std::make_unique<GSDumpDecompressZst>(fp, nullptr);
 	else
 		return std::make_unique<GSDumpRaw>(fp, nullptr);
 }
@@ -359,6 +361,95 @@ GSDumpLzma::~GSDumpLzma()
 		_aligned_free(m_area);
 }
 
+/******************************************************************/
+GSDumpDecompressZst::GSDumpDecompressZst(FILE* file, FILE* repack_file)
+	: GSDumpFile(file, repack_file)
+{
+	Initialize();
+}
+
+void GSDumpDecompressZst::Initialize()
+{
+	m_strm = ZSTD_createDStream();
+
+	m_area      = (uint8_t*)_aligned_malloc(OUTPUT_BUFFER_SIZE, 32);
+	m_inbuf.src = (uint8_t*)_aligned_malloc(INPUT_BUFFER_SIZE, 32);
+	m_inbuf.pos = 0;
+	m_inbuf.size = 0;
+	m_avail     = 0;
+	m_start     = 0;
+}
+
+void GSDumpDecompressZst::Decompress()
+{
+	ZSTD_outBuffer outbuf = { m_area, OUTPUT_BUFFER_SIZE, 0 };
+	while (outbuf.pos == 0)
+	{
+		// Nothing left in the input buffer. Read data from the file
+		if (m_inbuf.pos == m_inbuf.size && !feof(m_fp))
+		{
+			m_inbuf.size = fread((void*)m_inbuf.src, 1, INPUT_BUFFER_SIZE, m_fp);
+			m_inbuf.pos = 0;
+
+			if (ferror(m_fp))
+			{
+				fprintf(stderr, "Read error: %s\n", strerror(errno));
+				throw "BAD"; // Just exit the program
+			}
+		}
+
+		size_t ret = ZSTD_decompressStream(m_strm, &outbuf, &m_inbuf);
+		if (ZSTD_isError(ret))
+		{
+			fprintf(stderr, "Decoder error: (error code %s)\n", ZSTD_getErrorName(ret));
+			throw "BAD"; // Just exit the program
+		}
+	}
+
+	m_start = 0;
+	m_avail = outbuf.pos;
+}
+
+bool GSDumpDecompressZst::IsEof()
+{
+	return feof(m_fp) && m_avail == 0 && m_inbuf.pos == m_inbuf.size;
+}
+
+size_t GSDumpDecompressZst::Read(void* ptr, size_t size)
+{
+	size_t off = 0;
+	uint8_t* dst = (uint8_t*)ptr;
+	while (size && !IsEof())
+	{
+		if (m_avail == 0)
+		{
+			Decompress();
+		}
+
+		size_t l = std::min(size, m_avail);
+		memcpy(dst + off, m_area + m_start, l);
+		m_avail -= l;
+		size    -= l;
+		m_start += l;
+		off     += l;
+	}
+
+	if (off > 0)
+		Repack(ptr, off);
+
+	return off;
+}
+
+GSDumpDecompressZst::~GSDumpDecompressZst()
+{
+	ZSTD_freeDStream(m_strm);
+
+	if (m_inbuf.src)
+		_aligned_free((void*)m_inbuf.src);
+	if (m_area)
+		_aligned_free(m_area);
+}
+
 /******************************************************************/
 
 GSDumpRaw::GSDumpRaw(FILE* file, FILE* repack_file)

pcsx2/GS/GSLzma.h

@@ -15,11 +15,13 @@
 
 #pragma once
 
-#include <lzma.h>
 #include <memory>
 #include <string>
 #include <vector>
 
+#include <lzma.h>
+#include <zstd.h>
+
 #define GEN_REG_ENUM_CLASS_CONTENT(ClassName, EntryName, Value) \
 	EntryName = Value,
 
@@ -358,6 +360,30 @@ public:
 	size_t Read(void* ptr, size_t size) final;
 };
 
+class GSDumpDecompressZst : public GSDumpFile
+{
+	static constexpr u32 INPUT_BUFFER_SIZE = 512 * _1kb;
+	static constexpr u32 OUTPUT_BUFFER_SIZE = 2 * _1mb;
+
+	ZSTD_DStream* m_strm;
+	ZSTD_inBuffer m_inbuf;
+
+	uint8_t* m_area;
+
+	size_t m_avail;
+	size_t m_start;
+
+	void Decompress();
+	void Initialize();
+
+public:
+	GSDumpDecompressZst(FILE* file, FILE* repack_file);
+	virtual ~GSDumpDecompressZst();
+
+	bool IsEof() final;
+	size_t Read(void* ptr, size_t size) final;
+};
+
 class GSDumpRaw : public GSDumpFile
 {
 public:

pcsx2/GS/Renderers/Common/GSRenderer.cpp

@@ -564,7 +564,7 @@ void GSRenderer::VSync(u32 field, bool registers_written)
 					fd, m_regs));
 				compression_str = "with no compression";
 			}
-			else
+			else if (GSConfig.GSDumpCompression == GSDumpCompressionMethod::LZMA)
 			{
 				m_dump = std::unique_ptr<GSDumpBase>(new GSDumpXz(m_snapshot, GetDumpSerial(), m_crc,
 					DUMP_SCREENSHOT_WIDTH, DUMP_SCREENSHOT_HEIGHT,
@@ -572,6 +572,14 @@ void GSRenderer::VSync(u32 field, bool registers_written)
 					fd, m_regs));
 				compression_str = "with LZMA compression";
 			}
+			else
+			{
+				m_dump = std::unique_ptr<GSDumpBase>(new GSDumpZst(m_snapshot, GetDumpSerial(), m_crc,
+					DUMP_SCREENSHOT_WIDTH, DUMP_SCREENSHOT_HEIGHT,
+					screenshot_pixels.empty() ? nullptr : screenshot_pixels.data(),
+					fd, m_regs));
+				compression_str = "with Zstandard compression";
+			}
 
 			delete[] fd.data;
 
@@ -642,7 +650,7 @@ void GSRenderer::QueueSnapshot(const std::string& path, u32 gsdump_frames)
 		return;
 
 	// Allows for providing a complete path
-	if (path.size() > 4 && StringUtil::compareNoCase(path.substr(path.size() - 4, 4), ".png"))
+	if (path.size() > 4 && StringUtil::EndsWithNoCase(path, ".png"))
 	{
 		m_snapshot = path.substr(0, path.size() - 4);
 	}

pcsx2/VMManager.cpp

@@ -1111,7 +1111,9 @@ bool VMManager::IsElfFileName(const std::string& path)
 
 bool VMManager::IsGSDumpFileName(const std::string& path)
 {
-	return (StringUtil::EndsWithNoCase(path, ".gs") || StringUtil::EndsWithNoCase(path, ".gs.xz"));
+	return (StringUtil::EndsWithNoCase(path, ".gs") ||
+			StringUtil::EndsWithNoCase(path, ".gs.xz") ||
+			StringUtil::EndsWithNoCase(path, ".gs.zst"));
 }
 
 void VMManager::Execute()

pcsx2/gui/Dialogs/GSDumpDialog.cpp

@@ -222,6 +222,8 @@ void Dialogs::GSDumpDialog::GetDumpsList()
 			dumps.push_back(filename.substr(0, filename.length() - 3));
 		else if (filename.EndsWith(".gs.xz"))
 			dumps.push_back(filename.substr(0, filename.length() - 6));
+		else if (filename.EndsWith(".gs.zst"))
+			dumps.push_back(filename.substr(0, filename.length() - 7));
 		cont = snaps.GetNext(&filename);
 	}
 	std::sort(dumps.begin(), dumps.end(), [](const wxString& a, const wxString& b) { return a.CmpNoCase(b) < 0; });
@@ -249,6 +251,8 @@ void Dialogs::GSDumpDialog::SelectedDump(wxListEvent& evt)
 	wxString filename = g_Conf->Folders.Snapshots.ToAscii() + ("/" + evt.GetText()) + ".gs";
 	if (!wxFileExists(filename))
 		filename.append(".xz");
+	if (!wxFileExists(filename))
+		filename = filename.RemoveLast(3).append(".zst");
 	if (wxFileExists(filename_preview))
 	{
 		auto img = wxImage(filename_preview);

pcsx2/pcsx2.vcxproj

@@ -44,6 +44,7 @@
       <AdditionalIncludeDirectories>$(SolutionDir)3rdparty\imgui\imgui;$(SolutionDir)3rdparty\imgui\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>$(SolutionDir)3rdparty\libzip;$(SolutionDir)3rdparty\libzip\libzip\lib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>$(SolutionDir)3rdparty\d3d12memalloc\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\zstd\zstd\lib</AdditionalIncludeDirectories>
       <ExceptionHandling>Async</ExceptionHandling>
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>PrecompiledHeader.h</PrecompiledHeaderFile>

pcsx2/pcsx2core.vcxproj

@@ -47,6 +47,7 @@
       <AdditionalIncludeDirectories>$(SolutionDir)3rdparty\sdl2\include;$(SolutionDir)3rdparty\sdl2\SDL\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>$(SolutionDir)3rdparty\libzip;$(SolutionDir)3rdparty\libzip\libzip\lib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>$(SolutionDir)3rdparty\d3d12memalloc\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\zstd\zstd\lib</AdditionalIncludeDirectories>
       <ExceptionHandling>Async</ExceptionHandling>
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>PrecompiledHeader.h</PrecompiledHeaderFile>