CDVD: Add ZSO support (#10396)

* add zso support

* format and fixed typo

* fix typo in extension (duplicated .cso)

* format

* proper casting

* use regular casting; added lz4 to gitmodules

* use C++ style casting

* fix casts

* add lz4 submodule

* added windows build configuration

* add lz4 to cmake

* undo

* undo

* add lz4 to SearchForStuff

* undo

* add own lz4 source code

* cleanup

* fix

* add nwe sources to windows build

* cleanup

* don't use precompile headers on lz4

* stupid compiler

* add const. better logging.

* cast to std::string

* 3rdparty: Add lz4

* use 3rdparty lz4

* cleanup references to lz4.cpp

* format code

* add missing header

* use fmt::format

* don't call inflateReset on ZSO

* use LZ4_decompress_safe

* fix syntax

* fix call to LZ4_decompress_safe

* use LZ4_decompress_safe_partial to ignore padded data

* cleanup

* refactor

---------

Co-authored-by: Stenzek <stenzek@gmail.com>

.gitmodules

@@ -30,3 +30,6 @@
 [submodule "3rdparty/xz/xz"]
 	path = 3rdparty/xz/xz
 	url = https://github.com/tukaani-project/xz.git
+[submodule "3rdparty/lz4/lz4"]
+	path = 3rdparty/lz4/lz4
+	url = https://github.com/lz4/lz4

3rdparty/lz4/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_library(pcsx2-lz4
+	lz4/lib/lz4.c
+	lz4/lib/lz4.h
+)
+
+target_include_directories(pcsx2-lz4 PUBLIC lz4/lib)
+target_compile_definitions(pcsx2-lz4 PUBLIC
+	LZ4LIB_VISIBILITY=
+)
+
+add_library(LZ4::LZ4 ALIAS pcsx2-lz4)
+
+disable_compiler_warnings_for_target(pcsx2-lz4)

3rdparty/lz4/lz4

@@ -0,0 +1 @@
+Subproject commit b8fd2d15309dd4e605070bd4486e26b6ef814e29

3rdparty/lz4/lz4.vcxproj

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(SolutionDir)common\vsprops\BaseProjectConfig.props" />
+  <Import Project="$(SolutionDir)common\vsprops\WinSDK.props" />
+  <PropertyGroup Label="Globals">
+  <ProjectGuid>{39098635-446A-4FC3-9B1C-8609D94598A8}</ProjectGuid>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <PlatformToolset Condition="!$(Configuration.Contains(Clang))">$(DefaultPlatformToolset)</PlatformToolset>
+    <PlatformToolset Condition="$(Configuration.Contains(Clang))">ClangCL</PlatformToolset>
+    <WholeProgramOptimization Condition="$(Configuration.Contains(Release))">true</WholeProgramOptimization>
+    <UseDebugLibraries Condition="$(Configuration.Contains(Debug))">true</UseDebugLibraries>
+    <UseDebugLibraries Condition="!$(Configuration.Contains(Debug))">false</UseDebugLibraries>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings" />
+  <ImportGroup Label="PropertySheets">
+    <Import Project="..\DefaultProjectRootDir.props" />
+    <Import Project="..\3rdparty.props" />
+    <Import Condition="$(Configuration.Contains(Debug))" Project="..\..\common\vsprops\CodeGen_Debug.props" />
+    <Import Condition="$(Configuration.Contains(Devel))" Project="..\..\common\vsprops\CodeGen_Devel.props" />
+    <Import Condition="$(Configuration.Contains(Release))" Project="..\..\common\vsprops\CodeGen_Release.props" />
+    <Import Condition="!$(Configuration.Contains(Release))" Project="..\..\common\vsprops\IncrementalLinking.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PreprocessorDefinitions>LZ4LIB_VISIBILITY=;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(ProjectDir)\lz4\lib</AdditionalIncludeDirectories>
+      <WarningLevel>TurnOffAllWarnings</WarningLevel>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="lz4\lib\lz4.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="lz4\lib\lz4.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets" />
+</Project>

PCSX2_qt.sln

@@ -63,6 +63,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "demangler", "3rdparty\deman
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwebp", "3rdparty\libwebp\libwebp.vcxproj", "{522DAF2A-1F24-4742-B2C4-A956411F6AB2}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "3rdparty\lz4\lz4.vcxproj", "{39098635-446A-4FC3-9B1C-8609D94598A8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug AVX2|x64 = Debug AVX2|x64
@@ -763,6 +765,30 @@ Global
 		{522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release Clang|x64.Build.0 = Release Clang|x64
 		{522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release|x64.ActiveCfg = Release|x64
 		{522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release|x64.Build.0 = Release|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug AVX2|x64.ActiveCfg = Debug|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug AVX2|x64.Build.0 = Debug|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang AVX2|x64.ActiveCfg = Debug Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang AVX2|x64.Build.0 = Debug Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang|x64.ActiveCfg = Debug Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang|x64.Build.0 = Debug Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug|x64.ActiveCfg = Debug|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug|x64.Build.0 = Debug|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel AVX2|x64.ActiveCfg = Devel|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel AVX2|x64.Build.0 = Devel|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang AVX2|x64.ActiveCfg = Devel Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang AVX2|x64.Build.0 = Devel Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang|x64.ActiveCfg = Devel Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang|x64.Build.0 = Devel Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel|x64.ActiveCfg = Devel|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel|x64.Build.0 = Devel|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release AVX2|x64.ActiveCfg = Release|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release AVX2|x64.Build.0 = Release|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang AVX2|x64.ActiveCfg = Release Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang AVX2|x64.Build.0 = Release Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang|x64.ActiveCfg = Release Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang|x64.Build.0 = Release Clang|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release|x64.ActiveCfg = Release|x64
+		{39098635-446A-4FC3-9B1C-8609D94598A8}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -792,6 +818,7 @@ Global
 		{67D0160C-0FE4-44B9-AC2E-82BBCF4104DF} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
 		{1E3D706C-1D95-4E1B-BDF2-CA3D0007DF7F} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
 		{522DAF2A-1F24-4742-B2C4-A956411F6AB2} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
+		{39098635-446A-4FC3-9B1C-8609D94598A8} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0BC474EA-3628-45D3-9DBC-E22D0B7E0F77}

cmake/SearchForStuff.cmake

@@ -144,6 +144,7 @@ add_subdirectory(3rdparty/libzip EXCLUDE_FROM_ALL)
 add_subdirectory(3rdparty/rcheevos EXCLUDE_FROM_ALL)
 add_subdirectory(3rdparty/rapidjson EXCLUDE_FROM_ALL)
 add_subdirectory(3rdparty/discord-rpc EXCLUDE_FROM_ALL)
+add_subdirectory(3rdparty/lz4 EXCLUDE_FROM_ALL)
 
 if(USE_OPENGL)
 	add_subdirectory(3rdparty/glad EXCLUDE_FROM_ALL)

pcsx2-qt/MainWindow.cpp

@@ -67,24 +67,26 @@
 #endif
 
 const char* MainWindow::OPEN_FILE_FILTER =
-	QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.gz *.elf *.irx *.gs *.gs.xz *.gs.zst *.dump);;"
+	QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.zso *.gz *.elf *.irx *.gs *.gs.xz *.gs.zst *.dump);;"
 									"Single-Track Raw Images (*.bin *.iso);;"
 									"Cue Sheets (*.cue);;"
 									"Media Descriptor File (*.mdf);;"
 									"MAME CHD Images (*.chd);;"
 									"CSO Images (*.cso);;"
+									"ZSO Images (*.zso);;"
 									"GZ Images (*.gz);;"
 									"ELF Executables (*.elf);;"
 									"IRX Executables (*.irx);;"
 									"GS Dumps (*.gs *.gs.xz *.gs.zst);;"
 									"Block Dumps (*.dump)");
 
-const char* MainWindow::DISC_IMAGE_FILTER = QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.gz *.dump);;"
+const char* MainWindow::DISC_IMAGE_FILTER = QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.zso *.gz *.dump);;"
 																			"Single-Track Raw Images (*.bin *.iso);;"
 																			"Cue Sheets (*.cue);;"
 																			"Media Descriptor File (*.mdf);;"
 																			"MAME CHD Images (*.chd);;"
 																			"CSO Images (*.cso);;"
+																			"ZSO Images (*.zso);;"
 																			"GZ Images (*.gz);;"
 																			"Block Dumps (*.dump)");
 

pcsx2/CDVD/CsoFileReader.cpp

@@ -23,6 +23,7 @@
 #include "common/StringUtil.h"
 
 #include <zlib.h>
+#include "lz4.h"
 
 // Implementation of CSO compressed ISO reading, based on:
 // https://github.com/unknownbrackets/maxcso/blob/master/README_CSO.md
@@ -52,7 +53,7 @@ CsoFileReader::~CsoFileReader()
 bool CsoFileReader::CanHandle(const std::string& fileName, const std::string& displayName)
 {
 	bool supported = false;
-	if (StringUtil::EndsWith(displayName, ".cso"))
+	if (displayName.ends_with(".cso") || displayName.ends_with(".zso"))
 	{
 		FILE* fp = FileSystem::OpenCFile(fileName.c_str(), "rb");
 		CsoHeader hdr;
@@ -70,10 +71,10 @@ bool CsoFileReader::CanHandle(const std::string& fileName, const std::string& di
 
 bool CsoFileReader::ValidateHeader(const CsoHeader& hdr, Error* error)
 {
-	if (hdr.magic[0] != 'C' || hdr.magic[1] != 'I' || hdr.magic[2] != 'S' || hdr.magic[3] != 'O')
+	if ((hdr.magic[0] != 'C' && hdr.magic[0] != 'Z') || hdr.magic[1] != 'I' || hdr.magic[2] != 'S' || hdr.magic[3] != 'O')
 	{
 		// Invalid magic, definitely a bad file.
-		Error::SetString(error, "File is not a CHD.");
+		Error::SetString(error, "File is not a CSO or ZSO.");
 		return false;
 	}
 	if (hdr.ver > 1)
@@ -141,6 +142,9 @@ bool CsoFileReader::ReadFileHeader(Error* error)
 	m_indexShift = hdr.align;
 	m_totalSize = hdr.total_bytes;
 
+	// Check compression method (ZSO=lz4)
+	m_uselz4 = hdr.magic[0] == 'Z';
+
 	return true;
 }
 
@@ -167,14 +171,18 @@ bool CsoFileReader::InitializeBuffers(Error* error)
 		return false;
 	}
 
-	m_z_stream = std::make_unique<z_stream>();
+	// initialize zlib if not a ZSO
-	m_z_stream->zalloc = Z_NULL;
+	if (!m_uselz4)
-	m_z_stream->zfree = Z_NULL;
-	m_z_stream->opaque = Z_NULL;
-	if (inflateInit2(m_z_stream.get(), -15) != Z_OK)
 	{
-		Error::SetString(error, "Unable to initialize zlib for CSO decompression.");
+		m_z_stream = std::make_unique<z_stream>();
-		return false;
+		m_z_stream->zalloc = Z_NULL;
+		m_z_stream->zfree = Z_NULL;
+		m_z_stream->opaque = Z_NULL;
+		if (inflateInit2(m_z_stream.get(), -15) != Z_OK)
+		{
+			Error::SetString(error, "Unable to initialize zlib for CSO decompression.");
+			return false;
+		}
 	}
 
 	return true;
@@ -256,18 +264,34 @@ int CsoFileReader::ReadChunk(void* dst, s64 chunkID)
 		// This might be less bytes than frameRawSize in case of padding on the last frame.
 		// This is because the index positions must be aligned.
 		const u32 readRawBytes = fread(m_readBuffer.get(), 1, frameRawSize, m_src);
+		bool success = false;
 
-		m_z_stream->next_in = m_readBuffer.get();
+		if (m_uselz4)
-		m_z_stream->avail_in = readRawBytes;
+		{
-		m_z_stream->next_out = static_cast<Bytef*>(dst);
+			const int src_size = static_cast<int>(readRawBytes);
-		m_z_stream->avail_out = m_frameSize;
+			const int dst_size = static_cast<int>(m_frameSize);
+			const char* src_buf = reinterpret_cast<const char*>(m_readBuffer.get());
+			char* dst_buf = static_cast<char*>(dst);
+			
+			const int res = LZ4_decompress_safe_partial(src_buf, dst_buf, src_size, dst_size, dst_size);
+			success = (res > 0);
+		}
+		else
+		{
+			m_z_stream->next_in = m_readBuffer.get();
+			m_z_stream->avail_in = readRawBytes;
+			m_z_stream->next_out = static_cast<Bytef*>(dst);
+			m_z_stream->avail_out = m_frameSize;
 
-		int status = inflate(m_z_stream.get(), Z_FINISH);
+			const int status = inflate(m_z_stream.get(), Z_FINISH);
-		bool success = status == Z_STREAM_END && m_z_stream->total_out == m_frameSize;
+			success = (status == Z_STREAM_END && m_z_stream->total_out == m_frameSize);
+		}
 
 		if (!success)
-			Console.Error("Unable to decompress CSO frame using zlib.");
+			Console.Error(fmt::format("Unable to decompress CSO frame using {}", (m_uselz4)? "lz4":"zlib"));
-		inflateReset(m_z_stream.get());
+		
+		if (!m_uselz4)
+			inflateReset(m_z_stream.get());
 
 		return success ? m_frameSize : 0;
 	}

pcsx2/CDVD/CsoFileReader.h

@@ -60,6 +60,7 @@ private:
 	u32 m_frameSize = 0;
 	u8 m_frameShift = 0;
 	u8 m_indexShift = 0;
+	bool m_uselz4 = false; // flag to enable LZ4 decompression (ZSO files)
 	std::unique_ptr<u8[]> m_readBuffer;
 
 	std::unique_ptr<u32[]> m_index;

pcsx2/CMakeLists.txt

@@ -1136,6 +1136,7 @@ target_link_libraries(PCSX2_FLAGS INTERFACE
 	discord-rpc
 	SDL2::SDL2
 	ZLIB::ZLIB
+	LZ4::LZ4
 	SoundTouch::SoundTouch
 	PNG::PNG
 	LibLZMA::LibLZMA

pcsx2/ImGui/FullscreenUI.cpp

@@ -844,12 +844,12 @@ void FullscreenUI::DestroyResources()
 
 ImGuiFullscreen::FileSelectorFilters FullscreenUI::GetOpenFileFilters()
 {
-	return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.gz", "*.elf", "*.irx", "*.gs", "*.gs.xz", "*.gs.zst", "*.dump"};
+	return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.zso", "*.gz", "*.elf", "*.irx", "*.gs", "*.gs.xz", "*.gs.zst", "*.dump"};
 }
 
 ImGuiFullscreen::FileSelectorFilters FullscreenUI::GetDiscImageFilters()
 {
-	return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.gz"};
+	return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.zso", "*.gz"};
 }
 
 void FullscreenUI::DoStartPath(const std::string& path, std::optional<s32> state_index, std::optional<bool> fast_boot)

pcsx2/VMManager.cpp

@@ -2112,7 +2112,7 @@ bool VMManager::IsSaveStateFileName(const std::string_view& path)
 
 bool VMManager::IsDiscFileName(const std::string_view& path)
 {
-	static const char* extensions[] = {".iso", ".bin", ".img", ".mdf", ".gz", ".cso", ".chd"};
+	static const char* extensions[] = {".iso", ".bin", ".img", ".mdf", ".gz", ".cso", ".zso", ".chd"};
 
 	for (const char* test_extension : extensions)
 	{

pcsx2/pcsx2.vcxproj

@@ -38,6 +38,7 @@
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\wil\include</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\xz\xz\src\liblzma\api</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\zlib</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\lz4\lz4\lib</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\libpng</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\libchdr\include</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\cubeb\include</AdditionalIncludeDirectories>
@@ -64,7 +65,7 @@
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>PrecompiledHeader.h</PrecompiledHeaderFile>
       <ForcedIncludeFiles>PrecompiledHeader.h;%(ForcedIncludeFiles)</ForcedIncludeFiles>
-      <PreprocessorDefinitions>LZMA_API_STATIC;ST_NO_EXCEPTION_HANDLING;ENABLE_RAINTEGRATION;ENABLE_OPENGL;ENABLE_VULKAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>LZMA_API_STATIC;LZ4LIB_VISIBILITY=;ST_NO_EXCEPTION_HANDLING;ENABLE_RAINTEGRATION;ENABLE_OPENGL;ENABLE_VULKAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Platform)'=='x64'">XBYAK_NO_EXCEPTION;ZYCORE_STATIC_DEFINE;ZYDIS_STATIC_DEFINE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ObjectFileName>$(IntDir)%(RelativeDir)</ObjectFileName>
     </ClCompile>
@@ -855,6 +856,9 @@
     <ProjectReference Include="..\3rdparty\cpuinfo\cpuinfo.vcxproj">
       <Project>{7e183337-a7e9-460c-9d3d-568bc9f9bcc1}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\3rdparty\lz4\lz4.vcxproj">
+      <Project>{39098635-446a-4fc3-9b1c-8609d94598a8}</Project>
+    </ProjectReference>
     <ProjectReference Include="..\3rdparty\rainterface\rainterface.vcxproj">
       <Project>{95dd0a0c-d14d-4cff-a593-820ef26efcc8}</Project>
     </ProjectReference>