CDVD: Add ZSO support (#10396)

* add zso support

* format and fixed typo

* fix typo in extension (duplicated .cso)

* format

* proper casting

* use regular casting; added lz4 to gitmodules

* use C++ style casting

* fix casts

* add lz4 submodule

* added windows build configuration

* add lz4 to cmake

* undo

* undo

* add lz4 to SearchForStuff

* undo

* add own lz4 source code

* cleanup

* fix

* add nwe sources to windows build

* cleanup

* don't use precompile headers on lz4

* stupid compiler

* add const. better logging.

* cast to std::string

* 3rdparty: Add lz4

* use 3rdparty lz4

* cleanup references to lz4.cpp

* format code

* add missing header

* use fmt::format

* don't call inflateReset on ZSO

* use LZ4_decompress_safe

* fix syntax

* fix call to LZ4_decompress_safe

* use LZ4_decompress_safe_partial to ignore padded data

* cleanup

* refactor

---------

Co-authored-by: Stenzek <stenzek@gmail.com>
This commit is contained in:
JoseAaronLopezGarcia 2023-12-15 04:05:04 +01:00 committed by GitHub
parent c662dd8b04
commit 00e255ee3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 147 additions and 24 deletions

3
.gitmodules vendored
View File

@ -30,3 +30,6 @@
[submodule "3rdparty/xz/xz"]
path = 3rdparty/xz/xz
url = https://github.com/tukaani-project/xz.git
[submodule "3rdparty/lz4/lz4"]
path = 3rdparty/lz4/lz4
url = https://github.com/lz4/lz4

13
3rdparty/lz4/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,13 @@
add_library(pcsx2-lz4
lz4/lib/lz4.c
lz4/lib/lz4.h
)
target_include_directories(pcsx2-lz4 PUBLIC lz4/lib)
target_compile_definitions(pcsx2-lz4 PUBLIC
LZ4LIB_VISIBILITY=
)
add_library(LZ4::LZ4 ALIAS pcsx2-lz4)
disable_compiler_warnings_for_target(pcsx2-lz4)

1
3rdparty/lz4/lz4 vendored Submodule

@ -0,0 +1 @@
Subproject commit b8fd2d15309dd4e605070bd4486e26b6ef814e29

46
3rdparty/lz4/lz4.vcxproj vendored Normal file
View File

@ -0,0 +1,46 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(SolutionDir)common\vsprops\BaseProjectConfig.props" />
<Import Project="$(SolutionDir)common\vsprops\WinSDK.props" />
<PropertyGroup Label="Globals">
<ProjectGuid>{39098635-446A-4FC3-9B1C-8609D94598A8}</ProjectGuid>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<PlatformToolset Condition="!$(Configuration.Contains(Clang))">$(DefaultPlatformToolset)</PlatformToolset>
<PlatformToolset Condition="$(Configuration.Contains(Clang))">ClangCL</PlatformToolset>
<WholeProgramOptimization Condition="$(Configuration.Contains(Release))">true</WholeProgramOptimization>
<UseDebugLibraries Condition="$(Configuration.Contains(Debug))">true</UseDebugLibraries>
<UseDebugLibraries Condition="!$(Configuration.Contains(Debug))">false</UseDebugLibraries>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings" />
<ImportGroup Label="PropertySheets">
<Import Project="..\DefaultProjectRootDir.props" />
<Import Project="..\3rdparty.props" />
<Import Condition="$(Configuration.Contains(Debug))" Project="..\..\common\vsprops\CodeGen_Debug.props" />
<Import Condition="$(Configuration.Contains(Devel))" Project="..\..\common\vsprops\CodeGen_Devel.props" />
<Import Condition="$(Configuration.Contains(Release))" Project="..\..\common\vsprops\CodeGen_Release.props" />
<Import Condition="!$(Configuration.Contains(Release))" Project="..\..\common\vsprops\IncrementalLinking.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>LZ4LIB_VISIBILITY=;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(ProjectDir)\lz4\lib</AdditionalIncludeDirectories>
<WarningLevel>TurnOffAllWarnings</WarningLevel>
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="lz4\lib\lz4.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="lz4\lib\lz4.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets" />
</Project>

View File

@ -63,6 +63,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "demangler", "3rdparty\deman
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwebp", "3rdparty\libwebp\libwebp.vcxproj", "{522DAF2A-1F24-4742-B2C4-A956411F6AB2}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "3rdparty\lz4\lz4.vcxproj", "{39098635-446A-4FC3-9B1C-8609D94598A8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug AVX2|x64 = Debug AVX2|x64
@ -763,6 +765,30 @@ Global
{522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release Clang|x64.Build.0 = Release Clang|x64
{522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release|x64.ActiveCfg = Release|x64
{522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release|x64.Build.0 = Release|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug AVX2|x64.ActiveCfg = Debug|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug AVX2|x64.Build.0 = Debug|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang AVX2|x64.ActiveCfg = Debug Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang AVX2|x64.Build.0 = Debug Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang|x64.ActiveCfg = Debug Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang|x64.Build.0 = Debug Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug|x64.ActiveCfg = Debug|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Debug|x64.Build.0 = Debug|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel AVX2|x64.ActiveCfg = Devel|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel AVX2|x64.Build.0 = Devel|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang AVX2|x64.ActiveCfg = Devel Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang AVX2|x64.Build.0 = Devel Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang|x64.ActiveCfg = Devel Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang|x64.Build.0 = Devel Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel|x64.ActiveCfg = Devel|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Devel|x64.Build.0 = Devel|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release AVX2|x64.ActiveCfg = Release|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release AVX2|x64.Build.0 = Release|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang AVX2|x64.ActiveCfg = Release Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang AVX2|x64.Build.0 = Release Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang|x64.ActiveCfg = Release Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang|x64.Build.0 = Release Clang|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release|x64.ActiveCfg = Release|x64
{39098635-446A-4FC3-9B1C-8609D94598A8}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -792,6 +818,7 @@ Global
{67D0160C-0FE4-44B9-AC2E-82BBCF4104DF} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
{1E3D706C-1D95-4E1B-BDF2-CA3D0007DF7F} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
{522DAF2A-1F24-4742-B2C4-A956411F6AB2} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
{39098635-446A-4FC3-9B1C-8609D94598A8} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {0BC474EA-3628-45D3-9DBC-E22D0B7E0F77}

View File

@ -144,6 +144,7 @@ add_subdirectory(3rdparty/libzip EXCLUDE_FROM_ALL)
add_subdirectory(3rdparty/rcheevos EXCLUDE_FROM_ALL)
add_subdirectory(3rdparty/rapidjson EXCLUDE_FROM_ALL)
add_subdirectory(3rdparty/discord-rpc EXCLUDE_FROM_ALL)
add_subdirectory(3rdparty/lz4 EXCLUDE_FROM_ALL)
if(USE_OPENGL)
add_subdirectory(3rdparty/glad EXCLUDE_FROM_ALL)

View File

@ -67,24 +67,26 @@
#endif
const char* MainWindow::OPEN_FILE_FILTER =
QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.gz *.elf *.irx *.gs *.gs.xz *.gs.zst *.dump);;"
QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.zso *.gz *.elf *.irx *.gs *.gs.xz *.gs.zst *.dump);;"
"Single-Track Raw Images (*.bin *.iso);;"
"Cue Sheets (*.cue);;"
"Media Descriptor File (*.mdf);;"
"MAME CHD Images (*.chd);;"
"CSO Images (*.cso);;"
"ZSO Images (*.zso);;"
"GZ Images (*.gz);;"
"ELF Executables (*.elf);;"
"IRX Executables (*.irx);;"
"GS Dumps (*.gs *.gs.xz *.gs.zst);;"
"Block Dumps (*.dump)");
const char* MainWindow::DISC_IMAGE_FILTER = QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.gz *.dump);;"
const char* MainWindow::DISC_IMAGE_FILTER = QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.zso *.gz *.dump);;"
"Single-Track Raw Images (*.bin *.iso);;"
"Cue Sheets (*.cue);;"
"Media Descriptor File (*.mdf);;"
"MAME CHD Images (*.chd);;"
"CSO Images (*.cso);;"
"ZSO Images (*.zso);;"
"GZ Images (*.gz);;"
"Block Dumps (*.dump)");

View File

@ -23,6 +23,7 @@
#include "common/StringUtil.h"
#include <zlib.h>
#include "lz4.h"
// Implementation of CSO compressed ISO reading, based on:
// https://github.com/unknownbrackets/maxcso/blob/master/README_CSO.md
@ -52,7 +53,7 @@ CsoFileReader::~CsoFileReader()
bool CsoFileReader::CanHandle(const std::string& fileName, const std::string& displayName)
{
bool supported = false;
if (StringUtil::EndsWith(displayName, ".cso"))
if (displayName.ends_with(".cso") || displayName.ends_with(".zso"))
{
FILE* fp = FileSystem::OpenCFile(fileName.c_str(), "rb");
CsoHeader hdr;
@ -70,10 +71,10 @@ bool CsoFileReader::CanHandle(const std::string& fileName, const std::string& di
bool CsoFileReader::ValidateHeader(const CsoHeader& hdr, Error* error)
{
if (hdr.magic[0] != 'C' || hdr.magic[1] != 'I' || hdr.magic[2] != 'S' || hdr.magic[3] != 'O')
if ((hdr.magic[0] != 'C' && hdr.magic[0] != 'Z') || hdr.magic[1] != 'I' || hdr.magic[2] != 'S' || hdr.magic[3] != 'O')
{
// Invalid magic, definitely a bad file.
Error::SetString(error, "File is not a CHD.");
Error::SetString(error, "File is not a CSO or ZSO.");
return false;
}
if (hdr.ver > 1)
@ -141,6 +142,9 @@ bool CsoFileReader::ReadFileHeader(Error* error)
m_indexShift = hdr.align;
m_totalSize = hdr.total_bytes;
// Check compression method (ZSO=lz4)
m_uselz4 = hdr.magic[0] == 'Z';
return true;
}
@ -167,14 +171,18 @@ bool CsoFileReader::InitializeBuffers(Error* error)
return false;
}
m_z_stream = std::make_unique<z_stream>();
m_z_stream->zalloc = Z_NULL;
m_z_stream->zfree = Z_NULL;
m_z_stream->opaque = Z_NULL;
if (inflateInit2(m_z_stream.get(), -15) != Z_OK)
// initialize zlib if not a ZSO
if (!m_uselz4)
{
Error::SetString(error, "Unable to initialize zlib for CSO decompression.");
return false;
m_z_stream = std::make_unique<z_stream>();
m_z_stream->zalloc = Z_NULL;
m_z_stream->zfree = Z_NULL;
m_z_stream->opaque = Z_NULL;
if (inflateInit2(m_z_stream.get(), -15) != Z_OK)
{
Error::SetString(error, "Unable to initialize zlib for CSO decompression.");
return false;
}
}
return true;
@ -256,18 +264,34 @@ int CsoFileReader::ReadChunk(void* dst, s64 chunkID)
// This might be less bytes than frameRawSize in case of padding on the last frame.
// This is because the index positions must be aligned.
const u32 readRawBytes = fread(m_readBuffer.get(), 1, frameRawSize, m_src);
bool success = false;
m_z_stream->next_in = m_readBuffer.get();
m_z_stream->avail_in = readRawBytes;
m_z_stream->next_out = static_cast<Bytef*>(dst);
m_z_stream->avail_out = m_frameSize;
if (m_uselz4)
{
const int src_size = static_cast<int>(readRawBytes);
const int dst_size = static_cast<int>(m_frameSize);
const char* src_buf = reinterpret_cast<const char*>(m_readBuffer.get());
char* dst_buf = static_cast<char*>(dst);
int status = inflate(m_z_stream.get(), Z_FINISH);
bool success = status == Z_STREAM_END && m_z_stream->total_out == m_frameSize;
const int res = LZ4_decompress_safe_partial(src_buf, dst_buf, src_size, dst_size, dst_size);
success = (res > 0);
}
else
{
m_z_stream->next_in = m_readBuffer.get();
m_z_stream->avail_in = readRawBytes;
m_z_stream->next_out = static_cast<Bytef*>(dst);
m_z_stream->avail_out = m_frameSize;
const int status = inflate(m_z_stream.get(), Z_FINISH);
success = (status == Z_STREAM_END && m_z_stream->total_out == m_frameSize);
}
if (!success)
Console.Error("Unable to decompress CSO frame using zlib.");
inflateReset(m_z_stream.get());
Console.Error(fmt::format("Unable to decompress CSO frame using {}", (m_uselz4)? "lz4":"zlib"));
if (!m_uselz4)
inflateReset(m_z_stream.get());
return success ? m_frameSize : 0;
}

View File

@ -60,6 +60,7 @@ private:
u32 m_frameSize = 0;
u8 m_frameShift = 0;
u8 m_indexShift = 0;
bool m_uselz4 = false; // flag to enable LZ4 decompression (ZSO files)
std::unique_ptr<u8[]> m_readBuffer;
std::unique_ptr<u32[]> m_index;

View File

@ -1136,6 +1136,7 @@ target_link_libraries(PCSX2_FLAGS INTERFACE
discord-rpc
SDL2::SDL2
ZLIB::ZLIB
LZ4::LZ4
SoundTouch::SoundTouch
PNG::PNG
LibLZMA::LibLZMA

View File

@ -844,12 +844,12 @@ void FullscreenUI::DestroyResources()
ImGuiFullscreen::FileSelectorFilters FullscreenUI::GetOpenFileFilters()
{
return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.gz", "*.elf", "*.irx", "*.gs", "*.gs.xz", "*.gs.zst", "*.dump"};
return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.zso", "*.gz", "*.elf", "*.irx", "*.gs", "*.gs.xz", "*.gs.zst", "*.dump"};
}
ImGuiFullscreen::FileSelectorFilters FullscreenUI::GetDiscImageFilters()
{
return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.gz"};
return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.zso", "*.gz"};
}
void FullscreenUI::DoStartPath(const std::string& path, std::optional<s32> state_index, std::optional<bool> fast_boot)

View File

@ -2112,7 +2112,7 @@ bool VMManager::IsSaveStateFileName(const std::string_view& path)
bool VMManager::IsDiscFileName(const std::string_view& path)
{
static const char* extensions[] = {".iso", ".bin", ".img", ".mdf", ".gz", ".cso", ".chd"};
static const char* extensions[] = {".iso", ".bin", ".img", ".mdf", ".gz", ".cso", ".zso", ".chd"};
for (const char* test_extension : extensions)
{

View File

@ -38,6 +38,7 @@
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\wil\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\xz\xz\src\liblzma\api</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\zlib</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\lz4\lz4\lib</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\libpng</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\libchdr\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\cubeb\include</AdditionalIncludeDirectories>
@ -64,7 +65,7 @@
<PrecompiledHeader>Use</PrecompiledHeader>
<PrecompiledHeaderFile>PrecompiledHeader.h</PrecompiledHeaderFile>
<ForcedIncludeFiles>PrecompiledHeader.h;%(ForcedIncludeFiles)</ForcedIncludeFiles>
<PreprocessorDefinitions>LZMA_API_STATIC;ST_NO_EXCEPTION_HANDLING;ENABLE_RAINTEGRATION;ENABLE_OPENGL;ENABLE_VULKAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>LZMA_API_STATIC;LZ4LIB_VISIBILITY=;ST_NO_EXCEPTION_HANDLING;ENABLE_RAINTEGRATION;ENABLE_OPENGL;ENABLE_VULKAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions Condition="'$(Platform)'=='x64'">XBYAK_NO_EXCEPTION;ZYCORE_STATIC_DEFINE;ZYDIS_STATIC_DEFINE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ObjectFileName>$(IntDir)%(RelativeDir)</ObjectFileName>
</ClCompile>
@ -855,6 +856,9 @@
<ProjectReference Include="..\3rdparty\cpuinfo\cpuinfo.vcxproj">
<Project>{7e183337-a7e9-460c-9d3d-568bc9f9bcc1}</Project>
</ProjectReference>
<ProjectReference Include="..\3rdparty\lz4\lz4.vcxproj">
<Project>{39098635-446a-4fc3-9b1c-8609d94598a8}</Project>
</ProjectReference>
<ProjectReference Include="..\3rdparty\rainterface\rainterface.vcxproj">
<Project>{95dd0a0c-d14d-4cff-a593-820ef26efcc8}</Project>
</ProjectReference>