From 00e255ee3c713dfa5c85e714dc9120a5d2aa039e Mon Sep 17 00:00:00 2001 From: JoseAaronLopezGarcia Date: Fri, 15 Dec 2023 04:05:04 +0100 Subject: [PATCH] CDVD: Add ZSO support (#10396) * add zso support * format and fixed typo * fix typo in extension (duplicated .cso) * format * proper casting * use regular casting; added lz4 to gitmodules * use C++ style casting * fix casts * add lz4 submodule * added windows build configuration * add lz4 to cmake * undo * undo * add lz4 to SearchForStuff * undo * add own lz4 source code * cleanup * fix * add nwe sources to windows build * cleanup * don't use precompile headers on lz4 * stupid compiler * add const. better logging. * cast to std::string * 3rdparty: Add lz4 * use 3rdparty lz4 * cleanup references to lz4.cpp * format code * add missing header * use fmt::format * don't call inflateReset on ZSO * use LZ4_decompress_safe * fix syntax * fix call to LZ4_decompress_safe * use LZ4_decompress_safe_partial to ignore padded data * cleanup * refactor --------- Co-authored-by: Stenzek --- .gitmodules | 3 ++ 3rdparty/lz4/CMakeLists.txt | 13 ++++++++ 3rdparty/lz4/lz4 | 1 + 3rdparty/lz4/lz4.vcxproj | 46 +++++++++++++++++++++++++++ PCSX2_qt.sln | 27 ++++++++++++++++ cmake/SearchForStuff.cmake | 1 + pcsx2-qt/MainWindow.cpp | 6 ++-- pcsx2/CDVD/CsoFileReader.cpp | 60 +++++++++++++++++++++++++----------- pcsx2/CDVD/CsoFileReader.h | 1 + pcsx2/CMakeLists.txt | 1 + pcsx2/ImGui/FullscreenUI.cpp | 4 +-- pcsx2/VMManager.cpp | 2 +- pcsx2/pcsx2.vcxproj | 6 +++- 13 files changed, 147 insertions(+), 24 deletions(-) create mode 100644 3rdparty/lz4/CMakeLists.txt create mode 160000 3rdparty/lz4/lz4 create mode 100644 3rdparty/lz4/lz4.vcxproj diff --git a/.gitmodules b/.gitmodules index b62d8c35a0..04888bdf04 100644 --- a/.gitmodules +++ b/.gitmodules @@ -30,3 +30,6 @@ [submodule "3rdparty/xz/xz"] path = 3rdparty/xz/xz url = https://github.com/tukaani-project/xz.git +[submodule "3rdparty/lz4/lz4"] + path = 3rdparty/lz4/lz4 + url = https://github.com/lz4/lz4 diff --git a/3rdparty/lz4/CMakeLists.txt b/3rdparty/lz4/CMakeLists.txt new file mode 100644 index 0000000000..d06b487346 --- /dev/null +++ b/3rdparty/lz4/CMakeLists.txt @@ -0,0 +1,13 @@ +add_library(pcsx2-lz4 + lz4/lib/lz4.c + lz4/lib/lz4.h +) + +target_include_directories(pcsx2-lz4 PUBLIC lz4/lib) +target_compile_definitions(pcsx2-lz4 PUBLIC + LZ4LIB_VISIBILITY= +) + +add_library(LZ4::LZ4 ALIAS pcsx2-lz4) + +disable_compiler_warnings_for_target(pcsx2-lz4) diff --git a/3rdparty/lz4/lz4 b/3rdparty/lz4/lz4 new file mode 160000 index 0000000000..b8fd2d1530 --- /dev/null +++ b/3rdparty/lz4/lz4 @@ -0,0 +1 @@ +Subproject commit b8fd2d15309dd4e605070bd4486e26b6ef814e29 diff --git a/3rdparty/lz4/lz4.vcxproj b/3rdparty/lz4/lz4.vcxproj new file mode 100644 index 0000000000..65e8e9c67e --- /dev/null +++ b/3rdparty/lz4/lz4.vcxproj @@ -0,0 +1,46 @@ + + + + + + {39098635-446A-4FC3-9B1C-8609D94598A8} + + + + StaticLibrary + $(DefaultPlatformToolset) + ClangCL + true + true + false + + + + + + + + + + + + + + AllRules.ruleset + + + + LZ4LIB_VISIBILITY=;%(PreprocessorDefinitions) + %(AdditionalIncludeDirectories);$(ProjectDir)\lz4\lib + TurnOffAllWarnings + + + + + + + + + + + diff --git a/PCSX2_qt.sln b/PCSX2_qt.sln index bafbd7a08d..b553b4b0a4 100644 --- a/PCSX2_qt.sln +++ b/PCSX2_qt.sln @@ -63,6 +63,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "demangler", "3rdparty\deman EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwebp", "3rdparty\libwebp\libwebp.vcxproj", "{522DAF2A-1F24-4742-B2C4-A956411F6AB2}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "3rdparty\lz4\lz4.vcxproj", "{39098635-446A-4FC3-9B1C-8609D94598A8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug AVX2|x64 = Debug AVX2|x64 @@ -763,6 +765,30 @@ Global {522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release Clang|x64.Build.0 = Release Clang|x64 {522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release|x64.ActiveCfg = Release|x64 {522DAF2A-1F24-4742-B2C4-A956411F6AB2}.Release|x64.Build.0 = Release|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug AVX2|x64.ActiveCfg = Debug|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug AVX2|x64.Build.0 = Debug|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang AVX2|x64.ActiveCfg = Debug Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang AVX2|x64.Build.0 = Debug Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang|x64.ActiveCfg = Debug Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug Clang|x64.Build.0 = Debug Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug|x64.ActiveCfg = Debug|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Debug|x64.Build.0 = Debug|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel AVX2|x64.ActiveCfg = Devel|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel AVX2|x64.Build.0 = Devel|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang AVX2|x64.ActiveCfg = Devel Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang AVX2|x64.Build.0 = Devel Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang|x64.ActiveCfg = Devel Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel Clang|x64.Build.0 = Devel Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel|x64.ActiveCfg = Devel|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Devel|x64.Build.0 = Devel|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release AVX2|x64.ActiveCfg = Release|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release AVX2|x64.Build.0 = Release|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang AVX2|x64.ActiveCfg = Release Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang AVX2|x64.Build.0 = Release Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang|x64.ActiveCfg = Release Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release Clang|x64.Build.0 = Release Clang|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release|x64.ActiveCfg = Release|x64 + {39098635-446A-4FC3-9B1C-8609D94598A8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -792,6 +818,7 @@ Global {67D0160C-0FE4-44B9-AC2E-82BBCF4104DF} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38} {1E3D706C-1D95-4E1B-BDF2-CA3D0007DF7F} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38} {522DAF2A-1F24-4742-B2C4-A956411F6AB2} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38} + {39098635-446A-4FC3-9B1C-8609D94598A8} = {78EBE642-7A4D-4EA7-86BE-5639C6646C38} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0BC474EA-3628-45D3-9DBC-E22D0B7E0F77} diff --git a/cmake/SearchForStuff.cmake b/cmake/SearchForStuff.cmake index f605185b9e..7aa695032c 100644 --- a/cmake/SearchForStuff.cmake +++ b/cmake/SearchForStuff.cmake @@ -144,6 +144,7 @@ add_subdirectory(3rdparty/libzip EXCLUDE_FROM_ALL) add_subdirectory(3rdparty/rcheevos EXCLUDE_FROM_ALL) add_subdirectory(3rdparty/rapidjson EXCLUDE_FROM_ALL) add_subdirectory(3rdparty/discord-rpc EXCLUDE_FROM_ALL) +add_subdirectory(3rdparty/lz4 EXCLUDE_FROM_ALL) if(USE_OPENGL) add_subdirectory(3rdparty/glad EXCLUDE_FROM_ALL) diff --git a/pcsx2-qt/MainWindow.cpp b/pcsx2-qt/MainWindow.cpp index 26abd76889..2f4fc88819 100644 --- a/pcsx2-qt/MainWindow.cpp +++ b/pcsx2-qt/MainWindow.cpp @@ -67,24 +67,26 @@ #endif const char* MainWindow::OPEN_FILE_FILTER = - QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.gz *.elf *.irx *.gs *.gs.xz *.gs.zst *.dump);;" + QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.zso *.gz *.elf *.irx *.gs *.gs.xz *.gs.zst *.dump);;" "Single-Track Raw Images (*.bin *.iso);;" "Cue Sheets (*.cue);;" "Media Descriptor File (*.mdf);;" "MAME CHD Images (*.chd);;" "CSO Images (*.cso);;" + "ZSO Images (*.zso);;" "GZ Images (*.gz);;" "ELF Executables (*.elf);;" "IRX Executables (*.irx);;" "GS Dumps (*.gs *.gs.xz *.gs.zst);;" "Block Dumps (*.dump)"); -const char* MainWindow::DISC_IMAGE_FILTER = QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.gz *.dump);;" +const char* MainWindow::DISC_IMAGE_FILTER = QT_TRANSLATE_NOOP("MainWindow", "All File Types (*.bin *.iso *.cue *.mdf *.chd *.cso *.zso *.gz *.dump);;" "Single-Track Raw Images (*.bin *.iso);;" "Cue Sheets (*.cue);;" "Media Descriptor File (*.mdf);;" "MAME CHD Images (*.chd);;" "CSO Images (*.cso);;" + "ZSO Images (*.zso);;" "GZ Images (*.gz);;" "Block Dumps (*.dump)"); diff --git a/pcsx2/CDVD/CsoFileReader.cpp b/pcsx2/CDVD/CsoFileReader.cpp index 60c3241fcb..55ddc9d6e2 100644 --- a/pcsx2/CDVD/CsoFileReader.cpp +++ b/pcsx2/CDVD/CsoFileReader.cpp @@ -23,6 +23,7 @@ #include "common/StringUtil.h" #include +#include "lz4.h" // Implementation of CSO compressed ISO reading, based on: // https://github.com/unknownbrackets/maxcso/blob/master/README_CSO.md @@ -52,7 +53,7 @@ CsoFileReader::~CsoFileReader() bool CsoFileReader::CanHandle(const std::string& fileName, const std::string& displayName) { bool supported = false; - if (StringUtil::EndsWith(displayName, ".cso")) + if (displayName.ends_with(".cso") || displayName.ends_with(".zso")) { FILE* fp = FileSystem::OpenCFile(fileName.c_str(), "rb"); CsoHeader hdr; @@ -70,10 +71,10 @@ bool CsoFileReader::CanHandle(const std::string& fileName, const std::string& di bool CsoFileReader::ValidateHeader(const CsoHeader& hdr, Error* error) { - if (hdr.magic[0] != 'C' || hdr.magic[1] != 'I' || hdr.magic[2] != 'S' || hdr.magic[3] != 'O') + if ((hdr.magic[0] != 'C' && hdr.magic[0] != 'Z') || hdr.magic[1] != 'I' || hdr.magic[2] != 'S' || hdr.magic[3] != 'O') { // Invalid magic, definitely a bad file. - Error::SetString(error, "File is not a CHD."); + Error::SetString(error, "File is not a CSO or ZSO."); return false; } if (hdr.ver > 1) @@ -141,6 +142,9 @@ bool CsoFileReader::ReadFileHeader(Error* error) m_indexShift = hdr.align; m_totalSize = hdr.total_bytes; + // Check compression method (ZSO=lz4) + m_uselz4 = hdr.magic[0] == 'Z'; + return true; } @@ -167,14 +171,18 @@ bool CsoFileReader::InitializeBuffers(Error* error) return false; } - m_z_stream = std::make_unique(); - m_z_stream->zalloc = Z_NULL; - m_z_stream->zfree = Z_NULL; - m_z_stream->opaque = Z_NULL; - if (inflateInit2(m_z_stream.get(), -15) != Z_OK) + // initialize zlib if not a ZSO + if (!m_uselz4) { - Error::SetString(error, "Unable to initialize zlib for CSO decompression."); - return false; + m_z_stream = std::make_unique(); + m_z_stream->zalloc = Z_NULL; + m_z_stream->zfree = Z_NULL; + m_z_stream->opaque = Z_NULL; + if (inflateInit2(m_z_stream.get(), -15) != Z_OK) + { + Error::SetString(error, "Unable to initialize zlib for CSO decompression."); + return false; + } } return true; @@ -256,18 +264,34 @@ int CsoFileReader::ReadChunk(void* dst, s64 chunkID) // This might be less bytes than frameRawSize in case of padding on the last frame. // This is because the index positions must be aligned. const u32 readRawBytes = fread(m_readBuffer.get(), 1, frameRawSize, m_src); + bool success = false; - m_z_stream->next_in = m_readBuffer.get(); - m_z_stream->avail_in = readRawBytes; - m_z_stream->next_out = static_cast(dst); - m_z_stream->avail_out = m_frameSize; + if (m_uselz4) + { + const int src_size = static_cast(readRawBytes); + const int dst_size = static_cast(m_frameSize); + const char* src_buf = reinterpret_cast(m_readBuffer.get()); + char* dst_buf = static_cast(dst); + + const int res = LZ4_decompress_safe_partial(src_buf, dst_buf, src_size, dst_size, dst_size); + success = (res > 0); + } + else + { + m_z_stream->next_in = m_readBuffer.get(); + m_z_stream->avail_in = readRawBytes; + m_z_stream->next_out = static_cast(dst); + m_z_stream->avail_out = m_frameSize; - int status = inflate(m_z_stream.get(), Z_FINISH); - bool success = status == Z_STREAM_END && m_z_stream->total_out == m_frameSize; + const int status = inflate(m_z_stream.get(), Z_FINISH); + success = (status == Z_STREAM_END && m_z_stream->total_out == m_frameSize); + } if (!success) - Console.Error("Unable to decompress CSO frame using zlib."); - inflateReset(m_z_stream.get()); + Console.Error(fmt::format("Unable to decompress CSO frame using {}", (m_uselz4)? "lz4":"zlib")); + + if (!m_uselz4) + inflateReset(m_z_stream.get()); return success ? m_frameSize : 0; } diff --git a/pcsx2/CDVD/CsoFileReader.h b/pcsx2/CDVD/CsoFileReader.h index 297dedb02e..6d035d512e 100644 --- a/pcsx2/CDVD/CsoFileReader.h +++ b/pcsx2/CDVD/CsoFileReader.h @@ -60,6 +60,7 @@ private: u32 m_frameSize = 0; u8 m_frameShift = 0; u8 m_indexShift = 0; + bool m_uselz4 = false; // flag to enable LZ4 decompression (ZSO files) std::unique_ptr m_readBuffer; std::unique_ptr m_index; diff --git a/pcsx2/CMakeLists.txt b/pcsx2/CMakeLists.txt index 525f556a5a..42dc3735d6 100644 --- a/pcsx2/CMakeLists.txt +++ b/pcsx2/CMakeLists.txt @@ -1136,6 +1136,7 @@ target_link_libraries(PCSX2_FLAGS INTERFACE discord-rpc SDL2::SDL2 ZLIB::ZLIB + LZ4::LZ4 SoundTouch::SoundTouch PNG::PNG LibLZMA::LibLZMA diff --git a/pcsx2/ImGui/FullscreenUI.cpp b/pcsx2/ImGui/FullscreenUI.cpp index 144b0f6558..7ca3efb254 100644 --- a/pcsx2/ImGui/FullscreenUI.cpp +++ b/pcsx2/ImGui/FullscreenUI.cpp @@ -844,12 +844,12 @@ void FullscreenUI::DestroyResources() ImGuiFullscreen::FileSelectorFilters FullscreenUI::GetOpenFileFilters() { - return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.gz", "*.elf", "*.irx", "*.gs", "*.gs.xz", "*.gs.zst", "*.dump"}; + return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.zso", "*.gz", "*.elf", "*.irx", "*.gs", "*.gs.xz", "*.gs.zst", "*.dump"}; } ImGuiFullscreen::FileSelectorFilters FullscreenUI::GetDiscImageFilters() { - return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.gz"}; + return {"*.bin", "*.iso", "*.cue", "*.mdf", "*.chd", "*.cso", "*.zso", "*.gz"}; } void FullscreenUI::DoStartPath(const std::string& path, std::optional state_index, std::optional fast_boot) diff --git a/pcsx2/VMManager.cpp b/pcsx2/VMManager.cpp index d4027578ed..cd2056cc24 100644 --- a/pcsx2/VMManager.cpp +++ b/pcsx2/VMManager.cpp @@ -2112,7 +2112,7 @@ bool VMManager::IsSaveStateFileName(const std::string_view& path) bool VMManager::IsDiscFileName(const std::string_view& path) { - static const char* extensions[] = {".iso", ".bin", ".img", ".mdf", ".gz", ".cso", ".chd"}; + static const char* extensions[] = {".iso", ".bin", ".img", ".mdf", ".gz", ".cso", ".zso", ".chd"}; for (const char* test_extension : extensions) { diff --git a/pcsx2/pcsx2.vcxproj b/pcsx2/pcsx2.vcxproj index 4f520f1685..a659612d18 100644 --- a/pcsx2/pcsx2.vcxproj +++ b/pcsx2/pcsx2.vcxproj @@ -38,6 +38,7 @@ %(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\wil\include %(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\xz\xz\src\liblzma\api %(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\zlib + %(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\lz4\lz4\lib %(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\libpng %(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\libchdr\include %(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\cubeb\include @@ -64,7 +65,7 @@ Use PrecompiledHeader.h PrecompiledHeader.h;%(ForcedIncludeFiles) - LZMA_API_STATIC;ST_NO_EXCEPTION_HANDLING;ENABLE_RAINTEGRATION;ENABLE_OPENGL;ENABLE_VULKAN;%(PreprocessorDefinitions) + LZMA_API_STATIC;LZ4LIB_VISIBILITY=;ST_NO_EXCEPTION_HANDLING;ENABLE_RAINTEGRATION;ENABLE_OPENGL;ENABLE_VULKAN;%(PreprocessorDefinitions) XBYAK_NO_EXCEPTION;ZYCORE_STATIC_DEFINE;ZYDIS_STATIC_DEFINE;%(PreprocessorDefinitions) $(IntDir)%(RelativeDir) @@ -855,6 +856,9 @@ {7e183337-a7e9-460c-9d3d-568bc9f9bcc1} + + {39098635-446a-4fc3-9b1c-8609d94598a8} + {95dd0a0c-d14d-4cff-a593-820ef26efcc8}