ShuriZma
/
pcsx2
mirror of https://github.com/PCSX2/pcsx2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GS: Replace xz/liblzma with 7zip LZMA SDK

This commit is contained in:
Stenzek 2024-03-30 18:38:06 +10:00 committed by Connor McLaughlin
parent 907ae642d0
commit ad81318854
9 changed files with 752 additions and 533 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common/HeapArray.h
View File

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>, 2024 PCSX2 Dev Team
// SPDX-License-Identifier: GPL-3.0
// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
// SPDX-License-Identifier: LGPL-3.0+
#pragma once

3
pcsx2/CMakeLists.txt
View File

@ -47,7 +47,6 @@ if(WIN32)
WINVER=${MIN_WIN32}
_WIN32_WINNT=${MIN_WIN32}
WIN32_LEAN_AND_MEAN
LZMA_API_STATIC
WIL_SUPPRESS_EXCEPTIONS
)
endif(WIN32)
@ -1114,7 +1113,7 @@ target_link_libraries(PCSX2_FLAGS INTERFACE
LZ4::LZ4
SoundTouch::SoundTouch
PNG::PNG
LibLZMA::LibLZMA
LZMA::LZMA
Zstd::Zstd
${LIBC_LIBRARIES}
)

253
pcsx2/GS/GSDump.cpp
View File

@ -1,11 +1,20 @@
// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
// SPDX-License-Identifier: LGPL-3.0+
#include "GSDump.h"
#include "GSExtra.h"
#include "GSState.h"
#include "GS/GSDump.h"
#include "GS/GSExtra.h"
#include "GS/GSLzma.h"
#include "GS/GSState.h"
#include "common/Console.h"
#include "common/FileSystem.h"
#include "common/HeapArray.h"
#include "common/ScopedGuard.h"
#include <7zCrc.h>
#include <XzCrc64.h>
#include <XzEnc.h>
#include <zstd.h>
GSDumpBase::GSDumpBase(std::string fn)
: m_filename(std::move(fn))
@ -14,13 +23,13 @@ GSDumpBase::GSDumpBase(std::string fn)
{
m_gs = FileSystem::OpenCFile(m_filename.c_str(), "wb");
if (!m_gs)
Console.Error("GSDump: Error failed to open %s", m_filename.c_str());
Console.ErrorFmt("GSDump: Error failed to open {}", m_filename);
}
GSDumpBase::~GSDumpBase()
{
if (m_gs)
fclose(m_gs);
std::fclose(m_gs);
}
void GSDumpBase::AddHeader(const std::string& serial, u32 crc,
@ -104,13 +113,27 @@ void GSDumpBase::Write(const void* data, size_t size)
size_t written = fwrite(data, 1, size, m_gs);
if (written != size)
fprintf(stderr, "GSDump: Error failed to write data\n");
Console.Error("GSDump: Error failed to write data");
}
//////////////////////////////////////////////////////////////////////
// GSDump implementation
//////////////////////////////////////////////////////////////////////
namespace
{
class GSDumpUncompressed final : public GSDumpBase
{
void AppendRawData(const void* data, size_t size) final;
void AppendRawData(u8 c) final;
public:
GSDumpUncompressed(const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs);
virtual ~GSDumpUncompressed() = default;
};
GSDumpUncompressed::GSDumpUncompressed(const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs)
@ -128,96 +151,185 @@ void GSDumpUncompressed::AppendRawData(u8 c)
{
Write(&c, 1);
}
} // namespace
std::unique_ptr<GSDumpBase> GSDumpBase::CreateUncompressedDump(
const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs)
{
return std::make_unique<GSDumpUncompressed>(fn, serial, crc,
screenshot_width, screenshot_height, screenshot_pixels,
fd, regs);
}
namespace
{
class GsDumpBuffered : public GSDumpBase
{
protected:
void AppendRawData(const void* data, size_t size) override final;
void AppendRawData(u8 c) override final;
void EnsureSpace(size_t size);
DynamicHeapArray<u8, 64> m_buffer;
size_t m_buffer_size = 0;
public:
GsDumpBuffered(std::string fn);
virtual ~GsDumpBuffered() override = default;
};
GsDumpBuffered::GsDumpBuffered(std::string fn)
: GSDumpBase(std::move(fn))
{
m_buffer.resize(_1mb);
}
void GsDumpBuffered::AppendRawData(const void* data, size_t size)
{
if (size == 0) [[unlikely]]
return;
EnsureSpace(size);
std::memcpy(&m_buffer[m_buffer_size], data, size);
m_buffer_size += size;
}
void GsDumpBuffered::AppendRawData(u8 c)
{
EnsureSpace(1);
m_buffer[m_buffer_size++] = c;
}
void GsDumpBuffered::EnsureSpace(size_t size)
{
const size_t new_size = m_buffer_size + size;
if (new_size <= m_buffer.size())
return;
const size_t alloc_size = std::max(m_buffer.size() * 2, new_size);
m_buffer.resize(alloc_size);
}
} // namespace
//////////////////////////////////////////////////////////////////////
// GSDumpXz implementation
//////////////////////////////////////////////////////////////////////
namespace
{
class GSDumpXz final : public GsDumpBuffered
{
void Compress();
public:
GSDumpXz(const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs);
~GSDumpXz() override;
};
GSDumpXz::GSDumpXz(const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs)
: GSDumpBase(fn + ".gs.xz")
: GsDumpBuffered(fn + ".gs.xz")
{
m_strm = LZMA_STREAM_INIT;
lzma_ret ret = lzma_easy_encoder(&m_strm, 6 /*level*/, LZMA_CHECK_CRC64);
if (ret != LZMA_OK)
{
fprintf(stderr, "GSDumpXz: Error initializing LZMA encoder ! (error code %u)\n", ret);
return;
}
AddHeader(serial, crc, screenshot_width, screenshot_height, screenshot_pixels, fd, regs);
}
GSDumpXz::~GSDumpXz()
{
Flush();
// Finish the stream
m_strm.avail_in = 0;
Compress(LZMA_FINISH, LZMA_STREAM_END);
lzma_end(&m_strm);
Compress();
}
void GSDumpXz::AppendRawData(const void* data, size_t size)
void GSDumpXz::Compress()
{
size_t old_size = m_in_buff.size();
m_in_buff.resize(old_size + size);
memcpy(&m_in_buff[old_size], data, size);
// Enough data was accumulated, time to write/compress it. If compression
// is enabled, it will freeze PCSX2. 1GB should be enough for long dump.
//
// Note: long dumps are currently not supported so this path won't be executed
if (m_in_buff.size() > 1024 * 1024 * 1024)
Flush();
}
void GSDumpXz::AppendRawData(u8 c)
struct MemoryInStream
{
m_in_buff.push_back(c);
}
ISeqInStream vt;
const u8* buffer;
size_t buffer_size;
size_t read_pos;
};
MemoryInStream mis = {
{.Read = [](const ISeqInStream* p, void* buf, size_t* size) -> SRes {
MemoryInStream* mis = CONTAINER_FROM_VTBL(p, MemoryInStream, vt);
const size_t avail = mis->buffer_size - mis->read_pos;
const size_t copy = std::min(avail, *size);
void GSDumpXz::Flush()
std::memcpy(buf, &mis->buffer[mis->read_pos], copy);
mis->read_pos += copy;
*size = copy;
return SZ_OK;
}},
m_buffer.data(),
m_buffer_size,
0};
struct DumpOutStream
{
if (m_in_buff.empty())
return;
ISeqOutStream vt;
GSDumpXz* real;
};
DumpOutStream dos = {
{.Write = [](const ISeqOutStream* p, const void* buf, size_t size) -> size_t {
DumpOutStream* dos = CONTAINER_FROM_VTBL(p, DumpOutStream, vt);
dos->real->Write(buf, size);
return size;
}},
this};
m_strm.next_in = m_in_buff.data();
m_strm.avail_in = m_in_buff.size();
pxAssert(m_buffer_size > 0);
Compress(LZMA_RUN, LZMA_OK);
GSInit7ZCRCTables();
m_in_buff.clear();
}
void GSDumpXz::Compress(lzma_action action, lzma_ret expected_status)
CXzProps props;
XzProps_Init(&props);
const SRes res = Xz_Encode(&dos.vt, &mis.vt, &props, nullptr);
if (res != SZ_OK)
{
std::vector<u8> out_buff(1024 * 1024);
do
{
m_strm.next_out = out_buff.data();
m_strm.avail_out = out_buff.size();
lzma_ret ret = lzma_code(&m_strm, action);
if (ret != expected_status)
{
fprintf(stderr, "GSDumpXz: Error %d\n", (int)ret);
Console.ErrorFmt("Xz_Encode() failed: {}", static_cast<int>(res));
return;
}
}
} // namespace
size_t write_size = out_buff.size() - m_strm.avail_out;
Write(out_buff.data(), write_size);
} while (m_strm.avail_out == 0);
std::unique_ptr<GSDumpBase> GSDumpBase::CreateXzDump(
const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs)
{
return std::make_unique<GSDumpXz>(fn, serial, crc,
screenshot_width, screenshot_height, screenshot_pixels,
fd, regs);
}
//////////////////////////////////////////////////////////////////////
// GSDumpZstd implementation
//////////////////////////////////////////////////////////////////////
namespace
{
class GSDumpZst final : public GSDumpBase
{
ZSTD_CStream* m_strm;
std::vector<u8> m_in_buff;
std::vector<u8> m_out_buff;
void MayFlush();
void Compress(ZSTD_EndDirective action);
void AppendRawData(const void* data, size_t size);
void AppendRawData(u8 c);
public:
GSDumpZst(const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs);
virtual ~GSDumpZst();
};
GSDumpZst::GSDumpZst(const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs)
@ -276,7 +388,7 @@ void GSDumpZst::Compress(ZSTD_EndDirective action)
const size_t remaining = ZSTD_compressStream2(m_strm, &outbuf, &inbuf, action);
if (ZSTD_isError(remaining))
{
fprintf(stderr, "GSDumpZstd: Error %s\n", ZSTD_getErrorName(remaining));
Console.ErrorFmt("GSDumpZstd: Error {}", ZSTD_getErrorName(remaining));
return;
}
@ -302,3 +414,14 @@ void GSDumpZst::Compress(ZSTD_EndDirective action)
m_in_buff.clear();
}
} // namespace
std::unique_ptr<GSDumpBase> GSDumpBase::CreateZstDump(
const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs)
{
return std::make_unique<GSDumpZst>(fn, serial, crc,
screenshot_width, screenshot_height, screenshot_pixels,
fd, regs);
}

56
pcsx2/GS/GSDump.h
View File

@ -1,13 +1,11 @@
// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
// SPDX-License-Identifier: LGPL-3.0+
#pragma once
#include "SaveState.h"
#include "GSRegs.h"
#include "Renderers/SW/GSVertexSW.h"
#include <lzma.h>
#include <zstd.h>
#include "GS/GSRegs.h"
#include "GS/Renderers/SW/GSVertexSW.h"
/*
@ -68,53 +66,17 @@ public:
void ReadFIFO(u32 size);
void Transfer(int index, const u8* mem, size_t size);
bool VSync(int field, bool last, const GSPrivRegSet* regs);
};
class GSDumpUncompressed final : public GSDumpBase
{
void AppendRawData(const void* data, size_t size) final;
void AppendRawData(u8 c) final;
public:
GSDumpUncompressed(const std::string& fn, const std::string& serial, u32 crc,
static std::unique_ptr<GSDumpBase> CreateUncompressedDump(
const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs);
virtual ~GSDumpUncompressed() = default;
};
class GSDumpXz final : public GSDumpBase
{
lzma_stream m_strm;
std::vector<u8> m_in_buff;
void Flush();
void Compress(lzma_action action, lzma_ret expected_status);
void AppendRawData(const void* data, size_t size);
void AppendRawData(u8 c);
public:
GSDumpXz(const std::string& fn, const std::string& serial, u32 crc,
static std::unique_ptr<GSDumpBase> CreateXzDump(
const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs);
virtual ~GSDumpXz();
};
class GSDumpZst final : public GSDumpBase
{
ZSTD_CStream* m_strm;
std::vector<u8> m_in_buff;
std::vector<u8> m_out_buff;
void MayFlush();
void Compress(ZSTD_EndDirective action);
void AppendRawData(const void* data, size_t size);
void AppendRawData(u8 c);
public:
GSDumpZst(const std::string& fn, const std::string& serial, u32 crc,
static std::unique_ptr<GSDumpBase> CreateZstDump(
const std::string& fn, const std::string& serial, u32 crc,
u32 screenshot_width, u32 screenshot_height, const u32* screenshot_pixels,
const freezeData& fd, const GSPrivRegSet* regs);
virtual ~GSDumpZst();
};

496
pcsx2/GS/GSLzma.cpp
View File

@ -1,65 +1,32 @@
// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
// SPDX-License-Identifier: LGPL-3.0+
#include "common/AlignedMalloc.h"
#include "common/Console.h"
#include "common/FileSystem.h"
#include "common/ScopedGuard.h"
#include "common/StringUtil.h"
#include "common/BitUtils.h"
#include "common/Error.h"
#include "common/HeapArray.h"
#include "GS/GSDump.h"
#include "GS/GSLzma.h"
#include "GS/GSExtra.h"
#include <Alloc.h>
#include <7zCrc.h>
#include <Xz.h>
#include <XzCrc64.h>
#include <zstd.h>
#include <mutex>
using namespace GSDumpTypes;
GSDumpFile::GSDumpFile(FILE* file, FILE* repack_file)
: m_fp(file)
, m_repack_fp(repack_file)
{
}
GSDumpFile::GSDumpFile() = default;
void GSDumpFile::Repack(void* ptr, size_t size)
{
if (m_repack_fp == nullptr)
return;
size_t ret = fwrite(ptr, 1, size, m_repack_fp);
if (ret != size)
fprintf(stderr, "Failed to repack\n");
}
GSDumpFile::~GSDumpFile()
{
if (m_fp)
fclose(m_fp);
if (m_repack_fp)
fclose(m_repack_fp);
}
std::unique_ptr<GSDumpFile> GSDumpFile::OpenGSDump(const char* filename, const char* repack_filename /*= nullptr*/)
{
std::FILE* fp = FileSystem::OpenCFile(filename, "rb");
if (!fp)
return nullptr;
std::FILE* repack_fp = nullptr;
if (repack_filename && std::strlen(repack_filename) > 0)
{
repack_fp = FileSystem::OpenCFile(repack_filename, "wb");
if (!repack_fp)
{
std::fclose(fp);
return nullptr;
}
}
if (StringUtil::EndsWithNoCase(filename, ".xz"))
return std::make_unique<GSDumpLzma>(fp, nullptr);
else if (StringUtil::EndsWithNoCase(filename, ".zst"))
return std::make_unique<GSDumpDecompressZst>(fp, nullptr);
else
return std::make_unique<GSDumpRaw>(fp, nullptr);
}
GSDumpFile::~GSDumpFile() = default;
bool GSDumpFile::GetPreviewImageFromDump(const char* filename, u32* width, u32* height, std::vector<u32>* pixels)
{
@ -102,15 +69,21 @@ bool GSDumpFile::GetPreviewImageFromDump(const char* filename, u32* width, u32*
return true;
}
bool GSDumpFile::ReadFile()
bool GSDumpFile::ReadFile(Error* error)
{
u32 ss;
if (Read(&m_crc, sizeof(m_crc)) != sizeof(m_crc) || Read(&ss, sizeof(ss)) != sizeof(ss))
{
Error::SetString(error, "Failed to read header");
return false;
}
m_state_data.resize(ss);
if (Read(m_state_data.data(), ss) != ss)
{
Error::SetString(error, "Failed to read state data");
return false;
}
// Pull serial out of new header, if present.
if (m_crc == 0xFFFFFFFFu)
@ -118,7 +91,7 @@ bool GSDumpFile::ReadFile()
GSDumpHeader header;
if (m_state_data.size() < sizeof(header))
{
Console.Error("GSDump header is corrupted.");
Error::SetString(error, "GSDump header is corrupted.");
return false;
}
@ -130,7 +103,7 @@ bool GSDumpFile::ReadFile()
{
if (header.serial_offset > ss || (static_cast<u64>(header.serial_offset) + header.serial_size) > ss)
{
Console.Error("GSDump header is corrupted.");
Error::SetString(error, "GSDump header is corrupted.");
return false;
}
@ -141,12 +114,18 @@ bool GSDumpFile::ReadFile()
// Read the real state data
m_state_data.resize(header.state_size);
if (Read(m_state_data.data(), header.state_size) != header.state_size)
{
Error::SetString(error, "Failed to read real state data");
return false;
}
}
m_regs_data.resize(8192);
if (Read(m_regs_data.data(), m_regs_data.size()) != m_regs_data.size())
{
Error::SetString(error, "Failed to read regs data");
return false;
}
// read all the packet data in
// TODO: make this suck less by getting the full/extracted size and preallocating
@ -160,7 +139,10 @@ bool GSDumpFile::ReadFile()
if (read != read_size)
{
if (!IsEof())
{
Error::SetString(error, "Failed to read packet");
return false;
}
m_packet_data.resize(packet_data_size + read);
m_packet_data.shrink_to_fit();
@ -175,7 +157,10 @@ bool GSDumpFile::ReadFile()
do \
{ \
if (remaining < sizeof(u8)) \
{ \
Error::SetString(error, "Failed to read byte"); \
return false; \
} \
std::memcpy(dst, data, sizeof(u8)); \
data++; \
remaining--; \
@ -184,7 +169,10 @@ bool GSDumpFile::ReadFile()
do \
{ \
if (remaining < sizeof(u32)) \
{ \
Error::SetString(error, "Failed to read word"); \
return false; \
} \
std::memcpy(dst, data, sizeof(u32)); \
data += sizeof(u32); \
remaining -= sizeof(u32); \
@ -212,6 +200,7 @@ bool GSDumpFile::ReadFile()
packet.length = 8192;
break;
default:
Error::SetString(error, fmt::format("Unknown packet type {}", static_cast<u32>(packet.id)));
return false;
}
@ -242,121 +231,288 @@ bool GSDumpFile::ReadFile()
}
/******************************************************************/
GSDumpLzma::GSDumpLzma(FILE* file, FILE* repack_file)
: GSDumpFile(file, repack_file)
static std::once_flag s_lzma_crc_table_init;
void GSInit7ZCRCTables()
{
Initialize();
std::call_once(s_lzma_crc_table_init, []() {
CrcGenerateTable();
Crc64GenerateTable();
});
}
void GSDumpLzma::Initialize()
namespace
{
memset(&m_strm, 0, sizeof(lzma_stream));
lzma_ret ret = lzma_stream_decoder(&m_strm, UINT32_MAX, 0);
if (ret != LZMA_OK)
class GSDumpLzma final : public GSDumpFile
{
Console.Error("Error initializing the decoder! (error code %u)", ret);
pxFailRel("Failed to initialize LZMA decoder.");
public:
GSDumpLzma();
~GSDumpLzma() override;
protected:
bool Open(std::FILE* fp, Error* error) override;
bool IsEof() override;
size_t Read(void* ptr, size_t size) override;
private:
static constexpr size_t kInputBufSize = static_cast<size_t>(1) << 18;
struct Block
{
size_t file_offset;
size_t stream_offset;
size_t compressed_size;
size_t uncompressed_size;
CXzStreamFlags stream_flags;
};
bool DecompressNextBlock();
std::FILE* m_fp = nullptr;
std::vector<Block> m_blocks;
size_t m_stream_size = 0;
DynamicHeapArray<u8, 64> m_block_buffer;
size_t m_block_index = 0;
size_t m_block_size = 0;
size_t m_block_pos = 0;
DynamicHeapArray<u8, 64> m_block_read_buffer;
alignas(64) CXzUnpacker m_unpacker = {};
};
GSDumpLzma::GSDumpLzma() = default;
GSDumpLzma::~GSDumpLzma()
{
XzUnpacker_Free(&m_unpacker);
}
m_buff_size = 1024*1024;
m_area = (uint8_t*)_aligned_malloc(m_buff_size, VECTOR_ALIGNMENT);
m_inbuf = (uint8_t*)_aligned_malloc(BUFSIZ, VECTOR_ALIGNMENT);
m_avail = 0;
m_start = 0;
bool GSDumpLzma::Open(std::FILE* fp, Error* error)
{
m_fp = fp;
m_strm.avail_in = 0;
m_strm.next_in = m_inbuf;
GSInit7ZCRCTables();
m_strm.avail_out = m_buff_size;
m_strm.next_out = m_area;
struct MyFileInStream
{
ISeekInStream vt;
std::FILE* fp;
};
MyFileInStream fis = {
{.Read = [](const ISeekInStream* p, void* buf, size_t* size) -> SRes {
MyFileInStream* fis = CONTAINER_FROM_VTBL(p, MyFileInStream, vt);
const size_t size_to_read = *size;
const auto bytes_read = std::fread(buf, 1, size_to_read, fis->fp);
*size = (bytes_read >= 0) ? bytes_read : 0;
return (bytes_read == size_to_read) ? SZ_OK : SZ_ERROR_READ;
},
.Seek = [](const ISeekInStream* p, Int64* pos, ESzSeek origin) -> SRes {
MyFileInStream* fis = CONTAINER_FROM_VTBL(p, MyFileInStream, vt);
static_assert(SZ_SEEK_CUR == SEEK_CUR && SZ_SEEK_SET == SEEK_SET && SZ_SEEK_END == SEEK_END);
if (FileSystem::FSeek64(fis->fp, *pos, static_cast<int>(origin)) != 0)
return SZ_ERROR_READ;
const s64 new_pos = FileSystem::FTell64(fis->fp);
if (new_pos < 0)
return SZ_ERROR_READ;
*pos = new_pos;
return SZ_OK;
}},
m_fp};
CLookToRead2 look_stream = {};
LookToRead2_Init(&look_stream);
LookToRead2_CreateVTable(&look_stream, False);
look_stream.realStream = &fis.vt;
look_stream.bufSize = kInputBufSize;
look_stream.buf = static_cast<Byte*>(ISzAlloc_Alloc(&g_Alloc, kInputBufSize));
if (!look_stream.buf)
{
Error::SetString(error, "Failed to allocate lookahead buffer");
return false;
}
ScopedGuard guard = [&look_stream]() {
if (look_stream.buf)
ISzAlloc_Free(&g_Alloc, look_stream.buf);
};
// Read blocks
CXzs xzs;
Xzs_Construct(&xzs);
const ScopedGuard xzs_guard([&xzs]() {
Xzs_Free(&xzs, &g_Alloc);
});
const s64 file_size = FileSystem::FSize64(m_fp);
Int64 start_pos = file_size;
SRes res = Xzs_ReadBackward(&xzs, &look_stream.vt, &start_pos, nullptr, &g_Alloc);
if (res != SZ_OK)
{
Error::SetString(error, fmt::format("Xzs_ReadBackward() failed: {}", res));
return false;
}
void GSDumpLzma::Decompress()
const size_t num_blocks = Xzs_GetNumBlocks(&xzs);
if (num_blocks == 0)
{
lzma_action action = LZMA_RUN;
Error::SetString(error, "Stream has no blocks.");
return false;
}
m_strm.next_out = m_area;
m_strm.avail_out = m_buff_size;
// Nothing left in the input buffer. Read data from the file
if (m_strm.avail_in == 0 && !feof(m_fp))
m_blocks.reserve(num_blocks);
for (int sn = xzs.num - 1; sn >= 0; sn--)
{
m_strm.next_in = m_inbuf;
m_strm.avail_in = fread(m_inbuf, 1, BUFSIZ, m_fp);
if (ferror(m_fp))
const CXzStream& stream = xzs.streams[sn];
size_t src_offset = stream.startOffset + XZ_STREAM_HEADER_SIZE;
for (size_t bn = 0; bn < stream.numBlocks; bn++)
{
Console.Error("Read error: %s", strerror(errno));
pxFailRel("LZMA read error.");
const CXzBlockSizes& block = stream.blocks[bn];
Block out_block;
out_block.file_offset = src_offset;
out_block.stream_offset = m_stream_size;
out_block.compressed_size = std::min<size_t>(Common::AlignUpPow2(block.totalSize, 4),
static_cast<size_t>(file_size - static_cast<s64>(src_offset))); // LZMA blocks are 4 byte aligned?
out_block.uncompressed_size = block.unpackSize;
out_block.stream_flags = stream.flags;
m_stream_size += out_block.uncompressed_size;
src_offset += out_block.compressed_size;
m_blocks.push_back(std::move(out_block));
}
}
lzma_ret ret = lzma_code(&m_strm, action);
if (ret != LZMA_OK)
{
if (ret != LZMA_STREAM_END)
{
Console.Error("Decoder error: (error code %u)", ret);
pxFailRel("LZMA decoder error.");
}
DevCon.WriteLnFmt("XZ stream is {} bytes across {} blocks", m_stream_size, m_blocks.size());
XzUnpacker_Construct(&m_unpacker, &g_Alloc);
return true;
}
m_start = 0;
m_avail = m_buff_size - m_strm.avail_out;
bool GSDumpLzma::DecompressNextBlock()
{
if (m_block_index == m_blocks.size())
return false;
const Block& block = m_blocks[m_block_index];
if (block.compressed_size > m_block_read_buffer.size())
m_block_read_buffer.resize(Common::AlignUpPow2(block.compressed_size, _128kb));
if (FileSystem::FSeek64(m_fp, static_cast<s64>(block.file_offset), SEEK_SET) != 0 ||
std::fread(m_block_read_buffer.data(), block.compressed_size, 1, m_fp) != 1)
{
Console.ErrorFmt("Failed to read {} bytes from offset {}", block.file_offset, block.compressed_size);
return false;
}
if (block.uncompressed_size > m_block_buffer.size())
m_block_buffer.resize(Common::AlignUpPow2(block.uncompressed_size, _128kb));
XzUnpacker_Init(&m_unpacker);
m_unpacker.streamFlags = block.stream_flags;
XzUnpacker_PrepareToRandomBlockDecoding(&m_unpacker);
XzUnpacker_SetOutBuf(&m_unpacker, m_block_buffer.data(), block.uncompressed_size);
SizeT out_uncompressed_size = block.uncompressed_size;
SizeT out_compressed_size = block.compressed_size;
ECoderStatus status;
const SRes res = XzUnpacker_Code(&m_unpacker, nullptr, &out_uncompressed_size,
m_block_read_buffer.data(), &out_compressed_size, true, CODER_FINISH_END, &status);
if (res != SZ_OK || status != CODER_STATUS_FINISHED_WITH_MARK) [[unlikely]]
{
Console.ErrorFmt("XzUnpacker_Code() failed: {} (status {})", res, static_cast<unsigned>(status));
return false;
}
if (out_compressed_size != block.compressed_size || out_uncompressed_size != block.uncompressed_size)
{
Console.WarningFmt("Decompress size mismatch: {}/{} vs {}/{}", out_compressed_size, out_uncompressed_size,
block.compressed_size, block.uncompressed_size);
}
m_block_size = out_uncompressed_size;
m_block_pos = 0;
m_block_index++;
return true;
}
bool GSDumpLzma::IsEof()
{
return feof(m_fp) && m_avail == 0 && m_strm.avail_in == 0;
return (m_block_pos == m_block_size && m_block_index == m_blocks.size());
}
size_t GSDumpLzma::Read(void* ptr, size_t size)
{
size_t off = 0;
uint8_t* dst = (uint8_t*)ptr;
while (size && !IsEof())
u8* dst = static_cast<u8*>(ptr);
size_t remain = size;
while (remain > 0)
{
if (m_avail == 0)
{
Decompress();
if (m_block_size == m_block_pos && !DecompressNextBlock()) [[unlikely]]
break;
const size_t avail = (m_block_size - m_block_pos);
const size_t read = std::min(avail, remain);
pxAssert(avail > 0 && read > 0);
std::memcpy(dst, &m_block_buffer[m_block_pos], read);
dst += read;
remain -= read;
m_block_pos += read;
}
size_t l = std::min(size, m_avail);
memcpy(dst + off, m_area + m_start, l);
m_avail -= l;
size -= l;
m_start += l;
off += l;
}
if (off > 0)
Repack(ptr, off);
return off;
}
GSDumpLzma::~GSDumpLzma()
{
lzma_end(&m_strm);
if (m_inbuf)
_aligned_free(m_inbuf);
if (m_area)
_aligned_free(m_area);
return size - remain;
}
/******************************************************************/
GSDumpDecompressZst::GSDumpDecompressZst(FILE* file, FILE* repack_file)
: GSDumpFile(file, repack_file)
class GSDumpDecompressZst final : public GSDumpFile
{
Initialize();
static constexpr u32 INPUT_BUFFER_SIZE = 512 * _1kb;
static constexpr u32 OUTPUT_BUFFER_SIZE = 2 * _1mb;
std::FILE* m_fp = nullptr;
ZSTD_DStream* m_strm = nullptr;
ZSTD_inBuffer m_inbuf = {};
uint8_t* m_area = nullptr;
size_t m_avail = 0;
size_t m_start = 0;
bool Decompress();
public:
GSDumpDecompressZst();
~GSDumpDecompressZst() override;
bool Open(std::FILE* fp, Error* error) override;
bool IsEof() override;
size_t Read(void* ptr, size_t size) override;
};
GSDumpDecompressZst::GSDumpDecompressZst() = default;
GSDumpDecompressZst::~GSDumpDecompressZst()
{
if (m_strm)
ZSTD_freeDStream(m_strm);
if (m_inbuf.src)
_aligned_free((void*)m_inbuf.src);
if (m_area)
_aligned_free(m_area);
if (m_fp)
std::fclose(m_fp);
}
void GSDumpDecompressZst::Initialize()
bool GSDumpDecompressZst::Open(std::FILE* fp, Error* error)
{
m_fp = fp;
m_strm = ZSTD_createDStream();
m_area = (uint8_t*)_aligned_malloc(OUTPUT_BUFFER_SIZE, 32);
@ -365,15 +521,16 @@ void GSDumpDecompressZst::Initialize()
m_inbuf.size = 0;
m_avail = 0;
m_start = 0;
return true;
}
void GSDumpDecompressZst::Decompress()
bool GSDumpDecompressZst::Decompress()
{
ZSTD_outBuffer outbuf = {m_area, OUTPUT_BUFFER_SIZE, 0};
while (outbuf.pos == 0)
{
// Nothing left in the input buffer. Read data from the file
if (m_inbuf.pos == m_inbuf.size && !feof(m_fp))
if (m_inbuf.pos == m_inbuf.size && !std::feof(m_fp))
{
m_inbuf.size = fread((void*)m_inbuf.src, 1, INPUT_BUFFER_SIZE, m_fp);
m_inbuf.pos = 0;
@ -381,20 +538,21 @@ void GSDumpDecompressZst::Decompress()
if (ferror(m_fp))
{
Console.Error("Zst read error: %s", strerror(errno));
pxFailRel("Zst read error.");
return false;
}
}
size_t ret = ZSTD_decompressStream(m_strm, &outbuf, &m_inbuf);
const size_t ret = ZSTD_decompressStream(m_strm, &outbuf, &m_inbuf);
if (ZSTD_isError(ret))
{
Console.Error("Decoder error: (error code %s)", ZSTD_getErrorName(ret));
pxFailRel("Zst decoder error.");
return false;
}
}
m_start = 0;
m_avail = outbuf.pos;
return true;
}
bool GSDumpDecompressZst::IsEof()
@ -410,38 +568,48 @@ size_t GSDumpDecompressZst::Read(void* ptr, size_t size)
{
if (m_avail == 0)
{
Decompress();
if (!Decompress()) [[unlikely]]
break;
}
size_t l = std::min(size, m_avail);
memcpy(dst + off, m_area + m_start, l);
const size_t l = std::min(size, m_avail);
std::memcpy(dst + off, m_area + m_start, l);
m_avail -= l;
size -= l;
m_start += l;
off += l;
}
if (off > 0)
Repack(ptr, off);
return off;
}
GSDumpDecompressZst::~GSDumpDecompressZst()
{
ZSTD_freeDStream(m_strm);
if (m_inbuf.src)
_aligned_free((void*)m_inbuf.src);
if (m_area)
_aligned_free(m_area);
}
/******************************************************************/
GSDumpRaw::GSDumpRaw(FILE* file, FILE* repack_file)
: GSDumpFile(file, repack_file)
class GSDumpRaw final : public GSDumpFile
{
std::FILE* m_fp;
public:
GSDumpRaw();
~GSDumpRaw() override;
bool Open(std::FILE* fp, Error* error) override;
bool IsEof() override;
size_t Read(void* ptr, size_t size) override;
};
GSDumpRaw::GSDumpRaw() = default;
GSDumpRaw::~GSDumpRaw()
{
if (m_fp)
std::fclose(m_fp);
}
bool GSDumpRaw::Open(std::FILE* fp, Error* error)
{
m_fp = fp;
return true;
}
bool GSDumpRaw::IsEof()
@ -458,8 +626,28 @@ size_t GSDumpRaw::Read(void* ptr, size_t size)
return ret;
}
if (ret > 0)
Repack(ptr, ret);
return ret;
}
} // namespace
/******************************************************************/
std::unique_ptr<GSDumpFile> GSDumpFile::OpenGSDump(const char* filename, Error* error)
{
std::FILE* fp = FileSystem::OpenCFile(filename, "rb", error);
if (!fp)
return nullptr;
std::unique_ptr<GSDumpFile> file;
if (StringUtil::EndsWithNoCase(filename, ".xz"))
file = std::make_unique<GSDumpLzma>();
else if (StringUtil::EndsWithNoCase(filename, ".zst"))
file = std::make_unique<GSDumpDecompressZst>();
else
file = std::make_unique<GSDumpRaw>();
if (!file->Open(fp, error))
file = {};
return file;
}

75
pcsx2/GS/GSLzma.h
View File

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
// SPDX-License-Identifier: LGPL-3.0+
#pragma once
@ -7,8 +7,7 @@
#include <string>
#include <vector>
#include <lzma.h>
#include <zstd.h>
class Error;
#define GEN_REG_ENUM_CLASS_CONTENT(ClassName, EntryName, Value) \
EntryName = Value,
@ -291,7 +290,7 @@ public:
virtual ~GSDumpFile();
static std::unique_ptr<GSDumpFile> OpenGSDump(const char* filename, const char* repack_filename = nullptr);
static std::unique_ptr<GSDumpFile> OpenGSDump(const char* filename, Error* error = nullptr);
static bool GetPreviewImageFromDump(const char* filename, u32* width, u32* height, std::vector<u32>* pixels);
__fi const std::string& GetSerial() const { return m_serial; }
@ -301,21 +300,16 @@ public:
__fi const ByteArray& GetStateData() const { return m_state_data; }
__fi const GSDataArray& GetPackets() const { return m_dump_packets; }
bool ReadFile();
bool ReadFile(Error* error);
protected:
GSDumpFile(FILE* file, FILE* repack_file);
GSDumpFile();
virtual bool Open(std::FILE* fp, Error* error) = 0;
virtual bool IsEof() = 0;
virtual size_t Read(void* ptr, size_t size) = 0;
void Repack(void* ptr, size_t size);
FILE* m_fp = nullptr;
private:
FILE* m_repack_fp = nullptr;
std::string m_serial;
u32 m_crc = 0;
@ -326,58 +320,5 @@ private:
GSDataArray m_dump_packets;
};
class GSDumpLzma : public GSDumpFile
{
lzma_stream m_strm;
size_t m_buff_size;
uint8_t* m_area;
uint8_t* m_inbuf;
size_t m_avail;
size_t m_start;
void Decompress();
void Initialize();
public:
GSDumpLzma(FILE* file, FILE* repack_file);
virtual ~GSDumpLzma();
bool IsEof() final;
size_t Read(void* ptr, size_t size) final;
};
class GSDumpDecompressZst : public GSDumpFile
{
static constexpr u32 INPUT_BUFFER_SIZE = 512 * _1kb;
static constexpr u32 OUTPUT_BUFFER_SIZE = 2 * _1mb;
ZSTD_DStream* m_strm;
ZSTD_inBuffer m_inbuf;
uint8_t* m_area;
size_t m_avail;
size_t m_start;
void Decompress();
void Initialize();
public:
GSDumpDecompressZst(FILE* file, FILE* repack_file);
virtual ~GSDumpDecompressZst();
bool IsEof() final;
size_t Read(void* ptr, size_t size) final;
};
class GSDumpRaw : public GSDumpFile
{
public:
GSDumpRaw(FILE* file, FILE* repack_file);
virtual ~GSDumpRaw() = default;
bool IsEof() final;
size_t Read(void* ptr, size_t size) final;
};
// Initializes CRC tables used by LZMA SDK.
void GSInit7ZCRCTables();

16
pcsx2/GS/Renderers/Common/GSRenderer.cpp
View File

@ -676,23 +676,23 @@ void GSRenderer::VSync(u32 field, bool registers_written, bool idle_frame)
std::string_view compression_str;
if (GSConfig.GSDumpCompression == GSDumpCompressionMethod::Uncompressed)
{
m_dump = std::unique_ptr<GSDumpBase>(new GSDumpUncompressed(m_snapshot, VMManager::GetDiscSerial(),
m_dump = GSDumpBase::CreateUncompressedDump(m_snapshot, VMManager::GetDiscSerial(),
VMManager::GetDiscCRC(), screenshot_width, screenshot_height,
screenshot_pixels.empty() ? nullptr : screenshot_pixels.data(), fd, m_regs));
screenshot_pixels.empty() ? nullptr : screenshot_pixels.data(), fd, m_regs);
compression_str = TRANSLATE_SV("GS", "with no compression");
}
else if (GSConfig.GSDumpCompression == GSDumpCompressionMethod::LZMA)
{
m_dump = std::unique_ptr<GSDumpBase>(
new GSDumpXz(m_snapshot, VMManager::GetDiscSerial(), VMManager::GetDiscCRC(), screenshot_width,
screenshot_height, screenshot_pixels.empty() ? nullptr : screenshot_pixels.data(), fd, m_regs));
m_dump = GSDumpBase::CreateXzDump(m_snapshot, VMManager::GetDiscSerial(),
VMManager::GetDiscCRC(), screenshot_width, screenshot_height,
screenshot_pixels.empty() ? nullptr : screenshot_pixels.data(), fd, m_regs);
compression_str = TRANSLATE_SV("GS", "with LZMA compression");
}
else
{
m_dump = std::unique_ptr<GSDumpBase>(
new GSDumpZst(m_snapshot, VMManager::GetDiscSerial(), VMManager::GetDiscCRC(), screenshot_width,
screenshot_height, screenshot_pixels.empty() ? nullptr : screenshot_pixels.data(), fd, m_regs));
m_dump = GSDumpBase::CreateZstDump(m_snapshot, VMManager::GetDiscSerial(),
VMManager::GetDiscCRC(), screenshot_width, screenshot_height,
screenshot_pixels.empty() ? nullptr : screenshot_pixels.data(), fd, m_regs);
compression_str = TRANSLATE_SV("GS", "with Zstandard compression");
}

18
pcsx2/GSDumpReplayer.cpp
View File

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
// SPDX-License-Identifier: LGPL-3.0+
#include "GS.h"
@ -18,7 +18,9 @@
#include "fmt/core.h"
#include "common/Error.h"
#include "common/FileSystem.h"
#include "common/Path.h"
#include "common/StringUtil.h"
#include "common/Threading.h"
#include "common/Timer.h"
@ -87,10 +89,12 @@ bool GSDumpReplayer::Initialize(const char* filename)
Common::Timer timer;
Console.WriteLn("(GSDumpReplayer) Reading file '%s'...", filename);
s_dump_file = GSDumpFile::OpenGSDump(filename);
if (!s_dump_file || !s_dump_file->ReadFile())
Error error;
s_dump_file = GSDumpFile::OpenGSDump(filename, &error);
if (!s_dump_file || !s_dump_file->ReadFile(&error))
{
Host::ReportFormattedErrorAsync("GSDumpReplayer", "Failed to open or read '%s'.", filename);
Host::ReportErrorAsync("GSDumpReplayer", fmt::format("Failed to open or read '{}': {}",
Path::GetFileName(filename), error.GetDescription()));
s_dump_file.reset();
return false;
}
@ -119,10 +123,12 @@ bool GSDumpReplayer::ChangeDump(const char* filename)
return false;
}
Error error;
std::unique_ptr<GSDumpFile> new_dump(GSDumpFile::OpenGSDump(filename));
if (!new_dump || !new_dump->ReadFile())
if (!new_dump || !new_dump->ReadFile(&error))
{
Host::ReportFormattedErrorAsync("GSDumpReplayer", "Failed to open or read '%s'.", filename);
Host::ReportErrorAsync("GSDumpReplayer", fmt::format("Failed to open or read '{}': {}",
Path::GetFileName(filename), error.GetDescription()));
return false;
}

8
pcsx2/pcsx2.vcxproj
View File

@ -36,7 +36,6 @@
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\fmt\fmt\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\wil\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\xz\xz\src\liblzma\api</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\zlib</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\lz4\lz4\lib</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\libpng</AdditionalIncludeDirectories>
@ -60,6 +59,7 @@
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\glslang\glslang</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\vulkan-headers\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\d3d12memalloc\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\lzma\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\xbyak</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)3rdparty\zydis\include;$(SolutionDir)3rdparty\zydis\dependencies\zycore\include</AdditionalIncludeDirectories>
<PrecompiledHeader>Use</PrecompiledHeader>
@ -810,9 +810,6 @@
<ProjectReference Include="$(SolutionDir)3rdparty\soundtouch\SoundTouch.vcxproj">
<Project>{e9b51944-7e6d-4bcd-83f2-7bbd5a46182d}</Project>
</ProjectReference>
<ProjectReference Include="$(SolutionDir)3rdparty\xz\liblzma.vcxproj">
<Project>{12728250-16ec-4dc6-94d7-e21dd88947f8}</Project>
</ProjectReference>
<ProjectReference Include="$(SolutionDir)3rdparty\libchdr\libchdr.vcxproj">
<Project>{a0d2b3ad-1f72-4ee3-8b5c-f2c358da35f0}</Project>
</ProjectReference>
@ -849,6 +846,9 @@
<ProjectReference Include="..\3rdparty\lz4\lz4.vcxproj">
<Project>{39098635-446a-4fc3-9b1c-8609d94598a8}</Project>
</ProjectReference>
<ProjectReference Include="..\3rdparty\lzma\lzma.vcxproj">
<Project>{a4323327-3f2b-4271-83d9-7f9a3c66b6b2}</Project>
</ProjectReference>
<ProjectReference Include="..\3rdparty\rainterface\rainterface.vcxproj">
<Project>{95dd0a0c-d14d-4cff-a593-820ef26efcc8}</Project>
</ProjectReference>