From f3832a06c130d33661a1483dac1f1d7c0314b076 Mon Sep 17 00:00:00 2001 From: "Dr. Chat" Date: Fri, 20 Nov 2015 16:53:47 -0600 Subject: [PATCH 1/2] Add zlib repository --- .gitmodules | 3 +++ premake5.lua | 1 + third_party/zlib | 1 + third_party/zlib.lua | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+) create mode 160000 third_party/zlib create mode 100644 third_party/zlib.lua diff --git a/.gitmodules b/.gitmodules index 4ffdd7bae..8a55370ac 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "third_party/libav"] path = third_party/libav url = https://github.com/xenia-project/libav.git +[submodule "third_party/zlib"] + path = third_party/zlib + url = https://github.com/madler/zlib diff --git a/premake5.lua b/premake5.lua index 05fa0ae44..90dccfab3 100644 --- a/premake5.lua +++ b/premake5.lua @@ -164,6 +164,7 @@ solution("xenia") include("third_party/imgui.lua") include("third_party/libav.lua") include("third_party/xxhash.lua") + include("third_party/zlib.lua") include("build_tools/third_party/gflags.lua") include("src/xenia") diff --git a/third_party/zlib b/third_party/zlib new file mode 160000 index 000000000..508932916 --- /dev/null +++ b/third_party/zlib @@ -0,0 +1 @@ +Subproject commit 50893291621658f355bc5b4d450a8d06a563053d diff --git a/third_party/zlib.lua b/third_party/zlib.lua new file mode 100644 index 000000000..9ce7ee50a --- /dev/null +++ b/third_party/zlib.lua @@ -0,0 +1,41 @@ +group("third_party") +project("zlib") + uuid("AF89D75F-F723-47B1-9E29-29CDFA58CCAA") + kind("StaticLib") + language("C") + links({ + }) + defines({ + "_LIB", + }) + includedirs({ + "zlib", + }) + files({ + "zlib/adler32.c", + "zlib/compress.c", + "zlib/crc32.c", + "zlib/crc32.h", + "zlib/deflate.c", + "zlib/deflate.h", + "zlib/gzclose.c", + "zlib/gzguts.h", + "zlib/gzlib.c", + "zlib/gzread.c", + "zlib/gzwrite.c", + "zlib/infback.c", + "zlib/inffast.c", + "zlib/inffast.h", + "zlib/inffixed.h", + "zlib/inflate.c", + "zlib/inflate.h", + "zlib/inftrees.c", + "zlib/inftrees.h", + "zlib/trees.c", + "zlib/trees.h", + "zlib/uncompr.c", + "zlib/zconf.h", + "zlib/zlib.h", + "zlib/zutil.c", + "zlib/zutil.h", + }) From 0f2f4ab9b6bae03dc7b2559031b5a89d72b41df4 Mon Sep 17 00:00:00 2001 From: "Dr. Chat" Date: Fri, 20 Nov 2015 16:58:40 -0600 Subject: [PATCH 2/2] Compress/decompress memory automatically in trace files --- src/xenia/gpu/premake5.lua | 1 + src/xenia/gpu/trace_player.cc | 10 +++++-- src/xenia/gpu/trace_protocol.h | 2 ++ src/xenia/gpu/trace_reader.cc | 10 +++++++ src/xenia/gpu/trace_reader.h | 2 ++ src/xenia/gpu/trace_writer.cc | 55 ++++++++++++++++++++++++++++++---- src/xenia/gpu/trace_writer.h | 6 ++++ 7 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/xenia/gpu/premake5.lua b/src/xenia/gpu/premake5.lua index 521a04869..7889f320e 100644 --- a/src/xenia/gpu/premake5.lua +++ b/src/xenia/gpu/premake5.lua @@ -11,6 +11,7 @@ project("xenia-gpu") "xenia-base", "xenia-ui", "xxhash", + "zlib", }) defines({ }) diff --git a/src/xenia/gpu/trace_player.cc b/src/xenia/gpu/trace_player.cc index 49f91d82d..40291453f 100644 --- a/src/xenia/gpu/trace_player.cc +++ b/src/xenia/gpu/trace_player.cc @@ -150,8 +150,14 @@ void TracePlayer::PlayTraceOnThread(const uint8_t* trace_data, case TraceCommandType::kMemoryRead: { auto cmd = reinterpret_cast(trace_ptr); trace_ptr += sizeof(*cmd); - std::memcpy(memory->TranslatePhysical(cmd->base_ptr), trace_ptr, - cmd->length); + if (cmd->full_length) { + DecompressMemory(trace_ptr, cmd->length, + memory->TranslatePhysical(cmd->base_ptr), + cmd->full_length); + } else { + std::memcpy(memory->TranslatePhysical(cmd->base_ptr), trace_ptr, + cmd->length); + } trace_ptr += cmd->length; break; } diff --git a/src/xenia/gpu/trace_protocol.h b/src/xenia/gpu/trace_protocol.h index 1779b69f8..96ba70b54 100644 --- a/src/xenia/gpu/trace_protocol.h +++ b/src/xenia/gpu/trace_protocol.h @@ -61,12 +61,14 @@ struct MemoryReadCommand { TraceCommandType type; uint32_t base_ptr; uint32_t length; + uint32_t full_length; // Length after inflation. 0 if not deflated. }; struct MemoryWriteCommand { TraceCommandType type; uint32_t base_ptr; uint32_t length; + uint32_t full_length; // Length after inflation. 0 if not deflated. }; enum class EventType { diff --git a/src/xenia/gpu/trace_reader.cc b/src/xenia/gpu/trace_reader.cc index 8ed29e4ce..4f8da3e89 100644 --- a/src/xenia/gpu/trace_reader.cc +++ b/src/xenia/gpu/trace_reader.cc @@ -14,6 +14,8 @@ #include "xenia/gpu/trace_protocol.h" #include "xenia/memory.h" +#include "third_party/zlib/zlib.h" + namespace xe { namespace gpu { @@ -148,5 +150,13 @@ void TraceReader::ParseTrace() { } } +bool TraceReader::DecompressMemory(const uint8_t* src, size_t src_size, + uint8_t* dest, size_t dest_size) { + uLongf dest_len = uint32_t(dest_size); + int ret = uncompress(dest, &dest_len, src, uint32_t(src_size)); + assert_true(ret >= 0); + return ret >= 0; +} + } // namespace gpu } // namespace xe diff --git a/src/xenia/gpu/trace_reader.h b/src/xenia/gpu/trace_reader.h index 8b5798772..fff24279d 100644 --- a/src/xenia/gpu/trace_reader.h +++ b/src/xenia/gpu/trace_reader.h @@ -89,6 +89,8 @@ class TraceReader { protected: void ParseTrace(); + bool DecompressMemory(const uint8_t* src, size_t src_size, uint8_t* dest, + size_t dest_size); std::unique_ptr mmap_; const uint8_t* trace_data_ = nullptr; diff --git a/src/xenia/gpu/trace_writer.cc b/src/xenia/gpu/trace_writer.cc index ada89f379..5cebc2af6 100644 --- a/src/xenia/gpu/trace_writer.cc +++ b/src/xenia/gpu/trace_writer.cc @@ -9,8 +9,12 @@ #include "xenia/gpu/trace_writer.h" +#include "xenia/base/assert.h" +#include "xenia/base/logging.h" #include "xenia/base/string.h" +#include "third_party/zlib/zlib.h" + namespace xe { namespace gpu { @@ -22,6 +26,9 @@ TraceWriter::~TraceWriter() = default; bool TraceWriter::Open(const std::wstring& path) { Close(); + // Reserve 2 MB of space. + tmp_buff_.reserve(1024 * 2048); + auto canonical_path = xe::to_absolute_path(path); auto base_path = xe::find_base_path(canonical_path); xe::filesystem::CreateFolder(base_path); @@ -109,22 +116,48 @@ void TraceWriter::WriteMemoryRead(uint32_t base_ptr, size_t length) { if (!file_) { return; } + + bool compress = compress_output_ && length > compression_threshold_; + auto cmd = MemoryReadCommand({ - TraceCommandType::kMemoryRead, base_ptr, uint32_t(length), + TraceCommandType::kMemoryRead, base_ptr, uint32_t(length), 0, }); - fwrite(&cmd, 1, sizeof(cmd), file_); - fwrite(membase_ + base_ptr, 1, length, file_); + + if (compress) { + size_t written = WriteCompressed(membase_ + base_ptr, length); + cmd.length = uint32_t(written); + cmd.full_length = uint32_t(length); + + fwrite(&cmd, 1, sizeof(cmd), file_); + fwrite(tmp_buff_.data(), 1, written, file_); + } else { + fwrite(&cmd, 1, sizeof(cmd), file_); + fwrite(membase_ + base_ptr, 1, length, file_); + } } void TraceWriter::WriteMemoryWrite(uint32_t base_ptr, size_t length) { if (!file_) { return; } + + bool compress = compress_output_ && length > compression_threshold_; + auto cmd = MemoryWriteCommand({ - TraceCommandType::kMemoryWrite, base_ptr, uint32_t(length), + TraceCommandType::kMemoryWrite, base_ptr, uint32_t(length), 0, }); - fwrite(&cmd, 1, sizeof(cmd), file_); - fwrite(membase_ + base_ptr, 1, length, file_); + + if (compress) { + size_t written = WriteCompressed(membase_ + base_ptr, length); + cmd.length = uint32_t(written); + cmd.full_length = uint32_t(length); + + fwrite(&cmd, 1, sizeof(cmd), file_); + fwrite(tmp_buff_.data(), 1, written, file_); + } else { + fwrite(&cmd, 1, sizeof(cmd), file_); + fwrite(membase_ + base_ptr, 1, length, file_); + } } void TraceWriter::WriteEvent(EventType event_type) { @@ -137,5 +170,15 @@ void TraceWriter::WriteEvent(EventType event_type) { fwrite(&cmd, 1, sizeof(cmd), file_); } +size_t TraceWriter::WriteCompressed(void* buf, size_t length) { + tmp_buff_.resize(compressBound(uint32_t(length))); + + uLongf dest_len = (uint32_t)tmp_buff_.size(); + int ret = + compress(tmp_buff_.data(), &dest_len, (uint8_t*)buf, uint32_t(length)); + assert_true(ret >= 0); + return dest_len; +} + } // namespace gpu } // namespace xe diff --git a/src/xenia/gpu/trace_writer.h b/src/xenia/gpu/trace_writer.h index 915c18e65..cc4559c7f 100644 --- a/src/xenia/gpu/trace_writer.h +++ b/src/xenia/gpu/trace_writer.h @@ -40,8 +40,14 @@ class TraceWriter { void WriteEvent(EventType event_type); private: + size_t WriteCompressed(void* buf, size_t length); + uint8_t* membase_; FILE* file_; + + bool compress_output_ = true; + size_t compression_threshold_ = 0x1000; // min. number of bytes to compress. + std::vector tmp_buff_; }; } // namespace gpu