Merge pull request #466 from DrChat/trace_compression

GPU Tracefile Compression

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "third_party/libav"]
 	path = third_party/libav
 	url = https://github.com/xenia-project/libav.git
+[submodule "third_party/zlib"]
+	path = third_party/zlib
+	url = https://github.com/madler/zlib

premake5.lua

@@ -164,6 +164,7 @@ solution("xenia")
   include("third_party/imgui.lua")
   include("third_party/libav.lua")
   include("third_party/xxhash.lua")
+  include("third_party/zlib.lua")
   include("build_tools/third_party/gflags.lua")
 
   include("src/xenia")

src/xenia/gpu/premake5.lua

@@ -11,6 +11,7 @@ project("xenia-gpu")
     "xenia-base",
     "xenia-ui",
     "xxhash",
+    "zlib",
   })
   defines({
   })

src/xenia/gpu/trace_player.cc

@@ -150,8 +150,14 @@ void TracePlayer::PlayTraceOnThread(const uint8_t* trace_data,
       case TraceCommandType::kMemoryRead: {
         auto cmd = reinterpret_cast<const MemoryReadCommand*>(trace_ptr);
         trace_ptr += sizeof(*cmd);
-        std::memcpy(memory->TranslatePhysical(cmd->base_ptr), trace_ptr,
+        if (cmd->full_length) {
-                    cmd->length);
+          DecompressMemory(trace_ptr, cmd->length,
+                           memory->TranslatePhysical(cmd->base_ptr),
+                           cmd->full_length);
+        } else {
+          std::memcpy(memory->TranslatePhysical(cmd->base_ptr), trace_ptr,
+                      cmd->length);
+        }
         trace_ptr += cmd->length;
         break;
       }

src/xenia/gpu/trace_protocol.h

@@ -61,12 +61,14 @@ struct MemoryReadCommand {
   TraceCommandType type;
   uint32_t base_ptr;
   uint32_t length;
+  uint32_t full_length;  // Length after inflation. 0 if not deflated.
 };
 
 struct MemoryWriteCommand {
   TraceCommandType type;
   uint32_t base_ptr;
   uint32_t length;
+  uint32_t full_length;  // Length after inflation. 0 if not deflated.
 };
 
 enum class EventType {

src/xenia/gpu/trace_reader.cc

@@ -14,6 +14,8 @@
 #include "xenia/gpu/trace_protocol.h"
 #include "xenia/memory.h"
 
+#include "third_party/zlib/zlib.h"
+
 namespace xe {
 namespace gpu {
 
@@ -148,5 +150,13 @@ void TraceReader::ParseTrace() {
   }
 }
 
+bool TraceReader::DecompressMemory(const uint8_t* src, size_t src_size,
+                                   uint8_t* dest, size_t dest_size) {
+  uLongf dest_len = uint32_t(dest_size);
+  int ret = uncompress(dest, &dest_len, src, uint32_t(src_size));
+  assert_true(ret >= 0);
+  return ret >= 0;
+}
+
 }  // namespace gpu
 }  // namespace xe

src/xenia/gpu/trace_reader.h

@@ -89,6 +89,8 @@ class TraceReader {
 
  protected:
   void ParseTrace();
+  bool DecompressMemory(const uint8_t* src, size_t src_size, uint8_t* dest,
+                        size_t dest_size);
 
   std::unique_ptr<MappedMemory> mmap_;
   const uint8_t* trace_data_ = nullptr;

src/xenia/gpu/trace_writer.cc

@@ -9,8 +9,12 @@
 
 #include "xenia/gpu/trace_writer.h"
 
+#include "xenia/base/assert.h"
+#include "xenia/base/logging.h"
 #include "xenia/base/string.h"
 
+#include "third_party/zlib/zlib.h"
+
 namespace xe {
 namespace gpu {
 
@@ -22,6 +26,9 @@ TraceWriter::~TraceWriter() = default;
 bool TraceWriter::Open(const std::wstring& path) {
   Close();
 
+  // Reserve 2 MB of space.
+  tmp_buff_.reserve(1024 * 2048);
+
   auto canonical_path = xe::to_absolute_path(path);
   auto base_path = xe::find_base_path(canonical_path);
   xe::filesystem::CreateFolder(base_path);
@@ -109,22 +116,48 @@ void TraceWriter::WriteMemoryRead(uint32_t base_ptr, size_t length) {
   if (!file_) {
     return;
   }
+
+  bool compress = compress_output_ && length > compression_threshold_;
+
   auto cmd = MemoryReadCommand({
-      TraceCommandType::kMemoryRead, base_ptr, uint32_t(length),
+      TraceCommandType::kMemoryRead, base_ptr, uint32_t(length), 0,
   });
-  fwrite(&cmd, 1, sizeof(cmd), file_);
+
-  fwrite(membase_ + base_ptr, 1, length, file_);
+  if (compress) {
+    size_t written = WriteCompressed(membase_ + base_ptr, length);
+    cmd.length = uint32_t(written);
+    cmd.full_length = uint32_t(length);
+
+    fwrite(&cmd, 1, sizeof(cmd), file_);
+    fwrite(tmp_buff_.data(), 1, written, file_);
+  } else {
+    fwrite(&cmd, 1, sizeof(cmd), file_);
+    fwrite(membase_ + base_ptr, 1, length, file_);
+  }
 }
 
 void TraceWriter::WriteMemoryWrite(uint32_t base_ptr, size_t length) {
   if (!file_) {
     return;
   }
+
+  bool compress = compress_output_ && length > compression_threshold_;
+
   auto cmd = MemoryWriteCommand({
-      TraceCommandType::kMemoryWrite, base_ptr, uint32_t(length),
+      TraceCommandType::kMemoryWrite, base_ptr, uint32_t(length), 0,
   });
-  fwrite(&cmd, 1, sizeof(cmd), file_);
+
-  fwrite(membase_ + base_ptr, 1, length, file_);
+  if (compress) {
+    size_t written = WriteCompressed(membase_ + base_ptr, length);
+    cmd.length = uint32_t(written);
+    cmd.full_length = uint32_t(length);
+
+    fwrite(&cmd, 1, sizeof(cmd), file_);
+    fwrite(tmp_buff_.data(), 1, written, file_);
+  } else {
+    fwrite(&cmd, 1, sizeof(cmd), file_);
+    fwrite(membase_ + base_ptr, 1, length, file_);
+  }
 }
 
 void TraceWriter::WriteEvent(EventType event_type) {
@@ -137,5 +170,15 @@ void TraceWriter::WriteEvent(EventType event_type) {
   fwrite(&cmd, 1, sizeof(cmd), file_);
 }
 
+size_t TraceWriter::WriteCompressed(void* buf, size_t length) {
+  tmp_buff_.resize(compressBound(uint32_t(length)));
+
+  uLongf dest_len = (uint32_t)tmp_buff_.size();
+  int ret =
+      compress(tmp_buff_.data(), &dest_len, (uint8_t*)buf, uint32_t(length));
+  assert_true(ret >= 0);
+  return dest_len;
+}
+
 }  //  namespace gpu
 }  //  namespace xe

src/xenia/gpu/trace_writer.h

@@ -40,8 +40,14 @@ class TraceWriter {
   void WriteEvent(EventType event_type);
 
  private:
+  size_t WriteCompressed(void* buf, size_t length);
+
   uint8_t* membase_;
   FILE* file_;
+
+  bool compress_output_ = true;
+  size_t compression_threshold_ = 0x1000;  // min. number of bytes to compress.
+  std::vector<uint8_t> tmp_buff_;
 };
 
 }  // namespace gpu

third_party/zlib

@@ -0,0 +1 @@
+Subproject commit 50893291621658f355bc5b4d450a8d06a563053d

third_party/zlib.lua

@@ -0,0 +1,41 @@
+group("third_party")
+project("zlib")
+  uuid("AF89D75F-F723-47B1-9E29-29CDFA58CCAA")
+  kind("StaticLib")
+  language("C")
+  links({
+  })
+  defines({
+    "_LIB",
+  })
+  includedirs({
+    "zlib",
+  })
+  files({
+    "zlib/adler32.c",
+    "zlib/compress.c",
+    "zlib/crc32.c",
+    "zlib/crc32.h",
+    "zlib/deflate.c",
+    "zlib/deflate.h",
+    "zlib/gzclose.c",
+    "zlib/gzguts.h",
+    "zlib/gzlib.c",
+    "zlib/gzread.c",
+    "zlib/gzwrite.c",
+    "zlib/infback.c",
+    "zlib/inffast.c",
+    "zlib/inffast.h",
+    "zlib/inffixed.h",
+    "zlib/inflate.c",
+    "zlib/inflate.h",
+    "zlib/inftrees.c",
+    "zlib/inftrees.h",
+    "zlib/trees.c",
+    "zlib/trees.h",
+    "zlib/uncompr.c",
+    "zlib/zconf.h",
+    "zlib/zlib.h",
+    "zlib/zutil.c",
+    "zlib/zutil.h",
+  })