Replacing zlib with snappy in traces, adding trace header, docs.

.gitmodules

@@ -13,9 +13,6 @@
 [submodule "third_party/libav"]
 	path = third_party/libav
 	url = https://github.com/xenia-project/libav.git
-[submodule "third_party/zlib"]
-	path = third_party/zlib
-	url = https://github.com/madler/zlib
 [submodule "third_party/spirv-tools"]
 	path = third_party/spirv-tools
 	url = https://github.com/xenia-project/SPIRV-Tools.git

premake5.lua

@@ -166,7 +166,6 @@ solution("xenia")
   include("third_party/snappy.lua")
   include("third_party/spirv-tools.lua")
   include("third_party/xxhash.lua")
-  include("third_party/zlib.lua")
 
   include("src/xenia")
   include("src/xenia/app")

src/xenia/gpu/command_processor.cc

@@ -20,6 +20,8 @@
 #include "xenia/gpu/sampler_info.h"
 #include "xenia/gpu/texture_info.h"
 #include "xenia/gpu/xenos.h"
+#include "xenia/kernel/kernel_state.h"
+#include "xenia/kernel/user_module.h"
 
 namespace xe {
 namespace gpu {
@@ -86,9 +88,9 @@ void CommandProcessor::BeginTracing(const std::wstring& root_path) {
     XELOGE("Frame trace pending; ignoring streaming request.");
     return;
   }
-  std::wstring path = root_path + L"stream";
+  // Streaming starts on the next primary buffer execute.
   trace_state_ = TraceState::kStreaming;
-  trace_writer_.Open(path);
+  trace_stream_path_ = root_path;
 }
 
 void CommandProcessor::EndTracing() {
@@ -406,6 +408,18 @@ void CommandProcessor::ExecutePrimaryBuffer(uint32_t start_index,
                                             uint32_t end_index) {
   SCOPE_profile_cpu_f("gpu");
 
+  // If we have a pending trace stream open it now. That way we ensure we get
+  // all commands.
+  if (!trace_writer_.is_open() && trace_state_ == TraceState::kStreaming) {
+    uint32_t title_id = kernel_state_->GetExecutableModule()
+                            ? kernel_state_->GetExecutableModule()->title_id()
+                            : 0;
+    auto file_name =
+        xe::format_string(L"title_%8X_stream.xenia_gpu_trace", title_id);
+    auto path = trace_stream_path_ + file_name;
+    trace_writer_.Open(path, title_id);
+  }
+
   // Adjust pointer base.
   uint32_t start_ptr = primary_buffer_ptr_ + start_index * sizeof(uint32_t);
   start_ptr = (primary_buffer_ptr_ & ~0x1FFFFFFF) | (start_ptr & 0x1FFFFFFF);
@@ -718,7 +732,7 @@ bool CommandProcessor::ExecutePacketType3_XE_SWAP(RingbufferReader* reader,
   }
 
   if (trace_writer_.is_open()) {
-    trace_writer_.WriteEvent(EventType::kSwap);
+    trace_writer_.WriteEvent(EventCommand::Type::kSwap);
     trace_writer_.Flush();
     if (trace_state_ == TraceState::kSingleFrame) {
       trace_state_ = TraceState::kDisabled;
@@ -726,9 +740,11 @@ bool CommandProcessor::ExecutePacketType3_XE_SWAP(RingbufferReader* reader,
     }
   } else if (trace_state_ == TraceState::kSingleFrame) {
     // New trace request - we only start tracing at the beginning of a frame.
-    auto frame_number = L"frame_" + std::to_wstring(counter_);
+    uint32_t title_id = kernel_state_->GetExecutableModule()->title_id();
-    auto path = trace_frame_path_ + frame_number;
+    auto file_name = xe::format_string(L"title_%8X_frame_%u.xenia_gpu_trace",
-    trace_writer_.Open(path);
+                                       title_id, counter_);
+    auto path = trace_frame_path_ + file_name;
+    trace_writer_.Open(path, title_id);
   }
   ++counter_;
   return true;

src/xenia/gpu/command_processor.h

@@ -196,6 +196,7 @@ class CommandProcessor {
     kSingleFrame,
   };
   TraceState trace_state_ = TraceState::kDisabled;
+  std::wstring trace_stream_path_;
   std::wstring trace_frame_path_;
 
   std::atomic<bool> worker_running_;

src/xenia/gpu/gpu_flags.cc

@@ -9,7 +9,7 @@
 
 #include "xenia/gpu/gpu_flags.h"
 
-DEFINE_string(trace_gpu_prefix, "scratch/gpu/gpu_trace_",
+DEFINE_string(trace_gpu_prefix, "scratch/gpu/",
               "Prefix path for GPU trace files.");
 DEFINE_bool(trace_gpu_stream, false, "Trace all GPU packets.");
 

src/xenia/gpu/premake5.lua

@@ -7,12 +7,12 @@ project("xenia-gpu")
   kind("StaticLib")
   language("C++")
   links({
+    "snappy",
     "spirv-tools",
     "xenia-base",
     "xenia-ui",
     "xenia-ui-spirv",
     "xxhash",
-    "zlib",
   })
   defines({
   })

src/xenia/gpu/trace_dump.cc

@@ -60,7 +60,8 @@ int TraceDump::Main(const std::vector<std::wstring>& args) {
     file_picker->set_multi_selection(false);
     file_picker->set_title(L"Select Trace File");
     file_picker->set_extensions({
-        {L"Supported Files", L"*.*"}, {L"All Files (*.*)", L"*.*"},
+        {L"Supported Files", L"*.xenia_gpu_trace"},
+        {L"All Files (*.*)", L"*.*"},
     });
     if (file_picker->Show()) {
       auto selected_files = file_picker->selected_files();

src/xenia/gpu/trace_player.cc

@@ -162,31 +162,26 @@ void TracePlayer::PlayTraceOnThread(const uint8_t* trace_data,
         break;
       }
       case TraceCommandType::kMemoryRead: {
-        auto cmd = reinterpret_cast<const MemoryReadCommand*>(trace_ptr);
+        auto cmd = reinterpret_cast<const MemoryCommand*>(trace_ptr);
         trace_ptr += sizeof(*cmd);
-        if (cmd->full_length) {
+        DecompressMemory(cmd->encoding_format, trace_ptr, cmd->encoded_length,
-          DecompressMemory(trace_ptr, cmd->length,
+                         memory->TranslatePhysical(cmd->base_ptr),
-                           memory->TranslatePhysical(cmd->base_ptr),
+                         cmd->decoded_length);
-                           cmd->full_length);
+        trace_ptr += cmd->encoded_length;
-        } else {
-          std::memcpy(memory->TranslatePhysical(cmd->base_ptr), trace_ptr,
-                      cmd->length);
-        }
-        trace_ptr += cmd->length;
         break;
       }
       case TraceCommandType::kMemoryWrite: {
-        auto cmd = reinterpret_cast<const MemoryWriteCommand*>(trace_ptr);
+        auto cmd = reinterpret_cast<const MemoryCommand*>(trace_ptr);
         trace_ptr += sizeof(*cmd);
         // ?
-        trace_ptr += cmd->length;
+        trace_ptr += cmd->encoded_length;
         break;
       }
       case TraceCommandType::kEvent: {
         auto cmd = reinterpret_cast<const EventCommand*>(trace_ptr);
         trace_ptr += sizeof(*cmd);
         switch (cmd->event_type) {
-          case EventType::kSwap: {
+          case EventCommand::Type::kSwap: {
             if (playback_mode == TracePlaybackMode::kBreakOnSwap) {
               pending_break = true;
             }

src/xenia/gpu/trace_protocol.h

@@ -15,6 +15,30 @@
 namespace xe {
 namespace gpu {
 
+// Any byte changes to the files should bump this version.
+// Only builds with matching versions will work.
+// Other changes besides the file format may require bumps, such as
+// anything that changes what is recorded into the files (new GPU
+// command processor commands, etc).
+constexpr uint32_t kTraceFormatVersion = 1;
+
+// Trace file header identifying information about the trace.
+// This must be positioned at the start of the file and must only occur once.
+struct TraceHeader {
+  // Must be the first 4 bytes of the file.
+  // Set to kTraceFormatVersion.
+  uint32_t version;
+
+  // SHA1 of the commit used to record the trace.
+  char build_commit_sha[40];
+
+  // Title ID of game that was being recorded.
+  // May be 0 if not generated from a game or the ID could not be retrieved.
+  uint32_t title_id;
+};
+
+// Tags each command in the trace file stream as one of the *Command types.
+// Each command has this value as its first dword.
 enum class TraceCommandType : uint32_t {
   kPrimaryBufferStart,
   kPrimaryBufferEnd,
@@ -57,27 +81,41 @@ struct PacketEndCommand {
   TraceCommandType type;
 };
 
-struct MemoryReadCommand {
+// The compression format used for memory read/write buffers.
+// Note that not every memory read/write will have compressed data
+// (as it's silly to compress 4 byte buffers).
+enum class MemoryEncodingFormat {
+  // Data is in its raw form. encoded_length == decoded_length.
+  kNone,
+  // Data is compressed with third_party/snappy.
+  kSnappy,
+};
+
+// Represents the GPU reading or writing data from or to memory.
+// Used for both TraceCommandType::kMemoryRead and kMemoryWrite.
+struct MemoryCommand {
   TraceCommandType type;
+
+  // Base physical memory pointer this read starts at.
   uint32_t base_ptr;
-  uint32_t length;
+  // Encoding format of the data in the trace file.
-  uint32_t full_length;  // Length after inflation. 0 if not deflated.
+  MemoryEncodingFormat encoding_format;
-};
+  // Number of bytes the data occupies in the trace file in its encoded form.
-
+  uint32_t encoded_length;
-struct MemoryWriteCommand {
+  // Number of bytes the data occupies in memory after decoding.
-  TraceCommandType type;
+  // Note that if no encoding is used this will equal encoded_length.
-  uint32_t base_ptr;
+  uint32_t decoded_length;
-  uint32_t length;
-  uint32_t full_length;  // Length after inflation. 0 if not deflated.
-};
-
-enum class EventType {
-  kSwap,
 };
 
+// Represents a GPU event of EventCommand::Type.
 struct EventCommand {
   TraceCommandType type;
-  EventType event_type;
+
+  // Identifies the event that occurred.
+  enum class Type {
+    kSwap,
+  };
+  Type event_type;
 };
 
 }  // namespace gpu

src/xenia/gpu/trace_reader.cc

@@ -9,13 +9,17 @@
 
 #include "xenia/gpu/trace_reader.h"
 
+#include <cinttypes>
+
+#include "third_party/snappy/snappy.h"
+
+#include "xenia/base/logging.h"
 #include "xenia/base/mapped_memory.h"
+#include "xenia/base/math.h"
 #include "xenia/gpu/packet_disassembler.h"
 #include "xenia/gpu/trace_protocol.h"
 #include "xenia/memory.h"
 
-#include "third_party/zlib/zlib.h"
-
 namespace xe {
 namespace gpu {
 
@@ -30,6 +34,25 @@ bool TraceReader::Open(const std::wstring& path) {
   trace_data_ = reinterpret_cast<const uint8_t*>(mmap_->data());
   trace_size_ = mmap_->size();
 
+  // Verify version.
+  auto header = reinterpret_cast<const TraceHeader*>(trace_data_);
+  if (header->version != kTraceFormatVersion) {
+    XELOGE("Trace format version mismatch, code has %u, file has %u",
+           kTraceFormatVersion, header->version);
+    if (header->version < kTraceFormatVersion) {
+      XELOGE("You need to regenerate your trace for the latest version");
+    }
+    return false;
+  }
+
+  auto path_str = xe::to_string(path);
+  XELOGI("Mapped %" PRId64 "b trace from %s", trace_size_, path_str.c_str());
+  XELOGI("   Version: %u", header->version);
+  auto commit_str = std::string(header->build_commit_sha,
+                                xe::countof(header->build_commit_sha));
+  XELOGI("    Commit: %s", commit_str.c_str());
+  XELOGI("  Title ID: %u", header->title_id);
+
   ParseTrace();
 
   return true;
@@ -42,7 +65,10 @@ void TraceReader::Close() {
 }
 
 void TraceReader::ParseTrace() {
+  // Skip file header.
   auto trace_ptr = trace_data_;
+  trace_ptr += sizeof(TraceHeader);
+
   Frame current_frame;
   current_frame.start_ptr = trace_ptr;
   const PacketStartCommand* packet_start = nullptr;
@@ -117,20 +143,20 @@ void TraceReader::ParseTrace() {
         break;
       }
       case TraceCommandType::kMemoryRead: {
-        auto cmd = reinterpret_cast<const MemoryReadCommand*>(trace_ptr);
+        auto cmd = reinterpret_cast<const MemoryCommand*>(trace_ptr);
-        trace_ptr += sizeof(*cmd) + cmd->length;
+        trace_ptr += sizeof(*cmd) + cmd->encoded_length;
         break;
       }
       case TraceCommandType::kMemoryWrite: {
-        auto cmd = reinterpret_cast<const MemoryWriteCommand*>(trace_ptr);
+        auto cmd = reinterpret_cast<const MemoryCommand*>(trace_ptr);
-        trace_ptr += sizeof(*cmd) + cmd->length;
+        trace_ptr += sizeof(*cmd) + cmd->encoded_length;
         break;
       }
       case TraceCommandType::kEvent: {
         auto cmd = reinterpret_cast<const EventCommand*>(trace_ptr);
         trace_ptr += sizeof(*cmd);
         switch (cmd->event_type) {
-          case EventType::kSwap: {
+          case EventCommand::Type::kSwap: {
             pending_break = true;
             break;
           }
@@ -149,12 +175,21 @@ void TraceReader::ParseTrace() {
   }
 }
 
-bool TraceReader::DecompressMemory(const uint8_t* src, size_t src_size,
+bool TraceReader::DecompressMemory(MemoryEncodingFormat encoding_format,
+                                   const uint8_t* src, size_t src_size,
                                    uint8_t* dest, size_t dest_size) {
-  uLongf dest_len = uint32_t(dest_size);
+  switch (encoding_format) {
-  int ret = uncompress(dest, &dest_len, src, uint32_t(src_size));
+    case MemoryEncodingFormat::kNone:
-  assert_true(ret >= 0);
+      assert_true(src_size == dest_size);
-  return ret >= 0;
+      std::memcpy(dest, src, src_size);
+      return true;
+    case MemoryEncodingFormat::kSnappy:
+      return snappy::RawUncompress(reinterpret_cast<const char*>(src), src_size,
+                                   reinterpret_cast<char*>(dest));
+    default:
+      assert_unhandled_case(encoding_format);
+      return false;
+  }
 }
 
 }  // namespace gpu

src/xenia/gpu/trace_reader.h

@@ -80,6 +80,10 @@ class TraceReader {
   TraceReader() = default;
   virtual ~TraceReader() = default;
 
+  const TraceHeader* header() const {
+    return reinterpret_cast<const TraceHeader*>(trace_data_);
+  }
+
   const Frame* frame(int n) const { return &frames_[n]; }
   int frame_count() const { return int(frames_.size()); }
 
@@ -89,7 +93,8 @@ class TraceReader {
 
  protected:
   void ParseTrace();
-  bool DecompressMemory(const uint8_t* src, size_t src_size, uint8_t* dest,
+  bool DecompressMemory(MemoryEncodingFormat encoding_format,
+                        const uint8_t* src, size_t src_size, uint8_t* dest,
                         size_t dest_size);
 
   std::unique_ptr<MappedMemory> mmap_;

src/xenia/gpu/trace_viewer.cc

@@ -68,7 +68,8 @@ int TraceViewer::Main(const std::vector<std::wstring>& args) {
     file_picker->set_multi_selection(false);
     file_picker->set_title(L"Select Trace File");
     file_picker->set_extensions({
-        {L"Supported Files", L"*.*"}, {L"All Files (*.*)", L"*.*"},
+        {L"Supported Files", L"*.xenia_gpu_trace"},
+        {L"All Files (*.*)", L"*.*"},
     });
     if (file_picker->Show()) {
       auto selected_files = file_picker->selected_files();
@@ -358,14 +359,14 @@ void TraceViewer::DrawPacketDisassemblerUI() {
         break;
       }
       case TraceCommandType::kMemoryRead: {
-        auto cmd = reinterpret_cast<const MemoryReadCommand*>(trace_ptr);
+        auto cmd = reinterpret_cast<const MemoryCommand*>(trace_ptr);
-        trace_ptr += sizeof(*cmd) + cmd->length;
+        trace_ptr += sizeof(*cmd) + cmd->encoded_length;
         // ImGui::BulletText("MemoryRead");
         break;
       }
       case TraceCommandType::kMemoryWrite: {
-        auto cmd = reinterpret_cast<const MemoryWriteCommand*>(trace_ptr);
+        auto cmd = reinterpret_cast<const MemoryCommand*>(trace_ptr);
-        trace_ptr += sizeof(*cmd) + cmd->length;
+        trace_ptr += sizeof(*cmd) + cmd->encoded_length;
         // ImGui::BulletText("MemoryWrite");
         break;
       }
@@ -373,7 +374,7 @@ void TraceViewer::DrawPacketDisassemblerUI() {
         auto cmd = reinterpret_cast<const EventCommand*>(trace_ptr);
         trace_ptr += sizeof(*cmd);
         switch (cmd->event_type) {
-          case EventType::kSwap: {
+          case EventCommand::Type::kSwap: {
             ImGui::BulletText("<swap>");
             break;
           }

src/xenia/gpu/trace_writer.cc

@@ -9,12 +9,15 @@
 
 #include "xenia/gpu/trace_writer.h"
 
+#include "third_party/snappy/snappy-sinksource.h"
+#include "third_party/snappy/snappy.h"
+
+#include "build/version.h"
+
 #include "xenia/base/assert.h"
 #include "xenia/base/logging.h"
 #include "xenia/base/string.h"
 
-#include "third_party/zlib/zlib.h"
-
 namespace xe {
 namespace gpu {
 
@@ -23,18 +26,27 @@ TraceWriter::TraceWriter(uint8_t* membase)
 
 TraceWriter::~TraceWriter() = default;
 
-bool TraceWriter::Open(const std::wstring& path) {
+bool TraceWriter::Open(const std::wstring& path, uint32_t title_id) {
   Close();
 
-  // Reserve 2 MB of space.
-  tmp_buff_.reserve(1024 * 2048);
-
   auto canonical_path = xe::to_absolute_path(path);
   auto base_path = xe::find_base_path(canonical_path);
   xe::filesystem::CreateFolder(base_path);
 
   file_ = xe::filesystem::OpenFile(canonical_path, "wb");
-  return file_ != nullptr;
+  if (!file_) {
+    return false;
+  }
+
+  // Write header first. Must be at the top of the file.
+  TraceHeader header;
+  header.version = kTraceFormatVersion;
+  std::memcpy(header.build_commit_sha, XE_BUILD_COMMIT,
+              sizeof(header.build_commit_sha));
+  header.title_id = title_id;
+  fwrite(&header, sizeof(header), 1, file_);
+
+  return true;
 }
 
 void TraceWriter::Flush() {
@@ -55,9 +67,9 @@ void TraceWriter::WritePrimaryBufferStart(uint32_t base_ptr, uint32_t count) {
   if (!file_) {
     return;
   }
-  auto cmd = PrimaryBufferStartCommand({
+  PrimaryBufferStartCommand cmd = {
       TraceCommandType::kPrimaryBufferStart, base_ptr, 0,
-  });
+  };
   fwrite(&cmd, 1, sizeof(cmd), file_);
 }
 
@@ -65,9 +77,9 @@ void TraceWriter::WritePrimaryBufferEnd() {
   if (!file_) {
     return;
   }
-  auto cmd = PrimaryBufferEndCommand({
+  PrimaryBufferEndCommand cmd = {
       TraceCommandType::kPrimaryBufferEnd,
-  });
+  };
   fwrite(&cmd, 1, sizeof(cmd), file_);
 }
 
@@ -75,9 +87,9 @@ void TraceWriter::WriteIndirectBufferStart(uint32_t base_ptr, uint32_t count) {
   if (!file_) {
     return;
   }
-  auto cmd = IndirectBufferStartCommand({
+  IndirectBufferStartCommand cmd = {
       TraceCommandType::kIndirectBufferStart, base_ptr, 0,
-  });
+  };
   fwrite(&cmd, 1, sizeof(cmd), file_);
 }
 
@@ -85,9 +97,9 @@ void TraceWriter::WriteIndirectBufferEnd() {
   if (!file_) {
     return;
   }
-  auto cmd = IndirectBufferEndCommand({
+  IndirectBufferEndCommand cmd = {
       TraceCommandType::kIndirectBufferEnd,
-  });
+  };
   fwrite(&cmd, 1, sizeof(cmd), file_);
 }
 
@@ -95,9 +107,9 @@ void TraceWriter::WritePacketStart(uint32_t base_ptr, uint32_t count) {
   if (!file_) {
     return;
   }
-  auto cmd = PacketStartCommand({
+  PacketStartCommand cmd = {
       TraceCommandType::kPacketStart, base_ptr, count,
-  });
+  };
   fwrite(&cmd, 1, sizeof(cmd), file_);
   fwrite(membase_ + base_ptr, 4, count, file_);
 }
@@ -106,9 +118,9 @@ void TraceWriter::WritePacketEnd() {
   if (!file_) {
     return;
   }
-  auto cmd = PacketEndCommand({
+  PacketEndCommand cmd = {
       TraceCommandType::kPacketEnd,
-  });
+  };
   fwrite(&cmd, 1, sizeof(cmd), file_);
 }
 
@@ -116,69 +128,73 @@ void TraceWriter::WriteMemoryRead(uint32_t base_ptr, size_t length) {
   if (!file_) {
     return;
   }
-
+  WriteMemoryCommand(TraceCommandType::kMemoryRead, base_ptr, length);
-  bool compress = compress_output_ && length > compression_threshold_;
-
-  auto cmd = MemoryReadCommand({
-      TraceCommandType::kMemoryRead, base_ptr, uint32_t(length), 0,
-  });
-
-  if (compress) {
-    size_t written = WriteCompressed(membase_ + base_ptr, length);
-    cmd.length = uint32_t(written);
-    cmd.full_length = uint32_t(length);
-
-    fwrite(&cmd, 1, sizeof(cmd), file_);
-    fwrite(tmp_buff_.data(), 1, written, file_);
-  } else {
-    fwrite(&cmd, 1, sizeof(cmd), file_);
-    fwrite(membase_ + base_ptr, 1, length, file_);
-  }
 }
 
 void TraceWriter::WriteMemoryWrite(uint32_t base_ptr, size_t length) {
   if (!file_) {
     return;
   }
+  WriteMemoryCommand(TraceCommandType::kMemoryWrite, base_ptr, length);
+}
+
+class SnappySink : public snappy::Sink {
+ public:
+  SnappySink(FILE* file) : file_(file) {}
+
+  void Append(const char* bytes, size_t n) override {
+    fwrite(bytes, 1, n, file_);
+  }
+
+ private:
+  FILE* file_ = nullptr;
+};
+
+void TraceWriter::WriteMemoryCommand(TraceCommandType type, uint32_t base_ptr,
+                                     size_t length) {
+  MemoryCommand cmd;
+  cmd.type = type;
+  cmd.base_ptr = base_ptr;
+  cmd.encoding_format = MemoryEncodingFormat::kNone;
+  cmd.encoded_length = cmd.decoded_length = static_cast<uint32_t>(length);
 
   bool compress = compress_output_ && length > compression_threshold_;
-
-  auto cmd = MemoryWriteCommand({
-      TraceCommandType::kMemoryWrite, base_ptr, uint32_t(length), 0,
-  });
-
   if (compress) {
-    size_t written = WriteCompressed(membase_ + base_ptr, length);
+    // Write the header now so we reserve space in the buffer.
-    cmd.length = uint32_t(written);
+    long header_position = std::ftell(file_);
-    cmd.full_length = uint32_t(length);
+    cmd.encoding_format = MemoryEncodingFormat::kSnappy;
+    fwrite(&cmd, 1, sizeof(cmd), file_);
 
+    // Stream the content right to the buffer.
+    snappy::ByteArraySource snappy_source(
+        reinterpret_cast<const char*>(membase_ + cmd.base_ptr),
+        cmd.decoded_length);
+    SnappySink snappy_sink(file_);
+    cmd.encoded_length =
+        static_cast<uint32_t>(snappy::Compress(&snappy_source, &snappy_sink));
+
+    // Seek back and overwrite the header with our final size.
+    std::fseek(file_, header_position, SEEK_SET);
     fwrite(&cmd, 1, sizeof(cmd), file_);
-    fwrite(tmp_buff_.data(), 1, written, file_);
+    std::fseek(file_, header_position + sizeof(cmd) + cmd.encoded_length,
+               SEEK_SET);
   } else {
+    // Uncompressed - write buffer directly to the file.
+    cmd.encoding_format = MemoryEncodingFormat::kNone;
     fwrite(&cmd, 1, sizeof(cmd), file_);
-    fwrite(membase_ + base_ptr, 1, length, file_);
+    fwrite(membase_ + cmd.base_ptr, 1, cmd.decoded_length, file_);
   }
 }
 
-void TraceWriter::WriteEvent(EventType event_type) {
+void TraceWriter::WriteEvent(EventCommand::Type event_type) {
   if (!file_) {
     return;
   }
-  auto cmd = EventCommand({
+  EventCommand cmd = {
       TraceCommandType::kEvent, event_type,
-  });
+  };
   fwrite(&cmd, 1, sizeof(cmd), file_);
 }
 
-size_t TraceWriter::WriteCompressed(void* buf, size_t length) {
-  tmp_buff_.resize(compressBound(uint32_t(length)));
-
-  uLongf dest_len = (uint32_t)tmp_buff_.size();
-  int ret =
-      compress(tmp_buff_.data(), &dest_len, (uint8_t*)buf, uint32_t(length));
-  assert_true(ret >= 0);
-  return dest_len;
-}
-
 }  //  namespace gpu
 }  //  namespace xe

src/xenia/gpu/trace_writer.h

@@ -25,7 +25,7 @@ class TraceWriter {
 
   bool is_open() const { return file_ != nullptr; }
 
-  bool Open(const std::wstring& path);
+  bool Open(const std::wstring& path, uint32_t title_id);
   void Flush();
   void Close();
 
@@ -37,17 +37,17 @@ class TraceWriter {
   void WritePacketEnd();
   void WriteMemoryRead(uint32_t base_ptr, size_t length);
   void WriteMemoryWrite(uint32_t base_ptr, size_t length);
-  void WriteEvent(EventType event_type);
+  void WriteEvent(EventCommand::Type event_type);
 
  private:
-  size_t WriteCompressed(void* buf, size_t length);
+  void WriteMemoryCommand(TraceCommandType type, uint32_t base_ptr,
+                          size_t length);
 
   uint8_t* membase_;
   FILE* file_;
 
   bool compress_output_ = true;
-  size_t compression_threshold_ = 0x1000;  // min. number of bytes to compress.
+  size_t compression_threshold_ = 1024;  // Min. number of bytes to compress.
-  std::vector<uint8_t> tmp_buff_;
 };
 
 }  // namespace gpu

src/xenia/kernel/kernel_state.cc

@@ -354,7 +354,7 @@ object_ref<UserModule> KernelState::LoadUserModule(const char* raw_name,
 
   module->Dump();
 
-  if (module->dll_module() && module->entry_point() && call_entry) {
+  if (module->is_dll_module() && module->entry_point() && call_entry) {
     // Call DllMain(DLL_PROCESS_ATTACH):
     // https://msdn.microsoft.com/en-us/library/windows/desktop/ms682583%28v=vs.85%29.aspx
     uint64_t args[] = {
@@ -479,7 +479,7 @@ void KernelState::OnThreadExecute(XThread* thread) {
   // https://msdn.microsoft.com/en-us/library/windows/desktop/ms682583%28v=vs.85%29.aspx
   auto thread_state = thread->thread_state();
   for (auto user_module : user_modules_) {
-    if (user_module->dll_module() && user_module->entry_point()) {
+    if (user_module->is_dll_module() && user_module->entry_point()) {
       uint64_t args[] = {
           user_module->handle(),
           2,  // DLL_THREAD_ATTACH
@@ -501,7 +501,7 @@ void KernelState::OnThreadExit(XThread* thread) {
   // https://msdn.microsoft.com/en-us/library/windows/desktop/ms682583%28v=vs.85%29.aspx
   auto thread_state = thread->thread_state();
   for (auto user_module : user_modules_) {
-    if (user_module->dll_module() && user_module->entry_point()) {
+    if (user_module->is_dll_module() && user_module->entry_point()) {
       uint64_t args[] = {
           user_module->handle(),
           3,  // DLL_THREAD_DETACH

src/xenia/kernel/user_module.cc

@@ -28,6 +28,24 @@ UserModule::UserModule(KernelState* kernel_state)
 
 UserModule::~UserModule() { Unload(); }
 
+uint32_t UserModule::title_id() const {
+  if (module_format_ != kModuleFormatXex) {
+    return 0;
+  }
+  auto header = xex_header();
+  for (uint32_t i = 0; i < header->header_count; i++) {
+    auto& opt_header = header->headers[i];
+    if (opt_header.key == XEX_HEADER_EXECUTION_INFO) {
+      auto opt_header_ptr =
+          reinterpret_cast<const uint8_t*>(header) + opt_header.offset;
+      auto opt_exec_info =
+          reinterpret_cast<const xex2_opt_execution_info*>(opt_header_ptr);
+      return static_cast<uint32_t>(opt_exec_info->title_id);
+    }
+  }
+  return 0;
+}
+
 X_STATUS UserModule::LoadFromFile(std::string path) {
   X_STATUS result = X_STATUS_UNSUCCESSFUL;
 
@@ -134,7 +152,7 @@ X_STATUS UserModule::LoadFromMemory(const void* addr, const size_t length) {
     this->xex_module()->GetOptHeader(XEX_HEADER_ENTRY_POINT, &entry_point_);
     this->xex_module()->GetOptHeader(XEX_HEADER_DEFAULT_STACK_SIZE,
                                      &stack_size_);
-    dll_module_ = !!(header->module_flags & XEX_MODULE_DLL_MODULE);
+    is_dll_module_ = !!(header->module_flags & XEX_MODULE_DLL_MODULE);
   } else if (module_format_ == kModuleFormatElf) {
     auto elf_module =
         std::make_unique<cpu::ElfModule>(processor, kernel_state());
@@ -144,7 +162,7 @@ X_STATUS UserModule::LoadFromMemory(const void* addr, const size_t length) {
 
     entry_point_ = elf_module->entry_point();
     stack_size_ = 1024 * 1024;  // 1 MB
-    dll_module_ = false;        // Hardcoded not a DLL (for now)
+    is_dll_module_ = false;     // Hardcoded not a DLL (for now)
 
     processor_module_ = elf_module.get();
     if (!processor->AddModule(std::move(elf_module))) {

src/xenia/kernel/user_module.h

@@ -53,7 +53,9 @@ class UserModule : public XModule {
 
   const xex2_header* xex_header() const { return xex_module()->xex_header(); }
   uint32_t guest_xex_header() const { return guest_xex_header_; }
-  bool dll_module() const { return dll_module_; }
+  // The title ID in the xex header or 0 if this is not a xex.
+  uint32_t title_id() const;
+  bool is_dll_module() const { return is_dll_module_; }
 
   uint32_t entry_point() const { return entry_point_; }
   uint32_t stack_size() const { return stack_size_; }
@@ -95,7 +97,7 @@ class UserModule : public XModule {
   uint32_t guest_xex_header_ = 0;
   ModuleFormat module_format_ = kModuleFormatUndefined;
 
-  bool dll_module_ = false;
+  bool is_dll_module_ = false;
   uint32_t entry_point_ = 0;
   uint32_t stack_size_ = 0;
 };

third_party/zlib

@@ -1 +0,0 @@
-Subproject commit 50893291621658f355bc5b4d450a8d06a563053d

third_party/zlib.lua

@@ -1,41 +0,0 @@
-group("third_party")
-project("zlib")
-  uuid("AF89D75F-F723-47B1-9E29-29CDFA58CCAA")
-  kind("StaticLib")
-  language("C")
-  links({
-  })
-  defines({
-    "_LIB",
-  })
-  includedirs({
-    "zlib",
-  })
-  files({
-    "zlib/adler32.c",
-    "zlib/compress.c",
-    "zlib/crc32.c",
-    "zlib/crc32.h",
-    "zlib/deflate.c",
-    "zlib/deflate.h",
-    "zlib/gzclose.c",
-    "zlib/gzguts.h",
-    "zlib/gzlib.c",
-    "zlib/gzread.c",
-    "zlib/gzwrite.c",
-    "zlib/infback.c",
-    "zlib/inffast.c",
-    "zlib/inffast.h",
-    "zlib/inffixed.h",
-    "zlib/inflate.c",
-    "zlib/inflate.h",
-    "zlib/inftrees.c",
-    "zlib/inftrees.h",
-    "zlib/trees.c",
-    "zlib/trees.h",
-    "zlib/uncompr.c",
-    "zlib/zconf.h",
-    "zlib/zlib.h",
-    "zlib/zutil.c",
-    "zlib/zutil.h",
-  })