Initial support for xex patching

src/xenia/app/emulator_window.cc

@@ -1004,6 +1004,10 @@ void EmulatorWindow::UpdateTitle() {
     sb.Append(u8" (Preloading shaders\u2026)");
   }
 
+  patcher::Patcher* patcher = emulator()->patcher();
+  if (patcher && patcher->IsAnyPatchApplied()) {
+    sb.Append(u8" [Patches Applied]");
+  }
   window_->SetTitle(sb.to_string_view());
 }
 

src/xenia/app/xenia_main.cc

@@ -509,6 +509,10 @@ void EmulatorApp::EmulatorThread() {
         });
       });
 
+  emulator_->on_patch_apply.AddListener([this]() {
+    app_context().CallInUIThread([this]() { emulator_window_->UpdateTitle(); });
+  });
+
   emulator_->on_terminate.AddListener([]() {
     if (cvars::discord) {
       discord::DiscordPresence::NotPlaying();

src/xenia/emulator.cc

@@ -219,6 +219,8 @@ X_STATUS Emulator::Setup(
   // Bring up the virtual filesystem used by the kernel.
   file_system_ = std::make_unique<xe::vfs::VirtualFileSystem>();
 
+  patcher_ = std::make_unique<xe::patcher::Patcher>(storage_root_);
+
   // Shared kernel state.
   kernel_state_ = std::make_unique<xe::kernel::KernelState>(this);
 

src/xenia/emulator.h

@@ -21,6 +21,7 @@
 #include "xenia/base/exception_handler.h"
 #include "xenia/kernel/kernel_state.h"
 #include "xenia/memory.h"
+#include "xenia/patcher/patcher.h"
 #include "xenia/vfs/virtual_file_system.h"
 #include "xenia/xbox.h"
 
@@ -154,6 +155,8 @@ class Emulator {
   // This is effectively the guest operating system.
   kernel::KernelState* kernel_state() const { return kernel_state_.get(); }
 
+  patcher::Patcher* patcher() const { return patcher_.get(); }
+
   // Initializes the emulator and configures all components.
   // The given window is used for display and the provided functions are used
   // to create subsystems as required.
@@ -202,6 +205,7 @@ class Emulator {
  public:
   xe::Delegate<uint32_t, const std::string_view> on_launch;
   xe::Delegate<bool> on_shader_storage_initialization;
+  xe::Delegate<> on_patch_apply;
   xe::Delegate<> on_terminate;
   xe::Delegate<> on_exit;
 
@@ -237,6 +241,7 @@ class Emulator {
 
   std::unique_ptr<cpu::ExportResolver> export_resolver_;
   std::unique_ptr<vfs::VirtualFileSystem> file_system_;
+  std::unique_ptr<patcher::Patcher> patcher_;
 
   std::unique_ptr<kernel::KernelState> kernel_state_;
 

src/xenia/kernel/kernel_state.cc

@@ -406,6 +406,9 @@ object_ref<UserModule> KernelState::LoadUserModule(
   }
 
   module->Dump();
+  emulator_->patcher()->ApplyPatchesForTitle(memory_, module->title_id(),
+                                             module->hash());
+  emulator_->on_patch_apply();
 
   if (module->is_dll_module() && module->entry_point() && call_entry) {
     // Call DllMain(DLL_PROCESS_ATTACH):

src/xenia/kernel/user_module.cc

@@ -13,6 +13,7 @@
 
 #include "xenia/base/byte_stream.h"
 #include "xenia/base/logging.h"
+#include "xenia/base/xxhash.h"
 #include "xenia/cpu/elf_module.h"
 #include "xenia/cpu/processor.h"
 #include "xenia/cpu/xex_module.h"
@@ -72,6 +73,12 @@ X_STATUS UserModule::LoadFromFile(const std::string_view path) {
 
     // Load the module.
     result = LoadFromMemory(mmap->data(), mmap->size());
+    if (XSUCCEEDED(result)) {
+      XXH3_state_t hash_state;
+      XXH3_64bits_reset(&hash_state);
+      XXH3_64bits_update(&hash_state, mmap->data(), mmap->size());
+      hash_ = XXH3_64bits_digest(&hash_state);
+    }
   } else {
     std::vector<uint8_t> buffer(fs_entry->size());
 
@@ -93,6 +100,12 @@ X_STATUS UserModule::LoadFromFile(const std::string_view path) {
     // Load the module.
     result = LoadFromMemory(buffer.data(), bytes_read);
 
+    // Generate xex hash
+    XXH3_state_t hash_state;
+    XXH3_64bits_reset(&hash_state);
+    XXH3_64bits_update(&hash_state, buffer.data(), bytes_read);
+    hash_ = XXH3_64bits_digest(&hash_state);
+
     // Close the file.
     file->Destroy();
   }
@@ -128,6 +141,7 @@ X_STATUS UserModule::LoadFromFile(const std::string_view path) {
     }
   }
 
+  XELOGI("Module hash: {:16X} for {}", hash_, name_);
   return LoadXexContinue();
 }
 

src/xenia/kernel/user_module.h

@@ -38,6 +38,7 @@ class UserModule : public XModule {
 
   const std::string& path() const override { return path_; }
   const std::string& name() const override { return name_; }
+  uint64_t hash() const { return hash_; }
 
   enum ModuleFormat {
     kModuleFormatUndefined = 0,
@@ -100,6 +101,7 @@ class UserModule : public XModule {
 
   std::string name_;
   std::string path_;
+  uint64_t hash_ = -1;
 
   uint32_t guest_xex_header_ = 0;
   ModuleFormat module_format_ = kModuleFormatUndefined;

src/xenia/patcher/patch_db.cc

@@ -0,0 +1,164 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+#include <regex>
+
+#include "xenia/base/cvar.h"
+#include "xenia/base/filesystem.h"
+#include "xenia/base/logging.h"
+#include "xenia/memory.h"
+
+#include "xenia/patcher/patch_db.h"
+
+DEFINE_bool(apply_patches, true, "Enables patching functionality", "General");
+
+namespace xe {
+namespace patcher {
+
+PatchDB::PatchDB(const std::filesystem::path patches_root) {
+  patches_root_ = patches_root;
+  LoadPatches();
+}
+
+PatchDB::~PatchDB() {}
+
+void PatchDB::LoadPatches() {
+  if (!cvars::apply_patches) {
+    return;
+  }
+
+  const std::filesystem::path patches_directory = patches_root_ / "patches";
+  const std::vector<xe::filesystem::FileInfo>& patch_files =
+      filesystem::ListFiles(patches_directory);
+  const std::regex file_name_regex_match =
+      std::regex("^[A-Fa-f0-9]{8}.*\\.patch$");
+
+  for (const xe::filesystem::FileInfo& patch_file : patch_files) {
+    // Skip files that doesn't have only title_id as name and .patch as
+    // extension
+    if (!std::regex_match(path_to_utf8(patch_file.name),
+                          file_name_regex_match)) {
+      XELOGE("PatchDB: Skipped loading file {} due to incorrect filename",
+             path_to_utf8(patch_file.name));
+      continue;
+    }
+
+    const PatchFileEntry loaded_title_patches =
+        ReadPatchFile(patch_file.path / patch_file.name);
+    if (loaded_title_patches.title_id != -1) {
+      loaded_patches.push_back(loaded_title_patches);
+    }
+  }
+  XELOGI("PatchDB: Loaded patches for {} titles", loaded_patches.size());
+}
+
+PatchFileEntry PatchDB::ReadPatchFile(const std::filesystem::path& file_path) {
+  PatchFileEntry patchFile;
+  std::shared_ptr<cpptoml::table> patch_toml_fields;
+
+  try {
+    patch_toml_fields = cpptoml::parse_file(path_to_utf8(file_path));
+  } catch (...) {
+    XELOGE("PatchDB: Cannot load patch file: {}", path_to_utf8(file_path));
+    patchFile.title_id = -1;
+    return patchFile;
+  };
+
+  auto title_name = patch_toml_fields->get_as<std::string>("title_name");
+  auto title_id = patch_toml_fields->get_as<std::string>("title_id");
+  auto title_hash = patch_toml_fields->get_as<std::string>("hash");
+
+  patchFile.title_id = strtoul((*title_id).c_str(), NULL, 16);
+  patchFile.hash = strtoull((*title_hash).c_str(), NULL, 16);
+  patchFile.title_name = *title_name;
+
+  auto patch_table = patch_toml_fields->get_table_array("patch");
+
+  for (auto patch_table_entry : *patch_table) {
+    PatchInfoEntry patch = PatchInfoEntry();
+    auto patch_name = *patch_table_entry->get_as<std::string>("name");
+    auto patch_desc = *patch_table_entry->get_as<std::string>("desc");
+    auto patch_author = *patch_table_entry->get_as<std::string>("author");
+    auto is_enabled = *patch_table_entry->get_as<bool>("is_enabled");
+
+    patch.id = 0;  // Todo(Gliniak): Implement id for future GUI stuff
+    patch.patch_name = patch_name;
+    patch.patch_desc = patch_desc;
+    patch.patch_author = patch_author;
+    patch.is_enabled = is_enabled;
+
+    // Iterate through all available data sizes
+    for (const auto& type : patch_data_types_size) {
+      auto patch_entries = ReadPatchData(type.first, patch_table_entry);
+
+      for (const PatchDataEntry& patch_entry : *patch_entries) {
+        patch.patch_data.push_back(patch_entry);
+      }
+    }
+    patchFile.patch_info.push_back(patch);
+  }
+  return patchFile;
+}
+
+std::vector<PatchDataEntry>* PatchDB::ReadPatchData(
+    const std::string size_type,
+    const std::shared_ptr<cpptoml::table>& patch_table) {
+  std::vector<PatchDataEntry>* patch_data = new std::vector<PatchDataEntry>();
+  auto patch_data_tarr = patch_table->get_table_array(size_type);
+
+  if (!patch_data_tarr) {
+    return patch_data;
+  }
+
+  for (const auto& patch_data_table : *patch_data_tarr) {
+    uint32_t address = *patch_data_table->get_as<std::uint32_t>("address");
+    uint64_t value = *patch_data_table->get_as<std::uint64_t>("value");
+
+    if (size_type == "f32") {
+      float fvalue = float(*patch_data_table->get_as<double>("value"));
+      value = *(reinterpret_cast<uint32_t*>(&fvalue));
+    } else if (size_type == "f64") {
+      double dvalue = *patch_data_table->get_as<double>("value");
+      value = *(reinterpret_cast<uint64_t*>(&dvalue));
+    }
+
+    PatchDataEntry patchData = {GetAllocSize(size_type), address, value};
+    patch_data->push_back(patchData);
+  }
+  return patch_data;
+}
+
+std::vector<PatchFileEntry> PatchDB::GetTitlePatches(uint32_t title_id,
+                                                     const uint64_t hash) {
+  std::vector<PatchFileEntry> title_patches;
+
+  std::copy_if(loaded_patches.cbegin(), loaded_patches.cend(),
+               std::back_inserter(title_patches),
+               [=](const PatchFileEntry entry) {
+                 return entry.title_id == title_id &&
+                        (!entry.hash || entry.hash == hash);
+               });
+
+  return title_patches;
+}
+
+uint8_t PatchDB::GetAllocSize(const std::string provided_size) {
+  uint8_t alloc_size = 0;
+
+  auto itr = patch_data_types_size.find(provided_size);
+  if (itr == patch_data_types_size.cend()) {
+    XELOGW("PatchDB: Unsupported patch type!");
+    return alloc_size;
+  }
+
+  alloc_size = uint8_t(itr->second);
+  return alloc_size;
+}
+
+}  // namespace patcher
+}  // namespace xe

src/xenia/patcher/patch_db.h

@@ -0,0 +1,78 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_PATCH_DB_H_
+#define XENIA_PATCH_DB_H_
+
+#include <map>
+
+#include "third_party/cpptoml/include/cpptoml.h"
+
+namespace xe {
+namespace patcher {
+struct PatchDataEntry {
+  const uint8_t alloc_size_;
+  const uint32_t memory_address_;
+  const uint64_t new_value_;
+
+  PatchDataEntry(const uint8_t alloc_size, const uint32_t memory_address,
+                 const uint64_t new_value)
+      : alloc_size_(alloc_size),
+        memory_address_(memory_address),
+        new_value_(new_value){};
+};
+
+struct PatchInfoEntry {
+  uint32_t id;
+  std::string patch_name;
+  std::string patch_desc;
+  std::string patch_author;
+  std::vector<PatchDataEntry> patch_data;
+  bool is_enabled;
+};
+
+struct PatchFileEntry {
+  uint32_t title_id;
+  std::string title_name;
+  uint64_t hash = 0;
+  std::vector<PatchInfoEntry> patch_info;
+};
+
+class PatchDB {
+ public:
+  PatchDB(const std::filesystem::path patches_root);
+  ~PatchDB();
+
+  void LoadPatches();
+
+  PatchFileEntry ReadPatchFile(const std::filesystem::path& file_path);
+  std::vector<PatchDataEntry>* ReadPatchData(
+      const std::string size_type,
+      const std::shared_ptr<cpptoml::table>& patch_table);
+
+  std::vector<PatchFileEntry> GetTitlePatches(uint32_t title_id,
+                                              const uint64_t hash);
+  std::vector<PatchFileEntry>& GetAllPatches() { return loaded_patches; }
+
+ private:
+  std::vector<PatchFileEntry> loaded_patches;
+  std::filesystem::path patches_root_;
+
+  uint8_t GetAllocSize(const std::string provided_size);
+
+  const std::map<std::string, size_t> patch_data_types_size = {
+      {"f64", sizeof(uint64_t)},  {"f32", sizeof(uint32_t)},
+      {"be64", sizeof(uint64_t)}, {"be32", sizeof(uint32_t)},
+      {"be16", sizeof(uint16_t)}, {"be8", sizeof(uint8_t)}};
+};
+
+}  // namespace patcher
+}  // namespace xe
+
+#endif  // XENIA_PATCH_LOADER_H_

src/xenia/patcher/patcher.cc

@@ -0,0 +1,81 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+#include "xenia/patcher/patcher.h"
+
+#include "xenia/base/logging.h"
+
+namespace xe {
+namespace patcher {
+
+Patcher::Patcher(const std::filesystem::path patches_root) {
+  is_any_patch_applied_ = false;
+  patch_db = new PatchDB(patches_root);
+}
+
+Patcher::~Patcher() {}
+
+void Patcher::ApplyPatchesForTitle(Memory* memory, const uint32_t title_id,
+                                   const uint64_t hash) {
+  const auto title_patches = patch_db->GetTitlePatches(title_id, hash);
+
+  for (const PatchFileEntry& patchFile : title_patches) {
+    for (const PatchInfoEntry& patchEntry : patchFile.patch_info) {
+      if (!patchEntry.is_enabled) {
+        continue;
+      }
+      XELOGE("Patcher: Applying patch for: {}({:08X}) - {}",
+             patchFile.title_name, patchFile.title_id, patchEntry.patch_name);
+      ApplyPatch(memory, &patchEntry);
+    }
+  }
+}
+
+void Patcher::ApplyPatch(Memory* memory, const PatchInfoEntry* patch) {
+  for (const PatchDataEntry& patch_data_entry : patch->patch_data) {
+    uint32_t old_address_protect = 0;
+    auto address = memory->TranslateVirtual(patch_data_entry.memory_address_);
+    auto heap = memory->LookupHeap(patch_data_entry.memory_address_);
+    if (!heap) {
+      continue;
+    }
+
+    heap->QueryProtect(patch_data_entry.memory_address_, &old_address_protect);
+
+    heap->Protect(patch_data_entry.memory_address_,
+                  patch_data_entry.alloc_size_,
+                  kMemoryProtectRead | kMemoryProtectWrite);
+
+    switch (patch_data_entry.alloc_size_) {
+      case 1:
+        xe::store_and_swap(address, uint8_t(patch_data_entry.new_value_));
+        break;
+      case 2:
+        xe::store_and_swap(address, uint16_t(patch_data_entry.new_value_));
+        break;
+      case 4:
+        xe::store_and_swap(address, uint32_t(patch_data_entry.new_value_));
+        break;
+      case 8:
+        xe::store_and_swap(address, uint64_t(patch_data_entry.new_value_));
+        break;
+      default:
+        XELOGE("Patcher: Unsupported patch allocation size - {}",
+               patch_data_entry.alloc_size_);
+        break;
+    }
+    // Restore previous protection
+    heap->Protect(patch_data_entry.memory_address_,
+                  patch_data_entry.alloc_size_, old_address_protect);
+
+    is_any_patch_applied_ = true;
+  }
+}
+
+}  // namespace patcher
+}  // namespace xe

src/xenia/patcher/patcher.h

@@ -0,0 +1,37 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_PATCHER_H_
+#define XENIA_PATCHER_H_
+
+#include "xenia/memory.h"
+#include "xenia/patcher/patch_db.h"
+
+namespace xe {
+namespace patcher {
+
+class Patcher {
+ public:
+  Patcher(const std::filesystem::path patches_root);
+  ~Patcher();
+
+  void ApplyPatch(Memory* memory, const PatchInfoEntry* patch);
+  void ApplyPatchesForTitle(Memory* memory, const uint32_t title_id,
+                            const uint64_t hash);
+
+  bool IsAnyPatchApplied() { return is_any_patch_applied_; }
+
+ private:
+  PatchDB* patch_db;
+  bool is_any_patch_applied_;
+};
+
+}  // namespace patcher
+}  // namespace xe
+#endif