Initial support for xex patching

src/xenia/app/emulator_window.cc

@@ -1004,6 +1004,10 @@ void EmulatorWindow::UpdateTitle() {
     sb.Append(u8" (Preloading shaders\u2026)");
   }
 
+  patcher::Patcher* patcher = emulator()->patcher();
+  if (patcher && patcher->IsAnyPatchApplied()) {
+    sb.Append(u8" [Patches Applied]");
+  }
   window_->SetTitle(sb.to_string_view());
 }
 

src/xenia/app/xenia_main.cc

@@ -509,6 +509,10 @@ void EmulatorApp::EmulatorThread() {
         });
       });
 
+  emulator_->on_patch_apply.AddListener([this]() {
+    app_context().CallInUIThread([this]() { emulator_window_->UpdateTitle(); });
+  });
+
   emulator_->on_terminate.AddListener([]() {
     if (cvars::discord) {
       discord::DiscordPresence::NotPlaying();

src/xenia/base/string_util.h

@@ -111,6 +111,32 @@ inline size_t copy_and_swap_maybe_truncating(char16_t* dest,
   return chars_copied;
 }
 
+inline bool hex_string_to_array(std::vector<uint8_t>& output_array,
+                                const std::string_view value) {
+  output_array.reserve((value.size() + 1) / 2);
+
+  size_t remaining_length = value.size();
+  while (remaining_length > 0) {
+    uint8_t chars_to_read = remaining_length > 1 ? 2 : 1;
+    const char* substring_pointer =
+        value.data() + value.size() - remaining_length;
+
+    uint8_t string_value = 0;
+    std::from_chars_result result = std::from_chars(
+        substring_pointer, substring_pointer + chars_to_read, string_value, 16);
+
+    if (result.ec != std::errc() ||
+        result.ptr != substring_pointer + chars_to_read) {
+      output_array.clear();
+      return false;
+    }
+
+    output_array.push_back(string_value);
+    remaining_length -= chars_to_read;
+  }
+  return true;
+}
+
 inline std::string to_hex_string(uint32_t value) {
   return fmt::format("{:08X}", value);
 }

src/xenia/cpu/xex_module.cc

@@ -930,8 +930,6 @@ bool XexModule::Load(const std::string_view name, const std::string_view path,
   name_ = name;
   path_ = path;
 
-  uint8_t* data = memory()->TranslateVirtual(base_address_);
-
   // Load in the XEX basefile
   // We'll try using both XEX2 keys to see if any give a valid PE
   int result_code = ReadImage(xex_addr, xex_length, false);

src/xenia/emulator.cc

@@ -219,6 +219,8 @@ X_STATUS Emulator::Setup(
   // Bring up the virtual filesystem used by the kernel.
   file_system_ = std::make_unique<xe::vfs::VirtualFileSystem>();
 
+  patcher_ = std::make_unique<xe::patcher::Patcher>(storage_root_);
+
   // Shared kernel state.
   kernel_state_ = std::make_unique<xe::kernel::KernelState>(this);
 

src/xenia/emulator.h

@@ -21,6 +21,7 @@
 #include "xenia/base/exception_handler.h"
 #include "xenia/kernel/kernel_state.h"
 #include "xenia/memory.h"
+#include "xenia/patcher/patcher.h"
 #include "xenia/vfs/virtual_file_system.h"
 #include "xenia/xbox.h"
 
@@ -154,6 +155,8 @@ class Emulator {
   // This is effectively the guest operating system.
   kernel::KernelState* kernel_state() const { return kernel_state_.get(); }
 
+  patcher::Patcher* patcher() const { return patcher_.get(); }
+
   // Initializes the emulator and configures all components.
   // The given window is used for display and the provided functions are used
   // to create subsystems as required.
@@ -202,6 +205,7 @@ class Emulator {
  public:
   xe::Delegate<uint32_t, const std::string_view> on_launch;
   xe::Delegate<bool> on_shader_storage_initialization;
+  xe::Delegate<> on_patch_apply;
   xe::Delegate<> on_terminate;
   xe::Delegate<> on_exit;
 
@@ -237,6 +241,7 @@ class Emulator {
 
   std::unique_ptr<cpu::ExportResolver> export_resolver_;
   std::unique_ptr<vfs::VirtualFileSystem> file_system_;
+  std::unique_ptr<patcher::Patcher> patcher_;
 
   std::unique_ptr<kernel::KernelState> kernel_state_;
 

src/xenia/kernel/kernel_state.cc

@@ -406,6 +406,9 @@ object_ref<UserModule> KernelState::LoadUserModule(
   }
 
   module->Dump();
+  emulator_->patcher()->ApplyPatchesForTitle(memory_, module->title_id(),
+                                             module->hash());
+  emulator_->on_patch_apply();
 
   if (module->is_dll_module() && module->entry_point() && call_entry) {
     // Call DllMain(DLL_PROCESS_ATTACH):

src/xenia/kernel/user_module.cc

@@ -13,6 +13,7 @@
 
 #include "xenia/base/byte_stream.h"
 #include "xenia/base/logging.h"
+#include "xenia/base/xxhash.h"
 #include "xenia/cpu/elf_module.h"
 #include "xenia/cpu/processor.h"
 #include "xenia/cpu/xex_module.h"
@@ -128,6 +129,9 @@ X_STATUS UserModule::LoadFromFile(const std::string_view path) {
     }
   }
 
+  CalculateHash();
+
+  XELOGI("Module hash: {:016X} for {}", hash_.value_or(UINT64_MAX), name_);
   return LoadXexContinue();
 }
 
@@ -804,5 +808,46 @@ void UserModule::Dump() {
   xe::logging::AppendLogLine(xe::LogLevel::Info, 'i', sb.to_string_view());
 }
 
+void UserModule::CalculateHash() {
+  const BaseHeap* module_heap =
+      kernel_state_->memory()->LookupHeap(xex_module()->base_address());
+
+  if (!module_heap) {
+    XELOGE("Invalid heap for xex module! Address: {:08X}",
+           xex_module()->base_address());
+    return;
+  }
+
+  const uint32_t page_size = module_heap->page_size();
+  auto security_info = xex_module()->xex_security_info();
+
+  auto find_code_section_page = [&security_info](bool from_bottom) {
+    for (uint32_t i = 0; i < security_info->page_descriptor_count; i++) {
+      const uint32_t page_index =
+          from_bottom ? i : security_info->page_descriptor_count - i;
+      xex2_page_descriptor page_descriptor;
+      page_descriptor.value =
+          xe::byte_swap(security_info->page_descriptors[page_index].value);
+      if (page_descriptor.info != XEX_SECTION_CODE) {
+        continue;
+      }
+      return page_index;
+    }
+    return UINT32_MAX;
+  };
+
+  const uint32_t start_address =
+      xex_module()->base_address() + (find_code_section_page(true) * page_size);
+  const uint32_t end_address =
+      xex_module()->base_address() +
+      ((find_code_section_page(false) + 1) * page_size);
+
+  uint8_t* base_code_adr = memory()->TranslateVirtual(start_address);
+
+  XXH3_state_t hash_state;
+  XXH3_64bits_reset(&hash_state);
+  XXH3_64bits_update(&hash_state, base_code_adr, end_address - start_address);
+  hash_ = XXH3_64bits_digest(&hash_state);
+}
 }  // namespace kernel
 }  // namespace xe

src/xenia/kernel/user_module.h

@@ -38,6 +38,7 @@ class UserModule : public XModule {
 
   const std::string& path() const override { return path_; }
   const std::string& name() const override { return name_; }
+  std::optional<uint64_t> hash() const { return hash_; }
 
   enum ModuleFormat {
     kModuleFormatUndefined = 0,
@@ -97,9 +98,11 @@ class UserModule : public XModule {
 
  private:
   X_STATUS LoadXexContinue();
+  void CalculateHash();
 
   std::string name_;
   std::string path_;
+  std::optional<uint64_t> hash_ = std::nullopt;
 
   uint32_t guest_xex_header_ = 0;
   ModuleFormat module_format_ = kModuleFormatUndefined;

src/xenia/patcher/patch_db.cc

@@ -0,0 +1,223 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+#include "xenia/base/cvar.h"
+#include "xenia/base/filesystem.h"
+#include "xenia/base/logging.h"
+#include "xenia/memory.h"
+
+#include "xenia/patcher/patch_db.h"
+
+DEFINE_bool(apply_patches, true, "Enables custom patching functionality",
+            "General");
+
+namespace xe {
+namespace patcher {
+
+PatchDB::PatchDB(const std::filesystem::path patches_root) {
+  patches_root_ = patches_root;
+  LoadPatches();
+}
+
+PatchDB::~PatchDB() {}
+
+void PatchDB::LoadPatches() {
+  if (!cvars::apply_patches) {
+    return;
+  }
+
+  const std::filesystem::path patches_directory = patches_root_ / "patches";
+  const std::vector<xe::filesystem::FileInfo> patch_files =
+      filesystem::ListFiles(patches_directory);
+
+  for (const xe::filesystem::FileInfo& patch_file : patch_files) {
+    // Skip files that doesn't have only title_id as name and .patch as
+    // extension
+    if (!std::regex_match(path_to_utf8(patch_file.name),
+                          patch_filename_regex_)) {
+      XELOGE("PatchDB: Skipped loading file {} due to incorrect filename",
+             path_to_utf8(patch_file.name));
+      continue;
+    }
+
+    const PatchFileEntry loaded_title_patches =
+        ReadPatchFile(patch_file.path / patch_file.name);
+    if (loaded_title_patches.title_id != -1) {
+      loaded_patches_.push_back(loaded_title_patches);
+    }
+  }
+  XELOGI("PatchDB: Loaded patches for {} titles", loaded_patches_.size());
+}
+
+PatchFileEntry PatchDB::ReadPatchFile(const std::filesystem::path& file_path) {
+  PatchFileEntry patch_file;
+  std::shared_ptr<cpptoml::table> patch_toml_fields;
+
+  try {
+    patch_toml_fields = cpptoml::parse_file(path_to_utf8(file_path));
+  } catch (...) {
+    XELOGE("PatchDB: Cannot load patch file: {}", path_to_utf8(file_path));
+    patch_file.title_id = -1;
+    return patch_file;
+  };
+
+  auto title_name = patch_toml_fields->get_as<std::string>("title_name");
+  auto title_id = patch_toml_fields->get_as<std::string>("title_id");
+
+  patch_file.title_id = strtoul((*title_id).c_str(), NULL, 16);
+  patch_file.title_name = *title_name;
+  ReadHashes(patch_file, patch_toml_fields);
+
+  auto patch_table = patch_toml_fields->get_table_array("patch");
+
+  for (auto patch_table_entry : *patch_table) {
+    PatchInfoEntry patch = PatchInfoEntry();
+    auto patch_name = *patch_table_entry->get_as<std::string>("name");
+    auto patch_desc = *patch_table_entry->get_as<std::string>("desc");
+    auto patch_author = *patch_table_entry->get_as<std::string>("author");
+    auto is_enabled = *patch_table_entry->get_as<bool>("is_enabled");
+
+    patch.id = 0;  // Todo(Gliniak): Implement id for future GUI stuff
+    patch.patch_name = patch_name;
+    patch.patch_desc = patch_desc;
+    patch.patch_author = patch_author;
+    patch.is_enabled = is_enabled;
+
+    // Iterate through all available data sizes
+    for (const auto& patch_data_type : patch_data_types_size_) {
+      bool success =
+          ReadPatchData(patch.patch_data, patch_data_type, patch_table_entry);
+
+      if (!success) {
+        XELOGE("PatchDB: Cannot read patch {}", patch_name);
+        break;
+      }
+    }
+    patch_file.patch_info.push_back(patch);
+  }
+  return patch_file;
+}
+
+bool PatchDB::ReadPatchData(
+    std::vector<PatchDataEntry>& patch_data,
+    const std::pair<std::string, PatchData> data_type,
+    const std::shared_ptr<cpptoml::table>& patch_table) {
+  auto patch_data_tarr = patch_table->get_table_array(data_type.first);
+  if (!patch_data_tarr) {
+    return true;
+  }
+
+  for (const auto& patch_data_table : *patch_data_tarr) {
+    uint32_t address = *patch_data_table->get_as<std::uint32_t>("address");
+    size_t alloc_size = (size_t)data_type.second.size;
+
+    switch (data_type.second.type) {
+      case PatchDataType::kBE8: {
+        uint16_t value = *patch_data_table->get_as<uint8_t>("value");
+        patch_data.push_back({address, PatchDataValue(alloc_size, value)});
+        break;
+      }
+      case PatchDataType::kBE16: {
+        uint16_t value = *patch_data_table->get_as<uint16_t>("value");
+        patch_data.push_back(
+            {address, PatchDataValue(alloc_size, xe::byte_swap(value))});
+        break;
+      }
+      case PatchDataType::kBE32: {
+        uint32_t value = *patch_data_table->get_as<uint32_t>("value");
+        patch_data.push_back(
+            {address, PatchDataValue(alloc_size, xe::byte_swap(value))});
+        break;
+      }
+      case PatchDataType::kBE64: {
+        uint64_t value = *patch_data_table->get_as<uint64_t>("value");
+        patch_data.push_back(
+            {address, PatchDataValue(alloc_size, xe::byte_swap(value))});
+        break;
+      }
+      case PatchDataType::kF64: {
+        double val = *patch_data_table->get_as<double>("value");
+        uint64_t value = *reinterpret_cast<uint64_t*>(&val);
+        patch_data.push_back(
+            {address, PatchDataValue(alloc_size, xe::byte_swap(value))});
+        break;
+      }
+      case PatchDataType::kF32: {
+        float value = float(*patch_data_table->get_as<double>("value"));
+        patch_data.push_back(
+            {address, PatchDataValue(alloc_size, xe::byte_swap(value))});
+        break;
+      }
+      case PatchDataType::kString: {
+        std::string value = *patch_data_table->get_as<std::string>("value");
+        patch_data.push_back({address, PatchDataValue(value)});
+        break;
+      }
+      case PatchDataType::kU16String: {
+        std::u16string value =
+            xe::to_utf16(*patch_data_table->get_as<std::string>("value"));
+        patch_data.push_back({address, PatchDataValue(value)});
+        break;
+      }
+      case PatchDataType::kByteArray: {
+        std::vector<uint8_t> data;
+        const std::string value =
+            *patch_data_table->get_as<std::string>("value");
+
+        bool success = string_util::hex_string_to_array(data, value);
+        if (!success) {
+          XELOGW("PatchDB: Cannot convert hex string to byte array! Skipping",
+                 address);
+          return false;
+        }
+        patch_data.push_back({address, PatchDataValue(data)});
+        break;
+      }
+      default: {
+        XELOGW("PatchDB: Unknown patch data type for address {:08X}! Skipping",
+               address);
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+std::vector<PatchFileEntry> PatchDB::GetTitlePatches(
+    const uint32_t title_id, const std::optional<uint64_t> hash) {
+  std::vector<PatchFileEntry> title_patches;
+
+  std::copy_if(
+      loaded_patches_.cbegin(), loaded_patches_.cend(),
+      std::back_inserter(title_patches), [=](const PatchFileEntry entry) {
+        bool hash_exist = std::find(entry.hashes.cbegin(), entry.hashes.cend(),
+                                    hash) != entry.hashes.cend();
+
+        return entry.title_id == title_id &&
+               (entry.hashes.empty() || hash_exist);
+      });
+
+  return title_patches;
+}
+
+void PatchDB::ReadHashes(PatchFileEntry& patch_entry,
+                         std::shared_ptr<cpptoml::table> patch_toml_fields) {
+  auto title_hashes = patch_toml_fields->get_array_of<std::string>("hash");
+
+  for (const auto& hash : *title_hashes) {
+    patch_entry.hashes.push_back(strtoull(hash.c_str(), NULL, 16));
+  }
+
+  auto single_hash = patch_toml_fields->get_as<std::string>("hash");
+  if (single_hash) {
+    patch_entry.hashes.push_back(strtoull((*single_hash).c_str(), NULL, 16));
+  }
+}
+
+}  // namespace patcher
+}  // namespace xe

src/xenia/patcher/patch_db.h

@@ -0,0 +1,132 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_PATCH_DB_H_
+#define XENIA_PATCH_DB_H_
+
+#include <cstring>
+#include <map>
+#include <optional>
+#include <regex>
+
+#include "third_party/cpptoml/include/cpptoml.h"
+
+namespace xe {
+namespace patcher {
+
+struct PatchDataValue {
+  const size_t alloc_size;
+  std::vector<uint8_t> patch_data;
+
+  template <typename T>
+  PatchDataValue(const size_t size, const T value) : alloc_size(size) {
+    patch_data.resize(alloc_size);
+    std::memcpy(patch_data.data(), &value, alloc_size);
+  };
+
+  PatchDataValue(const std::vector<uint8_t> value) : alloc_size(value.size()) {
+    patch_data.resize(alloc_size);
+    std::memcpy(patch_data.data(), value.data(), alloc_size);
+  };
+
+  PatchDataValue(const std::string value) : alloc_size(value.size()) {
+    patch_data.resize(alloc_size);
+    std::memcpy(patch_data.data(), value.c_str(), alloc_size);
+  };
+
+  PatchDataValue(const std::u16string value) : alloc_size(value.size() * 2) {
+    patch_data.resize(alloc_size);
+    std::memcpy(patch_data.data(), value.c_str(), alloc_size);
+  };
+};
+
+struct PatchDataEntry {
+  const uint32_t address;
+  const PatchDataValue data;
+
+  PatchDataEntry(const uint32_t memory_address, const PatchDataValue patch_data)
+      : address(memory_address), data(patch_data){};
+};
+
+struct PatchInfoEntry {
+  uint32_t id;
+  std::string patch_name;
+  std::string patch_desc;
+  std::string patch_author;
+  std::vector<PatchDataEntry> patch_data;
+  bool is_enabled;
+};
+
+struct PatchFileEntry {
+  uint32_t title_id;
+  std::string title_name;
+  std::vector<uint64_t> hashes;
+  std::vector<PatchInfoEntry> patch_info;
+};
+
+enum class PatchDataType {
+  kBE8,
+  kBE16,
+  kBE32,
+  kBE64,
+  kF32,
+  kF64,
+  kString,
+  kU16String,
+  kByteArray
+};
+
+struct PatchData {
+  uint8_t size;
+  PatchDataType type;
+
+  PatchData(uint8_t size_, PatchDataType type_) : size(size_), type(type_){};
+};
+
+class PatchDB {
+ public:
+  PatchDB(const std::filesystem::path patches_root);
+  ~PatchDB();
+
+  void LoadPatches();
+
+  PatchFileEntry ReadPatchFile(const std::filesystem::path& file_path);
+  bool ReadPatchData(std::vector<PatchDataEntry>& patch_data,
+                     const std::pair<std::string, PatchData> data_type,
+                     const std::shared_ptr<cpptoml::table>& patch_table);
+
+  std::vector<PatchFileEntry> GetTitlePatches(
+      const uint32_t title_id, const std::optional<uint64_t> hash);
+  std::vector<PatchFileEntry>& GetAllPatches() { return loaded_patches_; }
+
+ private:
+  void ReadHashes(PatchFileEntry& patch_entry,
+                  std::shared_ptr<cpptoml::table> patch_toml_fields);
+
+  inline static const std::regex patch_filename_regex_ =
+      std::regex("^[A-Fa-f0-9]{8}.*\\.patch\\.toml$");
+
+  const std::map<std::string, PatchData> patch_data_types_size_ = {
+      {"string", PatchData(0, PatchDataType::kString)},
+      {"u16string", PatchData(0, PatchDataType::kU16String)},
+      {"array", PatchData(0, PatchDataType::kByteArray)},
+      {"f64", PatchData(sizeof(uint64_t), PatchDataType::kF64)},
+      {"f32", PatchData(sizeof(uint32_t), PatchDataType::kF32)},
+      {"be64", PatchData(sizeof(uint64_t), PatchDataType::kBE64)},
+      {"be32", PatchData(sizeof(uint32_t), PatchDataType::kBE32)},
+      {"be16", PatchData(sizeof(uint16_t), PatchDataType::kBE16)},
+      {"be8", PatchData(sizeof(uint8_t), PatchDataType::kBE8)}};
+
+  std::vector<PatchFileEntry> loaded_patches_;
+  std::filesystem::path patches_root_;
+};
+}  // namespace patcher
+}  // namespace xe
+
+#endif  // XENIA_PATCH_LOADER_H_

src/xenia/patcher/patcher.cc

@@ -0,0 +1,66 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+#include <cstring>
+
+#include "xenia/base/logging.h"
+#include "xenia/patcher/patcher.h"
+
+namespace xe {
+namespace patcher {
+
+Patcher::Patcher(const std::filesystem::path patches_root) {
+  is_any_patch_applied_ = false;
+  patch_db_ = new PatchDB(patches_root);
+}
+
+void Patcher::ApplyPatchesForTitle(Memory* memory, const uint32_t title_id,
+                                   const std::optional<uint64_t> hash) {
+  const auto title_patches = patch_db_->GetTitlePatches(title_id, hash);
+
+  for (const PatchFileEntry& patchFile : title_patches) {
+    for (const PatchInfoEntry& patchEntry : patchFile.patch_info) {
+      if (!patchEntry.is_enabled) {
+        continue;
+      }
+      XELOGE("Patcher: Applying patch for: {}({:08X}) - {}",
+             patchFile.title_name, patchFile.title_id, patchEntry.patch_name);
+      ApplyPatch(memory, &patchEntry);
+    }
+  }
+}
+
+void Patcher::ApplyPatch(Memory* memory, const PatchInfoEntry* patch) {
+  for (const PatchDataEntry& patch_data_entry : patch->patch_data) {
+    uint32_t old_address_protect = 0;
+    uint8_t* address = memory->TranslateVirtual(patch_data_entry.address);
+    xe::BaseHeap* heap = memory->LookupHeap(patch_data_entry.address);
+    if (!heap) {
+      continue;
+    }
+
+    heap->QueryProtect(patch_data_entry.address, &old_address_protect);
+
+    heap->Protect(patch_data_entry.address,
+                  (uint32_t)patch_data_entry.data.alloc_size,
+                  kMemoryProtectRead | kMemoryProtectWrite);
+
+    std::memcpy(address, patch_data_entry.data.patch_data.data(),
+                patch_data_entry.data.alloc_size);
+
+    // Restore previous protection
+    heap->Protect(patch_data_entry.address,
+                  (uint32_t)patch_data_entry.data.alloc_size,
+                  old_address_protect);
+
+    is_any_patch_applied_ = true;
+  }
+}
+
+}  // namespace patcher
+}  // namespace xe

src/xenia/patcher/patcher.h

@@ -0,0 +1,36 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2022 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_PATCHER_H_
+#define XENIA_PATCHER_H_
+
+#include "xenia/memory.h"
+#include "xenia/patcher/patch_db.h"
+
+namespace xe {
+namespace patcher {
+
+class Patcher {
+ public:
+  Patcher(const std::filesystem::path patches_root);
+
+  void ApplyPatch(Memory* memory, const PatchInfoEntry* patch);
+  void ApplyPatchesForTitle(Memory* memory, const uint32_t title_id,
+                            const std::optional<uint64_t> hash);
+
+  bool IsAnyPatchApplied() { return is_any_patch_applied_; }
+
+ private:
+  PatchDB* patch_db_;
+  bool is_any_patch_applied_;
+};
+
+}  // namespace patcher
+}  // namespace xe
+#endif