From 91f43a374d52248a28653945b2db2ebaed837536 Mon Sep 17 00:00:00 2001 From: Gliniak Date: Fri, 9 Oct 2020 21:02:32 +0200 Subject: [PATCH] Initial support for xex patching --- src/xenia/app/emulator_window.cc | 4 + src/xenia/app/xenia_main.cc | 4 + src/xenia/base/string_util.h | 26 ++++ src/xenia/cpu/xex_module.cc | 2 - src/xenia/emulator.cc | 2 + src/xenia/emulator.h | 5 + src/xenia/kernel/kernel_state.cc | 3 + src/xenia/kernel/user_module.cc | 45 +++++++ src/xenia/kernel/user_module.h | 3 + src/xenia/patcher/patch_db.cc | 223 +++++++++++++++++++++++++++++++ src/xenia/patcher/patch_db.h | 132 ++++++++++++++++++ src/xenia/patcher/patcher.cc | 66 +++++++++ src/xenia/patcher/patcher.h | 36 +++++ 13 files changed, 549 insertions(+), 2 deletions(-) create mode 100644 src/xenia/patcher/patch_db.cc create mode 100644 src/xenia/patcher/patch_db.h create mode 100644 src/xenia/patcher/patcher.cc create mode 100644 src/xenia/patcher/patcher.h diff --git a/src/xenia/app/emulator_window.cc b/src/xenia/app/emulator_window.cc index 7e6d9e6b7..fb60e3057 100644 --- a/src/xenia/app/emulator_window.cc +++ b/src/xenia/app/emulator_window.cc @@ -1004,6 +1004,10 @@ void EmulatorWindow::UpdateTitle() { sb.Append(u8" (Preloading shaders\u2026)"); } + patcher::Patcher* patcher = emulator()->patcher(); + if (patcher && patcher->IsAnyPatchApplied()) { + sb.Append(u8" [Patches Applied]"); + } window_->SetTitle(sb.to_string_view()); } diff --git a/src/xenia/app/xenia_main.cc b/src/xenia/app/xenia_main.cc index ed6f1e9f1..a4f4fbcb0 100644 --- a/src/xenia/app/xenia_main.cc +++ b/src/xenia/app/xenia_main.cc @@ -509,6 +509,10 @@ void EmulatorApp::EmulatorThread() { }); }); + emulator_->on_patch_apply.AddListener([this]() { + app_context().CallInUIThread([this]() { emulator_window_->UpdateTitle(); }); + }); + emulator_->on_terminate.AddListener([]() { if (cvars::discord) { discord::DiscordPresence::NotPlaying(); diff --git a/src/xenia/base/string_util.h b/src/xenia/base/string_util.h index 09575ff56..4cbbcc6b3 100644 --- a/src/xenia/base/string_util.h +++ b/src/xenia/base/string_util.h @@ -111,6 +111,32 @@ inline size_t copy_and_swap_maybe_truncating(char16_t* dest, return chars_copied; } +inline bool hex_string_to_array(std::vector& output_array, + const std::string_view value) { + output_array.reserve((value.size() + 1) / 2); + + size_t remaining_length = value.size(); + while (remaining_length > 0) { + uint8_t chars_to_read = remaining_length > 1 ? 2 : 1; + const char* substring_pointer = + value.data() + value.size() - remaining_length; + + uint8_t string_value = 0; + std::from_chars_result result = std::from_chars( + substring_pointer, substring_pointer + chars_to_read, string_value, 16); + + if (result.ec != std::errc() || + result.ptr != substring_pointer + chars_to_read) { + output_array.clear(); + return false; + } + + output_array.push_back(string_value); + remaining_length -= chars_to_read; + } + return true; +} + inline std::string to_hex_string(uint32_t value) { return fmt::format("{:08X}", value); } diff --git a/src/xenia/cpu/xex_module.cc b/src/xenia/cpu/xex_module.cc index 672d44666..b0b963467 100644 --- a/src/xenia/cpu/xex_module.cc +++ b/src/xenia/cpu/xex_module.cc @@ -930,8 +930,6 @@ bool XexModule::Load(const std::string_view name, const std::string_view path, name_ = name; path_ = path; - uint8_t* data = memory()->TranslateVirtual(base_address_); - // Load in the XEX basefile // We'll try using both XEX2 keys to see if any give a valid PE int result_code = ReadImage(xex_addr, xex_length, false); diff --git a/src/xenia/emulator.cc b/src/xenia/emulator.cc index 0fbf74d18..d0a7861b7 100644 --- a/src/xenia/emulator.cc +++ b/src/xenia/emulator.cc @@ -219,6 +219,8 @@ X_STATUS Emulator::Setup( // Bring up the virtual filesystem used by the kernel. file_system_ = std::make_unique(); + patcher_ = std::make_unique(storage_root_); + // Shared kernel state. kernel_state_ = std::make_unique(this); diff --git a/src/xenia/emulator.h b/src/xenia/emulator.h index 96d81d00a..0e3b9164b 100644 --- a/src/xenia/emulator.h +++ b/src/xenia/emulator.h @@ -21,6 +21,7 @@ #include "xenia/base/exception_handler.h" #include "xenia/kernel/kernel_state.h" #include "xenia/memory.h" +#include "xenia/patcher/patcher.h" #include "xenia/vfs/virtual_file_system.h" #include "xenia/xbox.h" @@ -154,6 +155,8 @@ class Emulator { // This is effectively the guest operating system. kernel::KernelState* kernel_state() const { return kernel_state_.get(); } + patcher::Patcher* patcher() const { return patcher_.get(); } + // Initializes the emulator and configures all components. // The given window is used for display and the provided functions are used // to create subsystems as required. @@ -202,6 +205,7 @@ class Emulator { public: xe::Delegate on_launch; xe::Delegate on_shader_storage_initialization; + xe::Delegate<> on_patch_apply; xe::Delegate<> on_terminate; xe::Delegate<> on_exit; @@ -237,6 +241,7 @@ class Emulator { std::unique_ptr export_resolver_; std::unique_ptr file_system_; + std::unique_ptr patcher_; std::unique_ptr kernel_state_; diff --git a/src/xenia/kernel/kernel_state.cc b/src/xenia/kernel/kernel_state.cc index 7d60aac64..8c514eb17 100644 --- a/src/xenia/kernel/kernel_state.cc +++ b/src/xenia/kernel/kernel_state.cc @@ -406,6 +406,9 @@ object_ref KernelState::LoadUserModule( } module->Dump(); + emulator_->patcher()->ApplyPatchesForTitle(memory_, module->title_id(), + module->hash()); + emulator_->on_patch_apply(); if (module->is_dll_module() && module->entry_point() && call_entry) { // Call DllMain(DLL_PROCESS_ATTACH): diff --git a/src/xenia/kernel/user_module.cc b/src/xenia/kernel/user_module.cc index a85a919c6..8bb5616fe 100644 --- a/src/xenia/kernel/user_module.cc +++ b/src/xenia/kernel/user_module.cc @@ -13,6 +13,7 @@ #include "xenia/base/byte_stream.h" #include "xenia/base/logging.h" +#include "xenia/base/xxhash.h" #include "xenia/cpu/elf_module.h" #include "xenia/cpu/processor.h" #include "xenia/cpu/xex_module.h" @@ -128,6 +129,9 @@ X_STATUS UserModule::LoadFromFile(const std::string_view path) { } } + CalculateHash(); + + XELOGI("Module hash: {:016X} for {}", hash_.value_or(UINT64_MAX), name_); return LoadXexContinue(); } @@ -804,5 +808,46 @@ void UserModule::Dump() { xe::logging::AppendLogLine(xe::LogLevel::Info, 'i', sb.to_string_view()); } +void UserModule::CalculateHash() { + const BaseHeap* module_heap = + kernel_state_->memory()->LookupHeap(xex_module()->base_address()); + + if (!module_heap) { + XELOGE("Invalid heap for xex module! Address: {:08X}", + xex_module()->base_address()); + return; + } + + const uint32_t page_size = module_heap->page_size(); + auto security_info = xex_module()->xex_security_info(); + + auto find_code_section_page = [&security_info](bool from_bottom) { + for (uint32_t i = 0; i < security_info->page_descriptor_count; i++) { + const uint32_t page_index = + from_bottom ? i : security_info->page_descriptor_count - i; + xex2_page_descriptor page_descriptor; + page_descriptor.value = + xe::byte_swap(security_info->page_descriptors[page_index].value); + if (page_descriptor.info != XEX_SECTION_CODE) { + continue; + } + return page_index; + } + return UINT32_MAX; + }; + + const uint32_t start_address = + xex_module()->base_address() + (find_code_section_page(true) * page_size); + const uint32_t end_address = + xex_module()->base_address() + + ((find_code_section_page(false) + 1) * page_size); + + uint8_t* base_code_adr = memory()->TranslateVirtual(start_address); + + XXH3_state_t hash_state; + XXH3_64bits_reset(&hash_state); + XXH3_64bits_update(&hash_state, base_code_adr, end_address - start_address); + hash_ = XXH3_64bits_digest(&hash_state); +} } // namespace kernel } // namespace xe diff --git a/src/xenia/kernel/user_module.h b/src/xenia/kernel/user_module.h index f904f8a4f..d78cc79bd 100644 --- a/src/xenia/kernel/user_module.h +++ b/src/xenia/kernel/user_module.h @@ -38,6 +38,7 @@ class UserModule : public XModule { const std::string& path() const override { return path_; } const std::string& name() const override { return name_; } + std::optional hash() const { return hash_; } enum ModuleFormat { kModuleFormatUndefined = 0, @@ -97,9 +98,11 @@ class UserModule : public XModule { private: X_STATUS LoadXexContinue(); + void CalculateHash(); std::string name_; std::string path_; + std::optional hash_ = std::nullopt; uint32_t guest_xex_header_ = 0; ModuleFormat module_format_ = kModuleFormatUndefined; diff --git a/src/xenia/patcher/patch_db.cc b/src/xenia/patcher/patch_db.cc new file mode 100644 index 000000000..6ebe48e6a --- /dev/null +++ b/src/xenia/patcher/patch_db.cc @@ -0,0 +1,223 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2022 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ +#include "xenia/base/cvar.h" +#include "xenia/base/filesystem.h" +#include "xenia/base/logging.h" +#include "xenia/memory.h" + +#include "xenia/patcher/patch_db.h" + +DEFINE_bool(apply_patches, true, "Enables custom patching functionality", + "General"); + +namespace xe { +namespace patcher { + +PatchDB::PatchDB(const std::filesystem::path patches_root) { + patches_root_ = patches_root; + LoadPatches(); +} + +PatchDB::~PatchDB() {} + +void PatchDB::LoadPatches() { + if (!cvars::apply_patches) { + return; + } + + const std::filesystem::path patches_directory = patches_root_ / "patches"; + const std::vector patch_files = + filesystem::ListFiles(patches_directory); + + for (const xe::filesystem::FileInfo& patch_file : patch_files) { + // Skip files that doesn't have only title_id as name and .patch as + // extension + if (!std::regex_match(path_to_utf8(patch_file.name), + patch_filename_regex_)) { + XELOGE("PatchDB: Skipped loading file {} due to incorrect filename", + path_to_utf8(patch_file.name)); + continue; + } + + const PatchFileEntry loaded_title_patches = + ReadPatchFile(patch_file.path / patch_file.name); + if (loaded_title_patches.title_id != -1) { + loaded_patches_.push_back(loaded_title_patches); + } + } + XELOGI("PatchDB: Loaded patches for {} titles", loaded_patches_.size()); +} + +PatchFileEntry PatchDB::ReadPatchFile(const std::filesystem::path& file_path) { + PatchFileEntry patch_file; + std::shared_ptr patch_toml_fields; + + try { + patch_toml_fields = cpptoml::parse_file(path_to_utf8(file_path)); + } catch (...) { + XELOGE("PatchDB: Cannot load patch file: {}", path_to_utf8(file_path)); + patch_file.title_id = -1; + return patch_file; + }; + + auto title_name = patch_toml_fields->get_as("title_name"); + auto title_id = patch_toml_fields->get_as("title_id"); + + patch_file.title_id = strtoul((*title_id).c_str(), NULL, 16); + patch_file.title_name = *title_name; + ReadHashes(patch_file, patch_toml_fields); + + auto patch_table = patch_toml_fields->get_table_array("patch"); + + for (auto patch_table_entry : *patch_table) { + PatchInfoEntry patch = PatchInfoEntry(); + auto patch_name = *patch_table_entry->get_as("name"); + auto patch_desc = *patch_table_entry->get_as("desc"); + auto patch_author = *patch_table_entry->get_as("author"); + auto is_enabled = *patch_table_entry->get_as("is_enabled"); + + patch.id = 0; // Todo(Gliniak): Implement id for future GUI stuff + patch.patch_name = patch_name; + patch.patch_desc = patch_desc; + patch.patch_author = patch_author; + patch.is_enabled = is_enabled; + + // Iterate through all available data sizes + for (const auto& patch_data_type : patch_data_types_size_) { + bool success = + ReadPatchData(patch.patch_data, patch_data_type, patch_table_entry); + + if (!success) { + XELOGE("PatchDB: Cannot read patch {}", patch_name); + break; + } + } + patch_file.patch_info.push_back(patch); + } + return patch_file; +} + +bool PatchDB::ReadPatchData( + std::vector& patch_data, + const std::pair data_type, + const std::shared_ptr& patch_table) { + auto patch_data_tarr = patch_table->get_table_array(data_type.first); + if (!patch_data_tarr) { + return true; + } + + for (const auto& patch_data_table : *patch_data_tarr) { + uint32_t address = *patch_data_table->get_as("address"); + size_t alloc_size = (size_t)data_type.second.size; + + switch (data_type.second.type) { + case PatchDataType::kBE8: { + uint16_t value = *patch_data_table->get_as("value"); + patch_data.push_back({address, PatchDataValue(alloc_size, value)}); + break; + } + case PatchDataType::kBE16: { + uint16_t value = *patch_data_table->get_as("value"); + patch_data.push_back( + {address, PatchDataValue(alloc_size, xe::byte_swap(value))}); + break; + } + case PatchDataType::kBE32: { + uint32_t value = *patch_data_table->get_as("value"); + patch_data.push_back( + {address, PatchDataValue(alloc_size, xe::byte_swap(value))}); + break; + } + case PatchDataType::kBE64: { + uint64_t value = *patch_data_table->get_as("value"); + patch_data.push_back( + {address, PatchDataValue(alloc_size, xe::byte_swap(value))}); + break; + } + case PatchDataType::kF64: { + double val = *patch_data_table->get_as("value"); + uint64_t value = *reinterpret_cast(&val); + patch_data.push_back( + {address, PatchDataValue(alloc_size, xe::byte_swap(value))}); + break; + } + case PatchDataType::kF32: { + float value = float(*patch_data_table->get_as("value")); + patch_data.push_back( + {address, PatchDataValue(alloc_size, xe::byte_swap(value))}); + break; + } + case PatchDataType::kString: { + std::string value = *patch_data_table->get_as("value"); + patch_data.push_back({address, PatchDataValue(value)}); + break; + } + case PatchDataType::kU16String: { + std::u16string value = + xe::to_utf16(*patch_data_table->get_as("value")); + patch_data.push_back({address, PatchDataValue(value)}); + break; + } + case PatchDataType::kByteArray: { + std::vector data; + const std::string value = + *patch_data_table->get_as("value"); + + bool success = string_util::hex_string_to_array(data, value); + if (!success) { + XELOGW("PatchDB: Cannot convert hex string to byte array! Skipping", + address); + return false; + } + patch_data.push_back({address, PatchDataValue(data)}); + break; + } + default: { + XELOGW("PatchDB: Unknown patch data type for address {:08X}! Skipping", + address); + return false; + } + } + } + return true; +} + +std::vector PatchDB::GetTitlePatches( + const uint32_t title_id, const std::optional hash) { + std::vector title_patches; + + std::copy_if( + loaded_patches_.cbegin(), loaded_patches_.cend(), + std::back_inserter(title_patches), [=](const PatchFileEntry entry) { + bool hash_exist = std::find(entry.hashes.cbegin(), entry.hashes.cend(), + hash) != entry.hashes.cend(); + + return entry.title_id == title_id && + (entry.hashes.empty() || hash_exist); + }); + + return title_patches; +} + +void PatchDB::ReadHashes(PatchFileEntry& patch_entry, + std::shared_ptr patch_toml_fields) { + auto title_hashes = patch_toml_fields->get_array_of("hash"); + + for (const auto& hash : *title_hashes) { + patch_entry.hashes.push_back(strtoull(hash.c_str(), NULL, 16)); + } + + auto single_hash = patch_toml_fields->get_as("hash"); + if (single_hash) { + patch_entry.hashes.push_back(strtoull((*single_hash).c_str(), NULL, 16)); + } +} + +} // namespace patcher +} // namespace xe diff --git a/src/xenia/patcher/patch_db.h b/src/xenia/patcher/patch_db.h new file mode 100644 index 000000000..f707c9281 --- /dev/null +++ b/src/xenia/patcher/patch_db.h @@ -0,0 +1,132 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2022 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_PATCH_DB_H_ +#define XENIA_PATCH_DB_H_ + +#include +#include +#include +#include + +#include "third_party/cpptoml/include/cpptoml.h" + +namespace xe { +namespace patcher { + +struct PatchDataValue { + const size_t alloc_size; + std::vector patch_data; + + template + PatchDataValue(const size_t size, const T value) : alloc_size(size) { + patch_data.resize(alloc_size); + std::memcpy(patch_data.data(), &value, alloc_size); + }; + + PatchDataValue(const std::vector value) : alloc_size(value.size()) { + patch_data.resize(alloc_size); + std::memcpy(patch_data.data(), value.data(), alloc_size); + }; + + PatchDataValue(const std::string value) : alloc_size(value.size()) { + patch_data.resize(alloc_size); + std::memcpy(patch_data.data(), value.c_str(), alloc_size); + }; + + PatchDataValue(const std::u16string value) : alloc_size(value.size() * 2) { + patch_data.resize(alloc_size); + std::memcpy(patch_data.data(), value.c_str(), alloc_size); + }; +}; + +struct PatchDataEntry { + const uint32_t address; + const PatchDataValue data; + + PatchDataEntry(const uint32_t memory_address, const PatchDataValue patch_data) + : address(memory_address), data(patch_data){}; +}; + +struct PatchInfoEntry { + uint32_t id; + std::string patch_name; + std::string patch_desc; + std::string patch_author; + std::vector patch_data; + bool is_enabled; +}; + +struct PatchFileEntry { + uint32_t title_id; + std::string title_name; + std::vector hashes; + std::vector patch_info; +}; + +enum class PatchDataType { + kBE8, + kBE16, + kBE32, + kBE64, + kF32, + kF64, + kString, + kU16String, + kByteArray +}; + +struct PatchData { + uint8_t size; + PatchDataType type; + + PatchData(uint8_t size_, PatchDataType type_) : size(size_), type(type_){}; +}; + +class PatchDB { + public: + PatchDB(const std::filesystem::path patches_root); + ~PatchDB(); + + void LoadPatches(); + + PatchFileEntry ReadPatchFile(const std::filesystem::path& file_path); + bool ReadPatchData(std::vector& patch_data, + const std::pair data_type, + const std::shared_ptr& patch_table); + + std::vector GetTitlePatches( + const uint32_t title_id, const std::optional hash); + std::vector& GetAllPatches() { return loaded_patches_; } + + private: + void ReadHashes(PatchFileEntry& patch_entry, + std::shared_ptr patch_toml_fields); + + inline static const std::regex patch_filename_regex_ = + std::regex("^[A-Fa-f0-9]{8}.*\\.patch\\.toml$"); + + const std::map patch_data_types_size_ = { + {"string", PatchData(0, PatchDataType::kString)}, + {"u16string", PatchData(0, PatchDataType::kU16String)}, + {"array", PatchData(0, PatchDataType::kByteArray)}, + {"f64", PatchData(sizeof(uint64_t), PatchDataType::kF64)}, + {"f32", PatchData(sizeof(uint32_t), PatchDataType::kF32)}, + {"be64", PatchData(sizeof(uint64_t), PatchDataType::kBE64)}, + {"be32", PatchData(sizeof(uint32_t), PatchDataType::kBE32)}, + {"be16", PatchData(sizeof(uint16_t), PatchDataType::kBE16)}, + {"be8", PatchData(sizeof(uint8_t), PatchDataType::kBE8)}}; + + std::vector loaded_patches_; + std::filesystem::path patches_root_; +}; +} // namespace patcher +} // namespace xe + +#endif // XENIA_PATCH_LOADER_H_ diff --git a/src/xenia/patcher/patcher.cc b/src/xenia/patcher/patcher.cc new file mode 100644 index 000000000..42ce42518 --- /dev/null +++ b/src/xenia/patcher/patcher.cc @@ -0,0 +1,66 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2022 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ +#include + +#include "xenia/base/logging.h" +#include "xenia/patcher/patcher.h" + +namespace xe { +namespace patcher { + +Patcher::Patcher(const std::filesystem::path patches_root) { + is_any_patch_applied_ = false; + patch_db_ = new PatchDB(patches_root); +} + +void Patcher::ApplyPatchesForTitle(Memory* memory, const uint32_t title_id, + const std::optional hash) { + const auto title_patches = patch_db_->GetTitlePatches(title_id, hash); + + for (const PatchFileEntry& patchFile : title_patches) { + for (const PatchInfoEntry& patchEntry : patchFile.patch_info) { + if (!patchEntry.is_enabled) { + continue; + } + XELOGE("Patcher: Applying patch for: {}({:08X}) - {}", + patchFile.title_name, patchFile.title_id, patchEntry.patch_name); + ApplyPatch(memory, &patchEntry); + } + } +} + +void Patcher::ApplyPatch(Memory* memory, const PatchInfoEntry* patch) { + for (const PatchDataEntry& patch_data_entry : patch->patch_data) { + uint32_t old_address_protect = 0; + uint8_t* address = memory->TranslateVirtual(patch_data_entry.address); + xe::BaseHeap* heap = memory->LookupHeap(patch_data_entry.address); + if (!heap) { + continue; + } + + heap->QueryProtect(patch_data_entry.address, &old_address_protect); + + heap->Protect(patch_data_entry.address, + (uint32_t)patch_data_entry.data.alloc_size, + kMemoryProtectRead | kMemoryProtectWrite); + + std::memcpy(address, patch_data_entry.data.patch_data.data(), + patch_data_entry.data.alloc_size); + + // Restore previous protection + heap->Protect(patch_data_entry.address, + (uint32_t)patch_data_entry.data.alloc_size, + old_address_protect); + + is_any_patch_applied_ = true; + } +} + +} // namespace patcher +} // namespace xe diff --git a/src/xenia/patcher/patcher.h b/src/xenia/patcher/patcher.h new file mode 100644 index 000000000..b32a9ba1d --- /dev/null +++ b/src/xenia/patcher/patcher.h @@ -0,0 +1,36 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2022 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_PATCHER_H_ +#define XENIA_PATCHER_H_ + +#include "xenia/memory.h" +#include "xenia/patcher/patch_db.h" + +namespace xe { +namespace patcher { + +class Patcher { + public: + Patcher(const std::filesystem::path patches_root); + + void ApplyPatch(Memory* memory, const PatchInfoEntry* patch); + void ApplyPatchesForTitle(Memory* memory, const uint32_t title_id, + const std::optional hash); + + bool IsAnyPatchApplied() { return is_any_patch_applied_; } + + private: + PatchDB* patch_db_; + bool is_any_patch_applied_; +}; + +} // namespace patcher +} // namespace xe +#endif