Merge remote-tracking branch 'GliniakRepo/patchingSystem' into canary_pr

This commit is contained in:
Gliniak 2022-05-19 10:01:33 +02:00
commit 5247220e73
18 changed files with 520 additions and 2 deletions

View File

@ -286,6 +286,7 @@ workspace("xenia")
include("src/xenia/hid")
include("src/xenia/hid/nop")
include("src/xenia/kernel")
include("src/xenia/patcher")
include("src/xenia/ui")
include("src/xenia/ui/spirv")
include("src/xenia/ui/vulkan")

View File

@ -1004,6 +1004,10 @@ void EmulatorWindow::UpdateTitle() {
sb.Append(u8" (Preloading shaders\u2026)");
}
patcher::Patcher* patcher = emulator()->patcher();
if (patcher && patcher->IsAnyPatchApplied()) {
sb.Append(u8" [Patches Applied]");
}
window_->SetTitle(sb.to_string_view());
}

View File

@ -28,6 +28,7 @@ project("xenia-app")
"xenia-ui",
"xenia-ui-spirv",
"xenia-ui-vulkan",
"xenia-patcher",
"xenia-vfs",
})
links({

View File

@ -509,6 +509,10 @@ void EmulatorApp::EmulatorThread() {
});
});
emulator_->on_patch_apply.AddListener([this]() {
app_context().CallInUIThread([this]() { emulator_window_->UpdateTitle(); });
});
emulator_->on_terminate.AddListener([]() {
if (cvars::discord) {
discord::DiscordPresence::NotPlaying();

View File

@ -111,6 +111,32 @@ inline size_t copy_and_swap_maybe_truncating(char16_t* dest,
return chars_copied;
}
inline bool hex_string_to_array(std::vector<uint8_t>& output_array,
const std::string_view value) {
output_array.reserve((value.size() + 1) / 2);
size_t remaining_length = value.size();
while (remaining_length > 0) {
uint8_t chars_to_read = remaining_length > 1 ? 2 : 1;
const char* substring_pointer =
value.data() + value.size() - remaining_length;
uint8_t string_value = 0;
std::from_chars_result result = std::from_chars(
substring_pointer, substring_pointer + chars_to_read, string_value, 16);
if (result.ec != std::errc() ||
result.ptr != substring_pointer + chars_to_read) {
output_array.clear();
return false;
}
output_array.push_back(string_value);
remaining_length -= chars_to_read;
}
return true;
}
inline std::string to_hex_string(uint32_t value) {
return fmt::format("{:08X}", value);
}

View File

@ -930,8 +930,6 @@ bool XexModule::Load(const std::string_view name, const std::string_view path,
name_ = name;
path_ = path;
uint8_t* data = memory()->TranslateVirtual(base_address_);
// Load in the XEX basefile
// We'll try using both XEX2 keys to see if any give a valid PE
int result_code = ReadImage(xex_addr, xex_length, false);

View File

@ -219,6 +219,8 @@ X_STATUS Emulator::Setup(
// Bring up the virtual filesystem used by the kernel.
file_system_ = std::make_unique<xe::vfs::VirtualFileSystem>();
patcher_ = std::make_unique<xe::patcher::Patcher>(storage_root_);
// Shared kernel state.
kernel_state_ = std::make_unique<xe::kernel::KernelState>(this);

View File

@ -21,6 +21,7 @@
#include "xenia/base/exception_handler.h"
#include "xenia/kernel/kernel_state.h"
#include "xenia/memory.h"
#include "xenia/patcher/patcher.h"
#include "xenia/vfs/virtual_file_system.h"
#include "xenia/xbox.h"
@ -154,6 +155,8 @@ class Emulator {
// This is effectively the guest operating system.
kernel::KernelState* kernel_state() const { return kernel_state_.get(); }
patcher::Patcher* patcher() const { return patcher_.get(); }
// Initializes the emulator and configures all components.
// The given window is used for display and the provided functions are used
// to create subsystems as required.
@ -202,6 +205,7 @@ class Emulator {
public:
xe::Delegate<uint32_t, const std::string_view> on_launch;
xe::Delegate<bool> on_shader_storage_initialization;
xe::Delegate<> on_patch_apply;
xe::Delegate<> on_terminate;
xe::Delegate<> on_exit;
@ -237,6 +241,7 @@ class Emulator {
std::unique_ptr<cpu::ExportResolver> export_resolver_;
std::unique_ptr<vfs::VirtualFileSystem> file_system_;
std::unique_ptr<patcher::Patcher> patcher_;
std::unique_ptr<kernel::KernelState> kernel_state_;

View File

@ -39,6 +39,7 @@ project("xenia-gpu-d3d12-trace-viewer")
"xenia-ui",
"xenia-ui-d3d12",
"xenia-vfs",
"xenia-patcher",
})
links({
"aes_128",
@ -86,6 +87,7 @@ project("xenia-gpu-d3d12-trace-dump")
"xenia-ui",
"xenia-ui-d3d12",
"xenia-vfs",
"xenia-patcher",
})
links({
"aes_128",

View File

@ -45,6 +45,7 @@ project("xenia-gpu-vulkan-trace-viewer")
"xenia-ui-spirv",
"xenia-ui-vulkan",
"xenia-vfs",
"xenia-patcher",
})
links({
"aes_128",
@ -112,6 +113,7 @@ project("xenia-gpu-vulkan-trace-dump")
"xenia-ui-spirv",
"xenia-ui-vulkan",
"xenia-vfs",
"xenia-patcher",
})
links({
"aes_128",

View File

@ -406,6 +406,9 @@ object_ref<UserModule> KernelState::LoadUserModule(
}
module->Dump();
emulator_->patcher()->ApplyPatchesForTitle(memory_, module->title_id(),
module->hash());
emulator_->on_patch_apply();
if (module->is_dll_module() && module->entry_point() && call_entry) {
// Call DllMain(DLL_PROCESS_ATTACH):

View File

@ -13,6 +13,7 @@
#include "xenia/base/byte_stream.h"
#include "xenia/base/logging.h"
#include "xenia/base/xxhash.h"
#include "xenia/cpu/elf_module.h"
#include "xenia/cpu/processor.h"
#include "xenia/cpu/xex_module.h"
@ -128,6 +129,9 @@ X_STATUS UserModule::LoadFromFile(const std::string_view path) {
}
}
CalculateHash();
XELOGI("Module hash: {:016X} for {}", hash_, name_);
return LoadXexContinue();
}
@ -804,5 +808,13 @@ void UserModule::Dump() {
xe::logging::AppendLogLine(xe::LogLevel::Info, 'i', sb.to_string_view());
}
void UserModule::CalculateHash() {
uint8_t* base_adr = memory()->TranslateVirtual(xex_module()->base_address());
XXH3_state_t hash_state;
XXH3_64bits_reset(&hash_state);
XXH3_64bits_update(&hash_state, base_adr, xex_module()->image_size());
hash_ = XXH3_64bits_digest(&hash_state);
}
} // namespace kernel
} // namespace xe

View File

@ -38,6 +38,7 @@ class UserModule : public XModule {
const std::string& path() const override { return path_; }
const std::string& name() const override { return name_; }
uint64_t hash() const { return hash_; }
enum ModuleFormat {
kModuleFormatUndefined = 0,
@ -97,9 +98,11 @@ class UserModule : public XModule {
private:
X_STATUS LoadXexContinue();
void CalculateHash();
std::string name_;
std::string path_;
uint64_t hash_ = -1;
uint32_t guest_xex_header_ = 0;
ModuleFormat module_format_ = kModuleFormatUndefined;

View File

@ -0,0 +1,200 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2022 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include <regex>
#include "xenia/base/cvar.h"
#include "xenia/base/filesystem.h"
#include "xenia/base/logging.h"
#include "xenia/memory.h"
#include "xenia/patcher/patch_db.h"
DEFINE_bool(apply_patches, true, "Enables custom patching functionality",
"General");
namespace xe {
namespace patcher {
PatchDB::PatchDB(const std::filesystem::path patches_root) {
patches_root_ = patches_root;
LoadPatches();
}
PatchDB::~PatchDB() {}
void PatchDB::LoadPatches() {
if (!cvars::apply_patches) {
return;
}
const std::filesystem::path patches_directory = patches_root_ / "patches";
const std::vector<xe::filesystem::FileInfo>& patch_files =
filesystem::ListFiles(patches_directory);
const std::regex file_name_regex_match = std::regex(patch_filename_regex);
for (const xe::filesystem::FileInfo& patch_file : patch_files) {
// Skip files that doesn't have only title_id as name and .patch as
// extension
if (!std::regex_match(path_to_utf8(patch_file.name),
file_name_regex_match)) {
XELOGE("PatchDB: Skipped loading file {} due to incorrect filename",
path_to_utf8(patch_file.name));
continue;
}
const PatchFileEntry loaded_title_patches =
ReadPatchFile(patch_file.path / patch_file.name);
if (loaded_title_patches.title_id != -1) {
loaded_patches.push_back(loaded_title_patches);
}
}
XELOGI("PatchDB: Loaded patches for {} titles", loaded_patches.size());
}
PatchFileEntry PatchDB::ReadPatchFile(const std::filesystem::path& file_path) {
PatchFileEntry patchFile;
std::shared_ptr<cpptoml::table> patch_toml_fields;
try {
patch_toml_fields = cpptoml::parse_file(path_to_utf8(file_path));
} catch (...) {
XELOGE("PatchDB: Cannot load patch file: {}", path_to_utf8(file_path));
patchFile.title_id = -1;
return patchFile;
};
auto title_name = patch_toml_fields->get_as<std::string>("title_name");
auto title_id = patch_toml_fields->get_as<std::string>("title_id");
patchFile.title_id = strtoul((*title_id).c_str(), NULL, 16);
patchFile.title_name = *title_name;
ReadHash(patchFile, patch_toml_fields);
auto patch_table = patch_toml_fields->get_table_array("patch");
for (auto patch_table_entry : *patch_table) {
PatchInfoEntry patch = PatchInfoEntry();
auto patch_name = *patch_table_entry->get_as<std::string>("name");
auto patch_desc = *patch_table_entry->get_as<std::string>("desc");
auto patch_author = *patch_table_entry->get_as<std::string>("author");
auto is_enabled = *patch_table_entry->get_as<bool>("is_enabled");
patch.id = 0; // Todo(Gliniak): Implement id for future GUI stuff
patch.patch_name = patch_name;
patch.patch_desc = patch_desc;
patch.patch_author = patch_author;
patch.is_enabled = is_enabled;
// Iterate through all available data sizes
for (const auto& patch_data_type : patch_data_types_size) {
bool success =
ReadPatchData(patch.patch_data, patch_data_type, patch_table_entry);
if (!success) {
XELOGE("PatchDB: Cannot read patch {}", patch_name);
break;
}
}
patchFile.patch_info.push_back(patch);
}
return patchFile;
}
bool PatchDB::ReadPatchData(
std::vector<PatchDataEntry>& patch_data,
const std::pair<std::string, PatchData> data_type,
const std::shared_ptr<cpptoml::table>& patch_table) {
auto patch_data_tarr = patch_table->get_table_array(data_type.first);
if (!patch_data_tarr) {
return true;
}
for (const auto& patch_data_table : *patch_data_tarr) {
uint32_t address = *patch_data_table->get_as<std::uint32_t>("address");
size_t alloc_size = (size_t)data_type.second.size;
switch (data_type.second.type) {
case PatchDataType::f64: {
double val = *patch_data_table->get_as<double>("value");
uint64_t value = *reinterpret_cast<uint64_t*>(&val);
patch_data.push_back({address, PatchDataValue(alloc_size, value)});
break;
}
case PatchDataType::f32: {
float value = float(*patch_data_table->get_as<double>("value"));
patch_data.push_back({address, PatchDataValue(alloc_size, value)});
break;
}
case PatchDataType::string: {
std::string value = *patch_data_table->get_as<std::string>("value");
patch_data.push_back({address, PatchDataValue(value)});
break;
}
case PatchDataType::u16string: {
std::u16string value =
xe::to_utf16(*patch_data_table->get_as<std::string>("value"));
patch_data.push_back({address, PatchDataValue(value)});
break;
}
case PatchDataType::byte_array: {
std::vector<uint8_t> data;
const std::string value =
*patch_data_table->get_as<std::string>("value");
bool success = string_util::hex_string_to_array(data, value);
if (!success) {
return false;
}
patch_data.push_back({address, PatchDataValue(value.size() / 2, data)});
break;
}
default: {
uint64_t value = *patch_data_table->get_as<uint64_t>("value");
patch_data.push_back({address, PatchDataValue(alloc_size, value)});
break;
}
}
}
return true;
}
std::vector<PatchFileEntry> PatchDB::GetTitlePatches(uint32_t title_id,
const uint64_t hash) {
std::vector<PatchFileEntry> title_patches;
std::copy_if(
loaded_patches.cbegin(), loaded_patches.cend(),
std::back_inserter(title_patches), [=](const PatchFileEntry entry) {
bool hash_exist = std::find(entry.hashes.cbegin(), entry.hashes.cend(),
hash) != entry.hashes.cend();
return entry.title_id == title_id &&
(entry.hashes.empty() || hash_exist);
});
return title_patches;
}
void PatchDB::ReadHash(PatchFileEntry& patchEntry,
std::shared_ptr<cpptoml::table> patch_toml_fields) {
auto title_hashes = patch_toml_fields->get_array_of<std::string>("hash");
for (const auto& hash : *title_hashes) {
patchEntry.hashes.push_back(strtoull(hash.c_str(), NULL, 16));
}
auto single_hash = patch_toml_fields->get_as<std::string>("hash");
if (single_hash) {
patchEntry.hashes.push_back(strtoull((*single_hash).c_str(), NULL, 16));
}
}
} // namespace patcher
} // namespace xe

View File

@ -0,0 +1,136 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2022 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_PATCH_DB_H_
#define XENIA_PATCH_DB_H_
#include <map>
#include "third_party/cpptoml/include/cpptoml.h"
namespace xe {
namespace patcher {
struct PatchDataValue {
const size_t alloc_size_;
const uint8_t* patch_data_ptr_;
PatchDataValue(const size_t alloc_size, const uint64_t value)
: alloc_size_(alloc_size) {
patch_data_ptr_ = new uint8_t[alloc_size_];
memcpy((void*)patch_data_ptr_, &value, alloc_size);
};
PatchDataValue(const size_t alloc_size, const float value)
: alloc_size_(alloc_size) {
patch_data_ptr_ = new uint8_t[alloc_size_];
memcpy((void*)patch_data_ptr_, &value, alloc_size);
};
PatchDataValue(const size_t alloc_size, const std::vector<uint8_t> value)
: alloc_size_(alloc_size) {
patch_data_ptr_ = new uint8_t[alloc_size_];
memcpy((void*)patch_data_ptr_, value.data(), alloc_size);
};
PatchDataValue(const std::string value) : alloc_size_(value.size()) {
patch_data_ptr_ = new uint8_t[alloc_size_];
memcpy((void*)patch_data_ptr_, value.c_str(), alloc_size_);
};
PatchDataValue(const std::u16string value) : alloc_size_(value.size() * 2) {
patch_data_ptr_ = new uint8_t[alloc_size_];
memcpy((void*)patch_data_ptr_, value.c_str(), alloc_size_);
};
};
struct PatchDataEntry {
const uint32_t memory_address_;
const PatchDataValue new_data_;
PatchDataEntry(const uint32_t memory_address, const PatchDataValue new_data)
: memory_address_(memory_address), new_data_(new_data){};
};
struct PatchInfoEntry {
uint32_t id;
std::string patch_name;
std::string patch_desc;
std::string patch_author;
std::vector<PatchDataEntry> patch_data;
bool is_enabled;
};
struct PatchFileEntry {
uint32_t title_id;
std::string title_name;
std::vector<uint64_t> hashes;
std::vector<PatchInfoEntry> patch_info;
};
enum class PatchDataType {
be8,
be16,
be32,
be64,
f32,
f64,
string,
u16string,
byte_array
};
struct PatchData {
uint8_t size;
PatchDataType type;
PatchData(uint8_t size_, PatchDataType type_) : size(size_), type(type_){};
};
class PatchDB {
public:
PatchDB(const std::filesystem::path patches_root);
~PatchDB();
void LoadPatches();
PatchFileEntry ReadPatchFile(const std::filesystem::path& file_path);
bool ReadPatchData(std::vector<PatchDataEntry>& patch_data,
const std::pair<std::string, PatchData> data_type,
const std::shared_ptr<cpptoml::table>& patch_table);
std::vector<PatchFileEntry> GetTitlePatches(uint32_t title_id,
const uint64_t hash);
std::vector<PatchFileEntry>& GetAllPatches() { return loaded_patches; }
private:
void ReadHash(PatchFileEntry& patchEntry,
std::shared_ptr<cpptoml::table> patch_toml_fields);
inline static const std::string patch_filename_regex =
"^[A-Fa-f0-9]{8}.*\\.patch\\.toml$";
const std::map<std::string, PatchData> patch_data_types_size = {
{"string", PatchData(0, PatchDataType::string)},
{"u16string", PatchData(0, PatchDataType::u16string)},
{"array", PatchData(0, PatchDataType::byte_array)},
{"f64", PatchData(sizeof(uint64_t), PatchDataType::f64)},
{"f32", PatchData(sizeof(uint32_t), PatchDataType::f32)},
{"be64", PatchData(sizeof(uint64_t), PatchDataType::be64)},
{"be32", PatchData(sizeof(uint32_t), PatchDataType::be32)},
{"be16", PatchData(sizeof(uint16_t), PatchDataType::be16)},
{"be8", PatchData(sizeof(uint8_t), PatchDataType::be8)}};
std::vector<PatchFileEntry> loaded_patches;
std::filesystem::path patches_root_;
};
} // namespace patcher
} // namespace xe
#endif // XENIA_PATCH_LOADER_H_

View File

@ -0,0 +1,68 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2022 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/patcher/patcher.h"
#include "xenia/base/logging.h"
namespace xe {
namespace patcher {
Patcher::Patcher(const std::filesystem::path patches_root) {
is_any_patch_applied_ = false;
patch_db = new PatchDB(patches_root);
}
Patcher::~Patcher() {}
void Patcher::ApplyPatchesForTitle(Memory* memory, const uint32_t title_id,
const uint64_t hash) {
const auto title_patches = patch_db->GetTitlePatches(title_id, hash);
for (const PatchFileEntry& patchFile : title_patches) {
for (const PatchInfoEntry& patchEntry : patchFile.patch_info) {
if (!patchEntry.is_enabled) {
continue;
}
XELOGE("Patcher: Applying patch for: {}({:08X}) - {}",
patchFile.title_name, patchFile.title_id, patchEntry.patch_name);
ApplyPatch(memory, &patchEntry);
}
}
}
void Patcher::ApplyPatch(Memory* memory, const PatchInfoEntry* patch) {
for (const PatchDataEntry& patch_data_entry : patch->patch_data) {
uint32_t old_address_protect = 0;
auto address = memory->TranslateVirtual(patch_data_entry.memory_address_);
auto heap = memory->LookupHeap(patch_data_entry.memory_address_);
if (!heap) {
continue;
}
heap->QueryProtect(patch_data_entry.memory_address_, &old_address_protect);
heap->Protect(patch_data_entry.memory_address_,
(uint32_t)patch_data_entry.new_data_.alloc_size_,
kMemoryProtectRead | kMemoryProtectWrite);
xe::copy_and_swap(address,
(uint8_t*)patch_data_entry.new_data_.patch_data_ptr_,
patch_data_entry.new_data_.alloc_size_);
// Restore previous protection
heap->Protect(patch_data_entry.memory_address_,
(uint32_t)patch_data_entry.new_data_.alloc_size_,
old_address_protect);
is_any_patch_applied_ = true;
}
}
} // namespace patcher
} // namespace xe

View File

@ -0,0 +1,37 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2022 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_PATCHER_H_
#define XENIA_PATCHER_H_
#include "xenia/memory.h"
#include "xenia/patcher/patch_db.h"
namespace xe {
namespace patcher {
class Patcher {
public:
Patcher(const std::filesystem::path patches_root);
~Patcher();
void ApplyPatch(Memory* memory, const PatchInfoEntry* patch);
void ApplyPatchesForTitle(Memory* memory, const uint32_t title_id,
const uint64_t hash);
bool IsAnyPatchApplied() { return is_any_patch_applied_; }
private:
PatchDB* patch_db;
bool is_any_patch_applied_;
};
} // namespace patcher
} // namespace xe
#endif

View File

@ -0,0 +1,14 @@
project_root = "../../.."
include(project_root.."/tools/build")
group("src")
project("xenia-patcher")
uuid("e1c75f76-9e7b-48f6-b17e-dbd20f7a1592")
kind("StaticLib")
language("C++")
links({
"xenia-base"
})
defines({
})
recursive_platform_files()