From dd3a049b2303a2b80759cbd76fbcbbce3ef89243 Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 14:59:47 -0600 Subject: [PATCH 01/11] [CPU] Oops. --- src/xenia/cpu/xex_module.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/xenia/cpu/xex_module.cc b/src/xenia/cpu/xex_module.cc index 6f8537970..e14d255bd 100644 --- a/src/xenia/cpu/xex_module.cc +++ b/src/xenia/cpu/xex_module.cc @@ -1144,9 +1144,8 @@ bool XexModule::LoadContinue() { } } - auto library_data = reinterpret_cast(opt_import_libraries) + - opt_import_libraries->string_table.size + 12; - uint32_t library_offset = 0; + auto library_data = reinterpret_cast(opt_import_libraries); + uint32_t library_offset = opt_import_libraries->string_table.size + 12; while (library_offset < opt_import_libraries->size) { auto library = reinterpret_cast(library_data + library_offset); From 5f764fc752c82674981a9f402f1bbd96b399112a Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 14:57:45 -0600 Subject: [PATCH 02/11] [CPU] Move LZX code to its own files. --- src/xenia/cpu/lzx.cc | 180 ++++++++++++++++++++++++++++++++++++ src/xenia/cpu/lzx.h | 29 ++++++ src/xenia/cpu/xex_module.cc | 162 +------------------------------- 3 files changed, 210 insertions(+), 161 deletions(-) create mode 100644 src/xenia/cpu/lzx.cc create mode 100644 src/xenia/cpu/lzx.h diff --git a/src/xenia/cpu/lzx.cc b/src/xenia/cpu/lzx.cc new file mode 100644 index 000000000..09602f2d1 --- /dev/null +++ b/src/xenia/cpu/lzx.cc @@ -0,0 +1,180 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2013 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/cpu/lzx.h" + +#include + +#include "xenia/base/byte_order.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/base/memory.h" +#include "xenia/kernel/util/xex2_info.h" + +#include "third_party/mspack/lzx.h" +#include "third_party/mspack/mspack.h" + +typedef struct mspack_memory_file_t { + struct mspack_system sys; + void* buffer; + off_t buffer_size; + off_t offset; +} mspack_memory_file; +mspack_memory_file* mspack_memory_open(struct mspack_system* sys, void* buffer, + const size_t buffer_size) { + assert_true(buffer_size < INT_MAX); + if (buffer_size >= INT_MAX) { + return NULL; + } + mspack_memory_file* memfile = + (mspack_memory_file*)calloc(1, sizeof(mspack_memory_file)); + if (!memfile) { + return NULL; + } + memfile->buffer = buffer; + memfile->buffer_size = (off_t)buffer_size; + memfile->offset = 0; + return memfile; +} +void mspack_memory_close(mspack_memory_file* file) { + mspack_memory_file* memfile = (mspack_memory_file*)file; + free(memfile); +} +int mspack_memory_read(struct mspack_file* file, void* buffer, int chars) { + mspack_memory_file* memfile = (mspack_memory_file*)file; + const off_t remaining = memfile->buffer_size - memfile->offset; + const off_t total = std::min(static_cast(chars), remaining); + memcpy(buffer, (uint8_t*)memfile->buffer + memfile->offset, total); + memfile->offset += total; + return (int)total; +} +int mspack_memory_write(struct mspack_file* file, void* buffer, int chars) { + mspack_memory_file* memfile = (mspack_memory_file*)file; + const off_t remaining = memfile->buffer_size - memfile->offset; + const off_t total = std::min(static_cast(chars), remaining); + memcpy((uint8_t*)memfile->buffer + memfile->offset, buffer, total); + memfile->offset += total; + return (int)total; +} +void* mspack_memory_alloc(struct mspack_system* sys, size_t chars) { + return calloc(chars, 1); +} +void mspack_memory_free(void* ptr) { free(ptr); } +void mspack_memory_copy(void* src, void* dest, size_t chars) { + memcpy(dest, src, chars); +} +struct mspack_system* mspack_memory_sys_create() { + struct mspack_system* sys = + (struct mspack_system*)calloc(1, sizeof(struct mspack_system)); + if (!sys) { + return NULL; + } + sys->read = mspack_memory_read; + sys->write = mspack_memory_write; + sys->alloc = mspack_memory_alloc; + sys->free = mspack_memory_free; + sys->copy = mspack_memory_copy; + return sys; +} +void mspack_memory_sys_destroy(struct mspack_system* sys) { free(sys); } + +int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest, + size_t dest_len, uint32_t window_size, void* window_data, + size_t window_data_len) { + uint32_t window_bits = 0; + uint32_t temp_sz = window_size; + for (size_t m = 0; m < 32; m++, window_bits++) { + temp_sz >>= 1; + if (temp_sz == 0x00000000) { + break; + } + } + + int result_code = 1; + + mspack_system* sys = mspack_memory_sys_create(); + mspack_memory_file* lzxsrc = + mspack_memory_open(sys, (void*)lzx_data, lzx_len); + mspack_memory_file* lzxdst = mspack_memory_open(sys, dest, dest_len); + lzxd_stream* lzxd = + lzxd_init(sys, (struct mspack_file*)lzxsrc, (struct mspack_file*)lzxdst, + window_bits, 0, 0x8000, (off_t)dest_len, 0); + + if (lzxd) { + if (window_data) { + // zero the window and then copy window_data to the end of it + memset(lzxd->window, 0, window_data_len); + memcpy(lzxd->window + (window_size - window_data_len), window_data, + window_data_len); + } + + result_code = lzxd_decompress(lzxd, (off_t)dest_len); + + lzxd_free(lzxd); + lzxd = NULL; + } + if (lzxsrc) { + mspack_memory_close(lzxsrc); + lzxsrc = NULL; + } + if (lzxdst) { + mspack_memory_close(lzxdst); + lzxdst = NULL; + } + if (sys) { + mspack_memory_sys_destroy(sys); + sys = NULL; + } + + return result_code; +} + +int lzxdelta_apply_patch(xe::xex2_delta_patch* patch, size_t patch_len, + uint32_t window_size, void* dest) { + void* patch_end = (char*)patch + patch_len; + auto* cur_patch = patch; + + while (patch_end > cur_patch) { + int patch_sz = -4; // 0 byte patches need us to remove 4 byte from next + // patch addr because of patch_data field + if (cur_patch->compressed_len == 0 && cur_patch->uncompressed_len == 0 && + cur_patch->new_addr == 0 && cur_patch->old_addr == 0) + break; + switch (cur_patch->compressed_len) { + case 0: // fill with 0 + memset((char*)dest + cur_patch->new_addr, 0, + cur_patch->uncompressed_len); + break; + case 1: // copy from old -> new + memcpy((char*)dest + cur_patch->new_addr, + (char*)dest + cur_patch->old_addr, cur_patch->uncompressed_len); + break; + default: // delta patch + patch_sz = + cur_patch->compressed_len - 4; // -4 because of patch_data field + + int result = lzx_decompress( + cur_patch->patch_data, cur_patch->compressed_len, + (char*)dest + cur_patch->new_addr, cur_patch->uncompressed_len, + window_size, (char*)dest + cur_patch->old_addr, + cur_patch->uncompressed_len); + + if (result) { + return result; + } + break; + } + + cur_patch++; + cur_patch = (xe::xex2_delta_patch*)((char*)cur_patch + + patch_sz); // TODO: make this less ugly + } + + return 0; +} diff --git a/src/xenia/cpu/lzx.h b/src/xenia/cpu/lzx.h new file mode 100644 index 000000000..ea47c220f --- /dev/null +++ b/src/xenia/cpu/lzx.h @@ -0,0 +1,29 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2013 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_CPU_LZX_H_ +#define XENIA_CPU_LZX_H_ + +#include +#include + +#include "xenia/cpu/module.h" + +namespace xe { +struct xex2_delta_patch; +} // namespace xe + +int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest, + size_t dest_len, uint32_t window_size, void* window_data, + size_t window_data_len); + +int lzxdelta_apply_patch(xe::xex2_delta_patch* patch, size_t patch_len, + uint32_t window_size, void* dest); + +#endif // XENIA_CPU_XEX_MODULE_H_ diff --git a/src/xenia/cpu/xex_module.cc b/src/xenia/cpu/xex_module.cc index e14d255bd..8f9e0be21 100644 --- a/src/xenia/cpu/xex_module.cc +++ b/src/xenia/cpu/xex_module.cc @@ -17,6 +17,7 @@ #include "xenia/base/memory.h" #include "xenia/cpu/cpu_flags.h" #include "xenia/cpu/export_resolver.h" +#include "xenia/cpu/lzx.h" #include "xenia/cpu/processor.h" #include "xenia/kernel/kernel_state.h" #include "xenia/kernel/xmodule.h" @@ -24,8 +25,6 @@ #include "third_party/crypto/TinySHA1.hpp" #include "third_party/crypto/rijndael-alg-fst.c" #include "third_party/crypto/rijndael-alg-fst.h" -#include "third_party/mspack/lzx.h" -#include "third_party/mspack/mspack.h" #include "third_party/pe/pe_image.h" static const uint8_t xe_xex2_retail_key[16] = { @@ -35,165 +34,6 @@ static const uint8_t xe_xex2_devkit_key[16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -typedef struct mspack_memory_file_t { - struct mspack_system sys; - void* buffer; - off_t buffer_size; - off_t offset; -} mspack_memory_file; -mspack_memory_file* mspack_memory_open(struct mspack_system* sys, void* buffer, - const size_t buffer_size) { - assert_true(buffer_size < INT_MAX); - if (buffer_size >= INT_MAX) { - return NULL; - } - mspack_memory_file* memfile = - (mspack_memory_file*)calloc(1, sizeof(mspack_memory_file)); - if (!memfile) { - return NULL; - } - memfile->buffer = buffer; - memfile->buffer_size = (off_t)buffer_size; - memfile->offset = 0; - return memfile; -} -void mspack_memory_close(mspack_memory_file* file) { - mspack_memory_file* memfile = (mspack_memory_file*)file; - free(memfile); -} -int mspack_memory_read(struct mspack_file* file, void* buffer, int chars) { - mspack_memory_file* memfile = (mspack_memory_file*)file; - const off_t remaining = memfile->buffer_size - memfile->offset; - const off_t total = std::min(static_cast(chars), remaining); - memcpy(buffer, (uint8_t*)memfile->buffer + memfile->offset, total); - memfile->offset += total; - return (int)total; -} -int mspack_memory_write(struct mspack_file* file, void* buffer, int chars) { - mspack_memory_file* memfile = (mspack_memory_file*)file; - const off_t remaining = memfile->buffer_size - memfile->offset; - const off_t total = std::min(static_cast(chars), remaining); - memcpy((uint8_t*)memfile->buffer + memfile->offset, buffer, total); - memfile->offset += total; - return (int)total; -} -void* mspack_memory_alloc(struct mspack_system* sys, size_t chars) { - return calloc(chars, 1); -} -void mspack_memory_free(void* ptr) { free(ptr); } -void mspack_memory_copy(void* src, void* dest, size_t chars) { - memcpy(dest, src, chars); -} -struct mspack_system* mspack_memory_sys_create() { - struct mspack_system* sys = - (struct mspack_system*)calloc(1, sizeof(struct mspack_system)); - if (!sys) { - return NULL; - } - sys->read = mspack_memory_read; - sys->write = mspack_memory_write; - sys->alloc = mspack_memory_alloc; - sys->free = mspack_memory_free; - sys->copy = mspack_memory_copy; - return sys; -} -void mspack_memory_sys_destroy(struct mspack_system* sys) { free(sys); } - -int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest, - size_t dest_len, uint32_t window_size, void* window_data, - size_t window_data_len) { - uint32_t window_bits = 0; - uint32_t temp_sz = window_size; - for (size_t m = 0; m < 32; m++, window_bits++) { - temp_sz >>= 1; - if (temp_sz == 0x00000000) { - break; - } - } - - int result_code = 1; - - mspack_system* sys = mspack_memory_sys_create(); - mspack_memory_file* lzxsrc = - mspack_memory_open(sys, (void*)lzx_data, lzx_len); - mspack_memory_file* lzxdst = mspack_memory_open(sys, dest, dest_len); - lzxd_stream* lzxd = - lzxd_init(sys, (struct mspack_file*)lzxsrc, (struct mspack_file*)lzxdst, - window_bits, 0, 0x8000, (off_t)dest_len, 0); - - if (lzxd) { - if (window_data) { - // zero the window and then copy window_data to the end of it - memset(lzxd->window, 0, window_data_len); - memcpy(lzxd->window + (window_size - window_data_len), window_data, - window_data_len); - } - - result_code = lzxd_decompress(lzxd, (off_t)dest_len); - - lzxd_free(lzxd); - lzxd = NULL; - } - if (lzxsrc) { - mspack_memory_close(lzxsrc); - lzxsrc = NULL; - } - if (lzxdst) { - mspack_memory_close(lzxdst); - lzxdst = NULL; - } - if (sys) { - mspack_memory_sys_destroy(sys); - sys = NULL; - } - - return result_code; -} - -int lzxdelta_apply_patch(xe::xex2_delta_patch* patch, size_t patch_len, - uint32_t window_size, void* dest) { - void* patch_end = (char*)patch + patch_len; - auto* cur_patch = patch; - - while (patch_end > cur_patch) { - int patch_sz = -4; // 0 byte patches need us to remove 4 byte from next - // patch addr because of patch_data field - if (cur_patch->compressed_len == 0 && cur_patch->uncompressed_len == 0 && - cur_patch->new_addr == 0 && cur_patch->old_addr == 0) - break; - switch (cur_patch->compressed_len) { - case 0: // fill with 0 - memset((char*)dest + cur_patch->new_addr, 0, - cur_patch->uncompressed_len); - break; - case 1: // copy from old -> new - memcpy((char*)dest + cur_patch->new_addr, - (char*)dest + cur_patch->old_addr, cur_patch->uncompressed_len); - break; - default: // delta patch - patch_sz = - cur_patch->compressed_len - 4; // -4 because of patch_data field - - int result = lzx_decompress( - cur_patch->patch_data, cur_patch->compressed_len, - (char*)dest + cur_patch->new_addr, cur_patch->uncompressed_len, - window_size, (char*)dest + cur_patch->old_addr, - cur_patch->uncompressed_len); - - if (result) { - return result; - } - break; - } - - cur_patch++; - cur_patch = (xe::xex2_delta_patch*)((char*)cur_patch + - patch_sz); // TODO: make this less ugly - } - - return 0; -} - void aes_decrypt_buffer(const uint8_t* session_key, const uint8_t* input_buffer, const size_t input_size, uint8_t* output_buffer, const size_t output_size) { From 77f9e1516e3b1407f057a337cbe9e7d97dfce4c7 Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 15:16:02 -0600 Subject: [PATCH 03/11] [CPU] Minor cleanup in LZX. --- src/xenia/cpu/lzx.cc | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/xenia/cpu/lzx.cc b/src/xenia/cpu/lzx.cc index 09602f2d1..98d263403 100644 --- a/src/xenia/cpu/lzx.cc +++ b/src/xenia/cpu/lzx.cc @@ -50,7 +50,7 @@ int mspack_memory_read(struct mspack_file* file, void* buffer, int chars) { mspack_memory_file* memfile = (mspack_memory_file*)file; const off_t remaining = memfile->buffer_size - memfile->offset; const off_t total = std::min(static_cast(chars), remaining); - memcpy(buffer, (uint8_t*)memfile->buffer + memfile->offset, total); + std::memcpy(buffer, (uint8_t*)memfile->buffer + memfile->offset, total); memfile->offset += total; return (int)total; } @@ -58,20 +58,20 @@ int mspack_memory_write(struct mspack_file* file, void* buffer, int chars) { mspack_memory_file* memfile = (mspack_memory_file*)file; const off_t remaining = memfile->buffer_size - memfile->offset; const off_t total = std::min(static_cast(chars), remaining); - memcpy((uint8_t*)memfile->buffer + memfile->offset, buffer, total); + std::memcpy((uint8_t*)memfile->buffer + memfile->offset, buffer, total); memfile->offset += total; return (int)total; } void* mspack_memory_alloc(struct mspack_system* sys, size_t chars) { - return calloc(chars, 1); + return std::calloc(chars, 1); } void mspack_memory_free(void* ptr) { free(ptr); } void mspack_memory_copy(void* src, void* dest, size_t chars) { - memcpy(dest, src, chars); + std::memcpy(dest, src, chars); } struct mspack_system* mspack_memory_sys_create() { struct mspack_system* sys = - (struct mspack_system*)calloc(1, sizeof(struct mspack_system)); + (struct mspack_system*)std::calloc(1, sizeof(struct mspack_system)); if (!sys) { return NULL; } @@ -109,9 +109,9 @@ int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest, if (lzxd) { if (window_data) { // zero the window and then copy window_data to the end of it - memset(lzxd->window, 0, window_data_len); - memcpy(lzxd->window + (window_size - window_data_len), window_data, - window_data_len); + std::memset(lzxd->window, 0, window_data_len); + std::memcpy(lzxd->window + (window_size - window_data_len), window_data, + window_data_len); } result_code = lzxd_decompress(lzxd, (off_t)dest_len); @@ -148,12 +148,13 @@ int lzxdelta_apply_patch(xe::xex2_delta_patch* patch, size_t patch_len, break; switch (cur_patch->compressed_len) { case 0: // fill with 0 - memset((char*)dest + cur_patch->new_addr, 0, - cur_patch->uncompressed_len); + std::memset((char*)dest + cur_patch->new_addr, 0, + cur_patch->uncompressed_len); break; case 1: // copy from old -> new - memcpy((char*)dest + cur_patch->new_addr, - (char*)dest + cur_patch->old_addr, cur_patch->uncompressed_len); + std::memcpy((char*)dest + cur_patch->new_addr, + (char*)dest + cur_patch->old_addr, + cur_patch->uncompressed_len); break; default: // delta patch patch_sz = From 5679b0f0c7d403149653a99ce7d612f23f6edbbd Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 15:16:39 -0600 Subject: [PATCH 04/11] [CPU] Fix XEX2 delta patching. --- src/xenia/cpu/lzx.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/xenia/cpu/lzx.cc b/src/xenia/cpu/lzx.cc index 98d263403..4ebd7b08d 100644 --- a/src/xenia/cpu/lzx.cc +++ b/src/xenia/cpu/lzx.cc @@ -112,6 +112,7 @@ int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest, std::memset(lzxd->window, 0, window_data_len); std::memcpy(lzxd->window + (window_size - window_data_len), window_data, window_data_len); + lzxd->ref_data_size = (uint32_t)window_data_len; } result_code = lzxd_decompress(lzxd, (off_t)dest_len); From 9162bc2af9dc643ab3a93d64281cbe412c57d9ec Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 16:20:46 -0600 Subject: [PATCH 05/11] [CPU] Fix Travis whining. --- src/xenia/cpu/lzx.cc | 1 + src/xenia/cpu/lzx.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/xenia/cpu/lzx.cc b/src/xenia/cpu/lzx.cc index 4ebd7b08d..f6fd77a9a 100644 --- a/src/xenia/cpu/lzx.cc +++ b/src/xenia/cpu/lzx.cc @@ -10,6 +10,7 @@ #include "xenia/cpu/lzx.h" #include +#include #include "xenia/base/byte_order.h" #include "xenia/base/logging.h" diff --git a/src/xenia/cpu/lzx.h b/src/xenia/cpu/lzx.h index ea47c220f..72515f944 100644 --- a/src/xenia/cpu/lzx.h +++ b/src/xenia/cpu/lzx.h @@ -26,4 +26,4 @@ int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest, int lzxdelta_apply_patch(xe::xex2_delta_patch* patch, size_t patch_len, uint32_t window_size, void* dest); -#endif // XENIA_CPU_XEX_MODULE_H_ +#endif // XENIA_CPU_LZX_H_ From c98f6af1d8b8d67f4e3919a80a6eb4f6734cf197 Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 17:47:08 -0600 Subject: [PATCH 06/11] [Kernel] Better stub XAM online schema. --- src/xenia/kernel/xam/xam_info.cc | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/xenia/kernel/xam/xam_info.cc b/src/xenia/kernel/xam/xam_info.cc index 997bbe33f..3065211b1 100644 --- a/src/xenia/kernel/xam/xam_info.cc +++ b/src/xenia/kernel/xam/xam_info.cc @@ -28,28 +28,30 @@ namespace xam { constexpr uint32_t X_LANGUAGE_ENGLISH = 1; constexpr uint32_t X_LANGUAGE_JAPANESE = 2; +// Empty stub schema binary. +uint8_t schema_bin[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x00, + 0x00, 0x2C, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x2C, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, +}; + dword_result_t XamGetOnlineSchema() { static uint32_t schema_guest = 0; - static uint32_t schema_ptr_guest = 0; if (!schema_guest) { - // create a dummy schema, 8 bytes of 0 seems to work fine - // (with another 8 bytes for schema ptr/schema size) - schema_guest = kernel_state()->memory()->SystemHeapAlloc(16); - schema_ptr_guest = schema_guest + 8; - + schema_guest = + kernel_state()->memory()->SystemHeapAlloc(8 + sizeof(schema_bin)); auto schema = kernel_state()->memory()->TranslateVirtual(schema_guest); - memset(schema, 0, 16); - - // store schema ptr + size - xe::store_and_swap(schema + 0x8, schema_guest); - xe::store_and_swap(schema + 0xC, 0x8); + std::memcpy(schema + 8, schema_bin, sizeof(schema_bin)); + xe::store_and_swap(schema + 0, schema_guest + 8); + xe::store_and_swap(schema + 4, sizeof(schema_bin)); } // return pointer to the schema ptr/schema size struct - return schema_ptr_guest; + return schema_guest; } -DECLARE_XAM_EXPORT2(XamGetOnlineSchema, kNone, kImplemented, kSketchy); +DECLARE_XAM_EXPORT1(XamGetOnlineSchema, kNone, kImplemented); void XamFormatDateString(dword_t unk, qword_t filetime, lpvoid_t buffer, dword_t buffer_length) { From 3baf6680550ee30fbafa9c343475e9814d187ec7 Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 19:04:42 -0600 Subject: [PATCH 07/11] [Kernel] Implement ExAcquireReadWriteLockExclusive, ExReleaseReadWriteLock. --- .../kernel/xboxkrnl/xboxkrnl_threading.cc | 101 +++++++++++++++--- 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.cc b/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.cc index 3a9d852b3..3a26b2ff1 100644 --- a/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.cc +++ b/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.cc @@ -373,8 +373,7 @@ void KeInitializeEvent(pointer_t event_ptr, dword_t event_type, } DECLARE_XBOXKRNL_EXPORT1(KeInitializeEvent, kThreading, kImplemented); -dword_result_t KeSetEvent(pointer_t event_ptr, dword_t increment, - dword_t wait) { +uint32_t keSetEvent(X_KEVENT* event_ptr, uint32_t increment, uint32_t wait) { auto ev = XObject::GetNativeObject(kernel_state(), event_ptr); if (!ev) { assert_always(); @@ -383,6 +382,11 @@ dword_result_t KeSetEvent(pointer_t event_ptr, dword_t increment, return ev->Set(increment, !!wait); } + +dword_result_t KeSetEvent(pointer_t event_ptr, dword_t increment, + dword_t wait) { + return keSetEvent(event_ptr, increment, wait); +} DECLARE_XBOXKRNL_EXPORT2(KeSetEvent, kThreading, kImplemented, kHighFrequency); dword_result_t KePulseEvent(pointer_t event_ptr, dword_t increment, @@ -508,9 +512,8 @@ void KeInitializeSemaphore(pointer_t semaphore_ptr, dword_t count, } DECLARE_XBOXKRNL_EXPORT1(KeInitializeSemaphore, kThreading, kImplemented); -dword_result_t KeReleaseSemaphore(pointer_t semaphore_ptr, - dword_t increment, dword_t adjustment, - dword_t wait) { +uint32_t keReleaseSemaphore(X_KSEMAPHORE* semaphore_ptr, uint32_t increment, + uint32_t adjustment, uint32_t wait) { auto sem = XObject::GetNativeObject(kernel_state(), semaphore_ptr); if (!sem) { @@ -523,6 +526,12 @@ dword_result_t KeReleaseSemaphore(pointer_t semaphore_ptr, return sem->ReleaseSemaphore(adjustment); } + +dword_result_t KeReleaseSemaphore(pointer_t semaphore_ptr, + dword_t increment, dword_t adjustment, + dword_t wait) { + return keReleaseSemaphore(semaphore_ptr, increment, adjustment, wait); +} DECLARE_XBOXKRNL_EXPORT1(KeReleaseSemaphore, kThreading, kImplemented); dword_result_t NtCreateSemaphore(lpdword_t handle_ptr, @@ -719,9 +728,9 @@ dword_result_t NtCancelTimer(dword_t timer_handle, } DECLARE_XBOXKRNL_EXPORT1(NtCancelTimer, kThreading, kImplemented); -dword_result_t KeWaitForSingleObject(lpvoid_t object_ptr, dword_t wait_reason, - dword_t processor_mode, dword_t alertable, - lpqword_t timeout_ptr) { +uint32_t keWaitForSingleObject(void* object_ptr, uint32_t wait_reason, + uint32_t processor_mode, uint32_t alertable, + uint64_t* timeout) { auto object = XObject::GetNativeObject(kernel_state(), object_ptr); if (!object) { @@ -730,12 +739,19 @@ dword_result_t KeWaitForSingleObject(lpvoid_t object_ptr, dword_t wait_reason, return X_STATUS_ABANDONED_WAIT_0; } - uint64_t timeout = timeout_ptr ? static_cast(*timeout_ptr) : 0u; - X_STATUS result = object->Wait(wait_reason, processor_mode, alertable, - timeout_ptr ? &timeout : nullptr); + X_STATUS result = + object->Wait(wait_reason, processor_mode, alertable, timeout); return result; } + +dword_result_t KeWaitForSingleObject(lpvoid_t object_ptr, dword_t wait_reason, + dword_t processor_mode, dword_t alertable, + lpqword_t timeout_ptr) { + uint64_t timeout = timeout_ptr ? static_cast(*timeout_ptr) : 0u; + return keWaitForSingleObject(object_ptr, wait_reason, processor_mode, + alertable, &timeout); +} DECLARE_XBOXKRNL_EXPORT3(KeWaitForSingleObject, kThreading, kImplemented, kBlocking, kHighFrequency); @@ -844,13 +860,12 @@ dword_result_t NtSignalAndWaitForSingleObjectEx(dword_t signal_handle, DECLARE_XBOXKRNL_EXPORT3(NtSignalAndWaitForSingleObjectEx, kThreading, kImplemented, kBlocking, kHighFrequency); -dword_result_t KfAcquireSpinLock(lpdword_t lock_ptr) { +uint32_t keKfAcquireSpinLock(uint32_t* lock) { // XELOGD( // "KfAcquireSpinLock(%.8X)", // lock_ptr); // Lock. - auto lock = reinterpret_cast(lock_ptr.host_address()); while (!xe::atomic_cas(0, 1, lock)) { // Spin! // TODO(benvanik): error on deadlock? @@ -863,18 +878,27 @@ dword_result_t KfAcquireSpinLock(lpdword_t lock_ptr) { return old_irql; } + +dword_result_t KfAcquireSpinLock(lpdword_t lock_ptr) { + auto lock = reinterpret_cast(lock_ptr.host_address()); + return keKfAcquireSpinLock(lock); +} DECLARE_XBOXKRNL_EXPORT3(KfAcquireSpinLock, kThreading, kImplemented, kBlocking, kHighFrequency); -void KfReleaseSpinLock(lpdword_t lock_ptr, dword_t old_irql) { +void keKfReleaseSpinLock(uint32_t* lock, dword_t old_irql) { // Restore IRQL. XThread* thread = XThread::GetCurrentThread(); thread->LowerIrql(old_irql); // Unlock. - auto lock = reinterpret_cast(lock_ptr.host_address()); xe::atomic_dec(lock); } + +void KfReleaseSpinLock(lpdword_t lock_ptr, dword_t old_irql) { + auto lock = reinterpret_cast(lock_ptr.host_address()); + keKfReleaseSpinLock(lock, old_irql); +} DECLARE_XBOXKRNL_EXPORT2(KfReleaseSpinLock, kThreading, kImplemented, kHighFrequency); @@ -1092,7 +1116,7 @@ struct X_ERWLOCK { be readers_entry_count; // 0xC X_KEVENT writer_event; // 0x10 X_KSEMAPHORE reader_semaphore; // 0x20 - be spin_lock; // 0x34 + uint32_t spin_lock; // 0x34 }; void ExInitializeReadWriteLock(pointer_t lock_ptr) { @@ -1105,6 +1129,51 @@ void ExInitializeReadWriteLock(pointer_t lock_ptr) { } DECLARE_XBOXKRNL_EXPORT1(ExInitializeReadWriteLock, kThreading, kImplemented); +void ExAcquireReadWriteLockExclusive(pointer_t lock_ptr) { + auto old_irql = keKfAcquireSpinLock(&lock_ptr->spin_lock); + lock_ptr->lock_count++; + keKfReleaseSpinLock(&lock_ptr->spin_lock, old_irql); + + if (!lock_ptr->lock_count) { + return; + } + + lock_ptr->writers_waiting_count++; + keWaitForSingleObject(&lock_ptr->writer_event, 0, 0, 0, nullptr); +} +DECLARE_XBOXKRNL_EXPORT4(ExAcquireReadWriteLockExclusive, kThreading, + kImplemented, kBlocking, kHighFrequency, kSketchy); + +void ExReleaseReadWriteLock(pointer_t lock_ptr) { + auto old_irql = keKfAcquireSpinLock(&lock_ptr->spin_lock); + lock_ptr->lock_count--; + + if (lock_ptr->lock_count < 0) { + keKfReleaseSpinLock(&lock_ptr->spin_lock, old_irql); + return; + } + + if (!lock_ptr->readers_entry_count) { + auto readers_waiting_count = lock_ptr->readers_waiting_count; + if (readers_waiting_count) { + lock_ptr->readers_waiting_count = 0; + lock_ptr->readers_entry_count = readers_waiting_count; + keKfReleaseSpinLock(&lock_ptr->spin_lock, old_irql); + keReleaseSemaphore(&lock_ptr->reader_semaphore, 1, readers_waiting_count, + 0); + return; + } + } + + auto count = lock_ptr->readers_entry_count--; + keKfReleaseSpinLock(&lock_ptr->spin_lock, old_irql); + if (!count) { + keSetEvent(&lock_ptr->writer_event, 1, 0); + } +} +DECLARE_XBOXKRNL_EXPORT2(ExReleaseReadWriteLock, kThreading, kImplemented, + kSketchy); + // NOTE: This function is very commonly inlined, and probably won't be called! pointer_result_t InterlockedPushEntrySList( pointer_t plist_ptr, pointer_t entry) { From 08fb15fcca7b04aab3eb005fdd501cee1fe6f358 Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 19:16:14 -0600 Subject: [PATCH 08/11] [Kernel] Oops. --- src/xenia/kernel/util/shim_utils.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/xenia/kernel/util/shim_utils.h b/src/xenia/kernel/util/shim_utils.h index a309dcdd1..0809a660e 100644 --- a/src/xenia/kernel/util/shim_utils.h +++ b/src/xenia/kernel/util/shim_utils.h @@ -564,6 +564,10 @@ using xe::cpu::ExportTag; DECLARE_EXPORT(xboxkrnl, name, category, \ xe::cpu::ExportTag::tag1 | xe::cpu::ExportTag::tag2 | \ xe::cpu::ExportTag::tag3) +#define DECLARE_XBOXKRNL_EXPORT4(name, category, tag1, tag2, tag3, tag4) \ + DECLARE_EXPORT(xboxkrnl, name, category, \ + xe::cpu::ExportTag::tag1 | xe::cpu::ExportTag::tag2 | \ + xe::cpu::ExportTag::tag3 | xe::cpu::ExportTag::tag4) } // namespace kernel } // namespace xe From 7f4f90b1ea877ad30b695d2e1a79d22a465ce9b4 Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 20:50:30 -0600 Subject: [PATCH 09/11] Emit mspack debug messages to Xenia. --- third_party/mspack.lua | 7 +++++-- third_party/mspack/logging.cc | 10 ++++++++++ third_party/mspack/lzxd.c | 5 +++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 third_party/mspack/logging.cc diff --git a/third_party/mspack.lua b/third_party/mspack.lua index 85b6bc08f..c1d1b44a5 100644 --- a/third_party/mspack.lua +++ b/third_party/mspack.lua @@ -2,8 +2,10 @@ group("third_party") project("mspack") uuid("0881692A-75A1-4E7B-87D8-BB9108CEDEA4") kind("StaticLib") - language("C") - + language("C++") + links({ + "xenia-base", + }) defines({ "_LIB", "HAVE_CONFIG_H", @@ -16,6 +18,7 @@ project("mspack") "mspack", }) files({ + "mspack/logging.cc", "mspack/lzx.h", "mspack/lzxd.c", "mspack/mspack.h", diff --git a/third_party/mspack/logging.cc b/third_party/mspack/logging.cc new file mode 100644 index 000000000..ae70478d0 --- /dev/null +++ b/third_party/mspack/logging.cc @@ -0,0 +1,10 @@ +#include "xenia/base/logging.h" + +extern "C" void xenia_log(const char* fmt, ...) { + char buffer[128]; + va_list args; + va_start(args, fmt); + vsnprintf(buffer, sizeof(buffer), fmt, args); + va_end(args); + XELOGW("mspack: %s", buffer); +} diff --git a/third_party/mspack/lzxd.c b/third_party/mspack/lzxd.c index 6cc33df08..548366409 100644 --- a/third_party/mspack/lzxd.c +++ b/third_party/mspack/lzxd.c @@ -15,6 +15,11 @@ #include #include +extern void xenia_log(const char*, ...); + +#undef D +#define D(x) do { xenia_log x; } while (0); + /* Microsoft's LZX document (in cab-sdk.exe) and their implementation * of the com.ms.util.cab Java package do not concur. * From 20315850dba1be29c81e58562b389866c8c13b75 Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 20:53:45 -0600 Subject: [PATCH 10/11] [Kernel] Stub XamFeatureEnabled. --- src/xenia/kernel/xam/xam_info.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/xenia/kernel/xam/xam_info.cc b/src/xenia/kernel/xam/xam_info.cc index 3065211b1..fd722a68c 100644 --- a/src/xenia/kernel/xam/xam_info.cc +++ b/src/xenia/kernel/xam/xam_info.cc @@ -28,6 +28,9 @@ namespace xam { constexpr uint32_t X_LANGUAGE_ENGLISH = 1; constexpr uint32_t X_LANGUAGE_JAPANESE = 2; +dword_result_t XamFeatureEnabled(dword_t unk) { return 0; } +DECLARE_XAM_EXPORT1(XamFeatureEnabled, kNone, kStub); + // Empty stub schema binary. uint8_t schema_bin[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, From 5fccc931eedc04379509939971f4e5ead2d770cd Mon Sep 17 00:00:00 2001 From: gibbed Date: Sat, 24 Nov 2018 21:32:19 -0600 Subject: [PATCH 11/11] Maybe fix Linux building mixed code projects? --- premake5.lua | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/premake5.lua b/premake5.lua index b8b1a07b5..891fbefa4 100644 --- a/premake5.lua +++ b/premake5.lua @@ -128,10 +128,6 @@ filter({"platforms:Linux", "toolset:gcc"}) end filter({"platforms:Linux", "language:C++", "toolset:clang"}) - buildoptions({ - "-std=c++14", - "-stdlib=libstdc++", - }) links({ "c++", "c++abi" @@ -139,6 +135,11 @@ filter({"platforms:Linux", "language:C++", "toolset:clang"}) disablewarnings({ "deprecated-register" }) +filter({"platforms:Linux", "language:C++", "toolset:clang", "files:*.cc or *.cpp"}) + buildoptions({ + "-std=c++14", + "-stdlib=libstdc++", + }) filter("platforms:Windows") system("windows")