From 147a70b9c1d3b3687a8890de4bc7fb2becd7ff0f Mon Sep 17 00:00:00 2001 From: Ben Vanik Date: Sat, 16 May 2015 00:23:13 -0700 Subject: [PATCH] Rewriting memory manager. --- src/xenia/apu/audio_system.cc | 9 +- src/xenia/apu/audio_system.h | 8 +- src/xenia/cpu/mmio_handler.cc | 107 +- src/xenia/cpu/mmio_handler.h | 32 +- src/xenia/cpu/mmio_handler_win.cc | 13 +- src/xenia/cpu/processor.cc | 5 + src/xenia/cpu/raw_module.cc | 5 +- src/xenia/cpu/test/util.h | 2 - src/xenia/cpu/thread_state.cc | 16 +- src/xenia/gpu/gl4/command_processor.cc | 98 +- src/xenia/gpu/gl4/gl4_graphics_system.cc | 6 +- src/xenia/gpu/gl4/gl4_graphics_system.h | 8 +- src/xenia/gpu/gl4/texture_cache.cc | 5 +- src/xenia/gpu/xe-gpu-trace-viewer.cc | 10 +- src/xenia/kernel/objects/xthread.cc | 3 + src/xenia/kernel/util/xex2.cc | 28 +- src/xenia/kernel/xam_info.cc | 2 +- src/xenia/kernel/xboxkrnl_memory.cc | 215 ++-- src/xenia/kernel/xboxkrnl_ob.cc | 5 + src/xenia/kernel/xboxkrnl_video.cc | 5 + src/xenia/kernel/xobject.cc | 1 + src/xenia/memory.cc | 1251 +++++++++++++++------- src/xenia/memory.h | 171 ++- 23 files changed, 1348 insertions(+), 657 deletions(-) diff --git a/src/xenia/apu/audio_system.cc b/src/xenia/apu/audio_system.cc index 8949173f9..d4e924890 100644 --- a/src/xenia/apu/audio_system.cc +++ b/src/xenia/apu/audio_system.cc @@ -74,7 +74,7 @@ X_STATUS AudioSystem::Setup() { processor_ = emulator_->processor(); // Let the processor know we want register access callbacks. - emulator_->memory()->AddMappedRange( + emulator_->memory()->AddVirtualMappedRange( 0x7FEA0000, 0xFFFF0000, 0x0000FFFF, this, reinterpret_cast(MMIOReadRegisterThunk), reinterpret_cast(MMIOWriteRegisterThunk)); @@ -94,6 +94,9 @@ X_STATUS AudioSystem::Setup() { thread_state_->set_name("Audio Worker"); thread_block_ = memory()->SystemHeapAlloc(2048); thread_state_->context()->r[13] = thread_block_; + XELOGI("Audio Worker Thread %X Stack: %.8X-%.8X", thread_state_->thread_id(), + thread_state_->stack_address(), + thread_state_->stack_address() + thread_state_->stack_size()); // Create worker thread. // This will initialize the audio system. @@ -252,7 +255,7 @@ void AudioSystem::UnregisterClient(size_t index) { // piece of hardware: // https://github.com/Free60Project/libxenon/blob/master/libxenon/drivers/xenon_sound/sound.c -uint64_t AudioSystem::ReadRegister(uint64_t addr) { +uint64_t AudioSystem::ReadRegister(uint32_t addr) { uint32_t r = addr & 0xFFFF; XELOGAPU("ReadRegister(%.4X)", r); // 1800h is read on startup and stored -- context? buffers? @@ -277,7 +280,7 @@ uint64_t AudioSystem::ReadRegister(uint64_t addr) { return value; } -void AudioSystem::WriteRegister(uint64_t addr, uint64_t value) { +void AudioSystem::WriteRegister(uint32_t addr, uint64_t value) { uint32_t r = addr & 0xFFFF; value = xe::byte_swap(uint32_t(value)); XELOGAPU("WriteRegister(%.4X, %.8X)", r, value); diff --git a/src/xenia/apu/audio_system.h b/src/xenia/apu/audio_system.h index b385f90a2..847698ac0 100644 --- a/src/xenia/apu/audio_system.h +++ b/src/xenia/apu/audio_system.h @@ -49,8 +49,8 @@ class AudioSystem { AudioDriver** out_driver) = 0; virtual void DestroyDriver(AudioDriver* driver) = 0; - virtual uint64_t ReadRegister(uint64_t addr); - virtual void WriteRegister(uint64_t addr, uint64_t value); + virtual uint64_t ReadRegister(uint32_t addr); + virtual void WriteRegister(uint32_t addr, uint64_t value); protected: virtual void Initialize(); @@ -58,10 +58,10 @@ class AudioSystem { private: void ThreadStart(); - static uint64_t MMIOReadRegisterThunk(AudioSystem* as, uint64_t addr) { + static uint64_t MMIOReadRegisterThunk(AudioSystem* as, uint32_t addr) { return as->ReadRegister(addr); } - static void MMIOWriteRegisterThunk(AudioSystem* as, uint64_t addr, + static void MMIOWriteRegisterThunk(AudioSystem* as, uint32_t addr, uint64_t value) { as->WriteRegister(addr, value); } diff --git a/src/xenia/cpu/mmio_handler.cc b/src/xenia/cpu/mmio_handler.cc index b9e58fff6..53b2e299f 100644 --- a/src/xenia/cpu/mmio_handler.cc +++ b/src/xenia/cpu/mmio_handler.cc @@ -23,9 +23,11 @@ namespace cpu { MMIOHandler* MMIOHandler::global_handler_ = nullptr; // Implemented in the platform cc file. -std::unique_ptr CreateMMIOHandler(uint8_t* mapping_base); +std::unique_ptr CreateMMIOHandler(uint8_t* virtual_membase, + uint8_t* physical_membase); -std::unique_ptr MMIOHandler::Install(uint8_t* mapping_base) { +std::unique_ptr MMIOHandler::Install(uint8_t* virtual_membase, + uint8_t* physical_membase) { // There can be only one handler at a time. assert_null(global_handler_); if (global_handler_) { @@ -33,7 +35,7 @@ std::unique_ptr MMIOHandler::Install(uint8_t* mapping_base) { } // Create the platform-specific handler. - auto handler = CreateMMIOHandler(mapping_base); + auto handler = CreateMMIOHandler(virtual_membase, physical_membase); // Platform-specific initialization for the handler. if (!handler->Initialize()) { @@ -49,45 +51,44 @@ MMIOHandler::~MMIOHandler() { global_handler_ = nullptr; } -bool MMIOHandler::RegisterRange(uint64_t address, uint64_t mask, uint64_t size, - void* context, MMIOReadCallback read_callback, +bool MMIOHandler::RegisterRange(uint32_t virtual_address, uint32_t mask, + uint32_t size, void* context, + MMIOReadCallback read_callback, MMIOWriteCallback write_callback) { mapped_ranges_.push_back({ - reinterpret_cast(mapping_base_) | address, - 0xFFFFFFFF00000000ull | mask, size, context, read_callback, - write_callback, + virtual_address, mask, size, context, read_callback, write_callback, }); return true; } -bool MMIOHandler::CheckLoad(uint64_t address, uint64_t* out_value) { +bool MMIOHandler::CheckLoad(uint32_t virtual_address, uint64_t* out_value) { for (const auto& range : mapped_ranges_) { - if (((address | (uint64_t)mapping_base_) & range.mask) == range.address) { - *out_value = static_cast(range.read(range.context, address)); + if ((virtual_address & range.mask) == range.address) { + *out_value = + static_cast(range.read(range.context, virtual_address)); return true; } } return false; } -bool MMIOHandler::CheckStore(uint64_t address, uint64_t value) { +bool MMIOHandler::CheckStore(uint32_t virtual_address, uint64_t value) { for (const auto& range : mapped_ranges_) { - if (((address | (uint64_t)mapping_base_) & range.mask) == range.address) { - range.write(range.context, address, value); + if ((virtual_address & range.mask) == range.address) { + range.write(range.context, virtual_address, value); return true; } } return false; } -uintptr_t MMIOHandler::AddWriteWatch(uint32_t guest_address, size_t length, - WriteWatchCallback callback, - void* callback_context, - void* callback_data) { +uintptr_t MMIOHandler::AddPhysicalWriteWatch(uint32_t guest_address, + size_t length, + WriteWatchCallback callback, + void* callback_context, + void* callback_data) { uint32_t base_address = guest_address; - if (base_address > 0xA0000000) { - base_address -= 0xA0000000; - } + assert_true(base_address < 0x1FFFFFFF); // Add to table. The slot reservation may evict a previous watch, which // could include our target, so we do it first. @@ -102,29 +103,33 @@ uintptr_t MMIOHandler::AddWriteWatch(uint32_t guest_address, size_t length, write_watch_mutex_.unlock(); // Make the desired range read only under all address spaces. - auto host_address = mapping_base_ + base_address; DWORD old_protect; - VirtualProtect(host_address, length, PAGE_READONLY, &old_protect); - VirtualProtect(host_address + 0xA0000000, length, PAGE_READONLY, - &old_protect); - VirtualProtect(host_address + 0xC0000000, length, PAGE_READONLY, - &old_protect); - VirtualProtect(host_address + 0xE0000000, length, PAGE_READONLY, - &old_protect); + VirtualProtect(physical_membase_ + entry->address, entry->length, + PAGE_READONLY, &old_protect); + VirtualProtect(virtual_membase_ + entry->address, entry->length, + PAGE_READONLY, &old_protect); + VirtualProtect(virtual_membase_ + 0xA0000000 + entry->address, entry->length, + PAGE_READONLY, &old_protect); + VirtualProtect(virtual_membase_ + 0xC0000000 + entry->address, entry->length, + PAGE_READONLY, &old_protect); + VirtualProtect(virtual_membase_ + 0xE0000000 + entry->address, entry->length, + PAGE_READONLY, &old_protect); return reinterpret_cast(entry); } void MMIOHandler::ClearWriteWatch(WriteWatchEntry* entry) { - auto host_address = mapping_base_ + entry->address; DWORD old_protect; - VirtualProtect(host_address, entry->length, PAGE_READWRITE, &old_protect); - VirtualProtect(host_address + 0xA0000000, entry->length, PAGE_READWRITE, - &old_protect); - VirtualProtect(host_address + 0xC0000000, entry->length, PAGE_READWRITE, - &old_protect); - VirtualProtect(host_address + 0xE0000000, entry->length, PAGE_READWRITE, - &old_protect); + VirtualProtect(physical_membase_ + entry->address, entry->length, + PAGE_READWRITE, &old_protect); + VirtualProtect(virtual_membase_ + entry->address, entry->length, + PAGE_READWRITE, &old_protect); + VirtualProtect(virtual_membase_ + 0xA0000000 + entry->address, entry->length, + PAGE_READWRITE, &old_protect); + VirtualProtect(virtual_membase_ + 0xC0000000 + entry->address, entry->length, + PAGE_READWRITE, &old_protect); + VirtualProtect(virtual_membase_ + 0xE0000000 + entry->address, entry->length, + PAGE_READWRITE, &old_protect); } void MMIOHandler::CancelWriteWatch(uintptr_t watch_handle) { @@ -145,17 +150,16 @@ void MMIOHandler::CancelWriteWatch(uintptr_t watch_handle) { } bool MMIOHandler::CheckWriteWatch(void* thread_state, uint64_t fault_address) { - uint32_t guest_address = uint32_t(fault_address - uintptr_t(mapping_base_)); - uint32_t base_address = guest_address; - if (base_address > 0xA0000000) { - base_address -= 0xA0000000; + uint32_t physical_address = uint32_t(fault_address); + if (physical_address > 0x1FFFFFFF) { + physical_address &= 0x1FFFFFFF; } std::list pending_invalidates; write_watch_mutex_.lock(); for (auto it = write_watches_.begin(); it != write_watches_.end();) { auto entry = *it; - if (entry->address <= base_address && - entry->address + entry->length > base_address) { + if (entry->address <= physical_address && + entry->address + entry->length > physical_address) { // Hit! pending_invalidates.push_back(entry); // TODO(benvanik): outside of lock? @@ -176,7 +180,7 @@ bool MMIOHandler::CheckWriteWatch(void* thread_state, uint64_t fault_address) { auto entry = pending_invalidates.back(); pending_invalidates.pop_back(); entry->callback(entry->callback_context, entry->callback_data, - guest_address); + physical_address); delete entry; } // Range was watched, so lets eat this access violation. @@ -185,18 +189,21 @@ bool MMIOHandler::CheckWriteWatch(void* thread_state, uint64_t fault_address) { bool MMIOHandler::HandleAccessFault(void* thread_state, uint64_t fault_address) { - if (fault_address < uint64_t(mapping_base_)) { + if (fault_address < uint64_t(virtual_membase_)) { // Quick kill anything below our mapping base. return false; } // Access violations are pretty rare, so we can do a linear search here. + // Only check if in the virtual range, as we only support virtual ranges. const MMIORange* range = nullptr; - for (const auto& test_range : mapped_ranges_) { - if ((fault_address & test_range.mask) == test_range.address) { - // Address is within the range of this mapping. - range = &test_range; - break; + if (fault_address < uint64_t(physical_membase_)) { + for (const auto& test_range : mapped_ranges_) { + if ((uint32_t(fault_address) & test_range.mask) == test_range.address) { + // Address is within the range of this mapping. + range = &test_range; + break; + } } } if (!range) { diff --git a/src/xenia/cpu/mmio_handler.h b/src/xenia/cpu/mmio_handler.h index b872a81cb..70646b85b 100644 --- a/src/xenia/cpu/mmio_handler.h +++ b/src/xenia/cpu/mmio_handler.h @@ -18,8 +18,8 @@ namespace xe { namespace cpu { -typedef uint64_t (*MMIOReadCallback)(void* context, uint64_t addr); -typedef void (*MMIOWriteCallback)(void* context, uint64_t addr, uint64_t value); +typedef uint64_t (*MMIOReadCallback)(void* context, uint32_t addr); +typedef void (*MMIOWriteCallback)(void* context, uint32_t addr, uint64_t value); typedef void (*WriteWatchCallback)(void* context_ptr, void* data_ptr, uint32_t address); @@ -29,19 +29,20 @@ class MMIOHandler { public: virtual ~MMIOHandler(); - static std::unique_ptr Install(uint8_t* mapping_base); + static std::unique_ptr Install(uint8_t* virtual_membase, + uint8_t* physical_membase); static MMIOHandler* global_handler() { return global_handler_; } - bool RegisterRange(uint64_t address, uint64_t mask, uint64_t size, + bool RegisterRange(uint32_t virtual_address, uint32_t mask, uint32_t size, void* context, MMIOReadCallback read_callback, MMIOWriteCallback write_callback); - bool CheckLoad(uint64_t address, uint64_t* out_value); - bool CheckStore(uint64_t address, uint64_t value); + bool CheckLoad(uint32_t virtual_address, uint64_t* out_value); + bool CheckStore(uint32_t virtual_address, uint64_t value); - uintptr_t AddWriteWatch(uint32_t guest_address, size_t length, - WriteWatchCallback callback, void* callback_context, - void* callback_data); + uintptr_t AddPhysicalWriteWatch(uint32_t guest_address, size_t length, + WriteWatchCallback callback, + void* callback_context, void* callback_data); void CancelWriteWatch(uintptr_t watch_handle); public: @@ -56,7 +57,9 @@ class MMIOHandler { void* callback_data; }; - MMIOHandler(uint8_t* mapping_base) : mapping_base_(mapping_base) {} + MMIOHandler(uint8_t* virtual_membase, uint8_t* physical_membase) + : virtual_membase_(virtual_membase), + physical_membase_(physical_membase) {} virtual bool Initialize() = 0; @@ -68,12 +71,13 @@ class MMIOHandler { virtual uint64_t* GetThreadStateRegPtr(void* thread_state_ptr, int32_t be_reg_index) = 0; - uint8_t* mapping_base_; + uint8_t* virtual_membase_; + uint8_t* physical_membase_; struct MMIORange { - uint64_t address; - uint64_t mask; - uint64_t size; + uint32_t address; + uint32_t mask; + uint32_t size; void* context; MMIOReadCallback read; MMIOWriteCallback write; diff --git a/src/xenia/cpu/mmio_handler_win.cc b/src/xenia/cpu/mmio_handler_win.cc index 2711ed167..86beca9ec 100644 --- a/src/xenia/cpu/mmio_handler_win.cc +++ b/src/xenia/cpu/mmio_handler_win.cc @@ -11,6 +11,10 @@ #include +namespace xe { +void CrashDump(); +} // namespace xe + namespace xe { namespace cpu { @@ -18,7 +22,8 @@ LONG CALLBACK MMIOExceptionHandler(PEXCEPTION_POINTERS ex_info); class WinMMIOHandler : public MMIOHandler { public: - WinMMIOHandler(uint8_t* mapping_base) : MMIOHandler(mapping_base) {} + WinMMIOHandler(uint8_t* virtual_membase, uint8_t* physical_membase) + : MMIOHandler(virtual_membase, physical_membase) {} ~WinMMIOHandler() override; protected: @@ -30,8 +35,9 @@ class WinMMIOHandler : public MMIOHandler { int32_t be_reg_index) override; }; -std::unique_ptr CreateMMIOHandler(uint8_t* mapping_base) { - return std::make_unique(mapping_base); +std::unique_ptr CreateMMIOHandler(uint8_t* virtual_membase, + uint8_t* physical_membase) { + return std::make_unique(virtual_membase, physical_membase); } bool WinMMIOHandler::Initialize() { @@ -67,6 +73,7 @@ LONG CALLBACK MMIOExceptionHandler(PEXCEPTION_POINTERS ex_info) { } else { // Failed to handle; continue search for a handler (and die if no other // handler is found). + xe::CrashDump(); return EXCEPTION_CONTINUE_SEARCH; } } diff --git a/src/xenia/cpu/processor.cc b/src/xenia/cpu/processor.cc index d811e2f07..806ee92a7 100644 --- a/src/xenia/cpu/processor.cc +++ b/src/xenia/cpu/processor.cc @@ -152,6 +152,11 @@ bool Processor::Setup() { interrupt_thread_state_->set_name("Interrupt"); interrupt_thread_block_ = memory_->SystemHeapAlloc(2048); interrupt_thread_state_->context()->r[13] = interrupt_thread_block_; + XELOGI("Interrupt Thread %X Stack: %.8X-%.8X", + interrupt_thread_state_->thread_id(), + interrupt_thread_state_->stack_address(), + interrupt_thread_state_->stack_address() + + interrupt_thread_state_->stack_size()); return true; } diff --git a/src/xenia/cpu/raw_module.cc b/src/xenia/cpu/raw_module.cc index e42b355f2..3142958f2 100644 --- a/src/xenia/cpu/raw_module.cc +++ b/src/xenia/cpu/raw_module.cc @@ -30,8 +30,11 @@ bool RawModule::LoadFile(uint32_t base_address, const std::wstring& path) { // Allocate memory. // Since we have no real heap just load it wherever. base_address_ = base_address; + memory_->LookupHeap(base_address_) + ->AllocFixed(base_address_, file_length, 0, + kMemoryAllocationReserve | kMemoryAllocationCommit, + kMemoryProtectRead | kMemoryProtectWrite); uint8_t* p = memory_->TranslateVirtual(base_address_); - std::memset(p, 0, file_length); // Read into memory. fread(p, file_length, 1, file); diff --git a/src/xenia/cpu/test/util.h b/src/xenia/cpu/test/util.h index 2a4819d6f..f9f55deb9 100644 --- a/src/xenia/cpu/test/util.h +++ b/src/xenia/cpu/test/util.h @@ -64,8 +64,6 @@ class TestFunction { void Run(std::function pre_call, std::function post_call) { for (auto& processor : processors) { - memory->Zero(0, memory_size); - xe::cpu::Function* fn; processor->ResolveFunction(0x1000, &fn); diff --git a/src/xenia/cpu/thread_state.cc b/src/xenia/cpu/thread_state.cc index e8c17c295..0ba1b2b0f 100644 --- a/src/xenia/cpu/thread_state.cc +++ b/src/xenia/cpu/thread_state.cc @@ -10,6 +10,7 @@ #include "xenia/cpu/thread_state.h" #include "xenia/base/assert.h" +#include "xenia/base/logging.h" #include "xenia/base/threading.h" #include "xenia/cpu/processor.h" #include "xenia/debug/debugger.h" @@ -49,12 +50,19 @@ ThreadState::ThreadState(Processor* processor, uint32_t thread_id, uint32_t stack_alignment = (stack_size & 0xF000) ? 0x1000 : 0x10000; uint32_t stack_padding = stack_alignment * 1; uint32_t actual_stack_size = stack_padding + stack_size; - stack_address_ = memory()->SystemHeapAlloc(actual_stack_size, stack_alignment); - assert_true(!(stack_address & 0xFFF)); // just to be safe + memory() + ->LookupHeapByType(false, 0x10000) + ->Alloc(actual_stack_size, stack_alignment, + kMemoryAllocationReserve | kMemoryAllocationCommit, + kMemoryProtectRead | kMemoryProtectWrite, true, + &stack_address_); + assert_true(!(stack_address_ & 0xFFF)); // just to be safe stack_position = stack_address_ + actual_stack_size; stack_allocated_ = true; memset(memory()->TranslateVirtual(stack_address_), 0xBE, actual_stack_size); - memory()->Protect(stack_address_, stack_padding, X_PAGE_NOACCESS); + memory() + ->LookupHeap(stack_address_) + ->Protect(stack_address_, stack_padding, kMemoryProtectNoAccess); } else { stack_address_ = stack_address; stack_position = stack_address_ + stack_size; @@ -100,7 +108,7 @@ ThreadState::~ThreadState() { _aligned_free(context_); if (stack_allocated_) { - memory()->SystemHeapFree(stack_address_); + memory()->LookupHeap(stack_address_)->Decommit(stack_address_, stack_size_); } } diff --git a/src/xenia/gpu/gl4/command_processor.cc b/src/xenia/gpu/gl4/command_processor.cc index aa80ac1aa..7308b35e9 100644 --- a/src/xenia/gpu/gl4/command_processor.cc +++ b/src/xenia/gpu/gl4/command_processor.cc @@ -456,7 +456,7 @@ void CommandProcessor::EnableReadPointerWriteBack(uint32_t ptr, uint32_t block_size) { // CP_RB_RPTR_ADDR Ring Buffer Read Pointer Address 0x70C // ptr = RB_RPTR_ADDR, pointer to write back the address to. - read_ptr_writeback_ptr_ = (primary_buffer_ptr_ & ~0x1FFFFFFF) + ptr; + read_ptr_writeback_ptr_ = ptr; // CP_RB_CNTL Ring Buffer Control 0x704 // block_size = RB_BLKSZ, number of quadwords read between updates of the // read pointer. @@ -966,7 +966,7 @@ bool CommandProcessor::ExecutePacketType3_XE_SWAP(RingbufferReader* reader, bool CommandProcessor::ExecutePacketType3_INDIRECT_BUFFER( RingbufferReader* reader, uint32_t packet, uint32_t count) { // indirect buffer dispatch - uint32_t list_ptr = reader->Read(); + uint32_t list_ptr = CpuToGpu(reader->Read()); uint32_t list_length = reader->Read(); ExecuteIndirectBuffer(GpuToCpu(list_ptr), list_length); return true; @@ -993,7 +993,7 @@ bool CommandProcessor::ExecutePacketType3_WAIT_REG_MEM(RingbufferReader* reader, poll_reg_addr &= ~0x3; value = xe::load(memory_->TranslatePhysical(poll_reg_addr)); value = GpuSwap(value, endianness); - trace_writer_.WriteMemoryRead(poll_reg_addr, 4); + trace_writer_.WriteMemoryRead(CpuToGpu(poll_reg_addr), 4); } else { // Register. assert_true(poll_reg_addr < RegisterFile::kRegisterCount); @@ -1093,7 +1093,7 @@ bool CommandProcessor::ExecutePacketType3_COND_WRITE(RingbufferReader* reader, // Memory. auto endianness = static_cast(poll_reg_addr & 0x3); poll_reg_addr &= ~0x3; - trace_writer_.WriteMemoryRead(poll_reg_addr, 4); + trace_writer_.WriteMemoryRead(CpuToGpu(poll_reg_addr), 4); value = xe::load(memory_->TranslatePhysical(poll_reg_addr)); value = GpuSwap(value, endianness); } else { @@ -1136,7 +1136,7 @@ bool CommandProcessor::ExecutePacketType3_COND_WRITE(RingbufferReader* reader, write_reg_addr &= ~0x3; write_data = GpuSwap(write_data, endianness); xe::store(memory_->TranslatePhysical(write_reg_addr), write_data); - trace_writer_.WriteMemoryWrite(write_reg_addr, 4); + trace_writer_.WriteMemoryWrite(CpuToGpu(write_reg_addr), 4); } else { // Register. WriteRegister(write_reg_addr, write_data); @@ -1182,7 +1182,7 @@ bool CommandProcessor::ExecutePacketType3_EVENT_WRITE_SHD( address &= ~0x3; data_value = GpuSwap(data_value, endianness); xe::store(memory_->TranslatePhysical(address), data_value); - trace_writer_.WriteMemoryWrite(address, 4); + trace_writer_.WriteMemoryWrite(CpuToGpu(address), 4); return true; } @@ -1208,7 +1208,7 @@ bool CommandProcessor::ExecutePacketType3_EVENT_WRITE_EXT( xe::copy_and_swap_16_aligned( reinterpret_cast(memory_->TranslatePhysical(address)), extents, xe::countof(extents)); - trace_writer_.WriteMemoryWrite(address, sizeof(extents)); + trace_writer_.WriteMemoryWrite(CpuToGpu(address), sizeof(extents)); return true; } @@ -1364,7 +1364,7 @@ bool CommandProcessor::ExecutePacketType3_LOAD_ALU_CONSTANT( assert_always(); return true; } - trace_writer_.WriteMemoryRead(address, size_dwords * 4); + trace_writer_.WriteMemoryRead(CpuToGpu(address), size_dwords * 4); for (uint32_t n = 0; n < size_dwords; n++, index++) { uint32_t data = xe::load_and_swap( memory_->TranslatePhysical(address + n * 4)); @@ -1395,7 +1395,7 @@ bool CommandProcessor::ExecutePacketType3_IM_LOAD(RingbufferReader* reader, uint32_t start = start_size >> 16; uint32_t size_dwords = start_size & 0xFFFF; // dwords assert_true(start == 0); - trace_writer_.WriteMemoryRead(addr, size_dwords * 4); + trace_writer_.WriteMemoryRead(CpuToGpu(addr), size_dwords * 4); LoadShader(shader_type, addr, memory_->TranslatePhysical(addr), size_dwords); return true; @@ -2106,29 +2106,31 @@ CommandProcessor::UpdateStatus CommandProcessor::UpdateBlendState() { draw_batcher_.Flush(DrawBatcher::FlushMode::kStateChange); - static const GLenum blend_map[] = {/* 0 */ GL_ZERO, - /* 1 */ GL_ONE, - /* 2 */ GL_ZERO, // ? - /* 3 */ GL_ZERO, // ? - /* 4 */ GL_SRC_COLOR, - /* 5 */ GL_ONE_MINUS_SRC_COLOR, - /* 6 */ GL_SRC_ALPHA, - /* 7 */ GL_ONE_MINUS_SRC_ALPHA, - /* 8 */ GL_DST_COLOR, - /* 9 */ GL_ONE_MINUS_DST_COLOR, - /* 10 */ GL_DST_ALPHA, - /* 11 */ GL_ONE_MINUS_DST_ALPHA, - /* 12 */ GL_CONSTANT_COLOR, - /* 13 */ GL_ONE_MINUS_CONSTANT_COLOR, - /* 14 */ GL_CONSTANT_ALPHA, - /* 15 */ GL_ONE_MINUS_CONSTANT_ALPHA, - /* 16 */ GL_SRC_ALPHA_SATURATE, + static const GLenum blend_map[] = { + /* 0 */ GL_ZERO, + /* 1 */ GL_ONE, + /* 2 */ GL_ZERO, // ? + /* 3 */ GL_ZERO, // ? + /* 4 */ GL_SRC_COLOR, + /* 5 */ GL_ONE_MINUS_SRC_COLOR, + /* 6 */ GL_SRC_ALPHA, + /* 7 */ GL_ONE_MINUS_SRC_ALPHA, + /* 8 */ GL_DST_COLOR, + /* 9 */ GL_ONE_MINUS_DST_COLOR, + /* 10 */ GL_DST_ALPHA, + /* 11 */ GL_ONE_MINUS_DST_ALPHA, + /* 12 */ GL_CONSTANT_COLOR, + /* 13 */ GL_ONE_MINUS_CONSTANT_COLOR, + /* 14 */ GL_CONSTANT_ALPHA, + /* 15 */ GL_ONE_MINUS_CONSTANT_ALPHA, + /* 16 */ GL_SRC_ALPHA_SATURATE, }; - static const GLenum blend_op_map[] = {/* 0 */ GL_FUNC_ADD, - /* 1 */ GL_FUNC_SUBTRACT, - /* 2 */ GL_MIN, - /* 3 */ GL_MAX, - /* 4 */ GL_FUNC_REVERSE_SUBTRACT, + static const GLenum blend_op_map[] = { + /* 0 */ GL_FUNC_ADD, + /* 1 */ GL_FUNC_SUBTRACT, + /* 2 */ GL_MIN, + /* 3 */ GL_MAX, + /* 4 */ GL_FUNC_REVERSE_SUBTRACT, }; for (int i = 0; i < xe::countof(regs.rb_blendcontrol); ++i) { uint32_t blend_control = regs.rb_blendcontrol[i]; @@ -2181,23 +2183,25 @@ CommandProcessor::UpdateStatus CommandProcessor::UpdateDepthStencilState() { draw_batcher_.Flush(DrawBatcher::FlushMode::kStateChange); - static const GLenum compare_func_map[] = {/* 0 */ GL_NEVER, - /* 1 */ GL_LESS, - /* 2 */ GL_EQUAL, - /* 3 */ GL_LEQUAL, - /* 4 */ GL_GREATER, - /* 5 */ GL_NOTEQUAL, - /* 6 */ GL_GEQUAL, - /* 7 */ GL_ALWAYS, + static const GLenum compare_func_map[] = { + /* 0 */ GL_NEVER, + /* 1 */ GL_LESS, + /* 2 */ GL_EQUAL, + /* 3 */ GL_LEQUAL, + /* 4 */ GL_GREATER, + /* 5 */ GL_NOTEQUAL, + /* 6 */ GL_GEQUAL, + /* 7 */ GL_ALWAYS, }; - static const GLenum stencil_op_map[] = {/* 0 */ GL_KEEP, - /* 1 */ GL_ZERO, - /* 2 */ GL_REPLACE, - /* 3 */ GL_INCR_WRAP, - /* 4 */ GL_DECR_WRAP, - /* 5 */ GL_INVERT, - /* 6 */ GL_INCR, - /* 7 */ GL_DECR, + static const GLenum stencil_op_map[] = { + /* 0 */ GL_KEEP, + /* 1 */ GL_ZERO, + /* 2 */ GL_REPLACE, + /* 3 */ GL_INCR_WRAP, + /* 4 */ GL_DECR_WRAP, + /* 5 */ GL_INVERT, + /* 6 */ GL_INCR, + /* 7 */ GL_DECR, }; // A2XX_RB_DEPTHCONTROL_Z_ENABLE if (regs.rb_depthcontrol & 0x00000002) { diff --git a/src/xenia/gpu/gl4/gl4_graphics_system.cc b/src/xenia/gpu/gl4/gl4_graphics_system.cc index a533cb091..bd2ee1f87 100644 --- a/src/xenia/gpu/gl4/gl4_graphics_system.cc +++ b/src/xenia/gpu/gl4/gl4_graphics_system.cc @@ -74,7 +74,7 @@ X_STATUS GL4GraphicsSystem::Setup(cpu::Processor* processor, [this](const SwapParameters& swap_params) { SwapHandler(swap_params); }); // Let the processor know we want register access callbacks. - memory_->AddMappedRange( + memory_->AddVirtualMappedRange( 0x7FC80000, 0xFFFF0000, 0x0000FFFF, this, reinterpret_cast(MMIOReadRegisterThunk), reinterpret_cast(MMIOWriteRegisterThunk)); @@ -275,7 +275,7 @@ void GL4GraphicsSystem::SwapHandler(const SwapParameters& swap_params) { }); } -uint64_t GL4GraphicsSystem::ReadRegister(uint64_t addr) { +uint64_t GL4GraphicsSystem::ReadRegister(uint32_t addr) { uint32_t r = addr & 0xFFFF; switch (r) { @@ -295,7 +295,7 @@ uint64_t GL4GraphicsSystem::ReadRegister(uint64_t addr) { return register_file_.values[r].u32; } -void GL4GraphicsSystem::WriteRegister(uint64_t addr, uint64_t value) { +void GL4GraphicsSystem::WriteRegister(uint32_t addr, uint64_t value) { uint32_t r = addr & 0xFFFF; switch (r) { diff --git a/src/xenia/gpu/gl4/gl4_graphics_system.h b/src/xenia/gpu/gl4/gl4_graphics_system.h index d1af6bcf0..558af2c1c 100644 --- a/src/xenia/gpu/gl4/gl4_graphics_system.h +++ b/src/xenia/gpu/gl4/gl4_graphics_system.h @@ -50,13 +50,13 @@ class GL4GraphicsSystem : public GraphicsSystem { private: void MarkVblank(); void SwapHandler(const SwapParameters& swap_params); - uint64_t ReadRegister(uint64_t addr); - void WriteRegister(uint64_t addr, uint64_t value); + uint64_t ReadRegister(uint32_t addr); + void WriteRegister(uint32_t addr, uint64_t value); - static uint64_t MMIOReadRegisterThunk(GL4GraphicsSystem* gs, uint64_t addr) { + static uint64_t MMIOReadRegisterThunk(GL4GraphicsSystem* gs, uint32_t addr) { return gs->ReadRegister(addr); } - static void MMIOWriteRegisterThunk(GL4GraphicsSystem* gs, uint64_t addr, + static void MMIOWriteRegisterThunk(GL4GraphicsSystem* gs, uint32_t addr, uint64_t value) { gs->WriteRegister(addr, value); } diff --git a/src/xenia/gpu/gl4/texture_cache.cc b/src/xenia/gpu/gl4/texture_cache.cc index adbda5f12..47e3776b1 100644 --- a/src/xenia/gpu/gl4/texture_cache.cc +++ b/src/xenia/gpu/gl4/texture_cache.cc @@ -490,7 +490,7 @@ TextureCache::TextureEntry* TextureCache::LookupOrInsertTexture( // Add a write watch. If any data in the given range is touched we'll get a // callback and evict the texture. We could reuse the storage, though the // driver is likely in a better position to pool that kind of stuff. - entry->write_watch_handle = memory_->AddWriteWatch( + entry->write_watch_handle = memory_->AddPhysicalWriteWatch( texture_info.guest_address, texture_info.input_length, [](void* context_ptr, void* data_ptr, uint32_t address) { auto self = reinterpret_cast(context_ptr); @@ -735,7 +735,8 @@ bool TextureCache::UploadTexture2D(GLuint texture, auto bpp = (bytes_per_block >> 2) + ((bytes_per_block >> 1) >> (bytes_per_block >> 2)); for (uint32_t y = 0, output_base_offset = 0; - y < texture_info.size_2d.block_height; + y < std::min(texture_info.size_2d.block_height, + texture_info.size_2d.logical_height); y++, output_base_offset += texture_info.size_2d.output_pitch) { auto input_base_offset = TextureInfo::TiledOffset2DOuter( offset_y + y, (texture_info.size_2d.input_width / diff --git a/src/xenia/gpu/xe-gpu-trace-viewer.cc b/src/xenia/gpu/xe-gpu-trace-viewer.cc index 39e94666d..5ca724c4d 100644 --- a/src/xenia/gpu/xe-gpu-trace-viewer.cc +++ b/src/xenia/gpu/xe-gpu-trace-viewer.cc @@ -771,7 +771,15 @@ class TracePlayer : public TraceReader { : loop_(loop), graphics_system_(graphics_system), current_frame_index_(0), - current_command_index_(-1) {} + current_command_index_(-1) { + // Need to allocate all of physical memory so that we can write to it + // during playback. + graphics_system_->memory() + ->LookupHeapByType(true, 4096) + ->AllocFixed(0, 0x1FFFFFFF, 4096, + kMemoryAllocationReserve | kMemoryAllocationCommit, + kMemoryProtectRead | kMemoryProtectWrite); + } ~TracePlayer() = default; GraphicsSystem* graphics_system() const { return graphics_system_; } diff --git a/src/xenia/kernel/objects/xthread.cc b/src/xenia/kernel/objects/xthread.cc index cc9f2fda4..c945970da 100644 --- a/src/xenia/kernel/objects/xthread.cc +++ b/src/xenia/kernel/objects/xthread.cc @@ -213,6 +213,9 @@ X_STATUS XThread::Create() { thread_state_ = new ThreadState(kernel_state()->processor(), thread_id_, 0, creation_params_.stack_size, thread_state_address_); + XELOGI("XThread%04X (%X) Stack: %.8X-%.8X", handle(), + thread_state_->thread_id(), thread_state_->stack_address(), + thread_state_->stack_address() + thread_state_->stack_size()); xe::store_and_swap( p + 0x05C, thread_state_->stack_address() + thread_state_->stack_size()); diff --git a/src/xenia/kernel/util/xex2.cc b/src/xenia/kernel/util/xex2.cc index c46687aba..7a38b57e8 100644 --- a/src/xenia/kernel/util/xex2.cc +++ b/src/xenia/kernel/util/xex2.cc @@ -536,8 +536,12 @@ int xe_xex2_read_image_uncompressed(const xe_xex2_header_t *header, // Allocate in-place the XEX memory. const uint32_t exe_length = xex_length - header->exe_offset; uint32_t uncompressed_size = exe_length; - uint32_t alloc_result = memory->HeapAlloc( - header->exe_address, uncompressed_size, xe::MEMORY_FLAG_ZERO); + bool alloc_result = + memory->LookupHeap(header->exe_address) + ->AllocFixed( + header->exe_address, uncompressed_size, 4096, + xe::kMemoryAllocationReserve | xe::kMemoryAllocationCommit, + xe::kMemoryProtectRead | xe::kMemoryProtectWrite); if (!alloc_result) { XELOGE("Unable to allocate XEX memory at %.8X-%.8X.", header->exe_address, uncompressed_size); @@ -588,14 +592,18 @@ int xe_xex2_read_image_basic_compressed(const xe_xex2_header_t *header, // Calculate the total size of the XEX image from its headers. uint32_t total_size = 0; for (uint32_t i = 0; i < header->section_count; i++) { - xe_xex2_section_t& section = header->sections[i]; + xe_xex2_section_t §ion = header->sections[i]; total_size += section.info.page_count * section.page_size; } // Allocate in-place the XEX memory. - uint32_t alloc_result = memory->HeapAlloc( - header->exe_address, total_size, xe::MEMORY_FLAG_ZERO); + bool alloc_result = + memory->LookupHeap(header->exe_address) + ->AllocFixed( + header->exe_address, total_size, 4096, + xe::kMemoryAllocationReserve | xe::kMemoryAllocationCommit, + xe::kMemoryProtectRead | xe::kMemoryProtectWrite); if (!alloc_result) { XELOGE("Unable to allocate XEX memory at %.8X-%.8X.", header->exe_address, uncompressed_size); @@ -731,8 +739,12 @@ int xe_xex2_read_image_compressed(const xe_xex2_header_t *header, } // Allocate in-place the XEX memory. - uint32_t alloc_result = memory->HeapAlloc( - header->exe_address, uncompressed_size, xe::MEMORY_FLAG_ZERO); + bool alloc_result = + memory->LookupHeap(header->exe_address) + ->AllocFixed( + header->exe_address, uncompressed_size, 4096, + xe::kMemoryAllocationReserve | xe::kMemoryAllocationCommit, + xe::kMemoryProtectRead | xe::kMemoryProtectWrite); if (!alloc_result) { XELOGE("Unable to allocate XEX memory at %.8X-%.8X.", header->exe_address, uncompressed_size); @@ -1084,4 +1096,4 @@ uint32_t xe_xex2_lookup_export(xe_xex2_ref xex, uint16_t ordinal) { // No match return 0; -} \ No newline at end of file +} diff --git a/src/xenia/kernel/xam_info.cc b/src/xenia/kernel/xam_info.cc index 54726765e..58c4f2453 100644 --- a/src/xenia/kernel/xam_info.cc +++ b/src/xenia/kernel/xam_info.cc @@ -41,7 +41,7 @@ SHIM_CALL XGetAVPack_shim(PPCContext* ppc_state, KernelState* state) { SHIM_CALL XGetGameRegion_shim(PPCContext* ppc_state, KernelState* state) { XELOGD("XGetGameRegion()"); - SHIM_SET_RETURN_64(XEX_REGION_ALL); + SHIM_SET_RETURN_64(0xFFFF); } SHIM_CALL XGetLanguage_shim(PPCContext* ppc_state, KernelState* state) { diff --git a/src/xenia/kernel/xboxkrnl_memory.cc b/src/xenia/kernel/xboxkrnl_memory.cc index c9e64a4b6..a1855c20f 100644 --- a/src/xenia/kernel/xboxkrnl_memory.cc +++ b/src/xenia/kernel/xboxkrnl_memory.cc @@ -17,19 +17,55 @@ namespace xe { namespace kernel { +uint32_t ToXdkProtectFlags(uint32_t protect) { + uint32_t result = 0; + if (!(protect & kMemoryProtectRead) && !(protect & kMemoryProtectWrite)) { + result = X_PAGE_NOACCESS; + } else if ((protect & kMemoryProtectRead) && + !(protect & kMemoryProtectWrite)) { + result = X_PAGE_READONLY; + } else { + result = X_PAGE_READWRITE; + } + if (protect & kMemoryProtectNoCache) { + result = X_PAGE_NOCACHE; + } + if (protect & kMemoryProtectWriteCombine) { + result = X_PAGE_WRITECOMBINE; + } + return result; +} + +uint32_t FromXdkProtectFlags(uint32_t protect) { + uint32_t result = 0; + if ((protect & X_PAGE_READONLY) | (protect & X_PAGE_EXECUTE_READ)) { + result |= kMemoryProtectRead; + } else if ((protect & X_PAGE_READWRITE) | + (protect & X_PAGE_EXECUTE_READWRITE)) { + result |= kMemoryProtectRead | kMemoryProtectWrite; + } + if (protect & X_PAGE_NOCACHE) { + result |= kMemoryProtectNoCache; + } + if (protect & X_PAGE_WRITECOMBINE) { + result |= kMemoryProtectWriteCombine; + } + return result; +} + SHIM_CALL NtAllocateVirtualMemory_shim(PPCContext* ppc_state, KernelState* state) { uint32_t base_addr_ptr = SHIM_GET_ARG_32(0); uint32_t base_addr_value = SHIM_MEM_32(base_addr_ptr); uint32_t region_size_ptr = SHIM_GET_ARG_32(1); uint32_t region_size_value = SHIM_MEM_32(region_size_ptr); - uint32_t allocation_type = SHIM_GET_ARG_32(2); // X_MEM_* bitmask - uint32_t protect_bits = SHIM_GET_ARG_32(3); // X_PAGE_* bitmask + uint32_t alloc_type = SHIM_GET_ARG_32(2); // X_MEM_* bitmask + uint32_t protect_bits = SHIM_GET_ARG_32(3); // X_PAGE_* bitmask uint32_t unknown = SHIM_GET_ARG_32(4); XELOGD("NtAllocateVirtualMemory(%.8X(%.8X), %.8X(%.8X), %.8X, %.8X, %.8X)", base_addr_ptr, base_addr_value, region_size_ptr, region_size_value, - allocation_type, protect_bits, unknown); + alloc_type, protect_bits, unknown); // NTSTATUS // _Inout_ PVOID *BaseAddress, @@ -52,12 +88,12 @@ SHIM_CALL NtAllocateVirtualMemory_shim(PPCContext* ppc_state, return; } // Check allocation type. - if (!(allocation_type & (X_MEM_COMMIT | X_MEM_RESET | X_MEM_RESERVE))) { + if (!(alloc_type & (X_MEM_COMMIT | X_MEM_RESET | X_MEM_RESERVE))) { SHIM_SET_RETURN_32(X_STATUS_INVALID_PARAMETER); return; } // If MEM_RESET is set only MEM_RESET can be set. - if (allocation_type & X_MEM_RESET && (allocation_type & ~X_MEM_RESET)) { + if (alloc_type & X_MEM_RESET && (alloc_type & ~X_MEM_RESET)) { SHIM_SET_RETURN_32(X_STATUS_INVALID_PARAMETER); return; } @@ -68,37 +104,60 @@ SHIM_CALL NtAllocateVirtualMemory_shim(PPCContext* ppc_state, } // Adjust size. - uint32_t adjusted_size = region_size_value; - // TODO(benvanik): adjust based on page size flags/etc? - - // TODO(benvanik): support different allocation types. - // Right now we treat everything as a commit and ignore allocations that have - // already happened. - if (base_addr_value) { - // Having a pointer already means that this is likely a follow-on COMMIT. - assert_true(!(allocation_type & X_MEM_RESERVE) && - (allocation_type & X_MEM_COMMIT)); - SHIM_SET_MEM_32(base_addr_ptr, base_addr_value); - SHIM_SET_MEM_32(region_size_ptr, adjusted_size); - SHIM_SET_RETURN_32(X_STATUS_SUCCESS); - return; + uint32_t page_size = 4096; + if (alloc_type & X_MEM_LARGE_PAGES) { + page_size = 64 * 1024; } + if (int32_t(region_size_value) < 0) { + // Some games pass in negative sizes. + region_size_value = -int32_t(region_size_value); + } + uint32_t adjusted_size = xe::round_up(region_size_value, page_size); // Allocate. - uint32_t flags = (allocation_type & X_MEM_NOZERO) ? 0 : MEMORY_FLAG_ZERO; - uint32_t addr = (uint32_t)state->memory()->HeapAlloc(base_addr_value, - adjusted_size, flags); - if (!addr) { + uint32_t allocation_type = 0; + if (alloc_type & X_MEM_RESERVE) { + allocation_type |= kMemoryAllocationReserve; + } + if (alloc_type & X_MEM_COMMIT) { + allocation_type |= kMemoryAllocationCommit; + } + if (alloc_type & X_MEM_RESET) { + XELOGE("X_MEM_RESET not implemented"); + assert_always(); + } + uint32_t protect = FromXdkProtectFlags(protect_bits); + uint32_t address = 0; + if (base_addr_value) { + auto heap = state->memory()->LookupHeap(base_addr_value); + if (heap->AllocFixed(base_addr_value, adjusted_size, page_size, + allocation_type, protect)) { + address = base_addr_value; + } + } else { + bool top_down = !!(alloc_type & X_MEM_TOP_DOWN); + auto heap = state->memory()->LookupHeapByType(false, page_size); + heap->Alloc(adjusted_size, page_size, allocation_type, protect, top_down, + &address); + } + if (!address) { // Failed - assume no memory available. SHIM_SET_RETURN_32(X_STATUS_NO_MEMORY); return; } - XELOGD("NtAllocateVirtualMemory = %.8X", addr); + // Zero memory, if needed. + if (address && !(alloc_type & X_MEM_NOZERO)) { + if (alloc_type & X_MEM_COMMIT) { + std::memset(SHIM_MEM_ADDR(address), 0, adjusted_size); + } + } + + XELOGD("NtAllocateVirtualMemory = %.8X", address); // Stash back. // Maybe set X_STATUS_ALREADY_COMMITTED if MEM_COMMIT? - SHIM_SET_MEM_32(base_addr_ptr, addr); + SHIM_SET_MEM_32(base_addr_ptr, address); SHIM_SET_MEM_32(region_size_ptr, adjusted_size); SHIM_SET_RETURN_32(X_STATUS_SUCCESS); } @@ -130,22 +189,24 @@ SHIM_CALL NtFreeVirtualMemory_shim(PPCContext* ppc_state, KernelState* state) { return; } - // TODO(benvanik): ignore decommits for now. + auto heap = state->memory()->LookupHeap(base_addr_value); + bool result = false; if (free_type == X_MEM_DECOMMIT) { - SHIM_SET_RETURN_32(X_STATUS_SUCCESS); - return; - } + // If zero, we may need to query size (free whole region). + assert_not_zero(region_size_value); - // Free. - uint32_t flags = 0; - uint32_t freed_size = state->memory()->HeapFree(base_addr_value, flags); - if (!freed_size) { + region_size_value = xe::round_up(region_size_value, heap->page_size()); + result = heap->Decommit(base_addr_value, region_size_value); + } else { + result = heap->Release(base_addr_value, ®ion_size_value); + } + if (!result) { SHIM_SET_RETURN_32(X_STATUS_UNSUCCESSFUL); return; } SHIM_SET_MEM_32(base_addr_ptr, base_addr_value); - SHIM_SET_MEM_32(region_size_ptr, freed_size); + SHIM_SET_MEM_32(region_size_ptr, region_size_value); SHIM_SET_RETURN_32(X_STATUS_SUCCESS); } @@ -168,9 +229,9 @@ SHIM_CALL NtQueryVirtualMemory_shim(PPCContext* ppc_state, KernelState* state) { XELOGD("NtQueryVirtualMemory(%.8X, %.8X)", base_address, memory_basic_information_ptr); - AllocationInfo alloc_info; - size_t result = state->memory()->QueryInformation(base_address, &alloc_info); - if (!result) { + auto heap = state->memory()->LookupHeap(base_address); + HeapAllocationInfo alloc_info; + if (!heap->QueryRegionInfo(base_address, &alloc_info)) { SHIM_SET_RETURN_32(X_STATUS_INVALID_PARAMETER); return; } @@ -179,15 +240,21 @@ SHIM_CALL NtQueryVirtualMemory_shim(PPCContext* ppc_state, KernelState* state) { static_cast(alloc_info.base_address); memory_basic_information->allocation_base = static_cast(alloc_info.allocation_base); - memory_basic_information->allocation_protect = alloc_info.allocation_protect; + memory_basic_information->allocation_protect = + ToXdkProtectFlags(alloc_info.allocation_protect); memory_basic_information->region_size = static_cast(alloc_info.region_size); - memory_basic_information->state = alloc_info.state; - memory_basic_information->protect = alloc_info.protect; + uint32_t x_state = 0; + if (alloc_info.state & kMemoryAllocationReserve) { + x_state |= X_MEM_RESERVE; + } + if (alloc_info.state & kMemoryAllocationCommit) { + x_state |= X_MEM_COMMIT; + } + memory_basic_information->state = x_state; + memory_basic_information->protect = ToXdkProtectFlags(alloc_info.protect); memory_basic_information->type = alloc_info.type; - XELOGE("NtQueryVirtualMemory NOT IMPLEMENTED"); - SHIM_SET_RETURN_32(X_STATUS_SUCCESS); } @@ -242,26 +309,20 @@ SHIM_CALL MmAllocatePhysicalMemoryEx_shim(PPCContext* ppc_state, assert_true(min_addr_range == 0); assert_true(max_addr_range == 0xFFFFFFFF); - // Allocate. - uint32_t flags = MEMORY_FLAG_PHYSICAL; - uint32_t base_address = (uint32_t)state->memory()->HeapAlloc( - 0, adjusted_size, flags, adjusted_alignment); - if (!base_address) { + uint32_t allocation_type = kMemoryAllocationReserve | kMemoryAllocationCommit; + uint32_t protect = FromXdkProtectFlags(protect_bits); + bool top_down = true; + auto heap = state->memory()->LookupHeapByType(true, page_size); + uint32_t base_address; + if (!heap->AllocRange(min_addr_range, max_addr_range, adjusted_size, + adjusted_alignment, allocation_type, protect, top_down, + &base_address)) { // Failed - assume no memory available. SHIM_SET_RETURN_32(0); return; } XELOGD("MmAllocatePhysicalMemoryEx = %.8X", base_address); - // Move the address into the right range. - // if (protect_bits & X_MEM_LARGE_PAGES) { - // base_address += 0xA0000000; - //} else if (protect_bits & X_MEM_16MB_PAGES) { - // base_address += 0xC0000000; - //} else { - // base_address += 0xE0000000; - //} - base_address += 0xA0000000; SHIM_SET_RETURN_64(base_address); } @@ -274,14 +335,10 @@ SHIM_CALL MmFreePhysicalMemory_shim(PPCContext* ppc_state, KernelState* state) { // base_address = result of MmAllocatePhysicalMemory. - // Strip off physical bits before passing down. - base_address &= ~0xE0000000; + assert_true((base_address & 0x1F) == 0); - // TODO(benvanik): free memory. - XELOGE("xeMmFreePhysicalMemory NOT IMPLEMENTED"); - // uint32_t size = ?; - // xe_memory_heap_free( - // state->memory(), base_address, size); + auto heap = state->memory()->LookupHeap(base_address); + heap->Release(base_address); } SHIM_CALL MmQueryAddressProtect_shim(PPCContext* ppc_state, @@ -290,7 +347,12 @@ SHIM_CALL MmQueryAddressProtect_shim(PPCContext* ppc_state, XELOGD("MmQueryAddressProtect(%.8X)", base_address); - uint32_t access = state->memory()->QueryProtect(base_address); + auto heap = state->memory()->LookupHeap(base_address); + uint32_t access; + if (!heap->QueryProtect(base_address, &access)) { + access = 0; + } + access = ToXdkProtectFlags(access); SHIM_SET_RETURN_32(access); } @@ -301,9 +363,13 @@ SHIM_CALL MmQueryAllocationSize_shim(PPCContext* ppc_state, XELOGD("MmQueryAllocationSize(%.8X)", base_address); - size_t size = state->memory()->QuerySize(base_address); + auto heap = state->memory()->LookupHeap(base_address); + uint32_t size; + if (!heap->QuerySize(base_address, &size)) { + size = 0; + } - SHIM_SET_RETURN_32(static_cast(size)); + SHIM_SET_RETURN_32(size); } SHIM_CALL MmQueryStatistics_shim(PPCContext* ppc_state, KernelState* state) { @@ -372,19 +438,12 @@ SHIM_CALL MmGetPhysicalAddress_shim(PPCContext* ppc_state, KernelState* state) { // ); // base_address = result of MmAllocatePhysicalMemory. - // We are always using virtual addresses, right now, since we don't need - // physical ones. We could munge up the address here to another mapped view - // of memory. + uint32_t physical_address = base_address & 0x1FFFFFFF; + if (base_address >= 0xE0000000) { + physical_address += 0x1000; + } - /*if (protect_bits & X_MEM_LARGE_PAGES) { - base_address |= 0xA0000000; - } else if (protect_bits & X_MEM_16MB_PAGES) { - base_address |= 0xC0000000; - } else { - base_address |= 0xE0000000; - }*/ - - SHIM_SET_RETURN_64(base_address); + SHIM_SET_RETURN_64(physical_address); } SHIM_CALL MmMapIoSpace_shim(PPCContext* ppc_state, KernelState* state) { diff --git a/src/xenia/kernel/xboxkrnl_ob.cc b/src/xenia/kernel/xboxkrnl_ob.cc index 8e8a874c5..71cbcc68c 100644 --- a/src/xenia/kernel/xboxkrnl_ob.cc +++ b/src/xenia/kernel/xboxkrnl_ob.cc @@ -83,6 +83,11 @@ SHIM_CALL ObReferenceObjectByHandle_shim(PPCContext* ppc_state, } break; } } break; + case 0xD017BEEF: { // ExSemaphoreObjectType + // TODO(benvanik): implement. + assert_unhandled_case(object_type_ptr); + native_ptr = 0xDEADF00D; + } break; case 0xD01BBEEF: { // ExThreadObjectType XThread* thread = (XThread*)object; native_ptr = thread->thread_state_ptr(); diff --git a/src/xenia/kernel/xboxkrnl_video.cc b/src/xenia/kernel/xboxkrnl_video.cc index dabf9741c..d660c16d5 100644 --- a/src/xenia/kernel/xboxkrnl_video.cc +++ b/src/xenia/kernel/xboxkrnl_video.cc @@ -380,6 +380,11 @@ SHIM_CALL VdPersistDisplay_shim(PPCContext* ppc_state, KernelState* state) { // unk1_ptr needs to be populated with a pointer passed to // MmFreePhysicalMemory(1, *unk1_ptr). + auto heap = state->memory()->LookupHeapByType(true, 16 * 1024); + uint32_t unk1_value; + heap->Alloc(64, 32, kMemoryAllocationReserve | kMemoryAllocationCommit, + kMemoryProtectNoAccess, false, &unk1_value); + SHIM_SET_MEM_32(unk1_ptr, unk1_value); // ? SHIM_SET_RETURN_64(1); diff --git a/src/xenia/kernel/xobject.cc b/src/xenia/kernel/xobject.cc index 88474ed87..689026d8e 100644 --- a/src/xenia/kernel/xobject.cc +++ b/src/xenia/kernel/xobject.cc @@ -113,6 +113,7 @@ X_STATUS XObject::Wait(uint32_t wait_reason, uint32_t processor_mode, // Or X_STATUS_ALERTED? return X_STATUS_USER_APC; case WAIT_TIMEOUT: + YieldProcessor(); return X_STATUS_TIMEOUT; default: case WAIT_FAILED: diff --git a/src/xenia/memory.cc b/src/xenia/memory.cc index 928f9af78..b557ac891 100644 --- a/src/xenia/memory.cc +++ b/src/xenia/memory.cc @@ -18,8 +18,6 @@ #include "xenia/base/math.h" #include "xenia/cpu/mmio_handler.h" -using namespace xe; - // TODO(benvanik): move xbox.h out #include "xenia/xbox.h" @@ -27,28 +25,15 @@ using namespace xe; #include #endif // WIN32 -#define MSPACES 1 -#define USE_LOCKS 0 -#define USE_DL_PREFIX 1 -#define HAVE_MORECORE 0 -#define HAVE_MREMAP 0 -#define malloc_getpagesize 4096 -#define DEFAULT_GRANULARITY 64 * 1024 -#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T -#define MALLOC_ALIGNMENT 32 -#define MALLOC_INSPECT_ALL 1 -#if XE_DEBUG -#define FOOTERS 0 -#endif // XE_DEBUG -#include "third_party/dlmalloc/malloc.c.h" - -DEFINE_bool(log_heap, false, "Log heap structure on alloc/free."); -DEFINE_uint64( - heap_guard_pages, 0, - "Allocate the given number of guard pages around all heap chunks."); DEFINE_bool(scribble_heap, false, "Scribble 0xCD into all allocated heap memory."); +namespace xe { + +uint32_t get_page_count(uint32_t value, uint32_t page_size) { + return xe::round_up(value, page_size) / page_size; +} + /** * Memory map: * 0x00000000 - 0x3FFFFFFF (1024mb) - virtual 4k pages @@ -81,40 +66,11 @@ DEFINE_bool(scribble_heap, false, * this. */ -const uint32_t kMemoryPhysicalHeapLow = 0x00010000; -const uint32_t kMemoryPhysicalHeapHigh = 0x20000000; -const uint32_t kMemoryVirtualHeapLow = 0x20000000; -const uint32_t kMemoryVirtualHeapHigh = 0x40000000; +static Memory* active_memory_ = nullptr; -class xe::MemoryHeap { - public: - MemoryHeap(Memory* memory, bool is_physical); - ~MemoryHeap(); - - int Initialize(uint32_t low, uint32_t high); - - uint32_t Alloc(uint32_t base_address, uint32_t size, uint32_t flags, - uint32_t alignment); - uint32_t Free(uint32_t address, uint32_t size); - uint32_t QuerySize(uint32_t base_address); - - void Dump(); - - private: - static uint32_t next_heap_id_; - static void DumpHandler(void* start, void* end, size_t used_bytes, - void* context); - - private: - Memory* memory_; - uint32_t heap_id_; - bool is_physical_; - std::mutex lock_; - uint32_t size_; - uint8_t* ptr_; - mspace space_; -}; -uint32_t MemoryHeap::next_heap_id_ = 1; +void CrashDump() { + active_memory_->DumpMap(); +} Memory::Memory() : virtual_membase_(nullptr), @@ -124,22 +80,26 @@ Memory::Memory() mapping_(0), mapping_base_(nullptr) { system_page_size_ = uint32_t(xe::page_size()); - virtual_heap_ = new MemoryHeap(this, false); - physical_heap_ = new MemoryHeap(this, true); + assert_zero(active_memory_); + active_memory_ = this; } Memory::~Memory() { + assert_true(active_memory_ == this); + active_memory_ = nullptr; + // Uninstall the MMIO handler, as we won't be able to service more // requests. mmio_handler_.reset(); - if (mapping_base_) { - // GPU writeback. - VirtualFree(TranslateVirtual(0xC0000000), 0x00100000, MEM_DECOMMIT); - } - - delete physical_heap_; - delete virtual_heap_; + heaps_.v00000000.Dispose(); + heaps_.v40000000.Dispose(); + heaps_.v80000000.Dispose(); + heaps_.v90000000.Dispose(); + heaps_.vA0000000.Dispose(); + heaps_.vC0000000.Dispose(); + heaps_.vE0000000.Dispose(); + heaps_.physical.Dispose(); // Unmap all views and close mapping. if (mapping_) { @@ -157,15 +117,15 @@ int Memory::Initialize() { // Create main page file-backed mapping. This is all reserved but // uncommitted (so it shouldn't expand page file). #if XE_PLATFORM_WIN32 - mapping_ = - CreateFileMapping(INVALID_HANDLE_VALUE, NULL, - PAGE_READWRITE | SEC_RESERVE, 1, 0, // entire 4gb space - NULL); + mapping_ = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE | SEC_RESERVE, + // entire 4gb space + 512mb physical: + 1, 0x1FFFFFFF, NULL); #else char mapping_path[] = "/xenia/mapping/XXXXXX"; mktemp(mapping_path); mapping_ = shm_open(mapping_path, O_CREAT, 0); - ftruncate(mapping_, 0x100000000); + ftruncate(mapping_, 0x11FFFFFFF); #endif // XE_PLATFORM_WIN32 if (!mapping_) { XELOGE("Unable to reserve the 4gb guest address space."); @@ -189,20 +149,42 @@ int Memory::Initialize() { return 1; } virtual_membase_ = mapping_base_; - physical_membase_ = virtual_membase_; + physical_membase_ = mapping_base_ + 0x100000000ull; - // Prepare heaps. - virtual_heap_->Initialize(kMemoryVirtualHeapLow, kMemoryVirtualHeapHigh); - physical_heap_->Initialize(kMemoryPhysicalHeapLow, - kMemoryPhysicalHeapHigh - 0x1000); + // Prepare virtual heaps. + heaps_.v00000000.Initialize(virtual_membase_, 0x00000000, 0x40000000, 4096); + heaps_.v40000000.Initialize(virtual_membase_, 0x40000000, + 0x40000000 - 0x01000000, 64 * 1024); + heaps_.v80000000.Initialize(virtual_membase_, 0x80000000, 0x10000000, + 64 * 1024); + heaps_.v90000000.Initialize(virtual_membase_, 0x90000000, 0x10000000, 4096); + + // Prepare physical heaps. + heaps_.physical.Initialize(physical_membase_, 0x00000000, 0x20000000, 4096); + heaps_.vA0000000.Initialize(virtual_membase_, 0xA0000000, 0x20000000, + 64 * 1024, &heaps_.physical); + heaps_.vC0000000.Initialize(virtual_membase_, 0xC0000000, 0x20000000, + 16 * 1024 * 1024, &heaps_.physical); + heaps_.vE0000000.Initialize(virtual_membase_, 0xE0000000, 0x1FD00000, 4096, + &heaps_.physical); + + // Take the first page at 0 so we can check for writes. + heaps_.v00000000.AllocFixed( + 0x00000000, 4096, 4096, + kMemoryAllocationReserve | kMemoryAllocationCommit, + // 0u); + kMemoryProtectRead | kMemoryProtectWrite); // GPU writeback. // 0xC... is physical, 0x7F... is virtual. We may need to overlay these. - VirtualAlloc(TranslatePhysical(0x00000000), 0x00100000, MEM_COMMIT, - PAGE_READWRITE); + heaps_.vC0000000.AllocFixed( + 0xC0000000, 0x01000000, 32, + kMemoryAllocationReserve | kMemoryAllocationCommit, + kMemoryProtectRead | kMemoryProtectWrite); // Add handlers for MMIO. - mmio_handler_ = cpu::MMIOHandler::Install(mapping_base_); + mmio_handler_ = + cpu::MMIOHandler::Install(virtual_membase_, physical_membase_); if (!mmio_handler_) { XELOGE("Unable to install MMIO handlers"); assert_always(); @@ -210,8 +192,10 @@ int Memory::Initialize() { } // I have no idea what this is, but games try to read/write there. - VirtualAlloc(TranslateVirtual(0x40000000), 0x00010000, MEM_COMMIT, - PAGE_READWRITE); + heaps_.v40000000.AllocFixed( + 0x40000000, 0x00010000, 32, + kMemoryAllocationReserve | kMemoryAllocationCommit, + kMemoryProtectRead | kMemoryProtectWrite); xe::store_and_swap(TranslateVirtual(0x40000000), 0x00C40000); xe::store_and_swap(TranslateVirtual(0x40000004), 0x00010000); @@ -219,36 +203,38 @@ int Memory::Initialize() { } const static struct { - uint32_t virtual_address_start; - uint32_t virtual_address_end; - uint32_t target_address; + uint64_t virtual_address_start; + uint64_t virtual_address_end; + uint64_t target_address; } map_info[] = { - 0x00000000, 0x3FFFFFFF, - 0x00000000, // (1024mb) - virtual 4k pages - 0x40000000, 0x7EFFFFFF, - 0x40000000, // (1024mb) - virtual 64k pages (cont) - 0x7F000000, 0x7F0FFFFF, - 0x00000000, // (1mb) - GPU writeback - 0x7F100000, 0x7FFFFFFF, - 0x00100000, // (15mb) - XPS? - 0x80000000, 0x8FFFFFFF, - 0x80000000, // (256mb) - xex 64k pages - 0x90000000, 0x9FFFFFFF, - 0x80000000, // (256mb) - xex 4k pages - 0xA0000000, 0xBFFFFFFF, - 0x00000000, // (512mb) - physical 64k pages - 0xC0000000, 0xDFFFFFFF, - 0x00000000, // - physical 16mb pages - 0xE0000000, 0xFFFFFFFF, - 0x00000000, // - physical 4k pages + // (1024mb) - virtual 4k pages + 0x00000000, 0x3FFFFFFF, 0x0000000000000000ull, + // (1024mb) - virtual 64k pages (cont) + 0x40000000, 0x7EFFFFFF, 0x0000000040000000ull, + // (16mb) - GPU writeback + 15mb of XPS? + 0x7F000000, 0x7FFFFFFF, 0x0000000100000000ull, + // (256mb) - xex 64k pages + 0x80000000, 0x8FFFFFFF, 0x0000000080000000ull, + // (256mb) - xex 4k pages + 0x90000000, 0x9FFFFFFF, 0x0000000080000000ull, + // (512mb) - physical 64k pages + 0xA0000000, 0xBFFFFFFF, 0x0000000100000000ull, + // - physical 16mb pages + 0xC0000000, 0xDFFFFFFF, 0x0000000100000000ull, + // - physical 4k pages + 0xE0000000, 0xFFFFFFFF, 0x0000000100000000ull, + // - physical raw + 0x100000000, 0x11FFFFFFF, 0x0000000100000000ull, }; int Memory::MapViews(uint8_t* mapping_base) { assert_true(xe::countof(map_info) == xe::countof(views_.all_views)); for (size_t n = 0; n < xe::countof(map_info); n++) { #if XE_PLATFORM_WIN32 + DWORD target_address_low = static_cast(map_info[n].target_address); + DWORD target_address_high = + static_cast(map_info[n].target_address >> 32); views_.all_views[n] = reinterpret_cast(MapViewOfFileEx( - mapping_, FILE_MAP_ALL_ACCESS, 0x00000000, - (DWORD)map_info[n].target_address, + mapping_, FILE_MAP_ALL_ACCESS, target_address_high, target_address_low, map_info[n].virtual_address_end - map_info[n].virtual_address_start + 1, mapping_base + map_info[n].virtual_address_start)); #else @@ -281,6 +267,44 @@ void Memory::UnmapViews() { } } +BaseHeap* Memory::LookupHeap(uint32_t address) { + if (address < 0x40000000) { + return &heaps_.v00000000; + } else if (address < 0x80000000) { + return &heaps_.v40000000; + } else if (address < 0x90000000) { + return &heaps_.v80000000; + } else if (address < 0xA0000000) { + return &heaps_.v90000000; + } else if (address < 0xC0000000) { + return &heaps_.vA0000000; + } else if (address < 0xE0000000) { + return &heaps_.vC0000000; + } else { + return &heaps_.vE0000000; + } +} + +BaseHeap* Memory::LookupHeapByType(bool physical, uint32_t page_size) { + if (physical) { + if (page_size <= 4096) { + // HACK + //return &heaps_.vE0000000; + return &heaps_.vA0000000; + } else if (page_size <= 64 * 1024) { + return &heaps_.vA0000000; + } else { + return &heaps_.vC0000000; + } + } else { + if (page_size <= 4096) { + return &heaps_.v00000000; + } else { + return &heaps_.v40000000; + } + } +} + void Memory::Zero(uint32_t address, uint32_t size) { std::memset(TranslateVirtual(address), 0, size); } @@ -319,23 +343,27 @@ uint32_t Memory::SearchAligned(uint32_t start, uint32_t end, return 0; } -bool Memory::AddMappedRange(uint32_t address, uint32_t mask, uint32_t size, - void* context, cpu::MMIOReadCallback read_callback, - cpu::MMIOWriteCallback write_callback) { +bool Memory::AddVirtualMappedRange(uint32_t virtual_address, uint32_t mask, + uint32_t size, void* context, + cpu::MMIOReadCallback read_callback, + cpu::MMIOWriteCallback write_callback) { DWORD protect = PAGE_NOACCESS; - if (!VirtualAlloc(TranslateVirtual(address), size, MEM_COMMIT, protect)) { + if (!VirtualAlloc(TranslateVirtual(virtual_address), size, MEM_COMMIT, + protect)) { XELOGE("Unable to map range; commit/protect failed"); return false; } - return mmio_handler_->RegisterRange(address, mask, size, context, + return mmio_handler_->RegisterRange(virtual_address, mask, size, context, read_callback, write_callback); } -uintptr_t Memory::AddWriteWatch(uint32_t guest_address, uint32_t length, - cpu::WriteWatchCallback callback, - void* callback_context, void* callback_data) { - return mmio_handler_->AddWriteWatch(guest_address, length, callback, - callback_context, callback_data); +uintptr_t Memory::AddPhysicalWriteWatch(uint32_t physical_address, + uint32_t length, + cpu::WriteWatchCallback callback, + void* callback_context, + void* callback_data) { + return mmio_handler_->AddPhysicalWriteWatch( + physical_address, length, callback, callback_context, callback_data); } void Memory::CancelWriteWatch(uintptr_t watch_handle) { @@ -346,11 +374,14 @@ uint32_t Memory::SystemHeapAlloc(uint32_t size, uint32_t alignment, uint32_t system_heap_flags) { // TODO(benvanik): lightweight pool. bool is_physical = !!(system_heap_flags & kSystemHeapPhysical); - uint32_t flags = MEMORY_FLAG_ZERO; - if (is_physical) { - flags |= MEMORY_FLAG_PHYSICAL; + auto heap = LookupHeapByType(is_physical, 4096); + uint32_t address; + if (!heap->Alloc(size, alignment, + kMemoryAllocationReserve | kMemoryAllocationCommit, + kMemoryProtectRead | kMemoryProtectWrite, false, &address)) { + return 0; } - return HeapAlloc(0, size, flags, alignment); + return address; } void Memory::SystemHeapFree(uint32_t address) { @@ -358,315 +389,731 @@ void Memory::SystemHeapFree(uint32_t address) { return; } // TODO(benvanik): lightweight pool. - HeapFree(address, 0); + auto heap = LookupHeapByType(false, 4096); + heap->Release(address); } -uint32_t Memory::HeapAlloc(uint32_t base_address, uint32_t size, uint32_t flags, - uint32_t alignment) { - // If we were given a base address we are outside of the normal heap and - // will place wherever asked (so long as it doesn't overlap the heap). - if (!base_address) { - // Normal allocation from the managed heap. - uint32_t result; - if (flags & MEMORY_FLAG_PHYSICAL) { - result = physical_heap_->Alloc(base_address, size, flags, alignment); - } else { - result = virtual_heap_->Alloc(base_address, size, flags, alignment); +void Memory::DumpMap() { + XELOGE("=================================================================="); + XELOGE("Memory Dump"); + XELOGE("=================================================================="); + XELOGE(" System Page Size: %d (%.8X)", system_page_size_, system_page_size_); + XELOGE(" Virtual Membase: %.16llX", virtual_membase_); + XELOGE(" Physical Membase: %.16llX", physical_membase_); + XELOGE(""); + XELOGE("------------------------------------------------------------------"); + XELOGE("Virtual Heaps"); + XELOGE("------------------------------------------------------------------"); + XELOGE(""); + heaps_.v00000000.DumpMap(); + heaps_.v40000000.DumpMap(); + heaps_.v80000000.DumpMap(); + heaps_.v90000000.DumpMap(); + XELOGE(""); + XELOGE("------------------------------------------------------------------"); + XELOGE("Physical Heaps"); + XELOGE("------------------------------------------------------------------"); + XELOGE(""); + heaps_.physical.DumpMap(); + heaps_.vA0000000.DumpMap(); + heaps_.vC0000000.DumpMap(); + heaps_.vE0000000.DumpMap(); + XELOGE(""); +} + +DWORD ToWin32ProtectFlags(uint32_t protect) { + DWORD result = 0; + if ((protect & kMemoryProtectRead) && !(protect & kMemoryProtectWrite)) { + result |= PAGE_READONLY; + } else if ((protect & kMemoryProtectRead) && + (protect & kMemoryProtectWrite)) { + result |= PAGE_READWRITE; + } else { + result |= PAGE_NOACCESS; + } + // if (protect & kMemoryProtectNoCache) { + // result |= PAGE_NOCACHE; + //} + // if (protect & kMemoryProtectWriteCombine) { + // result |= PAGE_WRITECOMBINE; + //} + return result; +} + +uint32_t FromWin32ProtectFlags(DWORD protect) { + uint32_t result = 0; + if (protect & PAGE_READONLY) { + result |= kMemoryProtectRead; + } else if (protect & PAGE_READWRITE) { + result |= kMemoryProtectRead | kMemoryProtectWrite; + } + if (protect & PAGE_NOCACHE) { + result |= kMemoryProtectNoCache; + } + if (protect & PAGE_WRITECOMBINE) { + result |= kMemoryProtectWriteCombine; + } + return result; +} + +BaseHeap::BaseHeap() + : membase_(nullptr), heap_base_(0), heap_size_(0), page_size_(0) {} + +BaseHeap::~BaseHeap() = default; + +void BaseHeap::Initialize(uint8_t* membase, uint32_t heap_base, + uint32_t heap_size, uint32_t page_size) { + membase_ = membase; + heap_base_ = heap_base; + heap_size_ = heap_size - 1; + page_size_ = page_size; + page_table_.resize(heap_size / page_size); +} + +void BaseHeap::Dispose() { + // Walk table and release all regions. + for (uint32_t page_number = 0; page_number < page_table_.size(); + ++page_number) { + auto& page_entry = page_table_[page_number]; + if (page_entry.state) { + VirtualFree(membase_ + heap_base_ + page_number * page_size_, 0, + MEM_RELEASE); + page_number += page_entry.region_page_count; } - if (result) { - if (flags & MEMORY_FLAG_ZERO) { - memset(TranslateVirtual(result), 0, size); + } +} + +void BaseHeap::DumpMap() { + std::lock_guard lock(heap_mutex_); + XELOGE("------------------------------------------------------------------"); + XELOGE("Heap: %.8X-%.8X", heap_base_, heap_base_ + heap_size_); + XELOGE("------------------------------------------------------------------"); + XELOGE(" Heap Base: %.8X", heap_base_); + XELOGE(" Heap Size: %d (%.8X)", heap_size_, heap_size_); + XELOGE(" Page Size: %d (%.8X)", page_size_, page_size_); + XELOGE(" Page Count: %lld", page_table_.size()); + bool is_empty_span = false; + uint32_t empty_span_start = 0; + for (uint32_t i = 0; i < uint32_t(page_table_.size()); ++i) { + auto& page = page_table_[i]; + if (!page.state) { + if (!is_empty_span) { + is_empty_span = true; + empty_span_start = i; } + continue; } - return result; - } else { - if (base_address >= kMemoryVirtualHeapLow && - base_address < kMemoryVirtualHeapHigh) { - // Overlapping managed heap. - assert_always(); - return 0; + if (is_empty_span) { + XELOGE(" %.8X-%.8X %6dp %10db unreserved", + heap_base_ + empty_span_start * page_size_, + heap_base_ + i * page_size_, i - empty_span_start, + (i - empty_span_start) * page_size_); + is_empty_span = false; } - if (base_address >= kMemoryPhysicalHeapLow && - base_address < kMemoryPhysicalHeapHigh) { - // Overlapping managed heap. - assert_always(); - return 0; + const char* state_name = " "; + if (page.state & kMemoryAllocationCommit) { + state_name = "COM"; + } else if (page.state & kMemoryAllocationReserve) { + state_name = "RES"; } - - uint8_t* p = TranslateVirtual(base_address); - // TODO(benvanik): check if address range is in use with a query. - - void* pv = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE); - if (!pv) { - // Failed. - assert_always(); - return 0; - } - - if (flags & MEMORY_FLAG_ZERO) { - memset(pv, 0, size); - } - - return base_address; + char access_r = (page.current_protect & kMemoryProtectRead) ? 'R' : ' '; + char access_w = (page.current_protect & kMemoryProtectWrite) ? 'W' : ' '; + XELOGE(" %.8X-%.8X %6dp %10db %s %c%c", heap_base_ + i * page_size_, + heap_base_ + (i + page.region_page_count) * page_size_, + page.region_page_count, page.region_page_count * page_size_, + state_name, access_r, access_w); + i += page.region_page_count; + } + if (is_empty_span) { + XELOGE(" %.8X-%.8X - %d unreserved pages)", + heap_base_ + empty_span_start * page_size_, heap_base_ + heap_size_, + page_table_.size() - empty_span_start); } } -int Memory::HeapFree(uint32_t address, uint32_t size) { - if (address >= kMemoryVirtualHeapLow && address < kMemoryVirtualHeapHigh) { - return virtual_heap_->Free(address, size) ? 0 : 1; - } else if (address >= kMemoryPhysicalHeapLow && - address < kMemoryPhysicalHeapHigh) { - return physical_heap_->Free(address, size) ? 0 : 1; - } else { - // A placed address. Decommit. - uint8_t* p = TranslateVirtual(address); - return VirtualFree(p, size, MEM_DECOMMIT) ? 0 : 1; - } +bool BaseHeap::Alloc(uint32_t size, uint32_t alignment, + uint32_t allocation_type, uint32_t protect, bool top_down, + uint32_t* out_address) { + *out_address = 0; + size = xe::round_up(size, page_size_); + alignment = xe::round_up(alignment, page_size_); + uint32_t low_address = heap_base_; + uint32_t high_address = heap_base_ + heap_size_; + return AllocRange(low_address, high_address, size, alignment, allocation_type, + protect, top_down, out_address); } -bool Memory::QueryInformation(uint32_t base_address, AllocationInfo* mem_info) { - uint8_t* p = TranslateVirtual(base_address); - MEMORY_BASIC_INFORMATION mbi; - if (!VirtualQuery(p, &mbi, sizeof(mbi))) { +bool BaseHeap::AllocFixed(uint32_t base_address, uint32_t size, + uint32_t alignment, uint32_t allocation_type, + uint32_t protect) { + alignment = xe::round_up(alignment, page_size_); + size = xe::align(size, alignment); + assert_true(base_address % alignment == 0); + uint32_t page_count = get_page_count(size, page_size_); + uint32_t start_page_number = (base_address - heap_base_) / page_size_; + uint32_t end_page_number = start_page_number + page_count - 1; + if (start_page_number >= page_table_.size() || + end_page_number > page_table_.size()) { + XELOGE("BaseHeap::Alloc passed out of range address range"); return false; } - mem_info->base_address = base_address; - mem_info->allocation_base = static_cast( - reinterpret_cast(mbi.AllocationBase) - virtual_membase_); - mem_info->allocation_protect = mbi.AllocationProtect; - mem_info->region_size = mbi.RegionSize; - mem_info->state = mbi.State; - mem_info->protect = mbi.Protect; - mem_info->type = mbi.Type; + + std::lock_guard lock(heap_mutex_); + + // - If we are reserving the entire range requested must not be already + // reserved. + // - If we are committing it's ok for pages within the range to already be + // committed. + for (uint32_t page_number = start_page_number; page_number <= end_page_number; + ++page_number) { + uint32_t state = page_table_[page_number].state; + if ((allocation_type == kMemoryAllocationReserve) && state) { + // Already reserved. + XELOGE("BaseHeap::Alloc attempting to reserve an already reserved range"); + return false; + } + if ((allocation_type == kMemoryAllocationCommit) && + !(state & kMemoryAllocationReserve)) { + // Attempting a commit-only op on an unreserved page. + XELOGE("BaseHeap::Alloc attempting commit on unreserved page"); + return false; + } + } + + // Allocate from host. + if (allocation_type == kMemoryAllocationReserve) { + // Reserve is not needed, as we are mapped already. + } else { + DWORD flAllocationType = 0; + if (allocation_type & kMemoryAllocationCommit) { + flAllocationType |= MEM_COMMIT; + } + LPVOID result = + VirtualAlloc(membase_ + heap_base_ + start_page_number * page_size_, + page_count * page_size_, flAllocationType, + ToWin32ProtectFlags(protect)); + if (!result) { + XELOGE("BaseHeap::Alloc failed to alloc range from host"); + return false; + } + } + + // Set page state. + for (uint32_t page_number = start_page_number; page_number <= end_page_number; + ++page_number) { + auto& page_entry = page_table_[page_number]; + if (allocation_type & kMemoryAllocationReserve) { + // Region is based on reservation. + page_entry.base_address = start_page_number; + page_entry.region_page_count = page_count; + } + page_entry.allocation_protect = protect; + page_entry.current_protect = protect; + page_entry.state = kMemoryAllocationReserve | allocation_type; + } + return true; } -uint32_t Memory::QuerySize(uint32_t base_address) { - if (base_address >= kMemoryVirtualHeapLow && - base_address < kMemoryVirtualHeapHigh) { - return virtual_heap_->QuerySize(base_address); - } else if (base_address >= kMemoryPhysicalHeapLow && - base_address < kMemoryPhysicalHeapHigh) { - return physical_heap_->QuerySize(base_address); +bool BaseHeap::AllocRange(uint32_t low_address, uint32_t high_address, + uint32_t size, uint32_t alignment, + uint32_t allocation_type, uint32_t protect, + bool top_down, uint32_t* out_address) { + *out_address = 0; + + alignment = xe::round_up(alignment, page_size_); + uint32_t page_count = get_page_count(size, page_size_); + low_address = std::max(heap_base_, xe::align(low_address, alignment)); + high_address = + std::min(heap_base_ + heap_size_, xe::align(high_address, alignment)); + uint32_t low_page_number = (low_address - heap_base_) / page_size_; + uint32_t high_page_number = (high_address - heap_base_) / page_size_; + low_page_number = std::min(uint32_t(page_table_.size()) - 1, low_page_number); + high_page_number = + std::min(uint32_t(page_table_.size()) - 1, high_page_number); + + std::lock_guard lock(heap_mutex_); + + // Find a free page range. + // The base page must match the requested alignment, so we first scan for + // a free aligned page and only then check for continuous free pages. + // TODO(benvanik): optimized searching (free list buckets, bitmap, etc). + uint32_t start_page_number = UINT_MAX; + uint32_t end_page_number = UINT_MAX; + uint32_t page_scan_stride = alignment / page_size_; + high_page_number = high_page_number - (high_page_number % page_scan_stride); + if (top_down) { + for (int64_t base_page_number = high_page_number - page_count; + base_page_number >= low_page_number; + base_page_number -= page_scan_stride) { + bool is_free = page_table_[base_page_number].state == 0; + if (page_table_[base_page_number].state != 0) { + // Base page not free, skip to next usable page. + continue; + } + // Check requested range to ensure free. + start_page_number = uint32_t(base_page_number); + end_page_number = uint32_t(base_page_number) + page_count - 1; + assert_true(end_page_number < page_table_.size()); + bool any_taken = false; + for (uint32_t page_number = uint32_t(base_page_number); + !any_taken && page_number <= end_page_number; ++page_number) { + bool is_free = page_table_[page_number].state == 0; + if (!is_free) { + // At least one page in the range is used, skip to next. + any_taken = true; + break; + } + } + if (!any_taken) { + // Found our place. + break; + } + // Retry. + start_page_number = end_page_number = UINT_MAX; + } } else { - // A placed address. - uint8_t* p = TranslateVirtual(base_address); - MEMORY_BASIC_INFORMATION mem_info; - if (VirtualQuery(p, &mem_info, sizeof(mem_info))) { - return uint32_t(mem_info.RegionSize); - } else { - // Error. - return 0; + for (uint32_t base_page_number = low_page_number; + base_page_number <= high_page_number; + base_page_number += page_scan_stride) { + bool is_free = page_table_[base_page_number].state == 0; + if (page_table_[base_page_number].state != 0) { + // Base page not free, skip to next usable page. + continue; + } + // Check requested range to ensure free. + start_page_number = base_page_number; + end_page_number = base_page_number + page_count - 1; + bool any_taken = false; + for (uint32_t page_number = base_page_number; + !any_taken && page_number <= end_page_number; ++page_number) { + bool is_free = page_table_[page_number].state == 0; + if (!is_free) { + // At least one page in the range is used, skip to next. + any_taken = true; + break; + } + } + if (!any_taken) { + // Found our place. + break; + } + // Retry. + start_page_number = end_page_number = UINT_MAX; } } -} - -int Memory::Protect(uint32_t address, uint32_t size, uint32_t access) { - uint8_t* p = TranslateVirtual(address); - - size_t heap_guard_size = FLAGS_heap_guard_pages * 4096; - p += heap_guard_size; - - DWORD new_protect = access; - new_protect = - new_protect & - (X_PAGE_NOACCESS | X_PAGE_READONLY | X_PAGE_READWRITE | X_PAGE_WRITECOPY | - X_PAGE_GUARD | X_PAGE_NOCACHE | X_PAGE_WRITECOMBINE); - - DWORD old_protect; - return VirtualProtect(p, size, new_protect, &old_protect) == TRUE ? 0 : 1; -} - -uint32_t Memory::QueryProtect(uint32_t address) { - uint8_t* p = TranslateVirtual(address); - MEMORY_BASIC_INFORMATION info; - size_t info_size = VirtualQuery((void*)p, &info, sizeof(info)); - if (!info_size) { - return 0; - } - return info.Protect; -} - -MemoryHeap::MemoryHeap(Memory* memory, bool is_physical) - : memory_(memory), is_physical_(is_physical) { - heap_id_ = next_heap_id_++; -} - -MemoryHeap::~MemoryHeap() { - if (space_) { - std::lock_guard guard(lock_); - destroy_mspace(space_); - space_ = NULL; + if (start_page_number == UINT_MAX || end_page_number == UINT_MAX) { + // Out of memory. + XELOGE("BaseHeap::Alloc failed to find contiguous range"); + assert_always("Heap exhausted!"); + return false; } - if (ptr_) { - VirtualFree(ptr_, 0, MEM_RELEASE); - } -} - -int MemoryHeap::Initialize(uint32_t low, uint32_t high) { - // Commit the memory where our heap will live and allocate it. - // TODO(benvanik): replace dlmalloc with an implementation that can commit - // as it goes. - size_ = high - low; - ptr_ = memory_->views_.v00000000 + low; - void* heap_result = VirtualAlloc(ptr_, size_, MEM_COMMIT, PAGE_READWRITE); - if (!heap_result) { - return 1; - } - space_ = create_mspace_with_base(ptr_, size_, 0); - - return 0; -} - -uint32_t MemoryHeap::Alloc(uint32_t base_address, uint32_t size, uint32_t flags, - uint32_t alignment) { - size_t alloc_size = size; - if (int32_t(alloc_size) < 0) { - alloc_size = uint32_t(-alloc_size); - } - size_t heap_guard_size = FLAGS_heap_guard_pages * 4096; - if (heap_guard_size) { - alignment = std::max(alignment, static_cast(heap_guard_size)); - alloc_size = - static_cast(xe::round_up(alloc_size, heap_guard_size)); - } - - lock_.lock(); - uint8_t* p = (uint8_t*)mspace_memalign(space_, alignment, - alloc_size + heap_guard_size * 2); - assert_true(reinterpret_cast(p) <= 0xFFFFFFFFFull); - if (FLAGS_heap_guard_pages) { - size_t real_size = mspace_usable_size(p); - DWORD old_protect; - VirtualProtect(p, heap_guard_size, PAGE_NOACCESS, &old_protect); - p += heap_guard_size; - VirtualProtect(p + alloc_size, heap_guard_size, PAGE_NOACCESS, - &old_protect); - } - if (FLAGS_log_heap) { - Dump(); - } - lock_.unlock(); - if (!p) { - return 0; - } - - if (is_physical_) { - // If physical, we need to commit the memory in the physical address ranges - // so that it can be accessed. - VirtualAlloc(memory_->views_.vA0000000 + (p - memory_->views_.v00000000), - alloc_size, MEM_COMMIT, PAGE_READWRITE); - VirtualAlloc(memory_->views_.vC0000000 + (p - memory_->views_.v00000000), - alloc_size, MEM_COMMIT, PAGE_READWRITE); - VirtualAlloc(memory_->views_.vE0000000 + (p - memory_->views_.v00000000), - alloc_size, MEM_COMMIT, PAGE_READWRITE); - } - - if (flags & MEMORY_FLAG_ZERO) { - memset(p, 0, alloc_size); - } else if (FLAGS_scribble_heap) { - // Trash the memory so that we can see bad read-before-write bugs easier. - memset(p, 0xCD, alloc_size); - } - - uint32_t address = - (uint32_t)((uintptr_t)p - (uintptr_t)memory_->mapping_base_); - - return address; -} - -uint32_t MemoryHeap::Free(uint32_t address, uint32_t size) { - uint8_t* p = memory_->TranslateVirtual(address); - - // Heap allocated address. - size_t heap_guard_size = FLAGS_heap_guard_pages * 4096; - p -= heap_guard_size; - size_t real_size = mspace_usable_size(p); - real_size -= heap_guard_size * 2; - if (!real_size) { - return 0; - } - - if (FLAGS_scribble_heap) { - // Trash the memory so that we can see bad read-before-write bugs easier. - memset(p + heap_guard_size, 0xDC, size); - } - - lock_.lock(); - if (FLAGS_heap_guard_pages) { - DWORD old_protect; - VirtualProtect(p, heap_guard_size, PAGE_READWRITE, &old_protect); - VirtualProtect(p + heap_guard_size + real_size, heap_guard_size, - PAGE_READWRITE, &old_protect); - } - mspace_free(space_, p); - if (FLAGS_log_heap) { - Dump(); - } - lock_.unlock(); - - if (is_physical_) { - // If physical, decommit from physical ranges too. - VirtualFree(memory_->views_.vA0000000 + (p - memory_->views_.v00000000), - size, MEM_DECOMMIT); - VirtualFree(memory_->views_.vC0000000 + (p - memory_->views_.v00000000), - size, MEM_DECOMMIT); - VirtualFree(memory_->views_.vE0000000 + (p - memory_->views_.v00000000), - size, MEM_DECOMMIT); - } - - return static_cast(real_size); -} - -uint32_t MemoryHeap::QuerySize(uint32_t base_address) { - uint8_t* p = memory_->TranslateVirtual(base_address); - - // Heap allocated address. - uint32_t heap_guard_size = uint32_t(FLAGS_heap_guard_pages * 4096); - p -= heap_guard_size; - uint32_t real_size = uint32_t(mspace_usable_size(p)); - real_size -= heap_guard_size * 2; - if (!real_size) { - return 0; - } - - return real_size; -} - -void MemoryHeap::Dump() { - XELOGI("MemoryHeap::Dump - %s", is_physical_ ? "physical" : "virtual"); - if (FLAGS_heap_guard_pages) { - XELOGI(" (heap guard pages enabled, stats will be wrong)"); - } - struct mallinfo info = mspace_mallinfo(space_); - XELOGI(" arena: %lld", info.arena); - XELOGI(" ordblks: %lld", info.ordblks); - XELOGI(" hblks: %lld", info.hblks); - XELOGI(" hblkhd: %lld", info.hblkhd); - XELOGI(" usmblks: %lld", info.usmblks); - XELOGI(" uordblks: %lld", info.uordblks); - XELOGI(" fordblks: %lld", info.fordblks); - XELOGI(" keepcost: %lld", info.keepcost); - mspace_inspect_all(space_, DumpHandler, this); -} - -void MemoryHeap::DumpHandler(void* start, void* end, size_t used_bytes, - void* context) { - MemoryHeap* heap = (MemoryHeap*)context; - Memory* memory = heap->memory_; - size_t heap_guard_size = FLAGS_heap_guard_pages * 4096; - uint64_t start_addr = (uint64_t)start + heap_guard_size; - uint64_t end_addr = (uint64_t)end - heap_guard_size; - uint32_t guest_start = - (uint32_t)(start_addr - (uintptr_t)memory->mapping_base_); - uint32_t guest_end = (uint32_t)(end_addr - (uintptr_t)memory->mapping_base_); - if (int32_t(end_addr - start_addr) > 0) { - XELOGI(" - %.8X-%.8X (%10db) %.16llX-%.16llX - %9db used", guest_start, - guest_end, (guest_end - guest_start), start_addr, end_addr, - used_bytes); + // Allocate from host. + if (allocation_type == kMemoryAllocationReserve) { + // Reserve is not needed, as we are mapped already. } else { - XELOGI(" - %.16llX-%.16llX - %9db used", - start, end, used_bytes); + DWORD flAllocationType = 0; + if (allocation_type & kMemoryAllocationCommit) { + flAllocationType |= MEM_COMMIT; + } + LPVOID result = + VirtualAlloc(membase_ + heap_base_ + start_page_number * page_size_, + page_count * page_size_, flAllocationType, + ToWin32ProtectFlags(protect)); + if (!result) { + XELOGE("BaseHeap::Alloc failed to alloc range from host"); + return false; + } } + + // Set page state. + for (uint32_t page_number = start_page_number; page_number <= end_page_number; + ++page_number) { + auto& page_entry = page_table_[page_number]; + page_entry.base_address = start_page_number; + page_entry.region_page_count = page_count; + page_entry.allocation_protect = protect; + page_entry.current_protect = protect; + page_entry.state = kMemoryAllocationReserve | allocation_type; + } + + *out_address = heap_base_ + start_page_number * page_size_; + return true; } + +bool BaseHeap::Decommit(uint32_t address, uint32_t size) { + uint32_t page_count = get_page_count(size, page_size_); + uint32_t start_page_number = (address - heap_base_) / page_size_; + uint32_t end_page_number = start_page_number + page_count - 1; + start_page_number = + std::min(uint32_t(page_table_.size()) - 1, start_page_number); + end_page_number = std::min(uint32_t(page_table_.size()) - 1, end_page_number); + + std::lock_guard lock(heap_mutex_); + + // Release from host. + // TODO(benvanik): find a way to actually decommit memory; + // mapped memory cannot be decommitted. + /*BOOL result = + VirtualFree(membase_ + heap_base_ + start_page_number * page_size_, + page_count * page_size_, MEM_DECOMMIT); + if (!result) { + PLOGW("BaseHeap::Decommit failed due to host VirtualFree failure"); + return false; + }*/ + + // Perform table change. + for (uint32_t page_number = start_page_number; page_number <= end_page_number; + ++page_number) { + auto& page_entry = page_table_[page_number]; + page_entry.state &= ~kMemoryAllocationCommit; + } + + return true; +} + +bool BaseHeap::Release(uint32_t base_address, uint32_t* out_region_size) { + std::lock_guard lock(heap_mutex_); + + // Given address must be a region base address. + uint32_t base_page_number = (base_address - heap_base_) / page_size_; + auto base_page_entry = page_table_[base_page_number]; + if (base_page_entry.base_address != base_page_number) { + XELOGE("BaseHeap::Release failed because address is not a region start"); + // return false; + } + + if (out_region_size) { + *out_region_size = base_page_entry.region_page_count * page_size_; + } + + // Release from host not needed as mapping reserves the range for us. + // TODO(benvanik): protect with NOACCESS? + /*BOOL result = VirtualFree( + membase_ + heap_base_ + base_page_number * page_size_, 0, MEM_RELEASE); + if (!result) { + PLOGE("BaseHeap::Release failed due to host VirtualFree failure"); + return false; + }*/ + // Instead, we just protect it. + DWORD old_protect; + if (!VirtualProtect(membase_ + heap_base_ + base_page_number * page_size_, + base_page_entry.region_page_count * page_size_, + PAGE_NOACCESS, &old_protect)) { + XELOGW("BaseHeap::Release failed due to host VirtualProtect failure"); + } + + // Perform table change. + uint32_t end_page_number = + base_page_number + base_page_entry.region_page_count - 1; + for (uint32_t page_number = base_page_number; page_number <= end_page_number; + ++page_number) { + auto& page_entry = page_table_[page_number]; + page_entry.qword = 0; + } + + return true; +} + +bool BaseHeap::Protect(uint32_t address, uint32_t size, uint32_t protect) { + uint32_t page_count = xe::round_up(size, page_size_) / page_size_; + uint32_t start_page_number = (address - heap_base_) / page_size_; + uint32_t end_page_number = start_page_number + page_count - 1; + start_page_number = + std::min(uint32_t(page_table_.size()) - 1, start_page_number); + end_page_number = std::min(uint32_t(page_table_.size()) - 1, end_page_number); + + std::lock_guard lock(heap_mutex_); + + // Ensure all pages are in the same reserved region and all are committed. + uint32_t first_base_address = UINT_MAX; + for (uint32_t page_number = start_page_number; page_number <= end_page_number; + ++page_number) { + auto page_entry = page_table_[page_number]; + if (first_base_address == UINT_MAX) { + first_base_address = page_entry.base_address; + } else if (first_base_address != page_entry.base_address) { + XELOGE("BaseHeap::Protect failed due to request spanning regions"); + return false; + } + if (!(page_entry.state & kMemoryAllocationCommit)) { + XELOGE("BaseHeap::Protect failed due to uncommitted page"); + return false; + } + } + + // Attempt host change (hopefully won't fail). + DWORD new_protect = ToWin32ProtectFlags(protect); + DWORD old_protect; + if (!VirtualProtect(membase_ + heap_base_ + start_page_number * page_size_, + page_count * page_size_, new_protect, &old_protect)) { + XELOGE("BaseHeap::Protect failed due to host VirtualProtect failure"); + return false; + } + + // Perform table change. + for (uint32_t page_number = start_page_number; page_number <= end_page_number; + ++page_number) { + auto& page_entry = page_table_[page_number]; + page_entry.current_protect = protect; + } + + return true; +} + +bool BaseHeap::QueryRegionInfo(uint32_t base_address, + HeapAllocationInfo* out_info) { + uint32_t start_page_number = (base_address - heap_base_) / page_size_; + if (start_page_number > page_table_.size()) { + XELOGE("BaseHeap::QueryRegionInfo base page out of range"); + return false; + } + + std::lock_guard lock(heap_mutex_); + + auto start_page_entry = page_table_[start_page_number]; + out_info->base_address = base_address; + out_info->allocation_base = 0; + out_info->allocation_protect = 0; + out_info->region_size = 0; + out_info->state = 0; + out_info->protect = 0; + out_info->type = 0; + if (start_page_entry.state) { + // Committed/reserved region. + out_info->allocation_base = start_page_entry.base_address * page_size_; + out_info->allocation_protect = start_page_entry.allocation_protect; + out_info->state = start_page_entry.state; + out_info->protect = start_page_entry.current_protect; + out_info->type = 0x20000; + for (uint32_t page_number = start_page_number; + page_number < start_page_number + start_page_entry.region_page_count; + ++page_number) { + auto page_entry = page_table_[page_number]; + if (page_entry.base_address != start_page_entry.base_address || + page_entry.state != start_page_entry.state || + page_entry.current_protect != start_page_entry.current_protect) { + // Different region or different properties within the region; done. + break; + } + out_info->region_size += page_size_; + } + } else { + // Free region. + for (uint32_t page_number = start_page_number; + page_number < page_table_.size(); ++page_number) { + auto page_entry = page_table_[page_number]; + if (page_entry.state) { + // First non-free page; done with region. + break; + } + out_info->region_size += page_size_; + } + } + return true; +} + +bool BaseHeap::QuerySize(uint32_t address, uint32_t* out_size) { + uint32_t page_number = (address - heap_base_) / page_size_; + if (page_number > page_table_.size()) { + XELOGE("BaseHeap::QuerySize base page out of range"); + *out_size = 0; + return false; + } + std::lock_guard lock(heap_mutex_); + auto page_entry = page_table_[page_number]; + *out_size = page_entry.region_page_count * page_size_; + return true; +} + +bool BaseHeap::QueryProtect(uint32_t address, uint32_t* out_protect) { + uint32_t page_number = (address - heap_base_) / page_size_; + if (page_number > page_table_.size()) { + XELOGE("BaseHeap::QueryProtect base page out of range"); + *out_protect = 0; + return false; + } + std::lock_guard lock(heap_mutex_); + auto page_entry = page_table_[page_number]; + *out_protect = page_entry.current_protect; + return true; +} + +uint32_t BaseHeap::GetPhysicalAddress(uint32_t address) { + // Only valid for memory in this range - will be bogus if the origin was + // outside of it. + uint32_t physical_address = address & 0x1FFFFFFF; + if (address >= 0xE0000000) { + physical_address += 0x1000; + } + return physical_address; +} + +VirtualHeap::VirtualHeap() = default; + +VirtualHeap::~VirtualHeap() = default; + +void VirtualHeap::Initialize(uint8_t* membase, uint32_t heap_base, + uint32_t heap_size, uint32_t page_size) { + BaseHeap::Initialize(membase, heap_base, heap_size, page_size); +} + +PhysicalHeap::PhysicalHeap() : parent_heap_(nullptr) {} + +PhysicalHeap::~PhysicalHeap() = default; + +void PhysicalHeap::Initialize(uint8_t* membase, uint32_t heap_base, + uint32_t heap_size, uint32_t page_size, + VirtualHeap* parent_heap) { + BaseHeap::Initialize(membase, heap_base, heap_size, page_size); + parent_heap_ = parent_heap; +} + +bool PhysicalHeap::Alloc(uint32_t size, uint32_t alignment, + uint32_t allocation_type, uint32_t protect, + bool top_down, uint32_t* out_address) { + *out_address = 0; + + // Default top-down. Since parent heap is bottom-up this prevents collisions. + top_down = true; + + // Adjust alignment size our page size differs from the parent. + size = xe::round_up(size, page_size_); + alignment = xe::round_up(alignment, page_size_); + + std::lock_guard lock(heap_mutex_); + + // Allocate from parent heap (gets our physical address in 0-512mb). + uint32_t parent_low_address = GetPhysicalAddress(heap_base_); + uint32_t parent_high_address = GetPhysicalAddress(heap_base_ + heap_size_); + uint32_t parent_address; + if (!parent_heap_->AllocRange(parent_low_address, parent_high_address, size, + alignment, allocation_type, protect, top_down, + &parent_address)) { + XELOGE( + "PhysicalHeap::Alloc unable to alloc physical memory in parent heap"); + return false; + } + + // Given the address we've reserved in the parent heap, pin that here. + // Shouldn't be possible for it to be allocated already. + uint32_t address = heap_base_ + parent_address; + if (!BaseHeap::AllocFixed(address, size, alignment, allocation_type, + protect)) { + XELOGE( + "PhysicalHeap::Alloc unable to pin physical memory in physical heap"); + // TODO(benvanik): don't leak parent memory. + return false; + } + *out_address = address; + return true; +} + +bool PhysicalHeap::AllocFixed(uint32_t base_address, uint32_t size, + uint32_t alignment, uint32_t allocation_type, + uint32_t protect) { + // Adjust alignment size our page size differs from the parent. + size = xe::round_up(size, page_size_); + alignment = xe::round_up(alignment, page_size_); + + std::lock_guard lock(heap_mutex_); + + // Allocate from parent heap (gets our physical address in 0-512mb). + // NOTE: this can potentially overwrite heap contents if there are already + // committed pages in the requested physical range. + // TODO(benvanik): flag for ensure-not-committed? + uint32_t parent_base_address = GetPhysicalAddress(base_address); + if (!parent_heap_->AllocFixed(parent_base_address, size, alignment, + allocation_type, protect)) { + XELOGE( + "PhysicalHeap::Alloc unable to alloc physical memory in parent heap"); + return false; + } + + // Given the address we've reserved in the parent heap, pin that here. + // Shouldn't be possible for it to be allocated already. + uint32_t address = heap_base_ + parent_base_address; + if (!BaseHeap::AllocFixed(address, size, alignment, allocation_type, + protect)) { + XELOGE( + "PhysicalHeap::Alloc unable to pin physical memory in physical heap"); + // TODO(benvanik): don't leak parent memory. + return false; + } + + return true; +} + +bool PhysicalHeap::AllocRange(uint32_t low_address, uint32_t high_address, + uint32_t size, uint32_t alignment, + uint32_t allocation_type, uint32_t protect, + bool top_down, uint32_t* out_address) { + *out_address = 0; + + // Adjust alignment size our page size differs from the parent. + size = xe::round_up(size, page_size_); + alignment = xe::round_up(alignment, page_size_); + + std::lock_guard lock(heap_mutex_); + + // Allocate from parent heap (gets our physical address in 0-512mb). + low_address = std::max(heap_base_, low_address); + high_address = std::min(heap_base_ + heap_size_, high_address); + uint32_t parent_low_address = GetPhysicalAddress(low_address); + uint32_t parent_high_address = GetPhysicalAddress(high_address); + uint32_t parent_address; + if (!parent_heap_->AllocRange(parent_low_address, parent_high_address, size, + alignment, allocation_type, protect, top_down, + &parent_address)) { + XELOGE( + "PhysicalHeap::Alloc unable to alloc physical memory in parent heap"); + return false; + } + + // Given the address we've reserved in the parent heap, pin that here. + // Shouldn't be possible for it to be allocated already. + uint32_t address = heap_base_ + parent_address; + if (!BaseHeap::AllocFixed(address, size, alignment, allocation_type, + protect)) { + XELOGE( + "PhysicalHeap::Alloc unable to pin physical memory in physical heap"); + // TODO(benvanik): don't leak parent memory. + return false; + } + *out_address = address; + return true; +} + +bool PhysicalHeap::Decommit(uint32_t address, uint32_t size) { + std::lock_guard lock(heap_mutex_); + uint32_t parent_address = GetPhysicalAddress(address); + if (!parent_heap_->Decommit(parent_address, size)) { + XELOGE("PhysicalHeap::Decommit failed due to parent heap failure"); + return false; + } + return BaseHeap::Decommit(address, size); +} + +bool PhysicalHeap::Release(uint32_t base_address, uint32_t* out_region_size) { + std::lock_guard lock(heap_mutex_); + uint32_t parent_base_address = GetPhysicalAddress(base_address); + if (!parent_heap_->Release(parent_base_address, out_region_size)) { + XELOGE("PhysicalHeap::Release failed due to parent heap failure"); + return false; + } + return BaseHeap::Release(base_address, out_region_size); +} + +bool PhysicalHeap::Protect(uint32_t address, uint32_t size, uint32_t protect) { + std::lock_guard lock(heap_mutex_); + uint32_t parent_address = GetPhysicalAddress(address); + bool parent_result = parent_heap_->Protect(parent_address, size, protect); + if (!parent_result) { + XELOGE("PhysicalHeap::Protect failed due to parent heap failure"); + return false; + } + return BaseHeap::Protect(address, size, protect); +} + +} // namespace xe diff --git a/src/xenia/memory.h b/src/xenia/memory.h index c8c164939..14bb09095 100644 --- a/src/xenia/memory.h +++ b/src/xenia/memory.h @@ -12,6 +12,7 @@ #include #include +#include #include #include "xenia/base/platform.h" @@ -25,25 +26,129 @@ enum SystemHeapFlag : uint32_t { kSystemHeapDefault = kSystemHeapVirtual, }; -class MemoryHeap; -// TODO(benvanik): move to heap. -enum { - MEMORY_FLAG_64KB_PAGES = (1 << 1), - MEMORY_FLAG_ZERO = (1 << 2), - MEMORY_FLAG_PHYSICAL = (1 << 3), +enum MemoryAllocationFlag : uint32_t { + kMemoryAllocationReserve = 1 << 0, + kMemoryAllocationCommit = 1 << 1, +}; + +enum MemoryProtectFlag : uint32_t { + kMemoryProtectRead = 1 << 0, + kMemoryProtectWrite = 1 << 1, + kMemoryProtectNoCache = 1 << 2, + kMemoryProtectWriteCombine = 1 << 3, + + kMemoryProtectNoAccess = 0, }; -// TODO(benvanik): move to heap. // Equivalent to the Win32 MEMORY_BASIC_INFORMATION struct. -struct AllocationInfo { +struct HeapAllocationInfo { + // A pointer to the base address of the region of pages. uint32_t base_address; + // A pointer to the base address of a range of pages allocated by the + // VirtualAlloc function. The page pointed to by the BaseAddress member is + // contained within this allocation range. uint32_t allocation_base; - uint32_t allocation_protect; // TBD - size_t region_size; - uint32_t state; // TBD - uint32_t protect; // TBD - uint32_t type; // TBD + // The memory protection option when the region was initially allocated. + uint32_t allocation_protect; + // The size of the region beginning at the base address in which all pages + // have identical attributes, in bytes. + uint32_t region_size; + // The state of the pages in the region (commit/free/reserve). + uint32_t state; + // The access protection of the pages in the region. + uint32_t protect; + // The type of pages in the region (private). + uint32_t type; +}; + +union PageEntry { + struct { + uint32_t base_address : 20; // in 4k pages + uint32_t region_page_count : 20; // in 4k pages + uint32_t allocation_protect : 4; + uint32_t current_protect : 4; + uint32_t state : 2; + uint32_t reserved : 14; + }; + uint64_t qword; +}; + +class BaseHeap { + public: + virtual ~BaseHeap(); + + uint32_t page_size() const { return page_size_; } + + virtual void Dispose(); + + void DumpMap(); + + virtual bool Alloc(uint32_t size, uint32_t alignment, + uint32_t allocation_type, uint32_t protect, bool top_down, + uint32_t* out_address); + virtual bool AllocFixed(uint32_t base_address, uint32_t size, + uint32_t alignment, uint32_t allocation_type, + uint32_t protect); + virtual bool AllocRange(uint32_t low_address, uint32_t high_address, + uint32_t size, uint32_t alignment, + uint32_t allocation_type, uint32_t protect, + bool top_down, uint32_t* out_address); + virtual bool Decommit(uint32_t address, uint32_t size); + virtual bool Release(uint32_t address, uint32_t* out_region_size = nullptr); + virtual bool Protect(uint32_t address, uint32_t size, uint32_t protect); + + bool QueryRegionInfo(uint32_t base_address, HeapAllocationInfo* out_info); + bool QuerySize(uint32_t address, uint32_t* out_size); + bool QueryProtect(uint32_t address, uint32_t* out_protect); + uint32_t GetPhysicalAddress(uint32_t address); + + protected: + BaseHeap(); + + void Initialize(uint8_t* membase, uint32_t heap_base, uint32_t heap_size, + uint32_t page_size); + + uint8_t* membase_; + uint32_t heap_base_; + uint32_t heap_size_; + uint32_t page_size_; + std::vector page_table_; + std::recursive_mutex heap_mutex_; +}; + +class VirtualHeap : public BaseHeap { + public: + VirtualHeap(); + ~VirtualHeap() override; + + void Initialize(uint8_t* membase, uint32_t heap_base, uint32_t heap_size, + uint32_t page_size); +}; + +class PhysicalHeap : public BaseHeap { + public: + PhysicalHeap(); + ~PhysicalHeap() override; + + void Initialize(uint8_t* membase, uint32_t heap_base, uint32_t heap_size, + uint32_t page_size, VirtualHeap* parent_heap); + + bool Alloc(uint32_t size, uint32_t alignment, uint32_t allocation_type, + uint32_t protect, bool top_down, uint32_t* out_address) override; + bool AllocFixed(uint32_t base_address, uint32_t size, uint32_t alignment, + uint32_t allocation_type, uint32_t protect) override; + bool AllocRange(uint32_t low_address, uint32_t high_address, uint32_t size, + uint32_t alignment, uint32_t allocation_type, + uint32_t protect, bool top_down, + uint32_t* out_address) override; + bool Decommit(uint32_t address, uint32_t size) override; + bool Release(uint32_t base_address, + uint32_t* out_region_size = nullptr) override; + bool Protect(uint32_t address, uint32_t size, uint32_t protect) override; + + protected: + VirtualHeap* parent_heap_; }; class Memory { @@ -82,27 +187,24 @@ class Memory { uint32_t SearchAligned(uint32_t start, uint32_t end, const uint32_t* values, size_t value_count); - bool AddMappedRange(uint32_t address, uint32_t mask, uint32_t size, - void* context, cpu::MMIOReadCallback read_callback, - cpu::MMIOWriteCallback write_callback); + bool AddVirtualMappedRange(uint32_t virtual_address, uint32_t mask, + uint32_t size, void* context, + cpu::MMIOReadCallback read_callback, + cpu::MMIOWriteCallback write_callback); - uintptr_t AddWriteWatch(uint32_t guest_address, uint32_t length, - cpu::WriteWatchCallback callback, - void* callback_context, void* callback_data); + uintptr_t AddPhysicalWriteWatch(uint32_t physical_address, uint32_t length, + cpu::WriteWatchCallback callback, + void* callback_context, void* callback_data); void CancelWriteWatch(uintptr_t watch_handle); uint32_t SystemHeapAlloc(uint32_t size, uint32_t alignment = 0x20, uint32_t system_heap_flags = kSystemHeapDefault); void SystemHeapFree(uint32_t address); - uint32_t HeapAlloc(uint32_t base_address, uint32_t size, uint32_t flags, - uint32_t alignment = 0x20); - int HeapFree(uint32_t address, uint32_t size); - bool QueryInformation(uint32_t base_address, AllocationInfo* mem_info); - uint32_t QuerySize(uint32_t base_address); + BaseHeap* LookupHeap(uint32_t address); + BaseHeap* LookupHeapByType(bool physical, uint32_t page_size); - int Protect(uint32_t address, uint32_t size, uint32_t access); - uint32_t QueryProtect(uint32_t address); + void DumpMap(); private: int MapViews(uint8_t* mapping_base); @@ -122,22 +224,31 @@ class Memory { uint8_t* v00000000; uint8_t* v40000000; uint8_t* v7F000000; - uint8_t* v7F100000; uint8_t* v80000000; uint8_t* v90000000; uint8_t* vA0000000; uint8_t* vC0000000; uint8_t* vE0000000; + uint8_t* physical; }; uint8_t* all_views[9]; } views_; std::unique_ptr mmio_handler_; - MemoryHeap* virtual_heap_; - MemoryHeap* physical_heap_; + struct { + VirtualHeap v00000000; + VirtualHeap v40000000; + VirtualHeap v80000000; + VirtualHeap v90000000; - friend class MemoryHeap; + VirtualHeap physical; + PhysicalHeap vA0000000; + PhysicalHeap vC0000000; + PhysicalHeap vE0000000; + } heaps_; + + friend class BaseHeap; }; } // namespace xe