From 76f6f8de808acd32402ac3ea06f58cae50f16c58 Mon Sep 17 00:00:00 2001 From: "Crunch (Chaz9)" Date: Sun, 29 Sep 2024 21:28:35 +0100 Subject: [PATCH] ok --- src/core/core_timing.cpp | 114 ++---- src/core/cpu_manager.cpp | 43 +- src/core/memory.cpp | 865 ++++----------------------------------- 3 files changed, 121 insertions(+), 901 deletions(-) diff --git a/src/core/core_timing.cpp b/src/core/core_timing.cpp index 1abfa920c4..6ac0007764 100644 --- a/src/core/core_timing.cpp +++ b/src/core/core_timing.cpp @@ -26,24 +26,6 @@ std::shared_ptr CreateEvent(std::string name, TimedCallback&& callbac return std::make_shared(std::move(callback), std::move(name)); } -struct CoreTiming::Event { - s64 time; - u64 fifo_order; - std::weak_ptr type; - s64 reschedule_time; - heap_t::handle_type handle{}; - - // Sort by time, unless the times are the same, in which case sort by - // the order added to the queue - friend bool operator>(const Event& left, const Event& right) { - return std::tie(left.time, left.fifo_order) > std::tie(right.time, right.fifo_order); - } - - friend bool operator<(const Event& left, const Event& right) { - return std::tie(left.time, left.fifo_order) < std::tie(right.time, right.fifo_order); - } -}; - CoreTiming::CoreTiming() : clock{Common::CreateOptimalClock()} {} CoreTiming::~CoreTiming() { @@ -87,7 +69,7 @@ void CoreTiming::Pause(bool is_paused) { } void CoreTiming::SyncPause(bool is_paused) { - if (is_paused == paused && paused_set == paused) { + if (is_paused == paused && paused_set == is_paused) { return; } @@ -112,7 +94,7 @@ bool CoreTiming::IsRunning() const { bool CoreTiming::HasPendingEvents() const { std::scoped_lock lock{basic_lock}; - return !(wait_set && event_queue.empty()); + return !event_queue.empty(); } void CoreTiming::ScheduleEvent(std::chrono::nanoseconds ns_into_future, @@ -121,8 +103,8 @@ void CoreTiming::ScheduleEvent(std::chrono::nanoseconds ns_into_future, std::scoped_lock scope{basic_lock}; const auto next_time{absolute_time ? ns_into_future : GetGlobalTimeNs() + ns_into_future}; - auto h{event_queue.emplace(Event{next_time.count(), event_fifo_id++, event_type, 0})}; - (*h).handle = h; + event_queue.emplace_back(Event{next_time.count(), event_fifo_id++, event_type}); + std::push_heap(event_queue.begin(), event_queue.end(), std::greater<>()); } event.Set(); @@ -136,9 +118,9 @@ void CoreTiming::ScheduleLoopingEvent(std::chrono::nanoseconds start_time, std::scoped_lock scope{basic_lock}; const auto next_time{absolute_time ? start_time : GetGlobalTimeNs() + start_time}; - auto h{event_queue.emplace( - Event{next_time.count(), event_fifo_id++, event_type, resched_time.count()})}; - (*h).handle = h; + event_queue.emplace_back( + Event{next_time.count(), event_fifo_id++, event_type, resched_time.count()}); + std::push_heap(event_queue.begin(), event_queue.end(), std::greater<>()); } event.Set(); @@ -149,17 +131,11 @@ void CoreTiming::UnscheduleEvent(const std::shared_ptr& event_type, { std::scoped_lock lk{basic_lock}; - std::vector to_remove; - for (auto itr = event_queue.begin(); itr != event_queue.end(); itr++) { - const Event& e = *itr; - if (e.type.lock().get() == event_type.get()) { - to_remove.push_back(itr->handle); - } - } - - for (auto& h : to_remove) { - event_queue.erase(h); - } + event_queue.erase( + std::remove_if(event_queue.begin(), event_queue.end(), + [&](const Event& e) { return e.type.lock().get() == event_type.get(); }), + event_queue.end()); + std::make_heap(event_queue.begin(), event_queue.end(), std::greater<>()); event_type->sequence_number++; } @@ -172,7 +148,7 @@ void CoreTiming::UnscheduleEvent(const std::shared_ptr& event_type, void CoreTiming::AddTicks(u64 ticks_to_add) { cpu_ticks += ticks_to_add; - downcount -= static_cast(cpu_ticks); + downcount -= static_cast(ticks_to_add); } void CoreTiming::Idle() { @@ -180,7 +156,7 @@ void CoreTiming::Idle() { } void CoreTiming::ResetTicks() { - downcount = MAX_SLICE_LENGTH; + downcount.store(MAX_SLICE_LENGTH, std::memory_order_release); } u64 CoreTiming::GetClockTicks() const { @@ -201,48 +177,38 @@ std::optional CoreTiming::Advance() { std::scoped_lock lock{advance_lock, basic_lock}; global_timer = GetGlobalTimeNs().count(); - while (!event_queue.empty() && event_queue.top().time <= global_timer) { - const Event& evt = event_queue.top(); + while (!event_queue.empty() && event_queue.front().time <= global_timer) { + Event evt = std::move(event_queue.front()); + std::pop_heap(event_queue.begin(), event_queue.end(), std::greater<>()); + event_queue.pop_back(); - if (const auto event_type{evt.type.lock()}) { + if (const auto event_type = evt.type.lock()) { const auto evt_time = evt.time; const auto evt_sequence_num = event_type->sequence_number; - if (evt.reschedule_time == 0) { - event_queue.pop(); + basic_lock.unlock(); - basic_lock.unlock(); + const auto new_schedule_time = event_type->callback( + evt_time, std::chrono::nanoseconds{GetGlobalTimeNs().count() - evt_time}); - event_type->callback( - evt_time, std::chrono::nanoseconds{GetGlobalTimeNs().count() - evt_time}); + basic_lock.lock(); - basic_lock.lock(); - } else { - basic_lock.unlock(); + if (evt_sequence_num != event_type->sequence_number) { + continue; + } - const auto new_schedule_time{event_type->callback( - evt_time, std::chrono::nanoseconds{GetGlobalTimeNs().count() - evt_time})}; + if (new_schedule_time.has_value() || evt.reschedule_time != 0) { + const auto next_schedule_time = new_schedule_time.value_or( + std::chrono::nanoseconds{evt.reschedule_time}); - basic_lock.lock(); - - if (evt_sequence_num != event_type->sequence_number) { - // Heap handle is invalidated after external modification. - continue; - } - - const auto next_schedule_time{new_schedule_time.has_value() - ? new_schedule_time.value().count() - : evt.reschedule_time}; - - // If this event was scheduled into a pause, its time now is going to be way - // behind. Re-set this event to continue from the end of the pause. - auto next_time{evt.time + next_schedule_time}; + auto next_time = evt.time + next_schedule_time.count(); if (evt.time < pause_end_time) { - next_time = pause_end_time + next_schedule_time; + next_time = pause_end_time + next_schedule_time.count(); } - event_queue.update(evt.handle, Event{next_time, event_fifo_id++, evt.type, - next_schedule_time, evt.handle}); + event_queue.emplace_back(Event{next_time, event_fifo_id++, evt.type, + next_schedule_time.count()}); + std::push_heap(event_queue.begin(), event_queue.end(), std::greater<>()); } } @@ -250,7 +216,7 @@ std::optional CoreTiming::Advance() { } if (!event_queue.empty()) { - return event_queue.top().time; + return event_queue.front().time; } else { return std::nullopt; } @@ -269,7 +235,7 @@ void CoreTiming::ThreadLoop() { #ifdef _WIN32 while (!paused && !event.IsSet() && wait_time > 0) { wait_time = *next_time - GetGlobalTimeNs().count(); - if (wait_time >= timer_resolution_ns) { + if (wait_time >= 1'000'000) { // 1ms Common::Windows::SleepForOneTick(); } else { #ifdef ARCHITECTURE_x86_64 @@ -290,10 +256,8 @@ void CoreTiming::ThreadLoop() { } else { // Queue is empty, wait until another event is scheduled and signals us to // continue. - wait_set = true; event.Wait(); } - wait_set = false; } paused_set = true; @@ -327,10 +291,4 @@ std::chrono::microseconds CoreTiming::GetGlobalTimeUs() const { return std::chrono::microseconds{Common::WallClock::CPUTickToUS(cpu_ticks)}; } -#ifdef _WIN32 -void CoreTiming::SetTimerResolutionNs(std::chrono::nanoseconds ns) { - timer_resolution_ns = ns.count(); -} -#endif - } // namespace Core::Timing diff --git a/src/core/cpu_manager.cpp b/src/core/cpu_manager.cpp index 9b1c773877..e7e341de16 100644 --- a/src/core/cpu_manager.cpp +++ b/src/core/cpu_manager.cpp @@ -1,6 +1,12 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include +#include +#include +#include +#include + #include "common/fiber.h" #include "common/microprofile.h" #include "common/scope_exit.h" @@ -24,6 +30,7 @@ void CpuManager::Initialize() { num_cores = is_multicore ? Core::Hardware::NUM_CPU_CORES : 1; gpu_barrier = std::make_unique(num_cores + 1); + core_data.resize(num_cores); for (std::size_t core = 0; core < num_cores; core++) { core_data[core].host_thread = std::jthread([this, core](std::stop_token token) { RunThread(token, core); }); @@ -31,10 +38,10 @@ void CpuManager::Initialize() { } void CpuManager::Shutdown() { - for (std::size_t core = 0; core < num_cores; core++) { - if (core_data[core].host_thread.joinable()) { - core_data[core].host_thread.request_stop(); - core_data[core].host_thread.join(); + for (auto& data : core_data) { + if (data.host_thread.joinable()) { + data.host_thread.request_stop(); + data.host_thread.join(); } } } @@ -66,12 +73,7 @@ void CpuManager::HandleInterrupt() { Kernel::KInterruptManager::HandleInterrupt(kernel, static_cast(core_index)); } -/////////////////////////////////////////////////////////////////////////////// -/// MultiCore /// -/////////////////////////////////////////////////////////////////////////////// - void CpuManager::MultiCoreRunGuestThread() { - // Similar to UserModeThreadStarter in HOS auto& kernel = system.Kernel(); auto* thread = Kernel::GetCurrentThreadPointer(kernel); kernel.CurrentScheduler()->OnThreadStart(); @@ -88,10 +90,6 @@ void CpuManager::MultiCoreRunGuestThread() { } void CpuManager::MultiCoreRunIdleThread() { - // Not accurate to HOS. Remove this entire method when singlecore is removed. - // See notes in KScheduler::ScheduleImpl for more information about why this - // is inaccurate. - auto& kernel = system.Kernel(); kernel.CurrentScheduler()->OnThreadStart(); @@ -105,10 +103,6 @@ void CpuManager::MultiCoreRunIdleThread() { } } -/////////////////////////////////////////////////////////////////////////////// -/// SingleCore /// -/////////////////////////////////////////////////////////////////////////////// - void CpuManager::SingleCoreRunGuestThread() { auto& kernel = system.Kernel(); auto* thread = Kernel::GetCurrentThreadPointer(kernel); @@ -154,19 +148,16 @@ void CpuManager::PreemptSingleCore(bool from_running_environment) { system.CoreTiming().Advance(); kernel.SetIsPhantomModeForSingleCore(false); } - current_core.store((current_core + 1) % Core::Hardware::NUM_CPU_CORES); + current_core.store((current_core + 1) % Core::Hardware::NUM_CPU_CORES, std::memory_order_release); system.CoreTiming().ResetTicks(); kernel.Scheduler(current_core).PreemptSingleCore(); - // We've now been scheduled again, and we may have exchanged schedulers. - // Reload the scheduler in case it's different. if (!kernel.Scheduler(current_core).IsIdle()) { idle_count = 0; } } void CpuManager::GuestActivate() { - // Similar to the HorizonKernelMain callback in HOS auto& kernel = system.Kernel(); auto* scheduler = kernel.CurrentScheduler(); @@ -184,27 +175,19 @@ void CpuManager::ShutdownThread() { } void CpuManager::RunThread(std::stop_token token, std::size_t core) { - /// Initialization system.RegisterCoreThread(core); - std::string name; - if (is_multicore) { - name = "CPUCore_" + std::to_string(core); - } else { - name = "CPUThread"; - } + std::string name = is_multicore ? "CPUCore_" + std::to_string(core) : "CPUThread"; MicroProfileOnThreadCreate(name.c_str()); Common::SetCurrentThreadName(name.c_str()); Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical); auto& data = core_data[core]; data.host_context = Common::Fiber::ThreadToFiber(); - // Cleanup SCOPE_EXIT { data.host_context->Exit(); MicroProfileOnThreadExit(); }; - // Running if (!gpu_barrier->Sync(token)) { return; } diff --git a/src/core/memory.cpp b/src/core/memory.cpp index f7eae9c598..6e53db3640 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "common/assert.h" #include "common/atomic_ops.h" @@ -32,17 +33,18 @@ namespace Core::Memory { namespace { -bool AddressSpaceContains(const Common::PageTable& table, const Common::ProcessAddress addr, - const std::size_t size) { +constexpr size_t PAGE_SIZE = 0x1000; +constexpr size_t PAGE_BITS = 12; +constexpr size_t PAGE_MASK = PAGE_SIZE - 1; + +inline bool AddressSpaceContains(const Common::PageTable& table, const Common::ProcessAddress addr, + const std::size_t size) { const Common::ProcessAddress max_addr = 1ULL << table.GetAddressSpaceBits(); return addr + size >= addr && addr + size <= max_addr; } -} // namespace +} // Anonymous namespace -// Implementation class used to keep the specifics of the memory subsystem hidden -// from outside classes. This also allows modification to the internals of the memory -// subsystem without needing to rebuild all files that make use of the memory interface. struct Memory::Impl { explicit Impl(Core::System& system_) : system{system_} {} @@ -66,12 +68,11 @@ struct Memory::Impl { void MapMemoryRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, Common::PhysicalAddress target, Common::MemoryPermission perms, bool separate_heap) { - ASSERT_MSG((size & SUYU_PAGEMASK) == 0, "non-page aligned size: {:016X}", size); - ASSERT_MSG((base & SUYU_PAGEMASK) == 0, "non-page aligned base: {:016X}", GetInteger(base)); + ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", GetInteger(base)); ASSERT_MSG(target >= DramMemoryMap::Base, "Out of bounds target: {:016X}", GetInteger(target)); - MapPages(page_table, base / SUYU_PAGESIZE, size / SUYU_PAGESIZE, target, - Common::PageType::Memory); + MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, target, Common::PageType::Memory); if (current_page_table->fastmem_arena) { buffer->Map(GetInteger(base), GetInteger(target) - DramMemoryMap::Base, size, perms, @@ -81,10 +82,9 @@ struct Memory::Impl { void UnmapRegion(Common::PageTable& page_table, Common::ProcessAddress base, u64 size, bool separate_heap) { - ASSERT_MSG((size & SUYU_PAGEMASK) == 0, "non-page aligned size: {:016X}", size); - ASSERT_MSG((base & SUYU_PAGEMASK) == 0, "non-page aligned base: {:016X}", GetInteger(base)); - MapPages(page_table, base / SUYU_PAGESIZE, size / SUYU_PAGESIZE, 0, - Common::PageType::Unmapped); + ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", GetInteger(base)); + MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, 0, Common::PageType::Unmapped); if (current_page_table->fastmem_arena) { buffer->Unmap(GetInteger(base), size, separate_heap); @@ -93,55 +93,28 @@ struct Memory::Impl { void ProtectRegion(Common::PageTable& page_table, VAddr vaddr, u64 size, Common::MemoryPermission perms) { - ASSERT_MSG((size & SUYU_PAGEMASK) == 0, "non-page aligned size: {:016X}", size); - ASSERT_MSG((vaddr & SUYU_PAGEMASK) == 0, "non-page aligned base: {:016X}", vaddr); + ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((vaddr & PAGE_MASK) == 0, "non-page aligned base: {:016X}", vaddr); if (!current_page_table->fastmem_arena) { return; } - u64 protect_bytes{}; - u64 protect_begin{}; - for (u64 addr = vaddr; addr < vaddr + size; addr += SUYU_PAGESIZE) { + for (u64 addr = vaddr; addr < vaddr + size; addr += PAGE_SIZE) { const Common::PageType page_type{ - current_page_table->pointers[addr >> SUYU_PAGEBITS].Type()}; - switch (page_type) { - case Common::PageType::RasterizerCachedMemory: - if (protect_bytes > 0) { - buffer->Protect(protect_begin, protect_bytes, perms); - protect_bytes = 0; - } - break; - default: - if (protect_bytes == 0) { - protect_begin = addr; - } - protect_bytes += SUYU_PAGESIZE; + current_page_table->pointers[addr >> PAGE_BITS].Type()}; + if (page_type != Common::PageType::RasterizerCachedMemory) { + buffer->Protect(addr, PAGE_SIZE, perms); } } - - if (protect_bytes > 0) { - buffer->Protect(protect_begin, protect_bytes, perms); - } } - [[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const { + u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const { const Common::PhysicalAddress paddr{ - current_page_table->backing_addr[vaddr >> SUYU_PAGEBITS]}; + current_page_table->backing_addr[vaddr >> PAGE_BITS]}; if (!paddr) { - return {}; - } - - return system.DeviceMemory().GetPointer(paddr + vaddr); - } - - [[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const { - const Common::PhysicalAddress paddr{ - current_page_table->backing_addr[vaddr >> SUYU_PAGEBITS]}; - - if (paddr == 0) { - return {}; + return nullptr; } return system.DeviceMemory().GetPointer(paddr + vaddr); @@ -155,9 +128,7 @@ struct Memory::Impl { if ((addr & 1) == 0) { return Read(addr); } else { - const u32 a{Read(addr)}; - const u32 b{Read(addr + sizeof(u8))}; - return static_cast((b << 8) | a); + return Read(addr) | static_cast(Read(addr + sizeof(u8))) << 8; } } @@ -165,9 +136,7 @@ struct Memory::Impl { if ((addr & 3) == 0) { return Read(addr); } else { - const u32 a{Read16(addr)}; - const u32 b{Read16(addr + sizeof(u16))}; - return (b << 16) | a; + return Read16(addr) | static_cast(Read16(addr + sizeof(u16))) << 16; } } @@ -175,9 +144,7 @@ struct Memory::Impl { if ((addr & 7) == 0) { return Read(addr); } else { - const u32 a{Read32(addr)}; - const u32 b{Read32(addr + sizeof(u32))}; - return (static_cast(b) << 32) | a; + return Read32(addr) | static_cast(Read32(addr + sizeof(u32))) << 32; } } @@ -232,7 +199,7 @@ struct Memory::Impl { std::string string; string.reserve(max_length); for (std::size_t i = 0; i < max_length; ++i) { - const char c = Read(vaddr); + const char c = Read(vaddr); if (c == '\0') { break; } @@ -243,648 +210,72 @@ struct Memory::Impl { return string; } - bool WalkBlock(const Common::ProcessAddress addr, const std::size_t size, auto on_unmapped, - auto on_memory, auto on_rasterizer, auto increment) { - const auto& page_table = *current_page_table; - std::size_t remaining_size = size; - std::size_t page_index = addr >> SUYU_PAGEBITS; - std::size_t page_offset = addr & SUYU_PAGEMASK; - bool user_accessible = true; - - if (!AddressSpaceContains(page_table, addr, size)) [[unlikely]] { - on_unmapped(size, addr); - return false; - } - - while (remaining_size) { - const std::size_t copy_amount = - std::min(static_cast(SUYU_PAGESIZE) - page_offset, remaining_size); - const auto current_vaddr = - static_cast((page_index << SUYU_PAGEBITS) + page_offset); - - const auto [pointer, type] = page_table.pointers[page_index].PointerType(); - switch (type) { - case Common::PageType::Unmapped: { - user_accessible = false; - on_unmapped(copy_amount, current_vaddr); - break; - } - case Common::PageType::Memory: { - u8* mem_ptr = - reinterpret_cast(pointer + page_offset + (page_index << SUYU_PAGEBITS)); - on_memory(copy_amount, mem_ptr); - break; - } - case Common::PageType::DebugMemory: { - u8* const mem_ptr{GetPointerFromDebugMemory(current_vaddr)}; - on_memory(copy_amount, mem_ptr); - break; - } - case Common::PageType::RasterizerCachedMemory: { - u8* const host_ptr{GetPointerFromRasterizerCachedMemory(current_vaddr)}; - on_rasterizer(current_vaddr, copy_amount, host_ptr); - break; - } - default: - UNREACHABLE(); - } - - page_index++; - page_offset = 0; - increment(copy_amount); - remaining_size -= copy_amount; - } - - return user_accessible; - } - - template - bool ReadBlockImpl(const Common::ProcessAddress src_addr, void* dest_buffer, - const std::size_t size) { - return WalkBlock( - src_addr, size, - [src_addr, size, &dest_buffer](const std::size_t copy_amount, - const Common::ProcessAddress current_vaddr) { - LOG_ERROR(HW_Memory, - "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - GetInteger(current_vaddr), GetInteger(src_addr), size); - std::memset(dest_buffer, 0, copy_amount); - }, - [&](const std::size_t copy_amount, const u8* const src_ptr) { - std::memcpy(dest_buffer, src_ptr, copy_amount); - }, - [&](const Common::ProcessAddress current_vaddr, const std::size_t copy_amount, - const u8* const host_ptr) { - if constexpr (!UNSAFE) { - HandleRasterizerDownload(GetInteger(current_vaddr), copy_amount); - } - std::memcpy(dest_buffer, host_ptr, copy_amount); - }, - [&](const std::size_t copy_amount) { - dest_buffer = static_cast(dest_buffer) + copy_amount; - }); - } - - bool ReadBlock(const Common::ProcessAddress src_addr, void* dest_buffer, - const std::size_t size) { - return ReadBlockImpl(src_addr, dest_buffer, size); - } - - bool ReadBlockUnsafe(const Common::ProcessAddress src_addr, void* dest_buffer, - const std::size_t size) { - return ReadBlockImpl(src_addr, dest_buffer, size); - } - - const u8* GetSpan(const VAddr src_addr, const std::size_t size) const { - if (current_page_table->blocks[src_addr >> SUYU_PAGEBITS] == - current_page_table->blocks[(src_addr + size) >> SUYU_PAGEBITS]) { - return GetPointerSilent(src_addr); - } - return nullptr; - } - - u8* GetSpan(const VAddr src_addr, const std::size_t size) { - if (current_page_table->blocks[src_addr >> SUYU_PAGEBITS] == - current_page_table->blocks[(src_addr + size) >> SUYU_PAGEBITS]) { - return GetPointerSilent(src_addr); - } - return nullptr; - } - - template - bool WriteBlockImpl(const Common::ProcessAddress dest_addr, const void* src_buffer, - const std::size_t size) { - return WalkBlock( - dest_addr, size, - [dest_addr, size](const std::size_t copy_amount, - const Common::ProcessAddress current_vaddr) { - LOG_ERROR(HW_Memory, - "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - GetInteger(current_vaddr), GetInteger(dest_addr), size); - }, - [&](const std::size_t copy_amount, u8* const dest_ptr) { - std::memcpy(dest_ptr, src_buffer, copy_amount); - }, - [&](const Common::ProcessAddress current_vaddr, const std::size_t copy_amount, - u8* const host_ptr) { - if constexpr (!UNSAFE) { - HandleRasterizerWrite(GetInteger(current_vaddr), copy_amount); - } - std::memcpy(host_ptr, src_buffer, copy_amount); - }, - [&](const std::size_t copy_amount) { - src_buffer = static_cast(src_buffer) + copy_amount; - }); - } - - bool WriteBlock(const Common::ProcessAddress dest_addr, const void* src_buffer, - const std::size_t size) { - return WriteBlockImpl(dest_addr, src_buffer, size); - } - - bool WriteBlockUnsafe(const Common::ProcessAddress dest_addr, const void* src_buffer, - const std::size_t size) { - return WriteBlockImpl(dest_addr, src_buffer, size); - } - - bool ZeroBlock(const Common::ProcessAddress dest_addr, const std::size_t size) { - return WalkBlock( - dest_addr, size, - [dest_addr, size](const std::size_t copy_amount, - const Common::ProcessAddress current_vaddr) { - LOG_ERROR(HW_Memory, - "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - GetInteger(current_vaddr), GetInteger(dest_addr), size); - }, - [](const std::size_t copy_amount, u8* const dest_ptr) { - std::memset(dest_ptr, 0, copy_amount); - }, - [&](const Common::ProcessAddress current_vaddr, const std::size_t copy_amount, - u8* const host_ptr) { - HandleRasterizerWrite(GetInteger(current_vaddr), copy_amount); - std::memset(host_ptr, 0, copy_amount); - }, - [](const std::size_t copy_amount) {}); - } - - bool CopyBlock(Common::ProcessAddress dest_addr, Common::ProcessAddress src_addr, - const std::size_t size) { - return WalkBlock( - dest_addr, size, - [&](const std::size_t copy_amount, const Common::ProcessAddress current_vaddr) { - LOG_ERROR(HW_Memory, - "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - GetInteger(current_vaddr), GetInteger(src_addr), size); - ZeroBlock(dest_addr, copy_amount); - }, - [&](const std::size_t copy_amount, const u8* const src_ptr) { - WriteBlockImpl(dest_addr, src_ptr, copy_amount); - }, - [&](const Common::ProcessAddress current_vaddr, const std::size_t copy_amount, - u8* const host_ptr) { - HandleRasterizerDownload(GetInteger(current_vaddr), copy_amount); - WriteBlockImpl(dest_addr, host_ptr, copy_amount); - }, - [&](const std::size_t copy_amount) { - dest_addr += copy_amount; - src_addr += copy_amount; - }); - } - - template - Result PerformCacheOperation(Common::ProcessAddress dest_addr, std::size_t size, - Callback&& cb) { - class InvalidMemoryException : public std::exception {}; - - try { - WalkBlock( - dest_addr, size, - [&](const std::size_t block_size, const Common::ProcessAddress current_vaddr) { - LOG_ERROR(HW_Memory, "Unmapped cache maintenance @ {:#018X}", - GetInteger(current_vaddr)); - throw InvalidMemoryException(); - }, - [&](const std::size_t block_size, u8* const host_ptr) {}, - [&](const Common::ProcessAddress current_vaddr, const std::size_t block_size, - u8* const host_ptr) { cb(current_vaddr, block_size); }, - [](const std::size_t block_size) {}); - } catch (InvalidMemoryException&) { - return Kernel::ResultInvalidCurrentMemory; - } - - return ResultSuccess; - } - - Result InvalidateDataCache(Common::ProcessAddress dest_addr, std::size_t size) { - auto on_rasterizer = [&](const Common::ProcessAddress current_vaddr, - const std::size_t block_size) { - // dc ivac: Invalidate to point of coherency - // GPU flush -> CPU invalidate - HandleRasterizerDownload(GetInteger(current_vaddr), block_size); - }; - return PerformCacheOperation(dest_addr, size, on_rasterizer); - } - - Result StoreDataCache(Common::ProcessAddress dest_addr, std::size_t size) { - auto on_rasterizer = [&](const Common::ProcessAddress current_vaddr, - const std::size_t block_size) { - // dc cvac: Store to point of coherency - // CPU flush -> GPU invalidate - HandleRasterizerWrite(GetInteger(current_vaddr), block_size); - }; - return PerformCacheOperation(dest_addr, size, on_rasterizer); - } - - Result FlushDataCache(Common::ProcessAddress dest_addr, std::size_t size) { - auto on_rasterizer = [&](const Common::ProcessAddress current_vaddr, - const std::size_t block_size) { - // dc civac: Store to point of coherency, and invalidate from cache - // CPU flush -> GPU invalidate - HandleRasterizerWrite(GetInteger(current_vaddr), block_size); - }; - return PerformCacheOperation(dest_addr, size, on_rasterizer); - } - - void MarkRegionDebug(u64 vaddr, u64 size, bool debug) { - if (vaddr == 0 || !AddressSpaceContains(*current_page_table, vaddr, size)) { - return; - } - - if (current_page_table->fastmem_arena) { - const auto perm{debug ? Common::MemoryPermission{} - : Common::MemoryPermission::ReadWrite}; - buffer->Protect(vaddr, size, perm); - } - - // Iterate over a contiguous CPU address space, marking/unmarking the region. - // The region is at a granularity of CPU pages. - - const u64 num_pages = ((vaddr + size - 1) >> SUYU_PAGEBITS) - (vaddr >> SUYU_PAGEBITS) + 1; - for (u64 i = 0; i < num_pages; ++i, vaddr += SUYU_PAGESIZE) { - const Common::PageType page_type{ - current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Type()}; - if (debug) { - // Switch page type to debug if now debug - switch (page_type) { - case Common::PageType::Unmapped: - ASSERT_MSG(false, "Attempted to mark unmapped pages as debug"); - break; - case Common::PageType::RasterizerCachedMemory: - case Common::PageType::DebugMemory: - // Page is already marked. - break; - case Common::PageType::Memory: - current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Store( - 0, Common::PageType::DebugMemory); - break; - default: - UNREACHABLE(); - } - } else { - // Switch page type to non-debug if now non-debug - switch (page_type) { - case Common::PageType::Unmapped: - ASSERT_MSG(false, "Attempted to mark unmapped pages as non-debug"); - break; - case Common::PageType::RasterizerCachedMemory: - case Common::PageType::Memory: - // Don't mess with already non-debug or rasterizer memory. - break; - case Common::PageType::DebugMemory: { - u8* const pointer{GetPointerFromDebugMemory(vaddr & ~SUYU_PAGEMASK)}; - current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Store( - reinterpret_cast(pointer) - (vaddr & ~SUYU_PAGEMASK), - Common::PageType::Memory); - break; - } - default: - UNREACHABLE(); - } - } - } - } - - void RasterizerMarkRegionCached(u64 vaddr, u64 size, bool cached) { - if (vaddr == 0 || !AddressSpaceContains(*current_page_table, vaddr, size)) { - return; - } - - if (current_page_table->fastmem_arena) { - Common::MemoryPermission perm{}; - if (!Settings::values.use_reactive_flushing.GetValue() || !cached) { - perm |= Common::MemoryPermission::Read; - } - if (!cached) { - perm |= Common::MemoryPermission::Write; - } - buffer->Protect(vaddr, size, perm); - } - - // Iterate over a contiguous CPU address space, which corresponds to the specified GPU - // address space, marking the region as un/cached. The region is marked un/cached at a - // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size - // is different). This assumes the specified GPU address region is contiguous as well. - - const u64 num_pages = ((vaddr + size - 1) >> SUYU_PAGEBITS) - (vaddr >> SUYU_PAGEBITS) + 1; - for (u64 i = 0; i < num_pages; ++i, vaddr += SUYU_PAGESIZE) { - const Common::PageType page_type{ - current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Type()}; - if (cached) { - // Switch page type to cached if now cached - switch (page_type) { - case Common::PageType::Unmapped: - // It is not necessary for a process to have this region mapped into its address - // space, for example, a system module need not have a VRAM mapping. - break; - case Common::PageType::DebugMemory: - case Common::PageType::Memory: - current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Store( - 0, Common::PageType::RasterizerCachedMemory); - break; - case Common::PageType::RasterizerCachedMemory: - // There can be more than one GPU region mapped per CPU region, so it's common - // that this area is already marked as cached. - break; - default: - UNREACHABLE(); - } - } else { - // Switch page type to uncached if now uncached - switch (page_type) { - case Common::PageType::Unmapped: // NOLINT(bugprone-branch-clone) - // It is not necessary for a process to have this region mapped into its address - // space, for example, a system module need not have a VRAM mapping. - break; - case Common::PageType::DebugMemory: - case Common::PageType::Memory: - // There can be more than one GPU region mapped per CPU region, so it's common - // that this area is already unmarked as cached. - break; - case Common::PageType::RasterizerCachedMemory: { - u8* const pointer{GetPointerFromRasterizerCachedMemory(vaddr & ~SUYU_PAGEMASK)}; - if (pointer == nullptr) { - // It's possible that this function has been called while updating the - // pagetable after unmapping a VMA. In that case the underlying VMA will no - // longer exist, and we should just leave the pagetable entry blank. - current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Store( - 0, Common::PageType::Unmapped); - } else { - current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Store( - reinterpret_cast(pointer) - (vaddr & ~SUYU_PAGEMASK), - Common::PageType::Memory); - } - break; - } - default: - UNREACHABLE(); - } - } - } - } - - /** - * Maps a region of pages as a specific type. - * - * @param page_table The page table to use to perform the mapping. - * @param base The base address to begin mapping at. - * @param size The total size of the range in bytes. - * @param target The target address to begin mapping from. - * @param type The page type to map the memory as. - */ - void MapPages(Common::PageTable& page_table, Common::ProcessAddress base_address, u64 size, - Common::PhysicalAddress target, Common::PageType type) { - auto base = GetInteger(base_address); - - LOG_DEBUG(HW_Memory, "Mapping {:016X} onto {:016X}-{:016X}", GetInteger(target), - base * SUYU_PAGESIZE, (base + size) * SUYU_PAGESIZE); - - const auto end = base + size; - ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}", - base + page_table.pointers.size()); - - if (!target) { - ASSERT_MSG(type != Common::PageType::Memory, - "Mapping memory page without a pointer @ {:016x}", base * SUYU_PAGESIZE); - - while (base != end) { - page_table.pointers[base].Store(0, type); - page_table.backing_addr[base] = 0; - page_table.blocks[base] = 0; - base += 1; - } - } else { - auto orig_base = base; - while (base != end) { - auto host_ptr = - reinterpret_cast(system.DeviceMemory().GetPointer(target)) - - (base << SUYU_PAGEBITS); - auto backing = GetInteger(target) - (base << SUYU_PAGEBITS); - page_table.pointers[base].Store(host_ptr, type); - page_table.backing_addr[base] = backing; - page_table.blocks[base] = orig_base << SUYU_PAGEBITS; - - ASSERT_MSG(page_table.pointers[base].Pointer(), - "memory mapping base yield a nullptr within the table"); - - base += 1; - target += SUYU_PAGESIZE; - } - } - } - - [[nodiscard]] u8* GetPointerImpl(u64 vaddr, auto on_unmapped, auto on_rasterizer) const { - // AARCH64 masks the upper 16 bit of all memory accesses - vaddr = vaddr & 0xffffffffffffULL; - - if (!AddressSpaceContains(*current_page_table, vaddr, 1)) [[unlikely]] { - on_unmapped(); - return nullptr; - } - - // Avoid adding any extra logic to this fast-path block - const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> SUYU_PAGEBITS].Raw(); - if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) { - return reinterpret_cast(pointer + vaddr); - } - switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) { - case Common::PageType::Unmapped: - on_unmapped(); - return nullptr; - case Common::PageType::Memory: - ASSERT_MSG(false, "Mapped memory page without a pointer @ 0x{:016X}", vaddr); - return nullptr; - case Common::PageType::DebugMemory: - return GetPointerFromDebugMemory(vaddr); - case Common::PageType::RasterizerCachedMemory: { - u8* const host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)}; - on_rasterizer(); - return host_ptr; - } - default: - UNREACHABLE(); - } - return nullptr; - } - - [[nodiscard]] u8* GetPointer(const Common::ProcessAddress vaddr) const { - return GetPointerImpl( - GetInteger(vaddr), - [vaddr]() { - LOG_ERROR(HW_Memory, "Unmapped GetPointer @ 0x{:016X}", GetInteger(vaddr)); - }, - []() {}); - } - - [[nodiscard]] u8* GetPointerSilent(const Common::ProcessAddress vaddr) const { - return GetPointerImpl( - GetInteger(vaddr), []() {}, []() {}); - } - - /** - * Reads a particular data type out of memory at the given virtual address. - * - * @param vaddr The virtual address to read the data type from. - * - * @tparam T The data type to read out of memory. This type *must* be - * trivially copyable, otherwise the behavior of this function - * is undefined. - * - * @returns The instance of T read from the specified virtual address. - */ template - T Read(Common::ProcessAddress vaddr) { - T result = 0; - const u8* const ptr = GetPointerImpl( - GetInteger(vaddr), - [vaddr]() { - LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:016X}", sizeof(T) * 8, - GetInteger(vaddr)); - }, - [&]() { HandleRasterizerDownload(GetInteger(vaddr), sizeof(T)); }); + T Read(const Common::ProcessAddress vaddr) { + T value; + const u8* const ptr = GetPointerFromRasterizerCachedMemory(GetInteger(vaddr)); if (ptr) { - std::memcpy(&result, ptr, sizeof(T)); + std::memcpy(&value, ptr, sizeof(T)); + } else { + LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:016X}", sizeof(T) * 8, GetInteger(vaddr)); + value = 0; } - return result; + return value; } - /** - * Writes a particular data type to memory at the given virtual address. - * - * @param vaddr The virtual address to write the data type to. - * - * @tparam T The data type to write to memory. This type *must* be - * trivially copyable, otherwise the behavior of this function - * is undefined. - */ template void Write(Common::ProcessAddress vaddr, const T data) { - u8* const ptr = GetPointerImpl( - GetInteger(vaddr), - [vaddr, data]() { - LOG_ERROR(HW_Memory, "Unmapped Write{} @ 0x{:016X} = 0x{:016X}", sizeof(T) * 8, - GetInteger(vaddr), static_cast(data)); - }, - [&]() { HandleRasterizerWrite(GetInteger(vaddr), sizeof(T)); }); + u8* const ptr = GetPointerFromRasterizerCachedMemory(GetInteger(vaddr)); if (ptr) { std::memcpy(ptr, &data, sizeof(T)); + system.GPU().InvalidateRegion(GetInteger(vaddr), sizeof(T)); + } else { + LOG_ERROR(HW_Memory, "Unmapped Write{} @ 0x{:016X} = 0x{:016X}", sizeof(T) * 8, + GetInteger(vaddr), static_cast(data)); } } template bool WriteExclusive(Common::ProcessAddress vaddr, const T data, const T expected) { - u8* const ptr = GetPointerImpl( - GetInteger(vaddr), - [vaddr, data]() { - LOG_ERROR(HW_Memory, "Unmapped WriteExclusive{} @ 0x{:016X} = 0x{:016X}", - sizeof(T) * 8, GetInteger(vaddr), static_cast(data)); - }, - [&]() { HandleRasterizerWrite(GetInteger(vaddr), sizeof(T)); }); + u8* const ptr = GetPointerFromRasterizerCachedMemory(GetInteger(vaddr)); if (ptr) { - return Common::AtomicCompareAndSwap(reinterpret_cast(ptr), data, expected); + const bool result = Common::AtomicCompareAndSwap(reinterpret_cast(ptr), data, expected); + if (result) { + system.GPU().InvalidateRegion(GetInteger(vaddr), sizeof(T)); + } + return result; + } else { + LOG_ERROR(HW_Memory, "Unmapped WriteExclusive{} @ 0x{:016X} = 0x{:016X}", sizeof(T) * 8, + GetInteger(vaddr), static_cast(data)); + return true; } - return true; } - bool WriteExclusive128(Common::ProcessAddress vaddr, const u128 data, const u128 expected) { - u8* const ptr = GetPointerImpl( - GetInteger(vaddr), - [vaddr, data]() { - LOG_ERROR(HW_Memory, "Unmapped WriteExclusive128 @ 0x{:016X} = 0x{:016X}{:016X}", - GetInteger(vaddr), static_cast(data[1]), static_cast(data[0])); - }, - [&]() { HandleRasterizerWrite(GetInteger(vaddr), sizeof(u128)); }); - if (ptr) { - return Common::AtomicCompareAndSwap(reinterpret_cast(ptr), data, expected); + bool ReadBlock(const Common::ProcessAddress src_addr, void* dest_buffer, + const std::size_t size) { + const u8* src_ptr = GetPointerFromRasterizerCachedMemory(GetInteger(src_addr)); + if (src_ptr) { + std::memcpy(dest_buffer, src_ptr, size); + return true; } - return true; + LOG_ERROR(HW_Memory, "Unmapped ReadBlock @ 0x{:016X}", GetInteger(src_addr)); + return false; } - void HandleRasterizerDownload(VAddr v_address, size_t size) { - const auto* p = GetPointerImpl( - v_address, []() {}, []() {}); - if (!gpu_device_memory) [[unlikely]] { - gpu_device_memory = &system.Host1x().MemoryManager(); + bool WriteBlock(const Common::ProcessAddress dest_addr, const void* src_buffer, + const std::size_t size) { + u8* const dest_ptr = GetPointerFromRasterizerCachedMemory(GetInteger(dest_addr)); + if (dest_ptr) { + std::memcpy(dest_ptr, src_buffer, size); + system.GPU().InvalidateRegion(GetInteger(dest_addr), size); + return true; } - const size_t core = system.GetCurrentHostThreadID(); - auto& current_area = rasterizer_read_areas[core]; - gpu_device_memory->ApplyOpOnPointer(p, scratch_buffers[core], [&](DAddr address) { - const DAddr end_address = address + size; - if (current_area.start_address <= address && end_address <= current_area.end_address) - [[likely]] { - return; - } - current_area = system.GPU().OnCPURead(address, size); - }); - } - - void HandleRasterizerWrite(VAddr v_address, size_t size) { - const auto* p = GetPointerImpl( - v_address, []() {}, []() {}); - constexpr size_t sys_core = Core::Hardware::NUM_CPU_CORES - 1; - const size_t core = std::min(system.GetCurrentHostThreadID(), - sys_core); // any other calls threads go to syscore. - if (!gpu_device_memory) [[unlikely]] { - gpu_device_memory = &system.Host1x().MemoryManager(); - } - // Guard on sys_core; - if (core == sys_core) [[unlikely]] { - sys_core_guard.lock(); - } - SCOPE_EXIT { - if (core == sys_core) [[unlikely]] { - sys_core_guard.unlock(); - } - }; - gpu_device_memory->ApplyOpOnPointer(p, scratch_buffers[core], [&](DAddr address) { - auto& current_area = rasterizer_write_areas[core]; - PAddr subaddress = address >> SUYU_PAGEBITS; - bool do_collection = current_area.last_address == subaddress; - if (!do_collection) [[unlikely]] { - do_collection = system.GPU().OnCPUWrite(address, size); - if (!do_collection) { - return; - } - current_area.last_address = subaddress; - } - gpu_dirty_managers[core].Collect(address, size); - }); - } - - struct GPUDirtyState { - PAddr last_address; - }; - - void InvalidateGPUMemory(u8* p, size_t size) { - constexpr size_t sys_core = Core::Hardware::NUM_CPU_CORES - 1; - const size_t core = std::min(system.GetCurrentHostThreadID(), - sys_core); // any other calls threads go to syscore. - if (!gpu_device_memory) [[unlikely]] { - gpu_device_memory = &system.Host1x().MemoryManager(); - } - // Guard on sys_core; - if (core == sys_core) [[unlikely]] { - sys_core_guard.lock(); - } - SCOPE_EXIT { - if (core == sys_core) [[unlikely]] { - sys_core_guard.unlock(); - } - }; - auto& gpu = system.GPU(); - gpu_device_memory->ApplyOpOnPointer( - p, scratch_buffers[core], [&](DAddr address) { gpu.InvalidateRegion(address, size); }); + LOG_ERROR(HW_Memory, "Unmapped WriteBlock @ 0x{:016X}", GetInteger(dest_addr)); + return false; } Core::System& system; - Tegra::MaxwellDeviceMemoryManager* gpu_device_memory{}; Common::PageTable* current_page_table = nullptr; - std::array - rasterizer_read_areas{}; - std::array rasterizer_write_areas{}; - std::array, Core::Hardware::NUM_CPU_CORES> scratch_buffers{}; - std::span gpu_dirty_managers; - std::mutex sys_core_guard; - std::optional heap_tracker; #ifdef __linux__ Common::HeapTracker* buffer{}; @@ -893,16 +284,10 @@ struct Memory::Impl { #endif }; -Memory::Memory(Core::System& system_) : system{system_} { - Reset(); -} +Memory::Memory(Core::System& system_) : impl{std::make_unique(system_)} {} Memory::~Memory() = default; -void Memory::Reset() { - impl = std::make_unique(system); -} - void Memory::SetCurrentPageTable(Kernel::KProcess& process) { impl->SetCurrentPageTable(process); } @@ -925,38 +310,20 @@ void Memory::ProtectRegion(Common::PageTable& page_table, Common::ProcessAddress bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const { const auto& page_table = *impl->current_page_table; - const size_t page = vaddr >> SUYU_PAGEBITS; + const size_t page = vaddr >> PAGE_BITS; if (page >= page_table.pointers.size()) { return false; } const auto [pointer, type] = page_table.pointers[page].PointerType(); - return pointer != 0 || type == Common::PageType::RasterizerCachedMemory || - type == Common::PageType::DebugMemory; -} - -bool Memory::IsValidVirtualAddressRange(Common::ProcessAddress base, u64 size) const { - Common::ProcessAddress end = base + size; - Common::ProcessAddress page = Common::AlignDown(GetInteger(base), SUYU_PAGESIZE); - - for (; page < end; page += SUYU_PAGESIZE) { - if (!IsValidVirtualAddress(page)) { - return false; - } - } - - return true; + return pointer != 0 || type == Common::PageType::RasterizerCachedMemory; } u8* Memory::GetPointer(Common::ProcessAddress vaddr) { - return impl->GetPointer(vaddr); -} - -u8* Memory::GetPointerSilent(Common::ProcessAddress vaddr) { - return impl->GetPointerSilent(vaddr); + return impl->GetPointerFromRasterizerCachedMemory(GetInteger(vaddr)); } const u8* Memory::GetPointer(Common::ProcessAddress vaddr) const { - return impl->GetPointer(vaddr); + return impl->GetPointerFromRasterizerCachedMemory(GetInteger(vaddr)); } u8 Memory::Read8(const Common::ProcessAddress addr) { @@ -1007,10 +374,6 @@ bool Memory::WriteExclusive64(Common::ProcessAddress addr, u64 data, u64 expecte return impl->WriteExclusive64(addr, data, expected); } -bool Memory::WriteExclusive128(Common::ProcessAddress addr, u128 data, u128 expected) { - return impl->WriteExclusive128(addr, data, expected); -} - std::string Memory::ReadCString(Common::ProcessAddress vaddr, std::size_t max_length) { return impl->ReadCString(vaddr, max_length); } @@ -1020,93 +383,9 @@ bool Memory::ReadBlock(const Common::ProcessAddress src_addr, void* dest_buffer, return impl->ReadBlock(src_addr, dest_buffer, size); } -bool Memory::ReadBlockUnsafe(const Common::ProcessAddress src_addr, void* dest_buffer, - const std::size_t size) { - return impl->ReadBlockUnsafe(src_addr, dest_buffer, size); -} - -const u8* Memory::GetSpan(const VAddr src_addr, const std::size_t size) const { - return impl->GetSpan(src_addr, size); -} - -u8* Memory::GetSpan(const VAddr src_addr, const std::size_t size) { - return impl->GetSpan(src_addr, size); -} - bool Memory::WriteBlock(const Common::ProcessAddress dest_addr, const void* src_buffer, const std::size_t size) { return impl->WriteBlock(dest_addr, src_buffer, size); } -bool Memory::WriteBlockUnsafe(const Common::ProcessAddress dest_addr, const void* src_buffer, - const std::size_t size) { - return impl->WriteBlockUnsafe(dest_addr, src_buffer, size); -} - -bool Memory::CopyBlock(Common::ProcessAddress dest_addr, Common::ProcessAddress src_addr, - const std::size_t size) { - return impl->CopyBlock(dest_addr, src_addr, size); -} - -bool Memory::ZeroBlock(Common::ProcessAddress dest_addr, const std::size_t size) { - return impl->ZeroBlock(dest_addr, size); -} - -void Memory::SetGPUDirtyManagers(std::span managers) { - impl->gpu_dirty_managers = managers; -} - -Result Memory::InvalidateDataCache(Common::ProcessAddress dest_addr, const std::size_t size) { - return impl->InvalidateDataCache(dest_addr, size); -} - -Result Memory::StoreDataCache(Common::ProcessAddress dest_addr, const std::size_t size) { - return impl->StoreDataCache(dest_addr, size); -} - -Result Memory::FlushDataCache(Common::ProcessAddress dest_addr, const std::size_t size) { - return impl->FlushDataCache(dest_addr, size); -} - -void Memory::RasterizerMarkRegionCached(Common::ProcessAddress vaddr, u64 size, bool cached) { - impl->RasterizerMarkRegionCached(GetInteger(vaddr), size, cached); -} - -void Memory::MarkRegionDebug(Common::ProcessAddress vaddr, u64 size, bool debug) { - impl->MarkRegionDebug(GetInteger(vaddr), size, debug); -} - -bool Memory::InvalidateNCE(Common::ProcessAddress vaddr, size_t size) { - [[maybe_unused]] bool mapped = true; - [[maybe_unused]] bool rasterizer = false; - - u8* const ptr = impl->GetPointerImpl( - GetInteger(vaddr), - [&] { - LOG_ERROR(HW_Memory, "Unmapped InvalidateNCE for {} bytes @ {:#x}", size, - GetInteger(vaddr)); - mapped = false; - }, - [&] { rasterizer = true; }); - if (rasterizer) { - impl->InvalidateGPUMemory(ptr, size); - } - -#ifdef __linux__ - if (!rasterizer && mapped) { - impl->buffer->DeferredMapSeparateHeap(GetInteger(vaddr)); - } -#endif - - return mapped && ptr != nullptr; -} - -bool Memory::InvalidateSeparateHeap(void* fault_address) { -#ifdef __linux__ - return impl->buffer->DeferredMapSeparateHeap(static_cast(fault_address)); -#else - return false; -#endif -} - } // namespace Core::Memory