diff --git a/Utilities/Thread.cpp b/Utilities/Thread.cpp index 44f0096e6b..a088028bdf 100644 --- a/Utilities/Thread.cpp +++ b/Utilities/Thread.cpp @@ -74,6 +74,7 @@ #include "sync.h" #include "util/logs.hpp" +#include "Emu/Memory/vm_locking.h" LOG_CHANNEL(sig_log, "SIG"); LOG_CHANNEL(sys_log, "SYS"); diff --git a/rpcs3/Emu/Cell/lv2/lv2.cpp b/rpcs3/Emu/Cell/lv2/lv2.cpp index 94f9f3117b..974a376d76 100644 --- a/rpcs3/Emu/Cell/lv2/lv2.cpp +++ b/rpcs3/Emu/Cell/lv2/lv2.cpp @@ -1,6 +1,7 @@ #include "stdafx.h" #include "Emu/System.h" #include "Emu/Memory/vm_ptr.h" +#include "Emu/Memory/vm_locking.h" #include "Emu/Cell/PPUFunction.h" #include "Emu/Cell/ErrorCodes.h" @@ -1114,6 +1115,26 @@ DECLARE(lv2_obj::g_waiting); thread_local DECLARE(lv2_obj::g_to_awake); +void lv2_obj::sleep(cpu_thread& cpu, const u64 timeout) +{ + vm::temporary_unlock(cpu); + std::lock_guard{g_mutex}, sleep_unlocked(cpu, timeout); + g_to_awake.clear(); +} + +bool lv2_obj::awake(cpu_thread* const thread, s32 prio) +{ + vm::temporary_unlock(); + std::lock_guard lock(g_mutex); + return awake_unlocked(thread, prio); +} + +bool lv2_obj::yield(cpu_thread& thread) +{ + vm::temporary_unlock(thread); + return awake(&thread, yield_cmd); +} + void lv2_obj::sleep_unlocked(cpu_thread& thread, u64 timeout) { const u64 start_time = get_guest_system_time(); diff --git a/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp b/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp index 8dc55262a4..af5117a801 100644 --- a/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp @@ -6,6 +6,7 @@ #include "Emu/Cell/ErrorCodes.h" #include "Emu/Cell/PPUThread.h" #include "Emu/Cell/PPUCallback.h" +#include "Emu/Memory/vm_locking.h" #include "sys_event.h" #include "sys_process.h" #include "sys_mmapper.h" diff --git a/rpcs3/Emu/Cell/lv2/sys_rsx.cpp b/rpcs3/Emu/Cell/lv2/sys_rsx.cpp index 06c7ef22f8..19d3c7a473 100644 --- a/rpcs3/Emu/Cell/lv2/sys_rsx.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_rsx.cpp @@ -3,6 +3,7 @@ #include "Emu/Cell/PPUModule.h" #include "Emu/Cell/ErrorCodes.h" +#include "Emu/Memory/vm_locking.h" #include "Emu/RSX/RSXThread.h" #include "sys_event.h" diff --git a/rpcs3/Emu/Cell/lv2/sys_sync.h b/rpcs3/Emu/Cell/lv2/sys_sync.h index 2d7adeda56..a503d3e6ac 100644 --- a/rpcs3/Emu/Cell/lv2/sys_sync.h +++ b/rpcs3/Emu/Cell/lv2/sys_sync.h @@ -4,7 +4,6 @@ #include "Utilities/sema.h" #include "Utilities/cond.h" -#include "Emu/Memory/vm_locking.h" #include "Emu/CPU/CPUThread.h" #include "Emu/Cell/ErrorCodes.h" #include "Emu/IdManager.h" @@ -156,26 +155,12 @@ private: static bool awake_unlocked(cpu_thread*, s32 prio = enqueue_cmd); public: - static void sleep(cpu_thread& cpu, const u64 timeout = 0) - { - vm::temporary_unlock(cpu); - std::lock_guard{g_mutex}, sleep_unlocked(cpu, timeout); - g_to_awake.clear(); - } + static void sleep(cpu_thread& cpu, const u64 timeout = 0); - static inline bool awake(cpu_thread* const thread, s32 prio = enqueue_cmd) - { - vm::temporary_unlock(); - std::lock_guard lock(g_mutex); - return awake_unlocked(thread, prio); - } + static bool awake(cpu_thread* const thread, s32 prio = enqueue_cmd); // Returns true on successful context switch, false otherwise - static bool yield(cpu_thread& thread) - { - vm::temporary_unlock(thread); - return awake(&thread, yield_cmd); - } + static bool yield(cpu_thread& thread); static void set_priority(cpu_thread& thread, s32 prio) { diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp index 55f5ff18f3..1f917a11a6 100644 --- a/rpcs3/Emu/Memory/vm.cpp +++ b/rpcs3/Emu/Memory/vm.cpp @@ -166,29 +166,29 @@ namespace vm for (u64 i = 0;; i++) { const u64 lock_val = g_range_lock.load(); + const u64 is_shared = g_shareable[begin >> 16].load(); const u64 lock_addr = static_cast(lock_val); // -> u64 const u32 lock_size = static_cast(lock_val >> 35); u64 addr = begin; - // See range_lock() - if (g_shareable[begin >> 16] | (((lock_val >> 32) & (range_full_mask >> 32)) ^ (range_locked >> 32))) + if (is_shared) { addr = addr & 0xffff; } - if (addr + size <= lock_addr || addr >= lock_addr + lock_size) [[likely]] + if ((lock_val & range_full_mask) != range_locked || addr + size <= lock_addr || addr >= lock_addr + lock_size) [[likely]] { range_lock->store(begin | (u64{size} << 32)); const u64 new_lock_val = g_range_lock.load(); - if (!(new_lock_val | (new_lock_val != lock_val))) [[likely]] + if (!new_lock_val || new_lock_val == lock_val) [[likely]] { break; } - range_lock->release(0); + range_lock->store(0); } std::shared_lock lock(g_mutex, std::try_to_lock); diff --git a/rpcs3/Emu/Memory/vm_locking.h b/rpcs3/Emu/Memory/vm_locking.h index 4a6196efc3..9d7865f0b2 100644 --- a/rpcs3/Emu/Memory/vm_locking.h +++ b/rpcs3/Emu/Memory/vm_locking.h @@ -43,6 +43,10 @@ namespace vm FORCE_INLINE void range_lock(atomic_t* range_lock, u32 begin, u32 size) { const u64 lock_val = g_range_lock.load(); + #ifndef _MSC_VER + __asm__(""); // Tiny barrier + #endif + const u64 is_shared = g_shareable[begin >> 16].load(); const u64 lock_addr = static_cast(lock_val); // -> u64 const u32 lock_size = static_cast(lock_val >> 35); @@ -50,24 +54,25 @@ namespace vm // Optimization: if range_locked is not used, the addr check will always pass // Otherwise, g_shareable is unchanged and its value is reliable to read - if (g_shareable[begin >> 16] | (((lock_val >> 32) & (range_full_mask >> 32)) ^ (range_locked >> 32))) + if (is_shared) { addr = addr & 0xffff; } - if (addr + size <= lock_addr || addr >= lock_addr + lock_size) [[likely]] + if (addr + size <= lock_addr || addr >= lock_addr + lock_size || ((lock_val >> 32) ^ (range_locked >> 32)) & (range_full_mask >> 32)) [[likely]] { - // Optimistic locking + // Optimistic locking. + // Note that we store the range we will be accessing, without any clamping. range_lock->store(begin | (u64{size} << 32)); const u64 new_lock_val = g_range_lock.load(); - if (!(new_lock_val | (new_lock_val != lock_val))) [[likely]] + if (!new_lock_val || new_lock_val == lock_val) [[likely]] { return; } - range_lock->release(0); + range_lock->store(0); } // Fallback to slow path diff --git a/rpcs3/Emu/RSX/GL/GLGSRender.cpp b/rpcs3/Emu/RSX/GL/GLGSRender.cpp index 340106dc61..816447a66d 100644 --- a/rpcs3/Emu/RSX/GL/GLGSRender.cpp +++ b/rpcs3/Emu/RSX/GL/GLGSRender.cpp @@ -4,6 +4,7 @@ #include "GLGSRender.h" #include "GLCompute.h" #include "GLVertexProgram.h" +#include "Emu/Memory/vm_locking.h" #define DUMP_VERTEX_DATA 0 diff --git a/rpcs3/Emu/RSX/VK/VKGSRender.cpp b/rpcs3/Emu/RSX/VK/VKGSRender.cpp index ba4e0f075e..cce40171c1 100644 --- a/rpcs3/Emu/RSX/VK/VKGSRender.cpp +++ b/rpcs3/Emu/RSX/VK/VKGSRender.cpp @@ -6,6 +6,7 @@ #include "VKRenderPass.h" #include "VKResourceManager.h" #include "VKCommandStream.h" +#include "Emu/Memory/vm_locking.h" namespace vk {