diff --git a/Utilities/Thread.cpp b/Utilities/Thread.cpp index 2539c791cd..b85b96691a 100644 --- a/Utilities/Thread.cpp +++ b/Utilities/Thread.cpp @@ -215,7 +215,8 @@ static const reg_table_t reg_table[17] = bool handle_access_violation(const u32 addr, x64_context* context) { - if (addr - RAW_SPU_BASE_ADDR < (6 * RAW_SPU_OFFSET) && (addr % RAW_SPU_OFFSET) >= RAW_SPU_PROB_OFFSET) // RawSPU MMIO registers + // check if address is RawSPU MMIO register + if (addr - RAW_SPU_BASE_ADDR < (6 * RAW_SPU_OFFSET) && (addr % RAW_SPU_OFFSET) >= RAW_SPU_PROB_OFFSET) { // one x64 instruction is manually decoded and interpreted x64_op_t op; @@ -277,6 +278,12 @@ bool handle_access_violation(const u32 addr, x64_context* context) return true; } + // check if fault is caused by reservation + if (vm::reservation_query(addr)) + { + return true; + } + // TODO: allow recovering from a page fault as a feature of PS3 virtual memory return false; } @@ -303,6 +310,24 @@ void _se_translator(unsigned int u, EXCEPTION_POINTERS* pExp) // else some fatal error (should crash) } +extern LPTOP_LEVEL_EXCEPTION_FILTER filter_set; + +LONG __stdcall exception_filter(_EXCEPTION_POINTERS* pExp) +{ + _se_translator(pExp->ExceptionRecord->ExceptionCode, pExp); + + if (filter_set) + { + return filter_set(pExp); + } + else + { + return EXCEPTION_CONTINUE_SEARCH; + } +} + +LPTOP_LEVEL_EXCEPTION_FILTER filter_set = SetUnhandledExceptionFilter(exception_filter); + #else void signal_handler(int sig, siginfo_t* info, void* uct) @@ -352,6 +377,11 @@ void SetCurrentNamedThread(NamedThreadBase* value) return; } + if (old_value) + { + vm::reservation_free(); + } + if (value && value->m_tls_assigned.exchange(true)) { LOG_ERROR(GENERAL, "Thread '%s' was already assigned to g_tls_this_thread of another thread", value->GetThreadName()); diff --git a/rpcs3/Emu/ARMv7/ARMv7Context.h b/rpcs3/Emu/ARMv7/ARMv7Context.h index f8ad2c8212..9200cd5752 100644 --- a/rpcs3/Emu/ARMv7/ARMv7Context.h +++ b/rpcs3/Emu/ARMv7/ARMv7Context.h @@ -114,9 +114,6 @@ struct ARMv7Context u32 TLS; - u32 R_ADDR; - u64 R_DATA; - struct perf_counter { u32 event; diff --git a/rpcs3/Emu/ARMv7/ARMv7Interpreter.cpp b/rpcs3/Emu/ARMv7/ARMv7Interpreter.cpp index dbe994c42b..e07bbaa00b 100644 --- a/rpcs3/Emu/ARMv7/ARMv7Interpreter.cpp +++ b/rpcs3/Emu/ARMv7/ARMv7Interpreter.cpp @@ -2657,10 +2657,11 @@ void ARMv7_instrs::LDREX(ARMv7Context& context, const ARMv7Code code, const ARMv if (ConditionPassed(context, cond)) { const u32 addr = context.read_gpr(n) + imm32; - const u32 value = vm::psv::read32(addr); - context.write_gpr(t, value); - context.R_ADDR = addr; - context.R_DATA = value; + + u32 value; + vm::reservation_acquire(&value, addr, sizeof(value)); + + context.write_gpr(t, value); } } @@ -4813,10 +4814,7 @@ void ARMv7_instrs::STREX(ARMv7Context& context, const ARMv7Code code, const ARMv { const u32 addr = context.read_gpr(n) + imm32; const u32 value = context.read_gpr(t); - - auto& sync_obj = vm::get_ref>(addr); - context.write_gpr(d, addr != context.R_ADDR || sync_obj.compare_and_swap((u32)context.R_DATA, value) != context.R_DATA); - context.R_ADDR = 0; + context.write_gpr(d, !vm::reservation_update(addr, &value, sizeof(value))); } } diff --git a/rpcs3/Emu/ARMv7/ARMv7Thread.cpp b/rpcs3/Emu/ARMv7/ARMv7Thread.cpp index 12b7c7b0b3..0a57e4abcf 100644 --- a/rpcs3/Emu/ARMv7/ARMv7Thread.cpp +++ b/rpcs3/Emu/ARMv7/ARMv7Thread.cpp @@ -118,7 +118,7 @@ void ARMv7Thread::InitRegs() context.ITSTATE.IT = 0; context.SP = m_stack_addr + m_stack_size; context.TLS = armv7_get_tls(GetId()); - context.R_ADDR = 0; + context.debug |= DF_DISASM | DF_PRINT; } void ARMv7Thread::InitStack() diff --git a/rpcs3/Emu/Cell/PPUInterpreter.h b/rpcs3/Emu/Cell/PPUInterpreter.h index 201385e827..be1611bf78 100644 --- a/rpcs3/Emu/Cell/PPUInterpreter.h +++ b/rpcs3/Emu/Cell/PPUInterpreter.h @@ -2500,39 +2500,16 @@ private: } void MFOCRF(u32 a, u32 rd, u32 crm) { - /* - if(a) - { - u32 n = 0, count = 0; - for(u32 i = 0; i < 8; ++i) - { - if(crm & (1 << i)) - { - n = i; - count++; - } - } - - if(count == 1) - { - //RD[32+4*n : 32+4*n+3] = CR[4*n : 4*n+3]; - u8 offset = n * 4; - CPU.GPR[rd] = (CPU.GPR[rd] & ~(0xf << offset)) | ((u32)CPU.GetCR(7 - n) << offset); - } - else - CPU.GPR[rd] = 0; - } - else - { - */ CPU.GPR[rd] = CPU.CR.CR; - //} } void LWARX(u32 rd, u32 ra, u32 rb) { - CPU.R_ADDR = ra ? CPU.GPR[ra] + CPU.GPR[rb] : CPU.GPR[rb]; - CPU.R_VALUE = vm::get_ref(vm::cast(CPU.R_ADDR)); - CPU.GPR[rd] = re32((u32)CPU.R_VALUE); + const u32 addr = ra ? CPU.GPR[ra] + CPU.GPR[rb] : CPU.GPR[rb]; + + be_t value; + vm::reservation_acquire(&value, vm::cast(addr), sizeof(value)); + + CPU.GPR[rd] = value; } void LDX(u32 rd, u32 ra, u32 rb) { @@ -2682,9 +2659,12 @@ private: } void LDARX(u32 rd, u32 ra, u32 rb) { - CPU.R_ADDR = ra ? CPU.GPR[ra] + CPU.GPR[rb] : CPU.GPR[rb]; - CPU.R_VALUE = vm::get_ref(vm::cast(CPU.R_ADDR)); - CPU.GPR[rd] = re64(CPU.R_VALUE); + const u64 addr = ra ? CPU.GPR[ra] + CPU.GPR[rb] : CPU.GPR[rb]; + + be_t value; + vm::reservation_acquire(&value, vm::cast(addr), sizeof(value)); + + CPU.GPR[rd] = value; } void DCBF(u32 ra, u32 rb) { @@ -2800,15 +2780,8 @@ private: { const u64 addr = ra ? CPU.GPR[ra] + CPU.GPR[rb] : CPU.GPR[rb]; - if (CPU.R_ADDR == addr) - { - CPU.SetCR_EQ(0, InterlockedCompareExchange(vm::get_ptr(vm::cast(CPU.R_ADDR)), re32((u32)CPU.GPR[rs]), (u32)CPU.R_VALUE) == (u32)CPU.R_VALUE); - } - else - { - CPU.SetCR_EQ(0, false); - } - CPU.R_ADDR = 0; + const be_t value = be_t::make((u32)CPU.GPR[rs]); + CPU.SetCR_EQ(0, vm::reservation_update(vm::cast(addr), &value, sizeof(value))); } void STWX(u32 rs, u32 ra, u32 rb) { @@ -2859,15 +2832,8 @@ private: { const u64 addr = ra ? CPU.GPR[ra] + CPU.GPR[rb] : CPU.GPR[rb]; - if (CPU.R_ADDR == addr) - { - CPU.SetCR_EQ(0, InterlockedCompareExchange(vm::get_ptr(vm::cast(CPU.R_ADDR)), re64(CPU.GPR[rs]), CPU.R_VALUE) == CPU.R_VALUE); - } - else - { - CPU.SetCR_EQ(0, false); - } - CPU.R_ADDR = 0; + const be_t value = be_t::make(CPU.GPR[rs]); + CPU.SetCR_EQ(0, vm::reservation_update(vm::cast(addr), &value, sizeof(value))); } void STBX(u32 rs, u32 ra, u32 rb) { diff --git a/rpcs3/Emu/Cell/PPUThread.h b/rpcs3/Emu/Cell/PPUThread.h index 820b244c2b..834200b1ae 100644 --- a/rpcs3/Emu/Cell/PPUThread.h +++ b/rpcs3/Emu/Cell/PPUThread.h @@ -536,11 +536,6 @@ public: //TBR : Time-Base Registers u64 TB; //TBR 0x10C - 0x10D - u64 cycle; - - u64 R_ADDR; // reservation address - u64 R_VALUE; // reservation value (BE) - u32 owned_mutexes; std::function custom_task; diff --git a/rpcs3/Emu/Cell/SPUThread.cpp b/rpcs3/Emu/Cell/SPUThread.cpp index f5faf9fff1..71033cbd9d 100644 --- a/rpcs3/Emu/Cell/SPUThread.cpp +++ b/rpcs3/Emu/Cell/SPUThread.cpp @@ -99,8 +99,6 @@ void SPUThread::InitRegs() m_event_mask = 0; m_events = 0; - - R_ADDR = 0; } void SPUThread::InitStack() @@ -437,103 +435,37 @@ void SPUThread::EnqMfcCmd(MFCReg& MFCArgs) if (op == MFC_GETLLAR_CMD) // get reservation { - if (R_ADDR) - { - m_events |= SPU_EVENT_LR; - } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack - R_ADDR = ea; - for (u32 i = 0; i < 16; i++) + vm::reservation_acquire(vm::get_ptr(ls_offset + lsa), ea, 128, [this]() { - R_DATA[i] = vm::get_ptr((u32)R_ADDR)[i]; - vm::get_ptr(ls_offset + lsa)[i] = R_DATA[i]; - } + m_events |= SPU_EVENT_LR; // TODO: atomic op + Notify(); + }); + MFCArgs.AtomicStat.PushUncond(MFC_GETLLAR_SUCCESS); } else if (op == MFC_PUTLLC_CMD) // store conditional { - MFCArgs.AtomicStat.PushUncond(MFC_PUTLLC_SUCCESS); - - if (R_ADDR == ea) + if (vm::reservation_update(ea, vm::get_ptr(ls_offset + lsa), 128)) { - u32 changed = 0, mask = 0; - u64 buf[16]; - for (u32 i = 0; i < 16; i++) - { - buf[i] = vm::get_ptr(ls_offset + lsa)[i]; - if (buf[i] != R_DATA[i]) - { - changed++; - mask |= (0x3 << (i * 2)); - if (vm::get_ptr((u32)R_ADDR)[i] != R_DATA[i]) - { - m_events |= SPU_EVENT_LR; - MFCArgs.AtomicStat.PushUncond(MFC_PUTLLC_FAILURE); - R_ADDR = 0; - return; - } - } - } - - for (u32 i = 0; i < 16; i++) - { - if (buf[i] != R_DATA[i]) - { - if (InterlockedCompareExchange(&vm::get_ptr((u32)R_ADDR)[i], buf[i], R_DATA[i]) != R_DATA[i]) - { - m_events |= SPU_EVENT_LR; - MFCArgs.AtomicStat.PushUncond(MFC_PUTLLC_FAILURE); - - if (changed > 1) - { - LOG_ERROR(Log::SPU, "MFC_PUTLLC_CMD: Memory corrupted (~x%d (mask=0x%x)) (opcode=0x%x, cmd=0x%x, lsa = 0x%x, ea = 0x%llx, tag = 0x%x, size = 0x%x)", - changed, mask, op, cmd, lsa, ea, tag, size); - Emu.Pause(); - } - - break; - } - } - } - - if (changed > 1) - { - LOG_WARNING(Log::SPU, "MFC_PUTLLC_CMD: Reservation impossibru (~x%d (mask=0x%x)) (opcode=0x%x, cmd=0x%x, lsa = 0x%x, ea = 0x%llx, tag = 0x%x, size = 0x%x)", - changed, mask, op, cmd, lsa, ea, tag, size); - - SPUDisAsm dis_asm(CPUDisAsm_InterpreterMode); - for (s32 i = (s32)PC; i < (s32)PC + 4 * 7; i += 4) - { - dis_asm.dump_pc = i; - dis_asm.offset = vm::get_ptr(ls_offset); - const u32 opcode = vm::read32(i + ls_offset); - (*SPU_instr::rrr_list)(&dis_asm, opcode); - if (i >= 0 && i < 0x40000) - { - LOG_NOTICE(Log::SPU, "*** %s", dis_asm.last_opcode.c_str()); - } - } - } + MFCArgs.AtomicStat.PushUncond(MFC_PUTLLC_SUCCESS); } else { MFCArgs.AtomicStat.PushUncond(MFC_PUTLLC_FAILURE); } - R_ADDR = 0; } - else // store unconditional + else // store unconditional (may be wrong) { - if (R_ADDR) // may be wrong - { - m_events |= SPU_EVENT_LR; - } + vm::reservation_break(ea); ProcessCmd(MFC_PUT_CMD, tag, lsa, ea, 128); + if (op == MFC_PUTLLUC_CMD) { MFCArgs.AtomicStat.PushUncond(MFC_PUTLLUC_SUCCESS); } - R_ADDR = 0; } break; } @@ -548,19 +480,6 @@ void SPUThread::EnqMfcCmd(MFCReg& MFCArgs) bool SPUThread::CheckEvents() { // checks events: - // SPU_EVENT_LR: - if (R_ADDR) - { - for (u32 i = 0; i < 16; i++) - { - if (vm::get_ptr((u32)R_ADDR)[i] != R_DATA[i]) - { - m_events |= SPU_EVENT_LR; - R_ADDR = 0; - break; - } - } - } return (m_events & m_event_mask) != 0; } diff --git a/rpcs3/Emu/Cell/SPUThread.h b/rpcs3/Emu/Cell/SPUThread.h index c2e2cc2f03..bab5f1aa2d 100644 --- a/rpcs3/Emu/Cell/SPUThread.h +++ b/rpcs3/Emu/Cell/SPUThread.h @@ -277,9 +277,6 @@ public: u32 SRR0; SPU_SNRConfig_hdr cfg; // Signal Notification Registers Configuration (OR-mode enabled: 0x1 for SNR1, 0x2 for SNR2) - u64 R_ADDR; // reservation address - u64 R_DATA[16]; // lock line data (BE) - std::shared_ptr SPUPs[64]; // SPU Thread Event Ports EventManager SPUQs; // SPU Queue Mapping std::shared_ptr group; // associated SPU Thread Group (null for raw spu) diff --git a/rpcs3/Emu/Memory/Memory.cpp b/rpcs3/Emu/Memory/Memory.cpp index 502694ff93..b892bff2b8 100644 --- a/rpcs3/Emu/Memory/Memory.cpp +++ b/rpcs3/Emu/Memory/Memory.cpp @@ -231,9 +231,11 @@ bool MemoryBase::Unmap(const u64 addr) MemBlockInfo::MemBlockInfo(u64 _addr, u32 _size) : MemInfo(_addr, PAGE_4K(_size)) { - void* real_addr = (void*)((u64)Memory.GetBaseAddr() + _addr); + void* real_addr = vm::get_ptr(vm::cast(_addr)); #ifdef _WIN32 - mem = VirtualAlloc(real_addr, size, MEM_COMMIT, PAGE_READWRITE); + void* priv_addr = vm::get_priv_ptr(vm::cast(_addr)); + void* priv_mem = VirtualAlloc(priv_addr, size, MEM_COMMIT, PAGE_READWRITE); + mem = priv_mem == priv_addr ? VirtualAlloc(real_addr, size, MEM_COMMIT, PAGE_READWRITE) : priv_mem; #else if (::mprotect(real_addr, size, PROT_READ | PROT_WRITE)) { @@ -262,7 +264,10 @@ void MemBlockInfo::Free() { Memory.UnregisterPages(addr, size); #ifdef _WIN32 - if (!VirtualFree(mem, size, MEM_DECOMMIT)) + DWORD old; + + if (!VirtualProtect(mem, size, PAGE_NOACCESS, &old) || !VirtualProtect(vm::get_priv_ptr(vm::cast(addr)), size, PAGE_NOACCESS, &old)) + //if (!VirtualFree(mem, size, MEM_DECOMMIT)) #else if (::mprotect(mem, size, PROT_NONE)) #endif @@ -715,4 +720,4 @@ bool VirtualMemoryBlock::Unreserve(u32 size) u32 VirtualMemoryBlock::GetReservedAmount() { return m_reserve_size; -} \ No newline at end of file +} diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp index 9a5f9eec9e..a32d176b2e 100644 --- a/rpcs3/Emu/Memory/vm.cpp +++ b/rpcs3/Emu/Memory/vm.cpp @@ -1,24 +1,194 @@ #include "stdafx.h" +#include "Utilities/Log.h" #include "Memory.h" +#include "Emu/System.h" #include "Emu/CPU/CPUThread.h" #include "Emu/Cell/PPUThread.h" #include "Emu/ARMv7/ARMv7Thread.h" +#include "Emu/SysCalls/lv2/sys_time.h" + +#ifdef _WIN32 +#include +#else +#include + +/* OS X uses MAP_ANON instead of MAP_ANONYMOUS */ +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif +#endif + namespace vm { - #ifdef _WIN32 - #include - void* const g_base_addr = VirtualAlloc(nullptr, 0x100000000, MEM_RESERVE, PAGE_NOACCESS); - #else - #include +#ifdef _WIN32 + HANDLE g_memory_handle; +#endif - /* OS X uses MAP_ANON instead of MAP_ANONYMOUS */ - #ifndef MAP_ANONYMOUS - #define MAP_ANONYMOUS MAP_ANON - #endif + void* g_priv_addr; - void* const g_base_addr = mmap(nullptr, 0x100000000, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); - #endif + void* initialize() + { +#ifdef _WIN32 + g_memory_handle = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE | SEC_RESERVE, 0x1, 0x0, NULL); + + void* base_addr = MapViewOfFile(g_memory_handle, PAGE_NOACCESS, 0, 0, 0x100000000); // main memory + g_priv_addr = MapViewOfFile(g_memory_handle, PAGE_NOACCESS, 0, 0, 0x100000000); // memory mirror for privileged access + + return base_addr; + //return VirtualAlloc(nullptr, 0x100000000, MEM_RESERVE, PAGE_NOACCESS); +#else + return mmap(nullptr, 0x100000000, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); +#endif + } + + void finalize() + { +#ifdef _WIN32 + UnmapViewOfFile(g_base_addr); + UnmapViewOfFile(g_priv_addr); + CloseHandle(g_memory_handle); +#else + munmap(g_base_addr, 0x100000000); +#endif + } + + void* const g_base_addr = (atexit(finalize), initialize()); + + void* g_reservation_owner = nullptr; + u32 g_reservation_addr = 0; + + std::function g_reservation_cb = nullptr; + + // break the reservation, return true if it was successfully broken + bool reservation_break(u32 addr) + { + LV2_LOCK(0); + + if (g_reservation_addr >> 12 == addr >> 12) + { + const auto stamp0 = get_time(); + +#ifdef _WIN32 + if (!VirtualAlloc(vm::get_ptr(addr & ~0xfff), 4096, MEM_COMMIT, PAGE_READWRITE)) +#else + +#endif + { + throw fmt::format("vm::reservation_break() failed (addr=0x%x)", addr); + } + + //LOG_NOTICE(MEMORY, "VirtualAlloc: %f us", (get_time() - stamp0) / 80.f); + + if (g_reservation_cb) + { + g_reservation_cb(); + g_reservation_cb = nullptr; + } + + g_reservation_owner = nullptr; + g_reservation_addr = 0; + + return true; + } + + return false; + } + + // read memory and reserve it for further atomic update, return true if the previous reservation was broken + bool reservation_acquire(void* data, u32 addr, u32 size, std::function callback) + { + const auto stamp0 = get_time(); + + bool broken = false; + + assert(size == 1 || size == 2 || size == 4 || size == 8 || size == 128); + assert((addr + size & ~0xfff) == (addr & ~0xfff)); + + { + LV2_LOCK(0); + + // break previous reservation + if (g_reservation_addr) + { + broken = reservation_break(g_reservation_addr); + } + + // change memory protection to read-only +#ifdef _WIN32 + DWORD old; + if (!VirtualProtect(vm::get_ptr(addr & ~0xfff), 4096, PAGE_READONLY, &old)) +#else + +#endif + { + throw fmt::format("vm::reservation_acquire() failed (addr=0x%x, size=%d)", addr, size); + } + + //LOG_NOTICE(MEMORY, "VirtualProtect: %f us", (get_time() - stamp0) / 80.f); + + // set the new reservation + g_reservation_addr = addr; + g_reservation_owner = GetCurrentNamedThread(); + g_reservation_cb = callback; + + // copy data + memcpy(data, vm::get_ptr(addr), size); + } + + return broken; + } + + // attempt to atomically update reserved memory + bool reservation_update(u32 addr, const void* data, u32 size) + { + assert(size == 1 || size == 2 || size == 4 || size == 8 || size == 128); + assert((addr + size & ~0xfff) == (addr & ~0xfff)); + + LV2_LOCK(0); + + if (g_reservation_addr != addr || g_reservation_owner != GetCurrentNamedThread()) + { + // atomic update failed + return false; + } + + // update memory using privileged access + memcpy(vm::get_priv_ptr(addr), data, size); + + // free the reservation and restore memory protection + reservation_break(addr); + + // atomic update succeeded + return true; + } + + // for internal use + bool reservation_query(u32 addr) + { + LV2_LOCK(0); + + if (!Memory.IsGoodAddr(addr)) + { + return false; + } + + // break the reservation + reservation_break(addr); + + return true; + } + + // for internal use + void reservation_free() + { + LV2_LOCK(0); + + if (g_reservation_owner == GetCurrentNamedThread()) + { + reservation_break(g_reservation_addr); + } + } bool check_addr(u32 addr) { diff --git a/rpcs3/Emu/Memory/vm.h b/rpcs3/Emu/Memory/vm.h index 7cc958c3ba..32bcf17859 100644 --- a/rpcs3/Emu/Memory/vm.h +++ b/rpcs3/Emu/Memory/vm.h @@ -21,7 +21,19 @@ namespace vm static void set_stack_size(u32 size) {} static void initialize_stack() {} +#ifdef _WIN32 + extern HANDLE g_memory_handle; +#endif + + extern void* g_priv_addr; extern void* const g_base_addr; + + bool reservation_break(u32 addr); + bool reservation_acquire(void* data, u32 addr, u32 size, std::function callback = nullptr); + bool reservation_update(u32 addr, const void* data, u32 size); + bool reservation_query(u32 addr); + void reservation_free(); + bool map(u32 addr, u32 size, u32 flags); bool unmap(u32 addr, u32 size = 0, u32 flags = 0); u32 alloc(u32 size, memory_location location = user_space); @@ -40,6 +52,18 @@ namespace vm return *get_ptr(addr); } + template + T* const get_priv_ptr(u32 addr) + { + return reinterpret_cast(static_cast(g_priv_addr) + addr); + } + + template + T& get_priv_ref(u32 addr) + { + return *get_priv_ptr(addr); + } + u32 get_addr(const void* real_pointer); __noinline void error(const u64 addr, const char* func);