mirror of https://github.com/RPCS3/rpcs3.git
Transactions: move loops inside
Rewrite loops in assembler (minor optimization)
This commit is contained in:
Nekotekina
parent
f6f45b8699
commit
33a1c743a4
|
|
@ -7,7 +7,7 @@ asmjit::JitRuntime& asmjit::get_global_runtime()
|
|||
return g_rt;
|
||||
}
|
||||
|
||||
void asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fallback)
|
||||
asmjit::Label asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fallback)
|
||||
{
|
||||
Label fall = c.newLabel();
|
||||
Label begin = c.newLabel();
|
||||
|
|
@ -18,6 +18,7 @@ void asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fall
|
|||
c.align(kAlignCode, 16);
|
||||
c.bind(begin);
|
||||
c.xbegin(fall);
|
||||
return begin;
|
||||
}
|
||||
|
||||
void asmjit::build_transaction_abort(asmjit::X86Assembler& c, unsigned char code)
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ namespace asmjit
|
|||
// Should only be used to build global functions
|
||||
JitRuntime& get_global_runtime();
|
||||
|
||||
// Emit xbegin and adjacent loop
|
||||
void build_transaction_enter(X86Assembler& c, Label fallback);
|
||||
// Emit xbegin and adjacent loop, return label at xbegin
|
||||
Label build_transaction_enter(X86Assembler& c, Label fallback);
|
||||
|
||||
// Emit xabort
|
||||
void build_transaction_abort(X86Assembler& c, unsigned char code);
|
||||
|
|
|
|||
|
|
@ -944,8 +944,23 @@ static T ppu_load_acquire_reservation(ppu_thread& ppu, u32 addr)
|
|||
|
||||
ppu.raddr = addr;
|
||||
|
||||
while (g_use_rtm)
|
||||
{
|
||||
ppu.rtime = vm::reservation_acquire(addr, sizeof(T));
|
||||
ppu.rdata = data;
|
||||
|
||||
if (LIKELY(vm::reservation_acquire(addr, sizeof(T)) == ppu.rtime))
|
||||
{
|
||||
return static_cast<T>(ppu.rdata);
|
||||
}
|
||||
else
|
||||
{
|
||||
_mm_pause();
|
||||
}
|
||||
}
|
||||
|
||||
// Do several attemps
|
||||
for (uint i = 0; g_use_rtm || i < 5; i++)
|
||||
for (uint i = 0; i < 5; i++)
|
||||
{
|
||||
ppu.rtime = vm::reservation_acquire(addr, sizeof(T));
|
||||
_mm_lfence();
|
||||
|
|
@ -981,7 +996,7 @@ extern u64 ppu_ldarx(ppu_thread& ppu, u32 addr)
|
|||
return ppu_load_acquire_reservation<u64>(ppu, addr);
|
||||
}
|
||||
|
||||
const auto ppu_stwcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rdata, u32 value)>([](asmjit::X86Assembler& c, auto& args)
|
||||
const auto ppu_stwcx_tx = build_function_asm<bool(*)(u32 raddr, u64 rtime, u64 rdata, u32 value)>([](asmjit::X86Assembler& c, auto& args)
|
||||
{
|
||||
using namespace asmjit;
|
||||
|
||||
|
|
@ -998,9 +1013,10 @@ const auto ppu_stwcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
|
|||
c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0], 3));
|
||||
c.bswap(args[2].r32());
|
||||
c.bswap(args[3].r32());
|
||||
c.mov(args[0].r32(), 5);
|
||||
|
||||
// Begin transaction
|
||||
build_transaction_enter(c, fall);
|
||||
Label begin = build_transaction_enter(c, fall);
|
||||
c.cmp(x86::qword_ptr(x86::r10), args[1]);
|
||||
c.jne(fail);
|
||||
c.cmp(x86::dword_ptr(x86::r11), args[2].r32());
|
||||
|
|
@ -1017,14 +1033,18 @@ const auto ppu_stwcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
|
|||
|
||||
// Touch memory after transaction failure
|
||||
c.bind(fall);
|
||||
c.sub(args[0].r32(), 1);
|
||||
c.jz(fail);
|
||||
c.sar(x86::eax, 24);
|
||||
c.js(fail);
|
||||
c.lock().add(x86::dword_ptr(x86::r11), 0);
|
||||
c.lock().add(x86::qword_ptr(x86::r10), 0);
|
||||
c.sar(x86::eax, 24);
|
||||
c.ret();
|
||||
c.jmp(begin);
|
||||
|
||||
c.bind(fail);
|
||||
build_transaction_abort(c, 0xff);
|
||||
c.int3();
|
||||
c.xor_(x86::eax, x86::eax);
|
||||
c.ret();
|
||||
});
|
||||
|
||||
extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value)
|
||||
|
|
@ -1039,27 +1059,14 @@ extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value)
|
|||
|
||||
if (g_use_rtm)
|
||||
{
|
||||
// Do several attempts (TODO)
|
||||
for (u32 i = 0; i < 5; i++)
|
||||
if (ppu_stwcx_tx(addr, ppu.rtime, ppu.rdata, reg_value))
|
||||
{
|
||||
const int r = ppu_stwcx_tx(addr, ppu.rtime, ppu.rdata, reg_value);
|
||||
|
||||
if (r > 0)
|
||||
{
|
||||
vm::reservation_notifier(addr, sizeof(u32)).notify_all();
|
||||
ppu.raddr = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (r < 0)
|
||||
{
|
||||
// Reservation lost
|
||||
ppu.raddr = 0;
|
||||
return false;
|
||||
}
|
||||
vm::reservation_notifier(addr, sizeof(u32)).notify_all();
|
||||
ppu.raddr = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Give up
|
||||
// Reservation lost
|
||||
ppu.raddr = 0;
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1078,7 +1085,7 @@ extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value)
|
|||
return result;
|
||||
}
|
||||
|
||||
const auto ppu_stdcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rdata, u64 value)>([](asmjit::X86Assembler& c, auto& args)
|
||||
const auto ppu_stdcx_tx = build_function_asm<bool(*)(u32 raddr, u64 rtime, u64 rdata, u64 value)>([](asmjit::X86Assembler& c, auto& args)
|
||||
{
|
||||
using namespace asmjit;
|
||||
|
||||
|
|
@ -1095,9 +1102,10 @@ const auto ppu_stdcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
|
|||
c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0], 3));
|
||||
c.bswap(args[2]);
|
||||
c.bswap(args[3]);
|
||||
c.mov(args[0].r32(), 5);
|
||||
|
||||
// Begin transaction
|
||||
build_transaction_enter(c, fall);
|
||||
Label begin = build_transaction_enter(c, fall);
|
||||
c.cmp(x86::qword_ptr(x86::r10), args[1]);
|
||||
c.jne(fail);
|
||||
c.cmp(x86::qword_ptr(x86::r11), args[2]);
|
||||
|
|
@ -1114,14 +1122,18 @@ const auto ppu_stdcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
|
|||
|
||||
// Touch memory after transaction failure
|
||||
c.bind(fall);
|
||||
c.sub(args[0].r32(), 1);
|
||||
c.jz(fail);
|
||||
c.sar(x86::eax, 24);
|
||||
c.js(fail);
|
||||
c.lock().add(x86::qword_ptr(x86::r11), 0);
|
||||
c.lock().add(x86::qword_ptr(x86::r10), 0);
|
||||
c.sar(x86::eax, 24);
|
||||
c.ret();
|
||||
c.jmp(begin);
|
||||
|
||||
c.bind(fail);
|
||||
build_transaction_abort(c, 0xff);
|
||||
c.int3();
|
||||
c.xor_(x86::eax, x86::eax);
|
||||
c.ret();
|
||||
});
|
||||
|
||||
extern bool ppu_stdcx(ppu_thread& ppu, u32 addr, u64 reg_value)
|
||||
|
|
@ -1136,27 +1148,14 @@ extern bool ppu_stdcx(ppu_thread& ppu, u32 addr, u64 reg_value)
|
|||
|
||||
if (g_use_rtm)
|
||||
{
|
||||
// Do several attempts (TODO)
|
||||
for (u32 i = 0; i < 5; i++)
|
||||
if (ppu_stdcx_tx(addr, ppu.rtime, ppu.rdata, reg_value))
|
||||
{
|
||||
const int r = ppu_stdcx_tx(addr, ppu.rtime, ppu.rdata, reg_value);
|
||||
|
||||
if (r > 0)
|
||||
{
|
||||
vm::reservation_notifier(addr, sizeof(u64)).notify_all();
|
||||
ppu.raddr = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (r < 0)
|
||||
{
|
||||
// Reservation lost
|
||||
ppu.raddr = 0;
|
||||
return false;
|
||||
}
|
||||
vm::reservation_notifier(addr, sizeof(u64)).notify_all();
|
||||
ppu.raddr = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Give up
|
||||
// Reservation lost
|
||||
ppu.raddr = 0;
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ namespace spu
|
|||
}
|
||||
}
|
||||
|
||||
const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const void* _old, const void* _new)>([](asmjit::X86Assembler& c, auto& args)
|
||||
const auto spu_putllc_tx = build_function_asm<bool(*)(u32 raddr, u64 rtime, const void* _old, const void* _new)>([](asmjit::X86Assembler& c, auto& args)
|
||||
{
|
||||
using namespace asmjit;
|
||||
|
||||
|
|
@ -228,6 +228,7 @@ const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const
|
|||
c.lea(x86::r11, x86::qword_ptr(x86::r11, args[0]));
|
||||
c.shr(args[0], 4);
|
||||
c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0]));
|
||||
c.mov(args[0].r32(), 3);
|
||||
|
||||
// Prepare data (Windows has only 6 volatile vector registers)
|
||||
c.vmovups(x86::ymm0, x86::yword_ptr(args[2], 0));
|
||||
|
|
@ -245,7 +246,7 @@ const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const
|
|||
#endif
|
||||
|
||||
// Begin transaction
|
||||
build_transaction_enter(c, fall);
|
||||
Label begin = build_transaction_enter(c, fall);
|
||||
c.cmp(x86::qword_ptr(x86::r10), args[1]);
|
||||
c.jne(fail);
|
||||
c.vxorps(x86::ymm0, x86::ymm0, x86::yword_ptr(x86::r11, 0));
|
||||
|
|
@ -282,14 +283,22 @@ const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const
|
|||
|
||||
// Touch memory after transaction failure
|
||||
c.bind(fall);
|
||||
c.sub(args[0].r32(), 1);
|
||||
c.jz(fail);
|
||||
c.sar(x86::eax, 24);
|
||||
c.js(fail);
|
||||
c.lock().add(x86::qword_ptr(x86::r11), 0);
|
||||
c.lock().add(x86::qword_ptr(x86::r10), 0);
|
||||
c.sar(x86::eax, 24);
|
||||
c.ret();
|
||||
#ifdef _WIN32
|
||||
c.vmovups(x86::ymm4, x86::yword_ptr(args[3], 0));
|
||||
c.vmovups(x86::ymm5, x86::yword_ptr(args[3], 96));
|
||||
#endif
|
||||
c.jmp(begin);
|
||||
|
||||
c.bind(fail);
|
||||
build_transaction_abort(c, 0xff);
|
||||
c.int3();
|
||||
c.xor_(x86::eax, x86::eax);
|
||||
c.ret();
|
||||
});
|
||||
|
||||
const auto spu_getll_tx = build_function_asm<u64(*)(u32 raddr, void* rdata)>([](asmjit::X86Assembler& c, auto& args)
|
||||
|
|
@ -308,7 +317,7 @@ const auto spu_getll_tx = build_function_asm<u64(*)(u32 raddr, void* rdata)>([](
|
|||
c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0]));
|
||||
|
||||
// Begin transaction
|
||||
build_transaction_enter(c, fall);
|
||||
Label begin = build_transaction_enter(c, fall);
|
||||
c.mov(x86::rax, x86::qword_ptr(x86::r10));
|
||||
c.vmovaps(x86::ymm0, x86::yword_ptr(x86::r11, 0));
|
||||
c.vmovaps(x86::ymm1, x86::yword_ptr(x86::r11, 32));
|
||||
|
|
@ -324,13 +333,13 @@ const auto spu_getll_tx = build_function_asm<u64(*)(u32 raddr, void* rdata)>([](
|
|||
|
||||
// Touch memory after transaction failure
|
||||
c.bind(fall);
|
||||
c.pause();
|
||||
c.mov(x86::rax, x86::qword_ptr(x86::r11));
|
||||
c.mov(x86::rax, x86::qword_ptr(x86::r10));
|
||||
c.mov(x86::eax, 1);
|
||||
c.ret();
|
||||
c.jmp(begin);
|
||||
});
|
||||
|
||||
const auto spu_putlluc_tx = build_function_asm<bool(*)(u32 raddr, const void* rdata)>([](asmjit::X86Assembler& c, auto& args)
|
||||
const auto spu_putlluc_tx = build_function_asm<void(*)(u32 raddr, const void* rdata)>([](asmjit::X86Assembler& c, auto& args)
|
||||
{
|
||||
using namespace asmjit;
|
||||
|
||||
|
|
@ -352,7 +361,7 @@ const auto spu_putlluc_tx = build_function_asm<bool(*)(u32 raddr, const void* rd
|
|||
c.vmovups(x86::ymm3, x86::yword_ptr(args[1], 96));
|
||||
|
||||
// Begin transaction
|
||||
build_transaction_enter(c, fall);
|
||||
Label begin = build_transaction_enter(c, fall);
|
||||
c.vmovaps(x86::yword_ptr(x86::r11, 0), x86::ymm0);
|
||||
c.vmovaps(x86::yword_ptr(x86::r11, 32), x86::ymm1);
|
||||
c.vmovaps(x86::yword_ptr(x86::r11, 64), x86::ymm2);
|
||||
|
|
@ -364,15 +373,14 @@ const auto spu_putlluc_tx = build_function_asm<bool(*)(u32 raddr, const void* rd
|
|||
c.mov(x86::qword_ptr(x86::r10), x86::rax);
|
||||
c.xend();
|
||||
c.vzeroupper();
|
||||
c.mov(x86::eax, 1);
|
||||
c.ret();
|
||||
|
||||
// Touch memory after transaction failure
|
||||
c.bind(fall);
|
||||
c.pause();
|
||||
c.lock().add(x86::qword_ptr(x86::r11), 0);
|
||||
c.lock().add(x86::qword_ptr(x86::r10), 0);
|
||||
c.xor_(x86::eax, x86::eax);
|
||||
c.ret();
|
||||
c.jmp(begin);
|
||||
});
|
||||
|
||||
void spu_int_ctrl_t::set(u64 ints)
|
||||
|
|
@ -1042,20 +1050,12 @@ void SPUThread::do_putlluc(const spu_mfc_cmd& args)
|
|||
auto& data = vm::_ref<decltype(rdata)>(addr);
|
||||
const auto to_write = _ref<decltype(rdata)>(args.lsa & 0x3ffff);
|
||||
|
||||
vm::reservation_acquire(addr, 128);
|
||||
|
||||
// Store unconditionally
|
||||
while (g_use_rtm)
|
||||
if (g_use_rtm)
|
||||
{
|
||||
if (spu_putlluc_tx(addr, to_write.data()))
|
||||
{
|
||||
vm::reservation_notifier(addr, 128).notify_all();
|
||||
tx_success++;
|
||||
return;
|
||||
}
|
||||
|
||||
busy_wait(300);
|
||||
tx_failure++;
|
||||
spu_putlluc_tx(addr, to_write.data());
|
||||
vm::reservation_notifier(addr, 128).notify_all();
|
||||
return;
|
||||
}
|
||||
|
||||
vm::writer_lock lock(0);
|
||||
|
|
@ -1248,19 +1248,9 @@ bool SPUThread::process_mfc_cmd(spu_mfc_cmd args)
|
|||
}
|
||||
}
|
||||
|
||||
while (g_use_rtm)
|
||||
if (g_use_rtm)
|
||||
{
|
||||
rtime = spu_getll_tx(raddr, rdata.data());
|
||||
|
||||
if (rtime & 1)
|
||||
{
|
||||
tx_failure++;
|
||||
busy_wait(300);
|
||||
continue;
|
||||
}
|
||||
|
||||
tx_success++;
|
||||
break;
|
||||
}
|
||||
|
||||
// Do several attemps
|
||||
|
|
@ -1312,28 +1302,13 @@ bool SPUThread::process_mfc_cmd(spu_mfc_cmd args)
|
|||
{
|
||||
if (g_use_rtm)
|
||||
{
|
||||
// Do several attempts (TODO)
|
||||
for (u32 i = 0; i < 3; i++)
|
||||
if (spu_putllc_tx(raddr, rtime, rdata.data(), to_write.data()))
|
||||
{
|
||||
const int r = spu_putllc_tx(raddr, rtime, rdata.data(), to_write.data());
|
||||
|
||||
if (r > 0)
|
||||
{
|
||||
vm::reservation_notifier(raddr, 128).notify_all();
|
||||
result = true;
|
||||
tx_success++;
|
||||
break;
|
||||
}
|
||||
|
||||
if (r < 0)
|
||||
{
|
||||
// Reservation lost
|
||||
break;
|
||||
}
|
||||
|
||||
// Don't fallback to heavyweight lock, just give up
|
||||
tx_failure++;
|
||||
vm::reservation_notifier(raddr, 128).notify_all();
|
||||
result = true;
|
||||
}
|
||||
|
||||
// Don't fallback to heavyweight lock, just give up
|
||||
}
|
||||
else if (rdata == data)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue