ShuriZma
/
dolphin
mirror of https://github.com/dolphin-emu/dolphin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9884 from JosJuice/jitarm64-paired-loadstore-addr 

JitArm64: Improve psq_l/psq_st address checking
This commit is contained in:
JMC47 2021-10-11 16:49:26 -04:00 committed by GitHub
parent eee302c040 15eb56142a
commit 3bfb3fa52b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 564 additions and 516 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Source/Core/Common/Arm64Emitter.h
View File

@ -1155,6 +1155,8 @@ public:
bool TryEORI2R(ARM64Reg Rd, ARM64Reg Rn, u64 imm); bool TryEORI2R(ARM64Reg Rd, ARM64Reg Rn, u64 imm);
// ABI related // ABI related
static constexpr BitSet32 CALLER_SAVED_GPRS = BitSet32(0x4007FFFF);
static constexpr BitSet32 CALLER_SAVED_FPRS = BitSet32(0xFFFF00FF);
void ABI_PushRegisters(BitSet32 registers); void ABI_PushRegisters(BitSet32 registers);
void ABI_PopRegisters(BitSet32 registers, BitSet32 ignore_mask = BitSet32(0)); void ABI_PopRegisters(BitSet32 registers, BitSet32 ignore_mask = BitSet32(0));

8
Source/Core/Core/PowerPC/JitArm64/Jit.h
View File

@ -225,10 +225,9 @@ protected:
void DumpCode(const u8* start, const u8* end); void DumpCode(const u8* start, const u8* end);
// Backpatching routines // Backpatching routines
bool DisasmLoadStore(const u8* ptr, u32* flags, Arm64Gen::ARM64Reg* reg);
void EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, Arm64Gen::ARM64Reg RS, void EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, Arm64Gen::ARM64Reg RS,
Arm64Gen::ARM64Reg addr, BitSet32 gprs_to_push = BitSet32(0), Arm64Gen::ARM64Reg addr, BitSet32 gprs_to_push = BitSet32(0),
BitSet32 fprs_to_push = BitSet32(0)); BitSet32 fprs_to_push = BitSet32(0), bool emitting_routine = false);
// Loadstore routines // Loadstore routines
void SafeLoadToReg(u32 dest, s32 addr, s32 offsetReg, u32 flags, s32 offset, bool update); void SafeLoadToReg(u32 dest, s32 addr, s32 offsetReg, u32 flags, s32 offset, bool update);
void SafeStoreFromReg(s32 dest, u32 value, s32 regOffset, u32 flags, s32 offset); void SafeStoreFromReg(s32 dest, u32 value, s32 regOffset, u32 flags, s32 offset);
@ -236,6 +235,8 @@ protected:
// jumps to the returned FixupBranch. Clobbers tmp and the 17 lower bits of addr_out. // jumps to the returned FixupBranch. Clobbers tmp and the 17 lower bits of addr_out.
Arm64Gen::FixupBranch BATAddressLookup(Arm64Gen::ARM64Reg addr_out, Arm64Gen::ARM64Reg addr_in, Arm64Gen::FixupBranch BATAddressLookup(Arm64Gen::ARM64Reg addr_out, Arm64Gen::ARM64Reg addr_in,
Arm64Gen::ARM64Reg tmp, const void* bat_table); Arm64Gen::ARM64Reg tmp, const void* bat_table);
Arm64Gen::FixupBranch CheckIfSafeAddress(Arm64Gen::ARM64Reg addr, Arm64Gen::ARM64Reg tmp1,
Arm64Gen::ARM64Reg tmp2);
void DoJit(u32 em_address, JitBlock* b, u32 nextPC); void DoJit(u32 em_address, JitBlock* b, u32 nextPC);
@ -253,7 +254,8 @@ protected:
void GenerateConvertDoubleToSingle(); void GenerateConvertDoubleToSingle();
void GenerateConvertSingleToDouble(); void GenerateConvertSingleToDouble();
void GenerateFPRF(bool single); void GenerateFPRF(bool single);
void GenerateQuantizedLoadStores(); void GenerateQuantizedLoads();
void GenerateQuantizedStores();
// Profiling // Profiling
void BeginTimeProfile(JitBlock* b); void BeginTimeProfile(JitBlock* b);

162
Source/Core/Core/PowerPC/JitArm64/JitArm64_BackPatch.cpp
View File

@ -3,12 +3,14 @@
#include <cinttypes> #include <cinttypes>
#include <cstddef> #include <cstddef>
#include <optional>
#include <string> #include <string>
#include "Common/BitSet.h" #include "Common/BitSet.h"
#include "Common/CommonFuncs.h" #include "Common/CommonFuncs.h"
#include "Common/CommonTypes.h" #include "Common/CommonTypes.h"
#include "Common/Logging/Log.h" #include "Common/Logging/Log.h"
#include "Common/MathUtil.h"
#include "Common/StringUtil.h" #include "Common/StringUtil.h"
#include "Common/Swap.h" #include "Common/Swap.h"
@ -51,48 +53,41 @@ void JitArm64::DoBacktrace(uintptr_t access_address, SContext* ctx)
} }
void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, ARM64Reg RS, void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, ARM64Reg RS,
ARM64Reg addr, BitSet32 gprs_to_push, BitSet32 fprs_to_push) ARM64Reg addr, BitSet32 gprs_to_push, BitSet32 fprs_to_push,
bool emitting_routine)
{ {
bool in_far_code = false; bool in_far_code = false;
const u8* fastmem_start = GetCodePtr(); const u8* fastmem_start = GetCodePtr();
std::optional<FixupBranch> slowmem_fixup;
if (fastmem) if (fastmem)
{ {
if (flags & BackPatchInfo::FLAG_STORE && flags & BackPatchInfo::FLAG_MASK_FLOAT) if (do_farcode && emitting_routine)
{ {
if (flags & BackPatchInfo::FLAG_SIZE_F32) const ARM64Reg temp1 = flags & BackPatchInfo::FLAG_STORE ? ARM64Reg::W0 : ARM64Reg::W3;
{ const ARM64Reg temp2 = ARM64Reg::W2;
m_float_emit.REV32(8, ARM64Reg::D0, RS);
m_float_emit.STR(32, ARM64Reg::D0, MEM_REG, addr); slowmem_fixup = CheckIfSafeAddress(addr, temp1, temp2);
} }
else if (flags & BackPatchInfo::FLAG_SIZE_F32X2)
if ((flags & BackPatchInfo::FLAG_STORE) && (flags & BackPatchInfo::FLAG_FLOAT))
{ {
m_float_emit.REV32(8, ARM64Reg::D0, RS); ARM64Reg temp = ARM64Reg::D0;
m_float_emit.STR(64, ARM64Reg::Q0, MEM_REG, addr); temp = ByteswapBeforeStore(this, &m_float_emit, temp, EncodeRegToDouble(RS), flags, true);
m_float_emit.STR(BackPatchInfo::GetFlagSize(flags), temp, MEM_REG, addr);
} }
else else if ((flags & BackPatchInfo::FLAG_LOAD) && (flags & BackPatchInfo::FLAG_FLOAT))
{ {
m_float_emit.REV64(8, ARM64Reg::Q0, RS); m_float_emit.LDR(BackPatchInfo::GetFlagSize(flags), EncodeRegToDouble(RS), MEM_REG, addr);
m_float_emit.STR(64, ARM64Reg::Q0, MEM_REG, addr);
} ByteswapAfterLoad(this, &m_float_emit, EncodeRegToDouble(RS), EncodeRegToDouble(RS), flags,
} true, false);
else if (flags & BackPatchInfo::FLAG_LOAD && flags & BackPatchInfo::FLAG_MASK_FLOAT)
{
if (flags & BackPatchInfo::FLAG_SIZE_F32)
{
m_float_emit.LDR(32, EncodeRegToDouble(RS), MEM_REG, addr);
m_float_emit.REV32(8, EncodeRegToDouble(RS), EncodeRegToDouble(RS));
}
else
{
m_float_emit.LDR(64, EncodeRegToDouble(RS), MEM_REG, addr);
m_float_emit.REV64(8, EncodeRegToDouble(RS), EncodeRegToDouble(RS));
}
} }
else if (flags & BackPatchInfo::FLAG_STORE) else if (flags & BackPatchInfo::FLAG_STORE)
{ {
ARM64Reg temp = ARM64Reg::W0; ARM64Reg temp = ARM64Reg::W0;
temp = ByteswapBeforeStore(this, temp, RS, flags, true); temp = ByteswapBeforeStore(this, &m_float_emit, temp, RS, flags, true);
if (flags & BackPatchInfo::FLAG_SIZE_32) if (flags & BackPatchInfo::FLAG_SIZE_32)
STR(temp, MEM_REG, addr); STR(temp, MEM_REG, addr);
@ -118,7 +113,7 @@ void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, AR
else if (flags & BackPatchInfo::FLAG_SIZE_8) else if (flags & BackPatchInfo::FLAG_SIZE_8)
LDRB(RS, MEM_REG, addr); LDRB(RS, MEM_REG, addr);
ByteswapAfterLoad(this, RS, RS, flags, true, false); ByteswapAfterLoad(this, &m_float_emit, RS, RS, flags, true, false);
} }
} }
const u8* fastmem_end = GetCodePtr(); const u8* fastmem_end = GetCodePtr();
@ -126,6 +121,13 @@ void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, AR
if (!fastmem || do_farcode) if (!fastmem || do_farcode)
{ {
if (fastmem && do_farcode) if (fastmem && do_farcode)
{
if (emitting_routine)
{
in_far_code = true;
SwitchToFarCode();
}
else
{ {
SlowmemHandler handler; SlowmemHandler handler;
handler.dest_reg = RS; handler.dest_reg = RS;
@ -154,56 +156,47 @@ void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, AR
return; return;
} }
} }
}
if (slowmem_fixup)
SetJumpTarget(*slowmem_fixup);
ABI_PushRegisters(gprs_to_push); ABI_PushRegisters(gprs_to_push);
m_float_emit.ABI_PushRegisters(fprs_to_push, ARM64Reg::X30); m_float_emit.ABI_PushRegisters(fprs_to_push, ARM64Reg::X30);
if (flags & BackPatchInfo::FLAG_STORE && flags & BackPatchInfo::FLAG_MASK_FLOAT) if (flags & BackPatchInfo::FLAG_STORE)
{ {
if (flags & BackPatchInfo::FLAG_SIZE_F32) const u32 access_size = BackPatchInfo::GetFlagSize(flags);
ARM64Reg src_reg = RS;
const ARM64Reg dst_reg = access_size == 64 ? ARM64Reg::X0 : ARM64Reg::W0;
if (flags & BackPatchInfo::FLAG_FLOAT)
{ {
m_float_emit.UMOV(32, ARM64Reg::W0, RS, 0); if (access_size == 64)
MOVP2R(ARM64Reg::X8, &PowerPC::Write_U32); m_float_emit.FMOV(dst_reg, EncodeRegToDouble(RS));
BLR(ARM64Reg::X8);
}
else if (flags & BackPatchInfo::FLAG_SIZE_F32X2)
{
m_float_emit.UMOV(64, ARM64Reg::X0, RS, 0);
MOVP2R(ARM64Reg::X8, &PowerPC::Write_U64);
ROR(ARM64Reg::X0, ARM64Reg::X0, 32);
BLR(ARM64Reg::X8);
}
else else
{ m_float_emit.FMOV(dst_reg, EncodeRegToSingle(RS));
m_float_emit.UMOV(64, ARM64Reg::X0, RS, 0);
MOVP2R(ARM64Reg::X8, &PowerPC::Write_U64); src_reg = dst_reg;
BLR(ARM64Reg::X8);
} }
}
else if (flags & BackPatchInfo::FLAG_LOAD && flags & BackPatchInfo::FLAG_MASK_FLOAT) if (flags & BackPatchInfo::FLAG_PAIR)
{ {
if (flags & BackPatchInfo::FLAG_SIZE_F32) // Compensate for the Write_ functions swapping the whole write instead of each pair
{ SwapPairs(this, dst_reg, src_reg, flags);
MOVP2R(ARM64Reg::X8, &PowerPC::Read_U32); src_reg = dst_reg;
BLR(ARM64Reg::X8);
m_float_emit.INS(32, RS, 0, ARM64Reg::X0);
} }
else
{ if (dst_reg != src_reg)
MOVP2R(ARM64Reg::X8, &PowerPC::Read_F64); MOV(dst_reg, src_reg);
BLR(ARM64Reg::X8);
m_float_emit.INS(64, RS, 0, ARM64Reg::X0);
}
}
else if (flags & BackPatchInfo::FLAG_STORE)
{
MOV(ARM64Reg::W0, RS);
const bool reverse = (flags & BackPatchInfo::FLAG_REVERSE) != 0; const bool reverse = (flags & BackPatchInfo::FLAG_REVERSE) != 0;
if (flags & BackPatchInfo::FLAG_SIZE_32) if (access_size == 64)
MOVP2R(ARM64Reg::X8, reverse ? &PowerPC::Write_U64_Swap : &PowerPC::Write_U64);
else if (access_size == 32)
MOVP2R(ARM64Reg::X8, reverse ? &PowerPC::Write_U32_Swap : &PowerPC::Write_U32); MOVP2R(ARM64Reg::X8, reverse ? &PowerPC::Write_U32_Swap : &PowerPC::Write_U32);
else if (flags & BackPatchInfo::FLAG_SIZE_16) else if (access_size == 16)
MOVP2R(ARM64Reg::X8, reverse ? &PowerPC::Write_U16_Swap : &PowerPC::Write_U16); MOVP2R(ARM64Reg::X8, reverse ? &PowerPC::Write_U16_Swap : &PowerPC::Write_U16);
else else
MOVP2R(ARM64Reg::X8, &PowerPC::Write_U8); MOVP2R(ARM64Reg::X8, &PowerPC::Write_U8);
@ -217,16 +210,40 @@ void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, AR
} }
else else
{ {
if (flags & BackPatchInfo::FLAG_SIZE_32) const u32 access_size = BackPatchInfo::GetFlagSize(flags);
if (access_size == 64)
MOVP2R(ARM64Reg::X8, &PowerPC::Read_U64);
else if (access_size == 32)
MOVP2R(ARM64Reg::X8, &PowerPC::Read_U32); MOVP2R(ARM64Reg::X8, &PowerPC::Read_U32);
else if (flags & BackPatchInfo::FLAG_SIZE_16) else if (access_size == 16)
MOVP2R(ARM64Reg::X8, &PowerPC::Read_U16); MOVP2R(ARM64Reg::X8, &PowerPC::Read_U16);
else if (flags & BackPatchInfo::FLAG_SIZE_8) else
MOVP2R(ARM64Reg::X8, &PowerPC::Read_U8); MOVP2R(ARM64Reg::X8, &PowerPC::Read_U8);
BLR(ARM64Reg::X8); BLR(ARM64Reg::X8);
ByteswapAfterLoad(this, RS, ARM64Reg::W0, flags, false, false); ARM64Reg src_reg = access_size == 64 ? ARM64Reg::X0 : ARM64Reg::W0;
if (flags & BackPatchInfo::FLAG_PAIR)
{
// Compensate for the Read_ functions swapping the whole read instead of each pair
const ARM64Reg dst_reg = flags & BackPatchInfo::FLAG_FLOAT ? src_reg : RS;
SwapPairs(this, dst_reg, src_reg, flags);
src_reg = dst_reg;
}
if (flags & BackPatchInfo::FLAG_FLOAT)
{
if (access_size == 64)
m_float_emit.FMOV(EncodeRegToDouble(RS), src_reg);
else
m_float_emit.FMOV(EncodeRegToSingle(RS), src_reg);
src_reg = RS;
}
ByteswapAfterLoad(this, &m_float_emit, RS, src_reg, flags, false, false);
} }
m_float_emit.ABI_PopRegisters(fprs_to_push, ARM64Reg::X30); m_float_emit.ABI_PopRegisters(fprs_to_push, ARM64Reg::X30);
@ -234,10 +251,19 @@ void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, AR
} }
if (in_far_code) if (in_far_code)
{
if (emitting_routine)
{
FixupBranch done = B();
SwitchToNearCode();
SetJumpTarget(done);
}
else
{ {
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
SwitchToNearCode(); SwitchToNearCode();
} }
}
} }
bool JitArm64::HandleFastmemFault(uintptr_t access_address, SContext* ctx) bool JitArm64::HandleFastmemFault(uintptr_t access_address, SContext* ctx)

16
Source/Core/Core/PowerPC/JitArm64/JitArm64_LoadStore.cpp
View File

@ -240,7 +240,7 @@ void JitArm64::SafeStoreFromReg(s32 dest, u32 value, s32 regOffset, u32 flags, s
LDR(IndexType::Unsigned, ARM64Reg::X0, PPC_REG, PPCSTATE_OFF(gather_pipe_ptr)); LDR(IndexType::Unsigned, ARM64Reg::X0, PPC_REG, PPCSTATE_OFF(gather_pipe_ptr));
ARM64Reg temp = ARM64Reg::W1; ARM64Reg temp = ARM64Reg::W1;
temp = ByteswapBeforeStore(this, temp, RS, flags, true); temp = ByteswapBeforeStore(this, &m_float_emit, temp, RS, flags, true);
if (accessSize == 32) if (accessSize == 32)
STR(IndexType::Post, temp, ARM64Reg::X0, 4); STR(IndexType::Post, temp, ARM64Reg::X0, 4);
@ -288,6 +288,20 @@ FixupBranch JitArm64::BATAddressLookup(ARM64Reg addr_out, ARM64Reg addr_in, ARM6
return fail; return fail;
} }
FixupBranch JitArm64::CheckIfSafeAddress(Arm64Gen::ARM64Reg addr, Arm64Gen::ARM64Reg tmp1,
Arm64Gen::ARM64Reg tmp2)
{
tmp2 = EncodeRegTo64(tmp2);
MOVP2R(tmp2, PowerPC::dbat_table.data());
LSR(tmp1, addr, PowerPC::BAT_INDEX_SHIFT);
LDR(tmp1, tmp2, ArithOption(tmp1, true));
FixupBranch pass = TBNZ(tmp1, IntLog2(PowerPC::BAT_PHYSICAL_BIT));
FixupBranch fail = B();
SetJumpTarget(pass);
return fail;
}
void JitArm64::lXX(UGeckoInstruction inst) void JitArm64::lXX(UGeckoInstruction inst)
{ {
INSTRUCTION_START INSTRUCTION_START

46
Source/Core/Core/PowerPC/JitArm64/JitArm64_LoadStoreFloating.cpp
View File

@ -26,7 +26,7 @@ void JitArm64::lfXX(UGeckoInstruction inst)
u32 a = inst.RA, b = inst.RB; u32 a = inst.RA, b = inst.RB;
s32 offset = inst.SIMM_16; s32 offset = inst.SIMM_16;
u32 flags = BackPatchInfo::FLAG_LOAD; u32 flags = BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT;
bool update = false; bool update = false;
s32 offset_reg = -1; s32 offset_reg = -1;
@ -36,38 +36,38 @@ void JitArm64::lfXX(UGeckoInstruction inst)
switch (inst.SUBOP10) switch (inst.SUBOP10)
{ {
case 567: // lfsux case 567: // lfsux
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
update = true; update = true;
offset_reg = b; offset_reg = b;
break; break;
case 535: // lfsx case 535: // lfsx
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
offset_reg = b; offset_reg = b;
break; break;
case 631: // lfdux case 631: // lfdux
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
update = true; update = true;
offset_reg = b; offset_reg = b;
break; break;
case 599: // lfdx case 599: // lfdx
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
offset_reg = b; offset_reg = b;
break; break;
} }
break; break;
case 49: // lfsu case 49: // lfsu
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
update = true; update = true;
break; break;
case 48: // lfs case 48: // lfs
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
break; break;
case 51: // lfdu case 51: // lfdu
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
update = true; update = true;
break; break;
case 50: // lfd case 50: // lfd
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
break; break;
} }
@ -75,7 +75,7 @@ void JitArm64::lfXX(UGeckoInstruction inst)
bool is_immediate = false; bool is_immediate = false;
const RegType type = const RegType type =
(flags & BackPatchInfo::FLAG_SIZE_F64) != 0 ? RegType::LowerPair : RegType::DuplicatedSingle; (flags & BackPatchInfo::FLAG_SIZE_64) != 0 ? RegType::LowerPair : RegType::DuplicatedSingle;
gpr.Lock(ARM64Reg::W0, ARM64Reg::W30); gpr.Lock(ARM64Reg::W0, ARM64Reg::W30);
fpr.Lock(ARM64Reg::Q0); fpr.Lock(ARM64Reg::Q0);
@ -190,7 +190,7 @@ void JitArm64::stfXX(UGeckoInstruction inst)
bool want_single = false; bool want_single = false;
s32 offset = inst.SIMM_16; s32 offset = inst.SIMM_16;
u32 flags = BackPatchInfo::FLAG_STORE; u32 flags = BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT;
bool update = false; bool update = false;
s32 offset_reg = -1; s32 offset_reg = -1;
@ -201,46 +201,46 @@ void JitArm64::stfXX(UGeckoInstruction inst)
{ {
case 663: // stfsx case 663: // stfsx
want_single = true; want_single = true;
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
offset_reg = b; offset_reg = b;
break; break;
case 695: // stfsux case 695: // stfsux
want_single = true; want_single = true;
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
update = true; update = true;
offset_reg = b; offset_reg = b;
break; break;
case 727: // stfdx case 727: // stfdx
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
offset_reg = b; offset_reg = b;
break; break;
case 759: // stfdux case 759: // stfdux
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
update = true; update = true;
offset_reg = b; offset_reg = b;
break; break;
case 983: // stfiwx case 983: // stfiwx
// This instruction writes the lower 32 bits of a double. want_single must be false // This instruction writes the lower 32 bits of a double. want_single must be false
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
offset_reg = b; offset_reg = b;
break; break;
} }
break; break;
case 53: // stfsu case 53: // stfsu
want_single = true; want_single = true;
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
update = true; update = true;
break; break;
case 52: // stfs case 52: // stfs
want_single = true; want_single = true;
flags |= BackPatchInfo::FLAG_SIZE_F32; flags |= BackPatchInfo::FLAG_SIZE_32;
break; break;
case 55: // stfdu case 55: // stfdu
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
update = true; update = true;
break; break;
case 54: // stfd case 54: // stfd
flags |= BackPatchInfo::FLAG_SIZE_F64; flags |= BackPatchInfo::FLAG_SIZE_64;
break; break;
} }
@ -361,16 +361,16 @@ void JitArm64::stfXX(UGeckoInstruction inst)
if (jo.optimizeGatherPipe && PowerPC::IsOptimizableGatherPipeWrite(imm_addr)) if (jo.optimizeGatherPipe && PowerPC::IsOptimizableGatherPipeWrite(imm_addr))
{ {
int accessSize; int accessSize;
if (flags & BackPatchInfo::FLAG_SIZE_F64) if (flags & BackPatchInfo::FLAG_SIZE_64)
accessSize = 64; accessSize = 64;
else else
accessSize = 32; accessSize = 32;
LDR(IndexType::Unsigned, ARM64Reg::X0, PPC_REG, PPCSTATE_OFF(gather_pipe_ptr)); LDR(IndexType::Unsigned, ARM64Reg::X0, PPC_REG, PPCSTATE_OFF(gather_pipe_ptr));
if (flags & BackPatchInfo::FLAG_SIZE_F64) if (flags & BackPatchInfo::FLAG_SIZE_64)
m_float_emit.REV64(8, ARM64Reg::Q0, V0); m_float_emit.REV64(8, ARM64Reg::Q0, V0);
else if (flags & BackPatchInfo::FLAG_SIZE_F32) else if (flags & BackPatchInfo::FLAG_SIZE_32)
m_float_emit.REV32(8, ARM64Reg::D0, V0); m_float_emit.REV32(8, ARM64Reg::D0, V0);
m_float_emit.STR(accessSize, IndexType::Post, accessSize == 64 ? ARM64Reg::Q0 : ARM64Reg::D0, m_float_emit.STR(accessSize, IndexType::Post, accessSize == 64 ? ARM64Reg::Q0 : ARM64Reg::D0,

115
Source/Core/Core/PowerPC/JitArm64/JitArm64_LoadStorePaired.cpp
View File

@ -19,14 +19,14 @@ void JitArm64::psq_lXX(UGeckoInstruction inst)
{ {
INSTRUCTION_START INSTRUCTION_START
JITDISABLE(bJITLoadStorePairedOff); JITDISABLE(bJITLoadStorePairedOff);
FALLBACK_IF(jo.memcheck || !jo.fastmem); FALLBACK_IF(jo.memcheck);
// The asm routines assume address translation is on. // If we have a fastmem arena, the asm routines assume address translation is on.
FALLBACK_IF(!MSR.DR); FALLBACK_IF(!js.assumeNoPairedQuantize && jo.fastmem_arena && !MSR.DR);
// X30 is LR // X30 is LR
// X0 contains the scale // X0 is the address
// X1 is the address // X1 contains the scale
// X2 is a temporary // X2 is a temporary
// Q0 is the return register // Q0 is the return register
// Q1 is a temporary // Q1 is a temporary
@ -36,13 +36,18 @@ void JitArm64::psq_lXX(UGeckoInstruction inst)
const int i = indexed ? inst.Ix : inst.I; const int i = indexed ? inst.Ix : inst.I;
const int w = indexed ? inst.Wx : inst.W; const int w = indexed ? inst.Wx : inst.W;
gpr.Lock(ARM64Reg::W0, ARM64Reg::W1, ARM64Reg::W2, ARM64Reg::W30); gpr.Lock(ARM64Reg::W0, ARM64Reg::W30);
fpr.Lock(ARM64Reg::Q0, ARM64Reg::Q1); fpr.Lock(ARM64Reg::Q0);
if (!js.assumeNoPairedQuantize)
{
gpr.Lock(ARM64Reg::W1, ARM64Reg::W2, ARM64Reg::W3);
fpr.Lock(ARM64Reg::Q1);
}
constexpr ARM64Reg scale_reg = ARM64Reg::W0; constexpr ARM64Reg addr_reg = ARM64Reg::W0;
constexpr ARM64Reg addr_reg = ARM64Reg::W1; constexpr ARM64Reg scale_reg = ARM64Reg::W1;
constexpr ARM64Reg type_reg = ARM64Reg::W2; constexpr ARM64Reg type_reg = ARM64Reg::W2;
ARM64Reg VS; ARM64Reg VS = fpr.RW(inst.RS, RegType::Single);
if (inst.RA || update) // Always uses the register on update if (inst.RA || update) // Always uses the register on update
{ {
@ -69,17 +74,20 @@ void JitArm64::psq_lXX(UGeckoInstruction inst)
if (js.assumeNoPairedQuantize) if (js.assumeNoPairedQuantize)
{ {
VS = fpr.RW(inst.RS, RegType::Single); BitSet32 gprs_in_use = gpr.GetCallerSavedUsed();
BitSet32 fprs_in_use = fpr.GetCallerSavedUsed();
// Wipe the registers we are using as temporaries
gprs_in_use[DecodeReg(ARM64Reg::W0)] = false;
fprs_in_use[DecodeReg(ARM64Reg::Q0)] = false;
fprs_in_use[DecodeReg(VS)] = 0;
u32 flags = BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_32;
if (!w) if (!w)
{ flags |= BackPatchInfo::FLAG_PAIR;
ADD(EncodeRegTo64(addr_reg), EncodeRegTo64(addr_reg), MEM_REG);
m_float_emit.LD1(32, 1, EncodeRegToDouble(VS), EncodeRegTo64(addr_reg)); EmitBackpatchRoutine(flags, jo.fastmem, jo.fastmem, VS, EncodeRegTo64(addr_reg), gprs_in_use,
} fprs_in_use);
else
{
m_float_emit.LDR(32, VS, EncodeRegTo64(addr_reg), MEM_REG);
}
m_float_emit.REV32(8, EncodeRegToDouble(VS), EncodeRegToDouble(VS));
} }
else else
{ {
@ -91,7 +99,6 @@ void JitArm64::psq_lXX(UGeckoInstruction inst)
LDR(EncodeRegTo64(type_reg), ARM64Reg::X30, ArithOption(EncodeRegTo64(type_reg), true)); LDR(EncodeRegTo64(type_reg), ARM64Reg::X30, ArithOption(EncodeRegTo64(type_reg), true));
BLR(EncodeRegTo64(type_reg)); BLR(EncodeRegTo64(type_reg));
VS = fpr.RW(inst.RS, RegType::Single);
m_float_emit.ORR(EncodeRegToDouble(VS), ARM64Reg::D0, ARM64Reg::D0); m_float_emit.ORR(EncodeRegToDouble(VS), ARM64Reg::D0, ARM64Reg::D0);
} }
@ -101,18 +108,23 @@ void JitArm64::psq_lXX(UGeckoInstruction inst)
m_float_emit.INS(32, VS, 1, ARM64Reg::Q0, 0); m_float_emit.INS(32, VS, 1, ARM64Reg::Q0, 0);
} }
gpr.Unlock(ARM64Reg::W0, ARM64Reg::W1, ARM64Reg::W2, ARM64Reg::W30); gpr.Unlock(ARM64Reg::W0, ARM64Reg::W30);
fpr.Unlock(ARM64Reg::Q0, ARM64Reg::Q1); fpr.Unlock(ARM64Reg::Q0);
if (!js.assumeNoPairedQuantize)
{
gpr.Unlock(ARM64Reg::W1, ARM64Reg::W2, ARM64Reg::W3);
fpr.Unlock(ARM64Reg::Q1);
}
} }
void JitArm64::psq_stXX(UGeckoInstruction inst) void JitArm64::psq_stXX(UGeckoInstruction inst)
{ {
INSTRUCTION_START INSTRUCTION_START
JITDISABLE(bJITLoadStorePairedOff); JITDISABLE(bJITLoadStorePairedOff);
FALLBACK_IF(jo.memcheck || !jo.fastmem); FALLBACK_IF(jo.memcheck);
// The asm routines assume address translation is on. // If we have a fastmem arena, the asm routines assume address translation is on.
FALLBACK_IF(!MSR.DR); FALLBACK_IF(!js.assumeNoPairedQuantize && jo.fastmem_arena && !MSR.DR);
// X30 is LR // X30 is LR
// X0 contains the scale // X0 contains the scale
@ -125,6 +137,7 @@ void JitArm64::psq_stXX(UGeckoInstruction inst)
const int i = indexed ? inst.Ix : inst.I; const int i = indexed ? inst.Ix : inst.I;
const int w = indexed ? inst.Wx : inst.W; const int w = indexed ? inst.Wx : inst.W;
if (!js.assumeNoPairedQuantize)
fpr.Lock(ARM64Reg::Q0, ARM64Reg::Q1); fpr.Lock(ARM64Reg::Q0, ARM64Reg::Q1);
const bool have_single = fpr.IsSingle(inst.RS); const bool have_single = fpr.IsSingle(inst.RS);
@ -160,7 +173,9 @@ void JitArm64::psq_stXX(UGeckoInstruction inst)
} }
} }
gpr.Lock(ARM64Reg::W0, ARM64Reg::W1, ARM64Reg::W2, ARM64Reg::W30); gpr.Lock(ARM64Reg::W0, ARM64Reg::W1, ARM64Reg::W30);
if (!js.assumeNoPairedQuantize)
gpr.Lock(ARM64Reg::W2);
constexpr ARM64Reg scale_reg = ARM64Reg::W0; constexpr ARM64Reg scale_reg = ARM64Reg::W0;
constexpr ARM64Reg addr_reg = ARM64Reg::W1; constexpr ARM64Reg addr_reg = ARM64Reg::W1;
@ -189,18 +204,18 @@ void JitArm64::psq_stXX(UGeckoInstruction inst)
MOV(gpr.R(inst.RA), addr_reg); MOV(gpr.R(inst.RA), addr_reg);
} }
if (js.assumeNoPairedQuantize)
{
BitSet32 gprs_in_use = gpr.GetCallerSavedUsed(); BitSet32 gprs_in_use = gpr.GetCallerSavedUsed();
BitSet32 fprs_in_use = fpr.GetCallerSavedUsed(); BitSet32 fprs_in_use = fpr.GetCallerSavedUsed();
// Wipe the registers we are using as temporaries // Wipe the registers we are using as temporaries
gprs_in_use &= BitSet32(~7); gprs_in_use[DecodeReg(ARM64Reg::W0)] = false;
fprs_in_use &= BitSet32(~3); gprs_in_use[DecodeReg(ARM64Reg::W1)] = false;
if (js.assumeNoPairedQuantize) u32 flags = BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_32;
{ if (!w)
u32 flags = BackPatchInfo::FLAG_STORE; flags |= BackPatchInfo::FLAG_PAIR;
flags |= (w ? BackPatchInfo::FLAG_SIZE_F32 : BackPatchInfo::FLAG_SIZE_F32X2);
EmitBackpatchRoutine(flags, jo.fastmem, jo.fastmem, VS, EncodeRegTo64(addr_reg), gprs_in_use, EmitBackpatchRoutine(flags, jo.fastmem, jo.fastmem, VS, EncodeRegTo64(addr_reg), gprs_in_use,
fprs_in_use); fprs_in_use);
@ -211,38 +226,18 @@ void JitArm64::psq_stXX(UGeckoInstruction inst)
UBFM(type_reg, scale_reg, 0, 2); // Type UBFM(type_reg, scale_reg, 0, 2); // Type
UBFM(scale_reg, scale_reg, 8, 13); // Scale UBFM(scale_reg, scale_reg, 8, 13); // Scale
// Inline address check MOVP2R(ARM64Reg::X30, w ? single_store_quantized : paired_store_quantized);
// FIXME: This doesn't correctly account for the BAT configuration.
TST(addr_reg, LogicalImm(0x0c000000, 32));
FixupBranch pass = B(CC_EQ);
FixupBranch fail = B();
SwitchToFarCode();
SetJumpTarget(fail);
// Slow
MOVP2R(ARM64Reg::X30, &paired_store_quantized[16 + w * 8]);
LDR(EncodeRegTo64(type_reg), ARM64Reg::X30, ArithOption(EncodeRegTo64(type_reg), true));
ABI_PushRegisters(gprs_in_use);
m_float_emit.ABI_PushRegisters(fprs_in_use, ARM64Reg::X30);
BLR(EncodeRegTo64(type_reg));
m_float_emit.ABI_PopRegisters(fprs_in_use, ARM64Reg::X30);
ABI_PopRegisters(gprs_in_use);
FixupBranch continue1 = B();
SwitchToNearCode();
SetJumpTarget(pass);
// Fast
MOVP2R(ARM64Reg::X30, &paired_store_quantized[w * 8]);
LDR(EncodeRegTo64(type_reg), ARM64Reg::X30, ArithOption(EncodeRegTo64(type_reg), true)); LDR(EncodeRegTo64(type_reg), ARM64Reg::X30, ArithOption(EncodeRegTo64(type_reg), true));
BLR(EncodeRegTo64(type_reg)); BLR(EncodeRegTo64(type_reg));
SetJumpTarget(continue1);
} }
if (js.assumeNoPairedQuantize && !have_single) if (js.assumeNoPairedQuantize && !have_single)
fpr.Unlock(VS); fpr.Unlock(VS);
gpr.Unlock(ARM64Reg::W0, ARM64Reg::W1, ARM64Reg::W2, ARM64Reg::W30); gpr.Unlock(ARM64Reg::W0, ARM64Reg::W1, ARM64Reg::W30);
if (!js.assumeNoPairedQuantize)
{
gpr.Unlock(ARM64Reg::W2);
fpr.Unlock(ARM64Reg::Q0, ARM64Reg::Q1); fpr.Unlock(ARM64Reg::Q0, ARM64Reg::Q1);
}
} }

23
Source/Core/Core/PowerPC/JitArm64/JitArm64_RegCache.cpp
View File

@ -149,15 +149,9 @@ void Arm64GPRCache::Start(PPCAnalyst::BlockRegStats& stats)
{ {
} }
bool Arm64GPRCache::IsCalleeSaved(ARM64Reg reg) const bool Arm64GPRCache::IsCallerSaved(ARM64Reg reg) const
{ {
static constexpr auto callee_regs = { return ARM64XEmitter::CALLER_SAVED_GPRS[DecodeReg(reg)];
ARM64Reg::X28, ARM64Reg::X27, ARM64Reg::X26, ARM64Reg::X25,
ARM64Reg::X24, ARM64Reg::X23, ARM64Reg::X22, ARM64Reg::X21,
ARM64Reg::X20, ARM64Reg::X19, ARM64Reg::INVALID_REG,
};
return std::find(callee_regs.begin(), callee_regs.end(), EncodeRegTo64(reg)) != callee_regs.end();
} }
const OpArg& Arm64GPRCache::GetGuestGPROpArg(size_t preg) const const OpArg& Arm64GPRCache::GetGuestGPROpArg(size_t preg) const
@ -416,7 +410,7 @@ BitSet32 Arm64GPRCache::GetCallerSavedUsed() const
BitSet32 registers(0); BitSet32 registers(0);
for (const auto& it : m_host_registers) for (const auto& it : m_host_registers)
{ {
if (it.IsLocked() && !IsCalleeSaved(it.GetReg())) if (it.IsLocked() && IsCallerSaved(it.GetReg()))
registers[DecodeReg(it.GetReg())] = true; registers[DecodeReg(it.GetReg())] = true;
} }
return registers; return registers;
@ -716,14 +710,9 @@ void Arm64FPRCache::FlushByHost(ARM64Reg host_reg, ARM64Reg tmp_reg)
} }
} }
bool Arm64FPRCache::IsCalleeSaved(ARM64Reg reg) const bool Arm64FPRCache::IsCallerSaved(ARM64Reg reg) const
{ {
static constexpr auto callee_regs = { return ARM64XEmitter::CALLER_SAVED_FPRS[DecodeReg(reg)];
ARM64Reg::Q8, ARM64Reg::Q9, ARM64Reg::Q10, ARM64Reg::Q11, ARM64Reg::Q12,
ARM64Reg::Q13, ARM64Reg::Q14, ARM64Reg::Q15, ARM64Reg::INVALID_REG,
};
return std::find(callee_regs.begin(), callee_regs.end(), reg) != callee_regs.end();
} }
bool Arm64FPRCache::IsTopHalfUsed(ARM64Reg reg) const bool Arm64FPRCache::IsTopHalfUsed(ARM64Reg reg) const
@ -841,7 +830,7 @@ BitSet32 Arm64FPRCache::GetCallerSavedUsed() const
BitSet32 registers(0); BitSet32 registers(0);
for (const auto& it : m_host_registers) for (const auto& it : m_host_registers)
{ {
if (it.IsLocked() && (!IsCalleeSaved(it.GetReg()) || IsTopHalfUsed(it.GetReg()))) if (it.IsLocked() && (IsCallerSaved(it.GetReg()) || IsTopHalfUsed(it.GetReg())))
registers[DecodeReg(it.GetReg())] = true; registers[DecodeReg(it.GetReg())] = true;
} }
return registers; return registers;

4
Source/Core/Core/PowerPC/JitArm64/JitArm64_RegCache.h
View File

@ -291,7 +291,7 @@ protected:
void FlushRegister(size_t index, bool maintain_state, Arm64Gen::ARM64Reg tmp_reg) override; void FlushRegister(size_t index, bool maintain_state, Arm64Gen::ARM64Reg tmp_reg) override;
private: private:
bool IsCalleeSaved(Arm64Gen::ARM64Reg reg) const; bool IsCallerSaved(Arm64Gen::ARM64Reg reg) const;
struct GuestRegInfo struct GuestRegInfo
{ {
@ -350,7 +350,7 @@ protected:
void FlushRegister(size_t preg, bool maintain_state, Arm64Gen::ARM64Reg tmp_reg) override; void FlushRegister(size_t preg, bool maintain_state, Arm64Gen::ARM64Reg tmp_reg) override;
private: private:
bool IsCalleeSaved(Arm64Gen::ARM64Reg reg) const; bool IsCallerSaved(Arm64Gen::ARM64Reg reg) const;
bool IsTopHalfUsed(Arm64Gen::ARM64Reg reg) const; bool IsTopHalfUsed(Arm64Gen::ARM64Reg reg) const;
void FlushRegisters(BitSet32 regs, bool maintain_state, Arm64Gen::ARM64Reg tmp_reg); void FlushRegisters(BitSet32 regs, bool maintain_state, Arm64Gen::ARM64Reg tmp_reg);

379
Source/Core/Core/PowerPC/JitArm64/JitAsm.cpp
View File

@ -223,7 +223,8 @@ void JitArm64::GenerateCommonAsm()
GenerateFPRF(false); GenerateFPRF(false);
JitRegister::Register(GetAsmRoutines()->fprf_single, GetCodePtr(), "JIT_FPRF"); JitRegister::Register(GetAsmRoutines()->fprf_single, GetCodePtr(), "JIT_FPRF");
GenerateQuantizedLoadStores(); GenerateQuantizedLoads();
GenerateQuantizedStores();
} }
// Input: X1 contains input, and D0 contains result of running the input through AArch64 FRECPE. // Input: X1 contains input, and D0 contains result of running the input through AArch64 FRECPE.
@ -483,17 +484,19 @@ void JitArm64::GenerateFPRF(bool single)
B(write_fprf_and_ret); B(write_fprf_and_ret);
} }
void JitArm64::GenerateQuantizedLoadStores() void JitArm64::GenerateQuantizedLoads()
{ {
// X0 is the scale // X0 is the address
// X1 is address // X1 is the scale
// X2 is a temporary on stores // X2 is a temporary
// X3 is a temporary (used in EmitBackpatchRoutine)
// X30 is LR // X30 is LR
// Q0 is the return for loads // Q0 is the return
// is the register for stores
// Q1 is a temporary // Q1 is a temporary
ARM64Reg addr_reg = ARM64Reg::X1; ARM64Reg addr_reg = ARM64Reg::X0;
ARM64Reg scale_reg = ARM64Reg::X0; ARM64Reg scale_reg = ARM64Reg::X1;
BitSet32 gprs_to_push = CALLER_SAVED_GPRS & ~BitSet32{0, 2, 3};
BitSet32 fprs_to_push = BitSet32(0xFFFFFFFF) & ~BitSet32{0, 1};
ARM64FloatEmitter float_emit(this); ARM64FloatEmitter float_emit(this);
const u8* start = GetCodePtr(); const u8* start = GetCodePtr();
@ -501,15 +504,22 @@ void JitArm64::GenerateQuantizedLoadStores()
BRK(100); BRK(100);
const u8* loadPairedFloatTwo = GetCodePtr(); const u8* loadPairedFloatTwo = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags = BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT |
float_emit.LD1(32, 1, ARM64Reg::D0, addr_reg); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_32;
float_emit.REV32(8, ARM64Reg::D0, ARM64Reg::D0);
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push & ~BitSet32{1}, fprs_to_push);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
} }
const u8* loadPairedU8Two = GetCodePtr(); const u8* loadPairedU8Two = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags = BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT |
float_emit.LDR(16, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_8;
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.UXTL(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.UXTL(8, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -522,8 +532,12 @@ void JitArm64::GenerateQuantizedLoadStores()
} }
const u8* loadPairedS8Two = GetCodePtr(); const u8* loadPairedS8Two = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags = BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT |
float_emit.LDR(16, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_8;
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.SXTL(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.SXTL(8, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -536,9 +550,12 @@ void JitArm64::GenerateQuantizedLoadStores()
} }
const u8* loadPairedU16Two = GetCodePtr(); const u8* loadPairedU16Two = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags = BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT |
float_emit.LD1(16, 1, ARM64Reg::D0, addr_reg); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_16;
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0);
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -550,9 +567,12 @@ void JitArm64::GenerateQuantizedLoadStores()
} }
const u8* loadPairedS16Two = GetCodePtr(); const u8* loadPairedS16Two = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags = BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT |
float_emit.LD1(16, 1, ARM64Reg::D0, addr_reg); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_16;
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0);
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -565,15 +585,22 @@ void JitArm64::GenerateQuantizedLoadStores()
const u8* loadPairedFloatOne = GetCodePtr(); const u8* loadPairedFloatOne = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags =
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0); BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_32;
float_emit.REV32(8, ARM64Reg::D0, ARM64Reg::D0);
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push & ~BitSet32{1}, fprs_to_push);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
} }
const u8* loadPairedU8One = GetCodePtr(); const u8* loadPairedU8One = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags =
float_emit.LDR(8, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0); BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_8;
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.UXTL(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.UXTL(8, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -586,8 +613,12 @@ void JitArm64::GenerateQuantizedLoadStores()
} }
const u8* loadPairedS8One = GetCodePtr(); const u8* loadPairedS8One = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags =
float_emit.LDR(8, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0); BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_8;
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.SXTL(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.SXTL(8, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -600,9 +631,12 @@ void JitArm64::GenerateQuantizedLoadStores()
} }
const u8* loadPairedU16One = GetCodePtr(); const u8* loadPairedU16One = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags =
float_emit.LDR(16, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0); BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_16;
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0);
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.UCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -614,9 +648,12 @@ void JitArm64::GenerateQuantizedLoadStores()
} }
const u8* loadPairedS16One = GetCodePtr(); const u8* loadPairedS16One = GetCodePtr();
{ {
ADD(addr_reg, addr_reg, MEM_REG); constexpr u32 flags =
float_emit.LDR(16, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0); BackPatchInfo::FLAG_LOAD | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_16;
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0);
EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SXTL(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.SCVTF(32, ARM64Reg::D0, ARM64Reg::D0);
@ -652,31 +689,37 @@ void JitArm64::GenerateQuantizedLoadStores()
single_load_quantized[5] = loadPairedU16One; single_load_quantized[5] = loadPairedU16One;
single_load_quantized[6] = loadPairedS8One; single_load_quantized[6] = loadPairedS8One;
single_load_quantized[7] = loadPairedS16One; single_load_quantized[7] = loadPairedS16One;
}
// Stores void JitArm64::GenerateQuantizedStores()
start = GetCodePtr(); {
// X0 is the scale
// X1 is the address
// X2 is a temporary
// X30 is LR
// Q0 is the register
// Q1 is a temporary
ARM64Reg scale_reg = ARM64Reg::X0;
ARM64Reg addr_reg = ARM64Reg::X1;
BitSet32 gprs_to_push = CALLER_SAVED_GPRS & ~BitSet32{0, 1, 2};
BitSet32 fprs_to_push = BitSet32(0xFFFFFFFF) & ~BitSet32{0, 1};
ARM64FloatEmitter float_emit(this);
const u8* start = GetCodePtr();
const u8* storePairedIllegal = GetCodePtr(); const u8* storePairedIllegal = GetCodePtr();
BRK(0x101); BRK(0x101);
const u8* storePairedFloat; const u8* storePairedFloat = GetCodePtr();
const u8* storePairedFloatSlow;
{ {
storePairedFloat = GetCodePtr(); constexpr u32 flags = BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT |
float_emit.REV32(8, ARM64Reg::D0, ARM64Reg::D0); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_32;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.ST1(64, ARM64Reg::Q0, 0, addr_reg, ARM64Reg::SP); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storePairedFloatSlow = GetCodePtr();
float_emit.UMOV(64, ARM64Reg::X0, ARM64Reg::Q0, 0);
ROR(ARM64Reg::X0, ARM64Reg::X0, 32);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U64);
BR(ARM64Reg::X2);
} }
const u8* storePairedU8 = GetCodePtr();
const u8* storePairedU8;
const u8* storePairedU8Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -685,25 +728,17 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UQXTN(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.UQXTN(8, ARM64Reg::D0, ARM64Reg::D0);
};
storePairedU8 = GetCodePtr(); constexpr u32 flags = BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT |
emit_quantize(); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_8;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.ST1(16, ARM64Reg::Q0, 0, addr_reg, ARM64Reg::SP); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storePairedU8Slow = GetCodePtr();
emit_quantize();
float_emit.UMOV(16, ARM64Reg::W0, ARM64Reg::Q0, 0);
REV16(ARM64Reg::W0, ARM64Reg::W0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U16);
BR(ARM64Reg::X2);
} }
const u8* storePairedS8; const u8* storePairedS8 = GetCodePtr();
const u8* storePairedS8Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -712,26 +747,17 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SQXTN(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.SQXTN(8, ARM64Reg::D0, ARM64Reg::D0);
};
storePairedS8 = GetCodePtr(); constexpr u32 flags = BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT |
emit_quantize(); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_8;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.ST1(16, ARM64Reg::Q0, 0, addr_reg, ARM64Reg::SP); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storePairedS8Slow = GetCodePtr();
emit_quantize();
float_emit.UMOV(16, ARM64Reg::W0, ARM64Reg::Q0, 0);
REV16(ARM64Reg::W0, ARM64Reg::W0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U16);
BR(ARM64Reg::X2);
} }
const u8* storePairedU16 = GetCodePtr();
const u8* storePairedU16;
const u8* storePairedU16Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -739,26 +765,17 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0);
};
storePairedU16 = GetCodePtr(); constexpr u32 flags = BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT |
emit_quantize(); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_16;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.ST1(32, ARM64Reg::Q0, 0, addr_reg, ARM64Reg::SP); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storePairedU16Slow = GetCodePtr();
emit_quantize();
float_emit.REV32(8, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UMOV(32, ARM64Reg::W0, ARM64Reg::Q0, 0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U32);
BR(ARM64Reg::X2);
} }
const u8* storePairedS16; // Used by Viewtiful Joe's intro movie const u8* storePairedS16 = GetCodePtr(); // Used by Viewtiful Joe's intro movie
const u8* storePairedS16Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -766,41 +783,28 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0);
};
storePairedS16 = GetCodePtr(); constexpr u32 flags = BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT |
emit_quantize(); BackPatchInfo::FLAG_PAIR | BackPatchInfo::FLAG_SIZE_16;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.ST1(32, ARM64Reg::Q0, 0, addr_reg, ARM64Reg::SP); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storePairedS16Slow = GetCodePtr();
emit_quantize();
float_emit.REV32(8, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UMOV(32, ARM64Reg::W0, ARM64Reg::Q0, 0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U32);
BR(ARM64Reg::X2);
} }
const u8* storeSingleFloat; const u8* storeSingleFloat = GetCodePtr();
const u8* storeSingleFloatSlow;
{ {
storeSingleFloat = GetCodePtr(); constexpr u32 flags =
float_emit.REV32(8, ARM64Reg::D0, ARM64Reg::D0); BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_32;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.STR(32, IndexType::Unsigned, ARM64Reg::D0, addr_reg, 0);
RET(ARM64Reg::X30);
storeSingleFloatSlow = GetCodePtr(); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
float_emit.UMOV(32, ARM64Reg::W0, ARM64Reg::Q0, 0); gprs_to_push, fprs_to_push);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U32);
BR(ARM64Reg::X2); RET(ARM64Reg::X30);
} }
const u8* storeSingleU8; // Used by MKWii const u8* storeSingleU8 = GetCodePtr(); // Used by MKWii
const u8* storeSingleU8Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -809,24 +813,17 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UQXTN(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.UQXTN(8, ARM64Reg::D0, ARM64Reg::D0);
};
storeSingleU8 = GetCodePtr(); constexpr u32 flags =
emit_quantize(); BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_8;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.ST1(8, ARM64Reg::Q0, 0, addr_reg); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storeSingleU8Slow = GetCodePtr();
emit_quantize();
float_emit.UMOV(8, ARM64Reg::W0, ARM64Reg::Q0, 0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U8);
BR(ARM64Reg::X2);
} }
const u8* storeSingleS8; const u8* storeSingleS8 = GetCodePtr();
const u8* storeSingleS8Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -835,24 +832,17 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SQXTN(8, ARM64Reg::D0, ARM64Reg::D0); float_emit.SQXTN(8, ARM64Reg::D0, ARM64Reg::D0);
};
storeSingleS8 = GetCodePtr(); constexpr u32 flags =
emit_quantize(); BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_8;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.ST1(8, ARM64Reg::Q0, 0, addr_reg); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storeSingleS8Slow = GetCodePtr();
emit_quantize();
float_emit.SMOV(8, ARM64Reg::W0, ARM64Reg::Q0, 0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U8);
BR(ARM64Reg::X2);
} }
const u8* storeSingleU16; // Used by MKWii const u8* storeSingleU16 = GetCodePtr(); // Used by MKWii
const u8* storeSingleU16Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -860,25 +850,17 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZU(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.UQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
};
storeSingleU16 = GetCodePtr(); constexpr u32 flags =
emit_quantize(); BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_16;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
float_emit.ST1(16, ARM64Reg::Q0, 0, addr_reg); gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storeSingleU16Slow = GetCodePtr();
emit_quantize();
float_emit.UMOV(16, ARM64Reg::W0, ARM64Reg::Q0, 0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U16);
BR(ARM64Reg::X2);
} }
const u8* storeSingleS16; const u8* storeSingleS16 = GetCodePtr();
const u8* storeSingleS16Slow;
{ {
auto emit_quantize = [this, &float_emit, scale_reg]() {
MOVP2R(ARM64Reg::X2, &m_quantizeTableS); MOVP2R(ARM64Reg::X2, &m_quantizeTableS);
ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3)); ADD(scale_reg, ARM64Reg::X2, scale_reg, ArithOption(scale_reg, ShiftType::LSL, 3));
float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0); float_emit.LDR(32, IndexType::Unsigned, ARM64Reg::D1, scale_reg, 0);
@ -886,28 +868,21 @@ void JitArm64::GenerateQuantizedLoadStores()
float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0); float_emit.FCVTZS(32, ARM64Reg::D0, ARM64Reg::D0);
float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0); float_emit.SQXTN(16, ARM64Reg::D0, ARM64Reg::D0);
};
storeSingleS16 = GetCodePtr(); constexpr u32 flags =
emit_quantize(); BackPatchInfo::FLAG_STORE | BackPatchInfo::FLAG_FLOAT | BackPatchInfo::FLAG_SIZE_16;
ADD(addr_reg, addr_reg, MEM_REG);
float_emit.REV16(8, ARM64Reg::D0, ARM64Reg::D0); EmitBackpatchRoutine(flags, jo.fastmem_arena, jo.fastmem_arena, ARM64Reg::D0, addr_reg,
float_emit.ST1(16, ARM64Reg::Q0, 0, addr_reg); gprs_to_push, fprs_to_push, true);
RET(ARM64Reg::X30); RET(ARM64Reg::X30);
storeSingleS16Slow = GetCodePtr();
emit_quantize();
float_emit.SMOV(16, ARM64Reg::W0, ARM64Reg::Q0, 0);
MOVP2R(ARM64Reg::X2, &PowerPC::Write_U16);
BR(ARM64Reg::X2);
} }
JitRegister::Register(start, GetCodePtr(), "JIT_QuantizedStore"); JitRegister::Register(start, GetCodePtr(), "JIT_QuantizedStore");
paired_store_quantized = reinterpret_cast<const u8**>(AlignCode16()); paired_store_quantized = reinterpret_cast<const u8**>(AlignCode16());
ReserveCodeSpace(32 * sizeof(u8*)); ReserveCodeSpace(8 * sizeof(u8*));
// Fast
paired_store_quantized[0] = storePairedFloat; paired_store_quantized[0] = storePairedFloat;
paired_store_quantized[1] = storePairedIllegal; paired_store_quantized[1] = storePairedIllegal;
paired_store_quantized[2] = storePairedIllegal; paired_store_quantized[2] = storePairedIllegal;
@ -917,31 +892,15 @@ void JitArm64::GenerateQuantizedLoadStores()
paired_store_quantized[6] = storePairedS8; paired_store_quantized[6] = storePairedS8;
paired_store_quantized[7] = storePairedS16; paired_store_quantized[7] = storePairedS16;
paired_store_quantized[8] = storeSingleFloat; single_store_quantized = reinterpret_cast<const u8**>(AlignCode16());
paired_store_quantized[9] = storePairedIllegal; ReserveCodeSpace(8 * sizeof(u8*));
paired_store_quantized[10] = storePairedIllegal;
paired_store_quantized[11] = storePairedIllegal;
paired_store_quantized[12] = storeSingleU8;
paired_store_quantized[13] = storeSingleU16;
paired_store_quantized[14] = storeSingleS8;
paired_store_quantized[15] = storeSingleS16;
// Slow single_store_quantized[0] = storeSingleFloat;
paired_store_quantized[16] = storePairedFloatSlow; single_store_quantized[1] = storePairedIllegal;
paired_store_quantized[17] = storePairedIllegal; single_store_quantized[2] = storePairedIllegal;
paired_store_quantized[18] = storePairedIllegal; single_store_quantized[3] = storePairedIllegal;
paired_store_quantized[19] = storePairedIllegal; single_store_quantized[4] = storeSingleU8;
paired_store_quantized[20] = storePairedU8Slow; single_store_quantized[5] = storeSingleU16;
paired_store_quantized[21] = storePairedU16Slow; single_store_quantized[6] = storeSingleS8;
paired_store_quantized[22] = storePairedS8Slow; single_store_quantized[7] = storeSingleS16;
paired_store_quantized[23] = storePairedS16Slow;
paired_store_quantized[24] = storeSingleFloatSlow;
paired_store_quantized[25] = storePairedIllegal;
paired_store_quantized[26] = storePairedIllegal;
paired_store_quantized[27] = storePairedIllegal;
paired_store_quantized[28] = storeSingleU8Slow;
paired_store_quantized[29] = storeSingleU16Slow;
paired_store_quantized[30] = storeSingleS8Slow;
paired_store_quantized[31] = storeSingleS16Slow;
} }

70
Source/Core/Core/PowerPC/JitArm64/Jit_Util.cpp
View File

@ -191,19 +191,47 @@ private:
bool m_sign_extend; bool m_sign_extend;
}; };
void ByteswapAfterLoad(ARM64XEmitter* emit, ARM64Reg dst_reg, ARM64Reg src_reg, u32 flags, void SwapPairs(ARM64XEmitter* emit, ARM64Reg dst_reg, ARM64Reg src_reg, u32 flags)
bool is_reversed, bool is_extended) {
if (flags & BackPatchInfo::FLAG_SIZE_32)
emit->ROR(dst_reg, src_reg, 32);
else if (flags & BackPatchInfo::FLAG_SIZE_16)
emit->ROR(dst_reg, src_reg, 16);
else
emit->REV16(dst_reg, src_reg);
}
void ByteswapAfterLoad(ARM64XEmitter* emit, Arm64Gen::ARM64FloatEmitter* float_emit,
ARM64Reg dst_reg, ARM64Reg src_reg, u32 flags, bool is_reversed,
bool is_extended)
{ {
if (is_reversed == !(flags & BackPatchInfo::FLAG_REVERSE)) if (is_reversed == !(flags & BackPatchInfo::FLAG_REVERSE))
{ {
if (flags & BackPatchInfo::FLAG_SIZE_32) if (flags & BackPatchInfo::FLAG_SIZE_64)
{ {
if (flags & BackPatchInfo::FLAG_FLOAT)
float_emit->REV64(8, dst_reg, src_reg);
else
emit->REV64(dst_reg, src_reg);
src_reg = dst_reg;
}
else if (flags & BackPatchInfo::FLAG_SIZE_32)
{
if (flags & BackPatchInfo::FLAG_FLOAT)
float_emit->REV32(8, dst_reg, src_reg);
else
emit->REV32(dst_reg, src_reg); emit->REV32(dst_reg, src_reg);
src_reg = dst_reg; src_reg = dst_reg;
} }
else if (flags & BackPatchInfo::FLAG_SIZE_16) else if (flags & BackPatchInfo::FLAG_SIZE_16)
{ {
if (flags & BackPatchInfo::FLAG_FLOAT)
float_emit->REV16(8, dst_reg, src_reg);
else
emit->REV16(dst_reg, src_reg); emit->REV16(dst_reg, src_reg);
src_reg = dst_reg; src_reg = dst_reg;
} }
} }
@ -215,24 +243,46 @@ void ByteswapAfterLoad(ARM64XEmitter* emit, ARM64Reg dst_reg, ARM64Reg src_reg,
} }
if (dst_reg != src_reg) if (dst_reg != src_reg)
{
if (flags & BackPatchInfo::FLAG_FLOAT)
float_emit->ORR(dst_reg, src_reg, src_reg);
else
emit->MOV(dst_reg, src_reg); emit->MOV(dst_reg, src_reg);
}
} }
ARM64Reg ByteswapBeforeStore(ARM64XEmitter* emit, ARM64Reg tmp_reg, ARM64Reg src_reg, u32 flags, ARM64Reg ByteswapBeforeStore(ARM64XEmitter* emit, Arm64Gen::ARM64FloatEmitter* float_emit,
bool want_reversed) ARM64Reg tmp_reg, ARM64Reg src_reg, u32 flags, bool want_reversed)
{ {
ARM64Reg dst_reg = src_reg; ARM64Reg dst_reg = src_reg;
if (want_reversed == !(flags & BackPatchInfo::FLAG_REVERSE)) if (want_reversed == !(flags & BackPatchInfo::FLAG_REVERSE))
{ {
if (flags & BackPatchInfo::FLAG_SIZE_32) if (flags & BackPatchInfo::FLAG_SIZE_64)
{ {
dst_reg = tmp_reg; dst_reg = tmp_reg;
if (flags & BackPatchInfo::FLAG_FLOAT)
float_emit->REV64(8, dst_reg, src_reg);
else
emit->REV64(dst_reg, src_reg);
}
else if (flags & BackPatchInfo::FLAG_SIZE_32)
{
dst_reg = tmp_reg;
if (flags & BackPatchInfo::FLAG_FLOAT)
float_emit->REV32(8, dst_reg, src_reg);
else
emit->REV32(dst_reg, src_reg); emit->REV32(dst_reg, src_reg);
} }
else if (flags & BackPatchInfo::FLAG_SIZE_16) else if (flags & BackPatchInfo::FLAG_SIZE_16)
{ {
dst_reg = tmp_reg; dst_reg = tmp_reg;
if (flags & BackPatchInfo::FLAG_FLOAT)
float_emit->REV16(8, dst_reg, src_reg);
else
emit->REV16(dst_reg, src_reg); emit->REV16(dst_reg, src_reg);
} }
} }
@ -243,6 +293,8 @@ ARM64Reg ByteswapBeforeStore(ARM64XEmitter* emit, ARM64Reg tmp_reg, ARM64Reg src
void MMIOLoadToReg(MMIO::Mapping* mmio, Arm64Gen::ARM64XEmitter* emit, BitSet32 gprs_in_use, void MMIOLoadToReg(MMIO::Mapping* mmio, Arm64Gen::ARM64XEmitter* emit, BitSet32 gprs_in_use,
BitSet32 fprs_in_use, ARM64Reg dst_reg, u32 address, u32 flags) BitSet32 fprs_in_use, ARM64Reg dst_reg, u32 address, u32 flags)
{ {
ASSERT(!(flags & BackPatchInfo::FLAG_FLOAT));
if (flags & BackPatchInfo::FLAG_SIZE_8) if (flags & BackPatchInfo::FLAG_SIZE_8)
{ {
MMIOReadCodeGenerator<u8> gen(emit, gprs_in_use, fprs_in_use, dst_reg, address, MMIOReadCodeGenerator<u8> gen(emit, gprs_in_use, fprs_in_use, dst_reg, address,
@ -262,13 +314,15 @@ void MMIOLoadToReg(MMIO::Mapping* mmio, Arm64Gen::ARM64XEmitter* emit, BitSet32
mmio->GetHandlerForRead<u32>(address).Visit(gen); mmio->GetHandlerForRead<u32>(address).Visit(gen);
} }
ByteswapAfterLoad(emit, dst_reg, dst_reg, flags, false, true); ByteswapAfterLoad(emit, nullptr, dst_reg, dst_reg, flags, false, true);
} }
void MMIOWriteRegToAddr(MMIO::Mapping* mmio, Arm64Gen::ARM64XEmitter* emit, BitSet32 gprs_in_use, void MMIOWriteRegToAddr(MMIO::Mapping* mmio, Arm64Gen::ARM64XEmitter* emit, BitSet32 gprs_in_use,
BitSet32 fprs_in_use, ARM64Reg src_reg, u32 address, u32 flags) BitSet32 fprs_in_use, ARM64Reg src_reg, u32 address, u32 flags)
{ {
src_reg = ByteswapBeforeStore(emit, ARM64Reg::W1, src_reg, flags, false); ASSERT(!(flags & BackPatchInfo::FLAG_FLOAT));
src_reg = ByteswapBeforeStore(emit, nullptr, ARM64Reg::W1, src_reg, flags, false);
if (flags & BackPatchInfo::FLAG_SIZE_8) if (flags & BackPatchInfo::FLAG_SIZE_8)
{ {

14
Source/Core/Core/PowerPC/JitArm64/Jit_Util.h
View File

@ -8,11 +8,17 @@
#include "Core/HW/MMIO.h" #include "Core/HW/MMIO.h"
void ByteswapAfterLoad(Arm64Gen::ARM64XEmitter* emit, Arm64Gen::ARM64Reg dst_reg, void SwapPairs(Arm64Gen::ARM64XEmitter* emit, Arm64Gen::ARM64Reg dst_reg,
Arm64Gen::ARM64Reg src_reg, u32 flags, bool is_reversed, bool is_extended); Arm64Gen::ARM64Reg src_reg, u32 flags);
Arm64Gen::ARM64Reg ByteswapBeforeStore(Arm64Gen::ARM64XEmitter* emit, Arm64Gen::ARM64Reg tmp_reg, void ByteswapAfterLoad(Arm64Gen::ARM64XEmitter* emit, Arm64Gen::ARM64FloatEmitter* float_emit,
Arm64Gen::ARM64Reg src_reg, u32 flags, bool want_reversed); Arm64Gen::ARM64Reg dst_reg, Arm64Gen::ARM64Reg src_reg, u32 flags,
bool is_reversed, bool is_extended);
Arm64Gen::ARM64Reg ByteswapBeforeStore(Arm64Gen::ARM64XEmitter* emit,
Arm64Gen::ARM64FloatEmitter* float_emit,
Arm64Gen::ARM64Reg tmp_reg, Arm64Gen::ARM64Reg src_reg,
u32 flags, bool want_reversed);
void MMIOLoadToReg(MMIO::Mapping* mmio, Arm64Gen::ARM64XEmitter* emit, BitSet32 gprs_in_use, void MMIOLoadToReg(MMIO::Mapping* mmio, Arm64Gen::ARM64XEmitter* emit, BitSet32 gprs_in_use,
BitSet32 fprs_in_use, Arm64Gen::ARM64Reg dst_reg, u32 address, u32 flags); BitSet32 fprs_in_use, Arm64Gen::ARM64Reg dst_reg, u32 address, u32 flags);

31
Source/Core/Core/PowerPC/JitArmCommon/BackPatch.h
View File

@ -13,31 +13,32 @@ struct BackPatchInfo
FLAG_SIZE_8 = (1 << 2), FLAG_SIZE_8 = (1 << 2),
FLAG_SIZE_16 = (1 << 3), FLAG_SIZE_16 = (1 << 3),
FLAG_SIZE_32 = (1 << 4), FLAG_SIZE_32 = (1 << 4),
FLAG_SIZE_F32 = (1 << 5), FLAG_SIZE_64 = (1 << 5),
FLAG_SIZE_F32X2 = (1 << 6), FLAG_FLOAT = (1 << 6),
FLAG_SIZE_F64 = (1 << 7), FLAG_PAIR = (1 << 7),
FLAG_REVERSE = (1 << 8), FLAG_REVERSE = (1 << 8),
FLAG_EXTEND = (1 << 9), FLAG_EXTEND = (1 << 9),
FLAG_ZERO_256 = (1 << 10), FLAG_ZERO_256 = (1 << 10),
FLAG_MASK_FLOAT = FLAG_SIZE_F32 | FLAG_SIZE_F32X2 | FLAG_SIZE_F64,
}; };
static u32 GetFlagSize(u32 flags) static u32 GetFlagSize(u32 flags)
{ {
u32 size = 0;
if (flags & FLAG_SIZE_8) if (flags & FLAG_SIZE_8)
return 8; size = 8;
if (flags & FLAG_SIZE_16) if (flags & FLAG_SIZE_16)
return 16; size = 16;
if (flags & FLAG_SIZE_32) if (flags & FLAG_SIZE_32)
return 32; size = 32;
if (flags & FLAG_SIZE_F32) if (flags & FLAG_SIZE_64)
return 32; size = 64;
if (flags & FLAG_SIZE_F32X2)
return 64;
if (flags & FLAG_SIZE_F64)
return 64;
if (flags & FLAG_ZERO_256) if (flags & FLAG_ZERO_256)
return 256; size = 256;
return 0;
if (flags & FLAG_PAIR)
size *= 2;
return size;
} }
}; };