ShuriZma
/
dolphin
mirror of https://github.com/dolphin-emu/dolphin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JIT: fix handling of PC in dispatcher/block cache. 

Specifically, don't make any assumptions about what effective addresses
are used for code, and correctly handle changes to MSR.DR/MSR.IR.

(Split off from dynamic-bat.)
This commit is contained in:
magumagu 2016-06-25 18:57:16 -07:00 committed by degasus
parent fa5a2474f4
commit 758e6406cd
16 changed files with 361 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Source/Core/Core/PowerPC/CachedInterpreter.cpp
View File

@ -41,10 +41,8 @@ void CachedInterpreter::Run()
void CachedInterpreter::SingleStep()
{
int block = GetBlockNumberFromStartAddress(PC);
if (block >= 0)
{
Instruction* code = (Instruction*)GetCompiledCodeFromBlock(block);
const u8* normalEntry = jit->GetBlockCache()->Dispatch();
const Instruction* code = reinterpret_cast<const Instruction*>(normalEntry);
while (true)
{
@ -68,9 +66,6 @@ void CachedInterpreter::SingleStep()
}
}
Jit(PC);
}
static void EndBlock(UGeckoInstruction data)
{
PC = NPC;

17
Source/Core/Core/PowerPC/Jit64/Jit.cpp
View File

@ -396,29 +396,12 @@ void Jit64::JustWriteExit(u32 destination, bool bl, u32 after)
linkData.exitAddress = destination;
linkData.linkStatus = false;
// Link opportunity!
int block;
if (jo.enableBlocklink && (block = blocks.GetBlockNumberFromStartAddress(destination)) >= 0)
{
// It exists! Joy of joy!
JitBlock* jb = blocks.GetBlock(block);
const u8* addr = jb->checkedEntry;
linkData.exitPtrs = GetWritableCodePtr();
if (bl)
CALL(addr);
else
JMP(addr, true);
linkData.linkStatus = true;
}
else
{
MOV(32, PPCSTATE(pc), Imm32(destination));
linkData.exitPtrs = GetWritableCodePtr();
if (bl)
CALL(asm_routines.dispatcher);
else
JMP(asm_routines.dispatcher, true);
}
b->linkData.push_back(linkData);

117
Source/Core/Core/PowerPC/Jit64/JitAsm.cpp
View File

@ -64,7 +64,7 @@ void Jit64AsmRoutineManager::Generate()
ABI_PopRegistersAndAdjustStack(1 << RSCRATCH2, 0);
#endif
ResetStack();
ResetStack(*this);
SUB(32, PPCSTATE(downcount), R(RSCRATCH2));
@ -102,31 +102,15 @@ void Jit64AsmRoutineManager::Generate()
MOV(64, R(RMEM), Imm64((u64)Memory::logical_base));
SetJumpTarget(membaseend);
// The following is an translation of JitBaseBlockCache::Dispatch into assembly.
// Fast block number lookup.
MOV(32, R(RSCRATCH), PPCSTATE(pc));
// TODO: We need to handle code which executes the same PC with
// different values of MSR.IR. It probably makes sense to handle
// MSR.DR here too, to allow IsOptimizableRAMAddress-based
// optimizations safe, because IR and DR are usually set/cleared together.
// TODO: Branching based on the 20 most significant bits of instruction
// addresses without translating them is wrong.
u64 icache = (u64)jit->GetBlockCache()->iCache.data();
u64 icacheVmem = (u64)jit->GetBlockCache()->iCacheVMEM.data();
u64 icacheEx = (u64)jit->GetBlockCache()->iCacheEx.data();
u32 mask = 0;
FixupBranch no_mem;
FixupBranch exit_mem;
FixupBranch exit_vmem;
if (SConfig::GetInstance().bWii)
mask = JIT_ICACHE_EXRAM_BIT;
mask |= JIT_ICACHE_VMEM_BIT;
TEST(32, R(RSCRATCH), Imm32(mask));
no_mem = J_CC(CC_NZ);
AND(32, R(RSCRATCH), Imm32(JIT_ICACHE_MASK));
u64 icache = reinterpret_cast<u64>(jit->GetBlockCache()->GetICache());
AND(32, R(RSCRATCH), Imm32(JitBaseBlockCache::iCache_Mask << 2));
if (icache <= INT_MAX)
{
MOV(32, R(RSCRATCH), MDisp(RSCRATCH, (s32)icache));
MOV(32, R(RSCRATCH), MDisp(RSCRATCH, static_cast<s32>(icache)));
}
else
{
@ -134,73 +118,46 @@ void Jit64AsmRoutineManager::Generate()
MOV(32, R(RSCRATCH), MRegSum(RSCRATCH2, RSCRATCH));
}
exit_mem = J();
SetJumpTarget(no_mem);
TEST(32, R(RSCRATCH), Imm32(JIT_ICACHE_VMEM_BIT));
FixupBranch no_vmem = J_CC(CC_Z);
AND(32, R(RSCRATCH), Imm32(JIT_ICACHE_MASK));
if (icacheVmem <= INT_MAX)
// Check whether the block we found matches the current state.
u64 blocks = reinterpret_cast<u64>(jit->GetBlockCache()->GetBlocks());
IMUL(32, RSCRATCH, R(RSCRATCH), Imm32(sizeof(JitBlock)));
if (blocks <= INT_MAX)
{
MOV(32, R(RSCRATCH), MDisp(RSCRATCH, (s32)icacheVmem));
ADD(64, R(RSCRATCH), Imm32(static_cast<s32>(blocks)));
}
else
{
MOV(64, R(RSCRATCH2), Imm64(icacheVmem));
MOV(32, R(RSCRATCH), MRegSum(RSCRATCH2, RSCRATCH));
}
if (SConfig::GetInstance().bWii)
exit_vmem = J();
SetJumpTarget(no_vmem);
if (SConfig::GetInstance().bWii)
{
TEST(32, R(RSCRATCH), Imm32(JIT_ICACHE_EXRAM_BIT));
FixupBranch no_exram = J_CC(CC_Z);
AND(32, R(RSCRATCH), Imm32(JIT_ICACHEEX_MASK));
if (icacheEx <= INT_MAX)
{
MOV(32, R(RSCRATCH), MDisp(RSCRATCH, (s32)icacheEx));
}
else
{
MOV(64, R(RSCRATCH2), Imm64(icacheEx));
MOV(32, R(RSCRATCH), MRegSum(RSCRATCH2, RSCRATCH));
}
SetJumpTarget(no_exram);
}
SetJumpTarget(exit_mem);
if (SConfig::GetInstance().bWii)
SetJumpTarget(exit_vmem);
TEST(32, R(RSCRATCH), R(RSCRATCH));
FixupBranch notfound = J_CC(CC_L);
// grab from list and jump to it
u64 codePointers = (u64)jit->GetBlockCache()->GetCodePointers();
if (codePointers <= INT_MAX)
{
JMPptr(MScaled(RSCRATCH, SCALE_8, (s32)codePointers));
}
else
{
MOV(64, R(RSCRATCH2), Imm64(codePointers));
JMPptr(MComplex(RSCRATCH2, RSCRATCH, SCALE_8, 0));
MOV(64, R(RSCRATCH2), Imm64(blocks));
ADD(64, R(RSCRATCH), R(RSCRATCH2));
}
// Check both block.effectiveAddress and block.msrBits.
MOV(32, R(RSCRATCH2), PPCSTATE(msr));
AND(32, R(RSCRATCH2), Imm32(JitBlock::JIT_CACHE_MSR_MASK));
SHL(64, R(RSCRATCH2), Imm8(32));
MOV(32, R(RSCRATCH_EXTRA), PPCSTATE(pc));
OR(64, R(RSCRATCH2), R(RSCRATCH_EXTRA));
CMP(64, R(RSCRATCH2), MDisp(RSCRATCH, static_cast<s32>(offsetof(JitBlock, effectiveAddress))));
FixupBranch notfound = J_CC(CC_NE);
// Success; branch to the block we found.
JMPptr(MDisp(RSCRATCH, static_cast<s32>(offsetof(JitBlock, normalEntry))));
SetJumpTarget(notfound);
// Failure; call into the block cache to update the state, then try again.
// (We need to loop because Jit() might not actually generate a block
// if we hit an ISI.)
// We reset the stack because Jit might clear the code cache.
// Also if we are in the middle of disabling BLR optimization on windows
// we need to reset the stack before _resetstkoflw() is called in Jit
// otherwise we will generate a second stack overflow exception during DoJit()
ResetStack();
ResetStack(*this);
// Ok, no block, let's jit
// Ok, no block, let's call the slow dispatcher
ABI_PushRegistersAndAdjustStack({}, 0);
ABI_CallFunctionA(32, (void*)&Jit, PPCSTATE(pc));
ABI_CallFunction(reinterpret_cast<void*>(&JitBase::Dispatch));
ABI_PopRegistersAndAdjustStack({}, 0);
JMP(dispatcherNoCheck, true); // no point in special casing this
// JMPptr(R(ABI_RETURN));
JMP(dispatcherNoCheck, true);
SetJumpTarget(bail);
doTiming = GetCodePtr();
@ -217,7 +174,7 @@ void Jit64AsmRoutineManager::Generate()
// Landing pad for drec space
if (SConfig::GetInstance().bEnableDebugging)
SetJumpTarget(dbg_exit);
ResetStack();
ResetStack(*this);
if (m_stack_top)
{
ADD(64, R(RSP), Imm8(0x18));
@ -232,12 +189,12 @@ void Jit64AsmRoutineManager::Generate()
GenerateCommon();
}
void Jit64AsmRoutineManager::ResetStack()
void Jit64AsmRoutineManager::ResetStack(X64CodeBlock& emitter)
{
if (m_stack_top)
MOV(64, R(RSP), Imm64((u64)m_stack_top - 0x20));
emitter.MOV(64, R(RSP), Imm64((u64)m_stack_top - 0x20));
else
MOV(64, R(RSP), M(&s_saved_rsp));
emitter.MOV(64, R(RSP), M(&s_saved_rsp));
}
void Jit64AsmRoutineManager::GenerateCommon()

2
Source/Core/Core/PowerPC/Jit64/JitAsm.h
View File

@ -25,7 +25,6 @@ class Jit64AsmRoutineManager : public CommonAsmRoutines
{
private:
void Generate();
void ResetStack();
void GenerateCommon();
u8* m_stack_top;
@ -41,4 +40,5 @@ public:
}
void Shutdown() { FreeCodeSpace(); }
void ResetStack(X64CodeBlock& emitter);
};

1
Source/Core/Core/PowerPC/Jit64/Jit_LoadStore.cpp
View File

@ -310,6 +310,7 @@ void Jit64::dcbx(UGeckoInstruction inst)
XOR(32, R(ABI_PARAM3), R(ABI_PARAM3));
ABI_CallFunction((void*)JitInterface::InvalidateICache);
ABI_PopRegistersAndAdjustStack(registersInUse, 0);
asm_routines.ResetStack(*this);
c = J(true);
SwitchToNearCode();
SetJumpTarget(c);

4
Source/Core/Core/PowerPC/Jit64/Jit_SystemRegisters.cpp
View File

@ -391,6 +391,10 @@ void Jit64::mtmsr(UGeckoInstruction inst)
gpr.Flush();
fpr.Flush();
// Our jit cache also stores some MSR bits, as they have changed, we either
// have to validate them in the BLR/RET check, or just flush the stack here.
asm_routines.ResetStack(*this);
// If some exceptions are pending and EE are now enabled, force checking
// external exceptions when going out of mtmsr in order to execute delayed
// interrupts as soon as possible.

12
Source/Core/Core/PowerPC/Jit64IL/JitIL.cpp
View File

@ -373,19 +373,9 @@ void JitIL::WriteExit(u32 destination)
linkData.exitPtrs = GetWritableCodePtr();
linkData.linkStatus = false;
// Link opportunity!
int block;
if (jo.enableBlocklink && (block = blocks.GetBlockNumberFromStartAddress(destination)) >= 0)
{
// It exists! Joy of joy!
JMP(blocks.GetBlock(block)->checkedEntry, true);
linkData.linkStatus = true;
}
else
{
MOV(32, PPCSTATE(pc), Imm32(destination));
JMP(asm_routines.dispatcher, true);
}
b->linkData.push_back(linkData);
}

2
Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp
View File

@ -17,7 +17,7 @@ void JitArm64BlockCache::WriteLinkBlock(u8* location, const JitBlock& block)
{
emit.B(CC_LE, block.normalEntry);
// We can't write DISPATCHER_PC here, as blink linking is only for 8bytes.
// We can't write DISPATCHER_PC here, as block linking may only use 8 bytes.
// So we'll hit two jumps when calling Advance.
emit.B(block.checkedEntry);
}

5
Source/Core/Core/PowerPC/JitArm64/JitArm64_LoadStore.cpp
View File

@ -791,11 +791,6 @@ void JitArm64::dcbz(UGeckoInstruction inst)
int a = inst.RA, b = inst.RB;
u32 mem_mask = Memory::ADDR_MASK_HW_ACCESS;
// The following masks the region used by the GC/Wii virtual memory lib
mem_mask |= Memory::ADDR_MASK_MEM1;
gpr.Lock(W0);
ARM64Reg addr_reg = W0;

78
Source/Core/Core/PowerPC/JitArm64/JitAsm.cpp
View File

@ -48,54 +48,54 @@ void JitArm64::GenerateAsm()
dispatcherNoCheck = GetCodePtr();
FixupBranch exram, vmem, not_exram, not_vmem;
bool assembly_dispatcher = true;
if (assembly_dispatcher)
{
// iCache[(address >> 2) & iCache_Mask];
ARM64Reg pc_masked = W25;
ARM64Reg cache_base = X27;
ARM64Reg block_num = W27;
ANDI2R(pc_masked, DISPATCHER_PC, JitBaseBlockCache::iCache_Mask << 2);
MOVP2R(cache_base, jit->GetBlockCache()->GetICache());
LDR(block_num, cache_base, EncodeRegTo64(pc_masked));
// VMEM
not_vmem = TBZ(DISPATCHER_PC, IntLog2(JIT_ICACHE_VMEM_BIT));
ANDI2R(pc_masked, DISPATCHER_PC, JIT_ICACHE_MASK);
MOVI2R(cache_base, (u64)jit->GetBlockCache()->iCacheVMEM.data());
vmem = B();
SetJumpTarget(not_vmem);
// blocks[block_num]
ARM64Reg block = X30;
ARM64Reg jit_block_size = W24;
MOVI2R(jit_block_size, sizeof(JitBlock));
MUL(block_num, block_num, jit_block_size);
MOVP2R(block, jit->GetBlockCache()->GetBlocks());
ADD(block, block, EncodeRegTo64(block_num));
if (SConfig::GetInstance().bWii)
{
// Wii EX-RAM
not_exram = TBZ(DISPATCHER_PC, IntLog2(JIT_ICACHE_EXRAM_BIT));
ANDI2R(pc_masked, DISPATCHER_PC, JIT_ICACHEEX_MASK);
MOVI2R(cache_base, (u64)jit->GetBlockCache()->iCacheEx.data());
exram = B();
SetJumpTarget(not_exram);
// b.effectiveAddress != addr || b.msrBits != msr
ARM64Reg pc_and_msr = W25;
ARM64Reg pc_and_msr2 = W24;
LDR(INDEX_UNSIGNED, pc_and_msr, block, offsetof(JitBlock, effectiveAddress));
CMP(pc_and_msr, DISPATCHER_PC);
FixupBranch pc_missmatch = B(CC_NEQ);
LDR(INDEX_UNSIGNED, pc_and_msr2, PPC_REG, PPCSTATE_OFF(msr));
ANDI2R(pc_and_msr2, pc_and_msr2, JitBlock::JIT_CACHE_MSR_MASK);
LDR(INDEX_UNSIGNED, pc_and_msr, block, offsetof(JitBlock, msrBits));
CMP(pc_and_msr, pc_and_msr2);
FixupBranch msr_missmatch = B(CC_NEQ);
// return blocks[block_num].normalEntry;
LDR(INDEX_UNSIGNED, block, block, offsetof(JitBlock, normalEntry));
BR(block);
SetJumpTarget(pc_missmatch);
SetJumpTarget(msr_missmatch);
}
// Common memory
ANDI2R(pc_masked, DISPATCHER_PC, JIT_ICACHE_MASK);
MOVI2R(cache_base, (u64)jit->GetBlockCache()->iCache.data());
SetJumpTarget(vmem);
if (SConfig::GetInstance().bWii)
SetJumpTarget(exram);
LDR(W27, cache_base, EncodeRegTo64(pc_masked));
FixupBranch JitBlock = TBNZ(W27, 7); // Test the 7th bit
// Success, it is our Jitblock.
MOVI2R(X30, (u64)jit->GetBlockCache()->GetCodePointers());
UBFM(X27, X27, 61, 60); // Same as X27 << 3
LDR(X30, X30, X27); // Load the block address in to R14
BR(X30);
// No need to jump anywhere after here, the block will go back to dispatcher start
SetJumpTarget(JitBlock);
// Call C version of Dispatch().
// FIXME: Implement this in inline assembly.
STR(INDEX_UNSIGNED, DISPATCHER_PC, PPC_REG, PPCSTATE_OFF(pc));
MOVI2R(X30, (u64) & ::Jit);
MOVP2R(X30, reinterpret_cast<void*>(&JitBase::Dispatch));
BLR(X30);
LDR(INDEX_UNSIGNED, DISPATCHER_PC, PPC_REG, PPCSTATE_OFF(pc));
B(dispatcherNoCheck);
// Jump to next block.
BR(X0);
SetJumpTarget(bail);
doTiming = GetCodePtr();

7
Source/Core/Core/PowerPC/JitCommon/JitBase.h
View File

@ -55,6 +55,10 @@
#define JITDISABLE(setting) \
FALLBACK_IF(SConfig::GetInstance().bJITOff || SConfig::GetInstance().setting)
class JitBase;
extern JitBase* jit;
class JitBase : public CPUCoreBase
{
protected:
@ -125,6 +129,7 @@ public:
JitOptions jo;
JitState js;
static const u8* Dispatch() { return jit->GetBlockCache()->Dispatch(); };
virtual JitBaseBlockCache* GetBlockCache() = 0;
virtual void Jit(u32 em_address) = 0;
@ -147,8 +152,6 @@ public:
bool HandleFault(uintptr_t access_address, SContext* ctx) override;
};
extern JitBase* jit;
void Jit(u32 em_address);
// Merged routines that should be moved somewhere better

172
Source/Core/Core/PowerPC/JitCommon/JitCache.cpp
View File

@ -34,26 +34,15 @@ bool JitBaseBlockCache::IsFull() const
void JitBaseBlockCache::Init()
{
if (m_initialized)
{
PanicAlert("JitBaseBlockCache::Init() - iCache is already initialized");
return;
}
JitRegister::Init(SConfig::GetInstance().m_perfDir);
iCache.fill(JIT_ICACHE_INVALID_BYTE);
iCacheEx.fill(JIT_ICACHE_INVALID_BYTE);
iCacheVMEM.fill(JIT_ICACHE_INVALID_BYTE);
iCache.fill(0);
Clear();
m_initialized = true;
}
void JitBaseBlockCache::Shutdown()
{
num_blocks = 0;
m_initialized = false;
JitRegister::Shutdown();
}
@ -80,7 +69,8 @@ void JitBaseBlockCache::Clear()
valid_block.ClearAll();
num_blocks = 0;
blockCodePointers.fill(nullptr);
blocks[0].msrBits = 0xFFFFFFFF;
blocks[0].invalid = true;
}
void JitBaseBlockCache::Reset()
@ -103,7 +93,9 @@ int JitBaseBlockCache::AllocateBlock(u32 em_address)
{
JitBlock& b = blocks[num_blocks];
b.invalid = false;
b.originalAddress = em_address;
b.effectiveAddress = em_address;
b.physicalAddress = PowerPC::JitCache_TranslateAddress(em_address).address;
b.msrBits = MSR & JitBlock::JIT_CACHE_MSR_MASK;
b.linkData.clear();
num_blocks++; // commit the current block
return num_blocks - 1;
@ -111,13 +103,23 @@ int JitBaseBlockCache::AllocateBlock(u32 em_address)
void JitBaseBlockCache::FinalizeBlock(int block_num, bool block_link, const u8* code_ptr)
{
blockCodePointers[block_num] = code_ptr;
JitBlock& b = blocks[block_num];
if (start_block_map.count(b.physicalAddress))
{
// We already have a block at this address; invalidate the old block.
// This should be very rare. This will only happen if the same block
// is called both with DR/IR enabled or disabled.
WARN_LOG(DYNA_REC, "Invalidating compiled block at same address %08x", b.physicalAddress);
int old_block_num = start_block_map[b.physicalAddress];
const JitBlock& old_b = blocks[old_block_num];
block_map.erase(
std::make_pair(old_b.physicalAddress + 4 * old_b.originalSize - 1, old_b.physicalAddress));
DestroyBlock(old_block_num, true);
}
start_block_map[b.physicalAddress] = block_num;
FastLookupEntryForAddress(b.effectiveAddress) = block_num;
std::memcpy(GetICachePtr(b.originalAddress), &block_num, sizeof(u32));
// Convert the logical address to a physical address for the block map
u32 pAddr = b.originalAddress & 0x1FFFFFFF;
u32 pAddr = b.physicalAddress;
for (u32 block = pAddr / 32; block <= (pAddr + (b.originalSize - 1) * 4) / 32; ++block)
valid_block.Set(block);
@ -135,46 +137,62 @@ void JitBaseBlockCache::FinalizeBlock(int block_num, bool block_link, const u8*
LinkBlockExits(block_num);
}
JitRegister::Register(blockCodePointers[block_num], b.codeSize, "JIT_PPC_%08x",
b.originalAddress);
JitRegister::Register(b.checkedEntry, b.codeSize, "JIT_PPC_%08x", b.physicalAddress);
}
const u8** JitBaseBlockCache::GetCodePointers()
int JitBaseBlockCache::GetBlockNumberFromStartAddress(u32 addr, u32 msr)
{
return blockCodePointers.data();
}
u8* JitBaseBlockCache::GetICachePtr(u32 addr)
u32 translated_addr = addr;
if (UReg_MSR(msr).IR)
{
if (addr & JIT_ICACHE_VMEM_BIT)
return &jit->GetBlockCache()->iCacheVMEM[addr & JIT_ICACHE_MASK];
if (addr & JIT_ICACHE_EXRAM_BIT)
return &jit->GetBlockCache()->iCacheEx[addr & JIT_ICACHEEX_MASK];
return &jit->GetBlockCache()->iCache[addr & JIT_ICACHE_MASK];
}
int JitBaseBlockCache::GetBlockNumberFromStartAddress(u32 addr)
auto translated = PowerPC::JitCache_TranslateAddress(addr);
if (!translated.valid)
{
u32 inst;
std::memcpy(&inst, GetICachePtr(addr), sizeof(u32));
if (inst & 0xfc000000) // definitely not a JIT block
return -1;
if ((int)inst >= num_blocks)
return -1;
if (blocks[inst].originalAddress != addr)
return -1;
return inst;
}
translated_addr = translated.address;
}
CompiledCode JitBaseBlockCache::GetCompiledCodeFromBlock(int block_num)
auto map_result = start_block_map.find(translated_addr);
if (map_result == start_block_map.end())
return -1;
int block_num = map_result->second;
const JitBlock& b = blocks[block_num];
if (b.invalid)
return -1;
if (b.effectiveAddress != addr)
return -1;
if (b.msrBits != (msr & JitBlock::JIT_CACHE_MSR_MASK))
return -1;
return block_num;
}
void JitBaseBlockCache::MoveBlockIntoFastCache(u32 addr, u32 msr)
{
return (CompiledCode)blockCodePointers[block_num];
int block_num = GetBlockNumberFromStartAddress(addr, msr);
if (block_num < 0)
{
Jit(addr);
}
else
{
FastLookupEntryForAddress(addr) = block_num;
LinkBlock(block_num);
}
}
const u8* JitBaseBlockCache::Dispatch()
{
int block_num = FastLookupEntryForAddress(PC);
while (blocks[block_num].effectiveAddress != PC ||
blocks[block_num].msrBits != (MSR & JitBlock::JIT_CACHE_MSR_MASK))
{
MoveBlockIntoFastCache(PC, MSR & JitBlock::JIT_CACHE_MSR_MASK);
block_num = FastLookupEntryForAddress(PC);
}
return blocks[block_num].normalEntry;
}
// Block linker
@ -195,7 +213,7 @@ void JitBaseBlockCache::LinkBlockExits(int i)
{
if (!e.linkStatus)
{
int destinationBlock = GetBlockNumberFromStartAddress(e.exitAddress);
int destinationBlock = GetBlockNumberFromStartAddress(e.exitAddress, b.msrBits);
if (destinationBlock != -1)
{
WriteLinkBlock(e.exitPtrs, blocks[destinationBlock]);
@ -208,17 +226,13 @@ void JitBaseBlockCache::LinkBlockExits(int i)
void JitBaseBlockCache::LinkBlock(int i)
{
LinkBlockExits(i);
JitBlock& b = blocks[i];
// equal_range(b) returns pair<iterator,iterator> representing the range
// of element with key b
auto ppp = links_to.equal_range(b.originalAddress);
if (ppp.first == ppp.second)
return;
const JitBlock& b = blocks[i];
auto ppp = links_to.equal_range(b.effectiveAddress);
for (auto iter = ppp.first; iter != ppp.second; ++iter)
{
// PanicAlert("Linking block %i to block %i", iter->second, i);
const JitBlock& b2 = blocks[iter->second];
if (b.msrBits == b2.msrBits)
LinkBlockExits(iter->second);
}
}
@ -226,21 +240,21 @@ void JitBaseBlockCache::LinkBlock(int i)
void JitBaseBlockCache::UnlinkBlock(int i)
{
JitBlock& b = blocks[i];
auto ppp = links_to.equal_range(b.originalAddress);
if (ppp.first == ppp.second)
return;
auto ppp = links_to.equal_range(b.effectiveAddress);
for (auto iter = ppp.first; iter != ppp.second; ++iter)
{
JitBlock& sourceBlock = blocks[iter->second];
if (sourceBlock.msrBits != b.msrBits)
continue;
for (auto& e : sourceBlock.linkData)
{
if (e.exitAddress == b.originalAddress)
if (e.exitAddress == b.effectiveAddress)
e.linkStatus = false;
}
}
links_to.erase(b.originalAddress);
links_to.erase(b.effectiveAddress);
}
void JitBaseBlockCache::DestroyBlock(int block_num, bool invalidate)
@ -258,20 +272,23 @@ void JitBaseBlockCache::DestroyBlock(int block_num, bool invalidate)
return;
}
b.invalid = true;
std::memcpy(GetICachePtr(b.originalAddress), &JIT_ICACHE_INVALID_WORD, sizeof(u32));
start_block_map.erase(b.physicalAddress);
FastLookupEntryForAddress(b.effectiveAddress) = 0;
UnlinkBlock(block_num);
// Send anyone who tries to run this block back to the dispatcher.
// Not entirely ideal, but .. pretty good.
// Spurious entrances from previously linked blocks can only come through checkedEntry
WriteDestroyBlock(b.checkedEntry, b.originalAddress);
WriteDestroyBlock(b.checkedEntry, b.effectiveAddress);
}
void JitBaseBlockCache::InvalidateICache(u32 address, const u32 length, bool forced)
{
// Convert the logical address to a physical address for the block map
u32 pAddr = address & 0x1FFFFFFF;
auto translated = PowerPC::JitCache_TranslateAddress(address);
if (!translated.valid)
return;
u32 pAddr = translated.address;
// Optimize the common case of length == 32 which is used by Interpreter::dcb*
bool destroy_block = true;
@ -288,20 +305,11 @@ void JitBaseBlockCache::InvalidateICache(u32 address, const u32 length, bool for
// address
if (destroy_block)
{
std::map<std::pair<u32, u32>, u32>::iterator it1 = block_map.lower_bound(
std::make_pair(pAddr, 0)),
it2 = it1;
while (it2 != block_map.end() && it2->first.second < pAddr + length)
auto it = block_map.lower_bound(std::make_pair(pAddr, 0));
while (it != block_map.end() && it->first.second < pAddr + length)
{
JitBlock& b = blocks[it2->second];
std::memcpy(GetICachePtr(b.originalAddress), &JIT_ICACHE_INVALID_WORD, sizeof(u32));
DestroyBlock(it2->second, true);
++it2;
}
if (it1 != it2)
{
block_map.erase(it1, it2);
DestroyBlock(it->second, true);
it = block_map.erase(it);
}
// If the code was actually modified, we need to clear the relevant entries from the

114
Source/Core/Core/PowerPC/JitCommon/JitCache.h
View File

@ -12,32 +12,55 @@
#include "Common/CommonTypes.h"
static const u32 JIT_ICACHE_SIZE = 0x2000000;
static const u32 JIT_ICACHE_MASK = 0x1ffffff;
static const u32 JIT_ICACHEEX_SIZE = 0x4000000;
static const u32 JIT_ICACHEEX_MASK = 0x3ffffff;
static const u32 JIT_ICACHE_EXRAM_BIT = 0x10000000;
static const u32 JIT_ICACHE_VMEM_BIT = 0x20000000;
// This corresponds to opcode 5 which is invalid in PowerPC
static const u32 JIT_ICACHE_INVALID_BYTE = 0x80;
static const u32 JIT_ICACHE_INVALID_WORD = 0x80808080;
// A JitBlock is block of compiled code which corresponds to the PowerPC
// code at a given address.
//
// The notion of the address of a block is a bit complicated because of the
// way address translation works, but basically it's the combination of an
// effective address, the address translation bits in MSR, and the physical
// address.
struct JitBlock
{
enum
{
// Mask for the MSR bits which determine whether a compiled block
// is valid (MSR.IR and MSR.DR, the address translation bits).
JIT_CACHE_MSR_MASK = 0x30,
};
// A special entry point for block linking; usually used to check the
// downcount.
const u8* checkedEntry;
// The normal entry point for the block, returned by Dispatch().
const u8* normalEntry;
u32 originalAddress;
// The effective address (PC) for the beginning of the block.
u32 effectiveAddress;
// The MSR bits expected for this block to be valid; see JIT_CACHE_MSR_MASK.
u32 msrBits;
// The physical address of the code represented by this block.
// Various maps in the cache are indexed by this (start_block_map,
// block_map, and valid_block in particular). This is useful because of
// of the way the instruction cache works on PowerPC.
u32 physicalAddress;
// The number of bytes of JIT'ed code contained in this block. Mostly
// useful for logging.
u32 codeSize;
// The number of PPC instructions represented by this block. Mostly
// useful for logging.
u32 originalSize;
int runCount; // for profiling.
// Whether this struct refers to a valid block. This is mostly useful as
// a debugging aid.
// FIXME: Change current users of invalid bit to assertions?
bool invalid;
// Information about exits to a known address from this block.
// This is used to implement block linking.
struct LinkData
{
u8* exitPtrs; // to be able to rewrite the exit jum
u8* exitPtrs; // to be able to rewrite the exit jump
u32 exitAddress;
bool linkStatus; // is it already linked?
};
@ -59,7 +82,12 @@ class ValidBlockBitSet final
public:
enum
{
VALID_BLOCK_MASK_SIZE = 0x20000000 / 32,
// ValidBlockBitSet covers the whole 32-bit address-space in 32-byte
// chunks.
// FIXME: Maybe we can get away with less? There isn't any actual
// RAM in most of this space.
VALID_BLOCK_MASK_SIZE = (1ULL << 32) / 32,
// The number of elements in the allocated array. Each u32 contains 32 bits.
VALID_BLOCK_ALLOC_ELEMENTS = VALID_BLOCK_MASK_SIZE / 32
};
// Directly accessed by Jit64.
@ -79,33 +107,53 @@ public:
class JitBaseBlockCache
{
enum
{
MAX_NUM_BLOCKS = 65536 * 2,
};
public:
static constexpr int MAX_NUM_BLOCKS = 65536 * 2;
static constexpr u32 iCache_Num_Elements = 0x10000;
static constexpr u32 iCache_Mask = iCache_Num_Elements - 1;
std::array<const u8*, MAX_NUM_BLOCKS> blockCodePointers;
std::array<JitBlock, MAX_NUM_BLOCKS> blocks;
private:
// We store the metadata of all blocks in a linear way within this array.
std::array<JitBlock, MAX_NUM_BLOCKS> blocks; // number -> JitBlock
int num_blocks;
std::multimap<u32, int> links_to;
// links_to hold all exit points of all valid blocks in a reverse way.
// It is used to query all blocks which links to an address.
std::multimap<u32, int> links_to; // destination_PC -> number
// Map indexed by the physical memory location.
// It is used to invalidate blocks based on memory location.
std::map<std::pair<u32, u32>, u32> block_map; // (end_addr, start_addr) -> number
// Map indexed by the physical address of the entry point.
// This is used to query the block based on the current PC in a slow way.
// TODO: This is redundant with block_map, and both should be a multimap.
std::map<u32, u32> start_block_map; // start_addr -> number
// This bitsets shows which cachelines overlap with any blocks.
// It is used to provide a fast way to query if no icache invalidation is needed.
ValidBlockBitSet valid_block;
bool m_initialized;
// This array is indexed with the masked PC and likely holds the correct block id.
// This is used as a fast cache of start_block_map used in the assembly dispatcher.
std::array<int, iCache_Num_Elements> iCache; // start_addr & mask -> number
void LinkBlockExits(int i);
void LinkBlock(int i);
void UnlinkBlock(int i);
u8* GetICachePtr(u32 addr);
void DestroyBlock(int block_num, bool invalidate);
void MoveBlockIntoFastCache(u32 em_address, u32 msr);
// Fast but risky block lookup based on iCache.
int& FastLookupEntryForAddress(u32 address) { return iCache[(address >> 2) & iCache_Mask]; }
// Virtual for overloaded
virtual void WriteLinkBlock(u8* location, const JitBlock& block) = 0;
virtual void WriteDestroyBlock(const u8* location, u32 address) = 0;
public:
JitBaseBlockCache() : num_blocks(0), m_initialized(false) {}
JitBaseBlockCache() : num_blocks(0) {}
virtual ~JitBaseBlockCache() {}
int AllocateBlock(u32 em_address);
void FinalizeBlock(int block_num, bool block_link, const u8* code_ptr);
@ -119,18 +167,20 @@ public:
// Code Cache
JitBlock* GetBlock(int block_num);
JitBlock* GetBlocks() { return blocks.data(); }
int* GetICache() { return iCache.data(); }
int GetNumBlocks() const;
const u8** GetCodePointers();
std::array<u8, JIT_ICACHE_SIZE> iCache;
std::array<u8, JIT_ICACHEEX_SIZE> iCacheEx;
std::array<u8, JIT_ICACHE_SIZE> iCacheVMEM;
// Fast way to get a block. Only works on the first ppc instruction of a block.
int GetBlockNumberFromStartAddress(u32 em_address);
// Look for the block in the slow but accurate way.
// This function shall be used if FastLookupEntryForAddress() failed.
int GetBlockNumberFromStartAddress(u32 em_address, u32 msr);
CompiledCode GetCompiledCodeFromBlock(int block_num);
// Get the normal entry for the block associated with the current program
// counter. This will JIT code if necessary. (This is the reference
// implementation; high-performance JITs will want to use a custom
// assembly version.)
const u8* Dispatch();
// DOES NOT WORK CORRECTLY WITH INLINING
void InvalidateICache(u32 address, const u32 length, bool forced);
u32* GetBlockBitSet() const { return valid_block.m_valid_block.get(); }

12
Source/Core/Core/PowerPC/JitInterface.cpp
View File

@ -150,7 +150,7 @@ void GetProfileResults(ProfileStats* prof_stats)
u64 timecost = block->ticCounter;
// Todo: tweak.
if (block->runCount >= 1)
prof_stats->block_stats.emplace_back(i, block->originalAddress, cost, timecost,
prof_stats->block_stats.emplace_back(i, block->effectiveAddress, cost, timecost,
block->runCount, block->codeSize);
prof_stats->cost_sum += cost;
prof_stats->timecost_sum += timecost;
@ -169,12 +169,12 @@ int GetHostCode(u32* address, const u8** code, u32* code_size)
return 1;
}
int block_num = jit->GetBlockCache()->GetBlockNumberFromStartAddress(*address);
int block_num = jit->GetBlockCache()->GetBlockNumberFromStartAddress(*address, MSR);
if (block_num < 0)
{
for (int i = 0; i < 500; i++)
{
block_num = jit->GetBlockCache()->GetBlockNumberFromStartAddress(*address - 4 * i);
block_num = jit->GetBlockCache()->GetBlockNumberFromStartAddress(*address - 4 * i, MSR);
if (block_num >= 0)
break;
}
@ -182,8 +182,8 @@ int GetHostCode(u32* address, const u8** code, u32* code_size)
if (block_num >= 0)
{
JitBlock* block = jit->GetBlockCache()->GetBlock(block_num);
if (!(block->originalAddress <= *address &&
block->originalSize + block->originalAddress >= *address))
if (!(block->effectiveAddress <= *address &&
block->originalSize + block->effectiveAddress >= *address))
block_num = -1;
}
@ -199,7 +199,7 @@ int GetHostCode(u32* address, const u8** code, u32* code_size)
*code = block->checkedEntry;
*code_size = block->codeSize;
*address = block->originalAddress;
*address = block->effectiveAddress;
return 0;
}

62
Source/Core/Core/PowerPC/MMU.cpp
View File

@ -76,7 +76,19 @@ enum XCheckTLBFlag
FLAG_READ,
FLAG_WRITE,
FLAG_OPCODE,
FLAG_OPCODE_NO_EXCEPTION
};
static bool IsOpcodeFlag(XCheckTLBFlag flag)
{
return flag == FLAG_OPCODE || flag == FLAG_OPCODE_NO_EXCEPTION;
}
static bool IsNoExceptionFlag(XCheckTLBFlag flag)
{
return flag == FLAG_NO_EXCEPTION || flag == FLAG_OPCODE_NO_EXCEPTION;
}
template <const XCheckTLBFlag flag>
static u32 TranslateAddress(const u32 address);
@ -836,6 +848,43 @@ bool IsOptimizableGatherPipeWrite(u32 address)
return address == 0xCC008000;
}
TranslateResult JitCache_TranslateAddress(u32 address)
{
if (!UReg_MSR(MSR).IR)
return TranslateResult{true, true, address};
bool from_bat = true;
int segment = address >> 28;
if (SConfig::GetInstance().bMMU && (address & Memory::ADDR_MASK_MEM1))
{
u32 tlb_addr = TranslateAddress<FLAG_OPCODE>(address);
if (tlb_addr == 0)
{
return TranslateResult{false, false, 0};
}
else
{
address = tlb_addr;
from_bat = false;
}
}
else
{
if ((segment == 0x8 || segment == 0x0) && (address & 0x0FFFFFFF) < Memory::REALRAM_SIZE)
address = address & 0x3FFFFFFF;
else if (Memory::m_pEXRAM && segment == 0x9 && (address & 0x0FFFFFFF) < Memory::EXRAM_SIZE)
address = address & 0x3FFFFFFF;
else if (Memory::bFakeVMEM && (segment == 0x7 || segment == 0x4))
address = 0x7E000000 | (address & Memory::FAKEVMEM_MASK);
else
return TranslateResult{false, false, 0};
}
return TranslateResult{true, from_bat, address};
}
// *********************************************************************************
// Warning: Test Area
//
@ -990,7 +1039,7 @@ static __forceinline TLBLookupResult LookupTLBPageAddress(const XCheckTLBFlag fl
u32* paddr)
{
u32 tag = vpa >> HW_PAGE_INDEX_SHIFT;
PowerPC::tlb_entry* tlbe = &PowerPC::ppcState.tlb[flag == FLAG_OPCODE][tag & HW_PAGE_INDEX_MASK];
PowerPC::tlb_entry* tlbe = &PowerPC::ppcState.tlb[IsOpcodeFlag(flag)][tag & HW_PAGE_INDEX_MASK];
if (tlbe->tag[0] == tag)
{
// Check if C bit requires updating
@ -1006,7 +1055,7 @@ static __forceinline TLBLookupResult LookupTLBPageAddress(const XCheckTLBFlag fl
}
}
if (flag != FLAG_NO_EXCEPTION)
if (!IsNoExceptionFlag(flag))
tlbe->recent = 0;
*paddr = tlbe->paddr[0] | (vpa & 0xfff);
@ -1028,7 +1077,7 @@ static __forceinline TLBLookupResult LookupTLBPageAddress(const XCheckTLBFlag fl
}
}
if (flag != FLAG_NO_EXCEPTION)
if (!IsNoExceptionFlag(flag))
tlbe->recent = 1;
*paddr = tlbe->paddr[1] | (vpa & 0xfff);
@ -1040,11 +1089,11 @@ static __forceinline TLBLookupResult LookupTLBPageAddress(const XCheckTLBFlag fl
static __forceinline void UpdateTLBEntry(const XCheckTLBFlag flag, UPTE2 PTE2, const u32 address)
{
if (flag == FLAG_NO_EXCEPTION)
if (IsNoExceptionFlag(flag))
return;
int tag = address >> HW_PAGE_INDEX_SHIFT;
PowerPC::tlb_entry* tlbe = &PowerPC::ppcState.tlb[flag == FLAG_OPCODE][tag & HW_PAGE_INDEX_MASK];
PowerPC::tlb_entry* tlbe = &PowerPC::ppcState.tlb[IsOpcodeFlag(flag)][tag & HW_PAGE_INDEX_MASK];
int index = tlbe->recent == 0 && tlbe->tag[0] != TLB_TAG_INVALID;
tlbe->recent = index;
tlbe->paddr[index] = PTE2.RPN << HW_PAGE_INDEX_SHIFT;
@ -1110,6 +1159,7 @@ static __forceinline u32 TranslatePageAddress(const u32 address, const XCheckTLB
switch (flag)
{
case FLAG_NO_EXCEPTION:
case FLAG_OPCODE_NO_EXCEPTION:
break;
case FLAG_READ:
PTE2.R = 1;
@ -1123,7 +1173,7 @@ static __forceinline u32 TranslatePageAddress(const u32 address, const XCheckTLB
break;
}
if (flag != FLAG_NO_EXCEPTION)
if (!IsNoExceptionFlag(flag))
*(u32*)&Memory::physical_base[pteg_addr + 4] = bswap(PTE2.Hex);
// We already updated the TLB entry if this was caused by a C bit.

7
Source/Core/Core/PowerPC/PowerPC.h
View File

@ -272,6 +272,13 @@ bool IsOptimizableRAMAddress(const u32 address);
u32 IsOptimizableMMIOAccess(u32 address, u32 accessSize);
bool IsOptimizableGatherPipeWrite(u32 address);
struct TranslateResult
{
bool valid;
bool from_bat;
u32 address;
};
TranslateResult JitCache_TranslateAddress(u32 address);
} // namespace
enum CRBits