diff --git a/Source/Core/Core/PowerPC/Jit64/Jit.cpp b/Source/Core/Core/PowerPC/Jit64/Jit.cpp
index 10362c56d4..e9b7333ed2 100644
--- a/Source/Core/Core/PowerPC/Jit64/Jit.cpp
+++ b/Source/Core/Core/PowerPC/Jit64/Jit.cpp
@@ -221,8 +221,6 @@ void Jit64::ClearCache()
 {
 	blocks.Clear();
 	trampolines.ClearCodeSpace();
-	jit->js.pcAtLoc.clear();
-	jit->js.registersInUseAtLoc.clear();
 	farcode.ClearCodeSpace();
 	ClearCodeSpace();
 	m_clear_cache_asap = false;
diff --git a/Source/Core/Core/PowerPC/JitCommon/JitBackpatch.cpp b/Source/Core/Core/PowerPC/JitCommon/JitBackpatch.cpp
index d1cfe7aac9..ea921817b8 100644
--- a/Source/Core/Core/PowerPC/JitCommon/JitBackpatch.cpp
+++ b/Source/Core/Core/PowerPC/JitCommon/JitBackpatch.cpp
@@ -65,21 +65,14 @@ bool Jitx86Base::BackPatch(u32 emAddress, SContext* ctx)
 		return false;
 	}
 
-	auto pc_it = jit->js.pcAtLoc.find(codePtr);
-	if (pc_it == jit->js.pcAtLoc.end())
+	auto it = registersInUseAtLoc.find(codePtr);
+	if (it == registersInUseAtLoc.end())
 	{
-		PanicAlert("BackPatch: no pc entry for address %p", codePtr);
-		return nullptr;
-	}
-	u32 pc = pc_it->second;
-
-	auto reguse_it = jit->js.registersInUseAtLoc.find(pc);
-	if (reguse_it == jit->js.registersInUseAtLoc.end())
-	{
-		PanicAlert("BackPatch: no register use entry for PC %x", pc);
+		PanicAlert("BackPatch: no register use entry for address %p", codePtr);
 		return false;
 	}
-	u32 registersInUse = reguse_it->second;
+
+	u32 registersInUse = it->second;
 
 	if (!info.isMemoryWrite)
 	{
@@ -104,7 +97,16 @@ bool Jitx86Base::BackPatch(u32 emAddress, SContext* ctx)
 	}
 	else
 	{
-		// TODO: special case FIFO writes
+		// TODO: special case FIFO writes. Also, support 32-bit mode.
+		it = pcAtLoc.find(codePtr);
+		if (it == pcAtLoc.end())
+		{
+			PanicAlert("BackPatch: no pc entry for address %p", codePtr);
+			return nullptr;
+		}
+
+		u32 pc = it->second;
+
 		u8 *start;
 		if (info.byteSwap || info.hasImmediate)
 		{
diff --git a/Source/Core/Core/PowerPC/JitCommon/JitBase.h b/Source/Core/Core/PowerPC/JitCommon/JitBase.h
index ebb29cb091..22c4ac9d2b 100644
--- a/Source/Core/Core/PowerPC/JitCommon/JitBase.h
+++ b/Source/Core/Core/PowerPC/JitCommon/JitBase.h
@@ -99,8 +99,6 @@ protected:
 		JitBlock *curBlock;
 
 		std::unordered_set<u32> fifoWriteAddresses;
-		std::unordered_map<u32, u32> registersInUseAtLoc;
-		std::unordered_map<u8 *, u32> pcAtLoc;
 	};
 
 	PPCAnalyst::CodeBlock code_block;
diff --git a/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp b/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp
index 464c59b5a6..08bbb1474d 100644
--- a/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp
+++ b/Source/Core/Core/PowerPC/JitCommon/JitCache.cpp
@@ -339,19 +339,14 @@ using namespace Gen;
 				block_map.erase(it1, it2);
 			}
 
+			// If the code was actually modified, we need to clear the relevant entries from the
+			// FIFO write address cache, so we don't end up with FIFO checks in places they shouldn't
+			// be (this can clobber flags, and thus break any optimization that relies on flags
+			// being in the right place between instructions).
 			if (!forced)
 			{
 				for (u32 i = address; i < address + length; i += 4)
-				{
-					// If the code was actually modified, we need to clear the relevant entries from the
-					// FIFO write address cache, so we don't end up with FIFO checks in places they shouldn't
-					// be (this can clobber flags, and thus break any optimization that relies on flags
-					// being in the right place between instructions).
 					jit->js.fifoWriteAddresses.erase(i);
-					// We use these entries to determine whether there's a fastmem fault at this location,
-					// so clear it since the block is being replaced.
-					jit->js.registersInUseAtLoc.erase(i);
-				}
 			}
 		}
 	}
diff --git a/Source/Core/Core/PowerPC/JitCommon/Jit_Util.cpp b/Source/Core/Core/PowerPC/JitCommon/Jit_Util.cpp
index 6f01e846c0..0e9853948a 100644
--- a/Source/Core/Core/PowerPC/JitCommon/Jit_Util.cpp
+++ b/Source/Core/Core/PowerPC/JitCommon/Jit_Util.cpp
@@ -299,8 +299,7 @@ void EmuCodeBlock::SafeLoadToReg(X64Reg reg_value, const Gen::OpArg & opAddress,
 	if (!SConfig::GetInstance().m_LocalCoreStartupParameter.bMMU &&
 	    SConfig::GetInstance().m_LocalCoreStartupParameter.bFastmem &&
 	    !opAddress.IsImm() &&
-	    !(flags & (SAFE_LOADSTORE_NO_SWAP | SAFE_LOADSTORE_NO_FASTMEM)) &&
-		jit->js.registersInUseAtLoc.find(jit->js.compilerPC) == jit->js.registersInUseAtLoc.end()
+	    !(flags & (SAFE_LOADSTORE_NO_SWAP | SAFE_LOADSTORE_NO_FASTMEM))
 #ifdef ENABLE_MEM_CHECK
 	    && !SConfig::GetInstance().m_LocalCoreStartupParameter.bEnableDebugging
 #endif
@@ -308,8 +307,7 @@ void EmuCodeBlock::SafeLoadToReg(X64Reg reg_value, const Gen::OpArg & opAddress,
 	{
 		u8 *mov = UnsafeLoadToReg(reg_value, opAddress, accessSize, offset, signExtend);
 
-		jit->js.pcAtLoc[mov] = jit->js.compilerPC;
-		jit->js.registersInUseAtLoc[jit->js.compilerPC] = registersInUse;
+		registersInUseAtLoc[mov] = registersInUse;
 	}
 	else
 	{
@@ -484,8 +482,7 @@ void EmuCodeBlock::SafeWriteRegToReg(OpArg reg_value, X64Reg reg_addr, int acces
 	if (!SConfig::GetInstance().m_LocalCoreStartupParameter.bMMU &&
 	    SConfig::GetInstance().m_LocalCoreStartupParameter.bFastmem &&
 	    !(flags & SAFE_LOADSTORE_NO_FASTMEM) &&
-		(reg_value.IsImm() || !(flags & SAFE_LOADSTORE_NO_SWAP)) &&
-		jit->js.registersInUseAtLoc.find(jit->js.compilerPC) == jit->js.registersInUseAtLoc.end()
+		(reg_value.IsImm() || !(flags & SAFE_LOADSTORE_NO_SWAP))
 #ifdef ENABLE_MEM_CHECK
 	    && !SConfig::GetInstance().m_LocalCoreStartupParameter.bEnableDebugging
 #endif
@@ -499,8 +496,8 @@ void EmuCodeBlock::SafeWriteRegToReg(OpArg reg_value, X64Reg reg_addr, int acces
 			NOP(padding);
 		}
 
-		jit->js.pcAtLoc[mov] = jit->js.compilerPC;
-		jit->js.registersInUseAtLoc[jit->js.compilerPC] = registersInUse;
+		registersInUseAtLoc[mov] = registersInUse;
+		pcAtLoc[mov] = jit->js.compilerPC;
 		return;
 	}
 
diff --git a/Source/Core/Core/PowerPC/JitCommon/Jit_Util.h b/Source/Core/Core/PowerPC/JitCommon/Jit_Util.h
index 35549d0616..43b54debd9 100644
--- a/Source/Core/Core/PowerPC/JitCommon/Jit_Util.h
+++ b/Source/Core/Core/PowerPC/JitCommon/Jit_Util.h
@@ -136,4 +136,7 @@ public:
 	void ConvertSingleToDouble(Gen::X64Reg dst, Gen::X64Reg src, bool src_is_gpr = false);
 	void ConvertDoubleToSingle(Gen::X64Reg dst, Gen::X64Reg src);
 	void SetFPRF(Gen::X64Reg xmm);
+protected:
+	std::unordered_map<u8 *, u32> registersInUseAtLoc;
+	std::unordered_map<u8 *, u32> pcAtLoc;
 };