diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp
index a68fc078ee..c9c0b02224 100644
--- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp
+++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp
@@ -123,7 +123,6 @@ bool JitArm64::HandleStackFault()
 void JitArm64::ClearCache()
 {
   m_fault_to_handler.clear();
-  m_handler_to_loc.clear();
 
   blocks.Clear();
   const Common::ScopedJITPageWriteAndNoExecute enable_jit_page_writes;
diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.h b/Source/Core/Core/PowerPC/JitArm64/Jit.h
index bc831360c1..c6205281d1 100644
--- a/Source/Core/Core/PowerPC/JitArm64/Jit.h
+++ b/Source/Core/Core/PowerPC/JitArm64/Jit.h
@@ -178,21 +178,6 @@ public:
   bool IsFPRStoreSafe(size_t guest_reg) const;
 
 protected:
-  struct SlowmemHandler
-  {
-    Arm64Gen::ARM64Reg dest_reg;
-    Arm64Gen::ARM64Reg addr_reg;
-    BitSet32 gprs;
-    BitSet32 fprs;
-    u32 flags;
-
-    bool operator<(const SlowmemHandler& rhs) const
-    {
-      return std::tie(dest_reg, addr_reg, gprs, fprs, flags) <
-             std::tie(rhs.dest_reg, rhs.addr_reg, rhs.gprs, rhs.fprs, rhs.flags);
-    }
-  };
-
   struct FastmemArea
   {
     const u8* fastmem_code;
@@ -296,7 +281,6 @@ protected:
 
   // <Fastmem fault location, slowmem handler location>
   std::map<const u8*, FastmemArea> m_fault_to_handler;
-  std::map<SlowmemHandler, const u8*> m_handler_to_loc;
   Arm64GPRCache gpr;
   Arm64FPRCache fpr;
 
diff --git a/Source/Core/Core/PowerPC/JitArm64/JitArm64_BackPatch.cpp b/Source/Core/Core/PowerPC/JitArm64/JitArm64_BackPatch.cpp
index 11da7bb923..53770830bb 100644
--- a/Source/Core/Core/PowerPC/JitArm64/JitArm64_BackPatch.cpp
+++ b/Source/Core/Core/PowerPC/JitArm64/JitArm64_BackPatch.cpp
@@ -122,39 +122,14 @@ void JitArm64::EmitBackpatchRoutine(u32 flags, bool fastmem, bool do_farcode, AR
   {
     if (fastmem && do_farcode)
     {
-      if (emitting_routine)
-      {
-        in_far_code = true;
-        SwitchToFarCode();
-      }
-      else
-      {
-        SlowmemHandler handler;
-        handler.dest_reg = RS;
-        handler.addr_reg = addr;
-        handler.gprs = gprs_to_push;
-        handler.fprs = fprs_to_push;
-        handler.flags = flags;
+      in_far_code = true;
+      SwitchToFarCode();
 
+      if (!emitting_routine)
+      {
         FastmemArea* fastmem_area = &m_fault_to_handler[fastmem_end];
-        auto handler_loc_iter = m_handler_to_loc.find(handler);
-
-        if (handler_loc_iter == m_handler_to_loc.end())
-        {
-          in_far_code = true;
-          SwitchToFarCode();
-          const u8* handler_loc = GetCodePtr();
-          m_handler_to_loc[handler] = handler_loc;
-          fastmem_area->fastmem_code = fastmem_start;
-          fastmem_area->slowmem_code = handler_loc;
-        }
-        else
-        {
-          const u8* handler_loc = handler_loc_iter->second;
-          fastmem_area->fastmem_code = fastmem_start;
-          fastmem_area->slowmem_code = handler_loc;
-          return;
-        }
+        fastmem_area->fastmem_code = fastmem_start;
+        fastmem_area->slowmem_code = GetCodePtr();
       }
     }