diff --git a/Source/Core/VideoCommon/RenderBase.cpp b/Source/Core/VideoCommon/RenderBase.cpp
index 67721caabc..00bb5d24fc 100644
--- a/Source/Core/VideoCommon/RenderBase.cpp
+++ b/Source/Core/VideoCommon/RenderBase.cpp
@@ -639,11 +639,10 @@ void Renderer::Swap(u32 xfbAddr, u32 fbWidth, u32 fbStride, u32 fbHeight, const
     auto* xfb_entry = g_texture_cache->GetTexture(xfbAddr, fbWidth, fbHeight, TextureFormat::XFB,
                                                   force_safe_texture_cache_hash);
 
-    if (xfb_entry)
+    if (xfb_entry && xfb_entry->id != m_last_xfb_id)
     {
-      // TODO, check if xfb_entry is a duplicate of the previous frame and skip SwapImpl
-
       m_last_xfb_texture = xfb_entry->texture.get();
+      m_last_xfb_id = xfb_entry->id;
 
       // TODO: merge more generic parts into VideoCommon
       g_renderer->SwapImpl(xfb_entry->texture.get(), rc, ticks, xfb_entry->gamma);
diff --git a/Source/Core/VideoCommon/RenderBase.h b/Source/Core/VideoCommon/RenderBase.h
index 75b6b12509..11a73536e5 100644
--- a/Source/Core/VideoCommon/RenderBase.h
+++ b/Source/Core/VideoCommon/RenderBase.h
@@ -207,6 +207,7 @@ private:
   } m_frame_dump_config;
 
   AbstractTexture * m_last_xfb_texture;
+  u64 m_last_xfb_id = 0;
 
   // Note: Only used for auto-ir
   u32 m_last_xfb_width = MAX_XFB_WIDTH;
diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp
index c6df251723..89fbb6ac59 100644
--- a/Source/Core/VideoCommon/TextureCacheBase.cpp
+++ b/Source/Core/VideoCommon/TextureCacheBase.cpp
@@ -1611,6 +1611,7 @@ TextureCacheBase::TCacheEntry* TextureCacheBase::AllocateCacheEntry(const Textur
   }
   TCacheEntry* cacheEntry = new TCacheEntry(std::move(texture));
   cacheEntry->textures_by_hash_iter = textures_by_hash.end();
+  cacheEntry->id = last_entry_id++;
   return cacheEntry;
 }
 
diff --git a/Source/Core/VideoCommon/TextureCacheBase.h b/Source/Core/VideoCommon/TextureCacheBase.h
index 76eb1b58db..bec67c042a 100644
--- a/Source/Core/VideoCommon/TextureCacheBase.h
+++ b/Source/Core/VideoCommon/TextureCacheBase.h
@@ -89,6 +89,7 @@ public:
     bool is_xfb_copy = false;
     float y_scale = 1.0f;
     float gamma = 1.0f;
+    u64 id;
 
     unsigned int native_width,
         native_height;  // Texture dimensions from the GameCube's point of view
@@ -268,6 +269,7 @@ private:
   TexAddrCache textures_by_address;
   TexHashCache textures_by_hash;
   TexPool texture_pool;
+  u64 last_entry_id = 0;
 
   // Backup configuration values
   struct BackupConfig