diff --git a/src/xenia/gpu/vulkan/texture_cache.cc b/src/xenia/gpu/vulkan/texture_cache.cc
index 32bfef36f..2ea69875a 100644
--- a/src/xenia/gpu/vulkan/texture_cache.cc
+++ b/src/xenia/gpu/vulkan/texture_cache.cc
@@ -376,7 +376,7 @@ void TextureCache::WatchCallback(void* context_ptr, void* data_ptr,
 
   // Add to pending list so Scavenge will clean it up.
   self->invalidated_textures_mutex_.lock();
-  self->invalidated_textures_->push_back(touched_texture);
+  self->invalidated_textures_->insert(touched_texture);
   self->invalidated_textures_mutex_.unlock();
 }
 
@@ -1467,13 +1467,16 @@ bool TextureCache::SetupTextureBinding(VkCommandBuffer command_buffer,
 void TextureCache::RemoveInvalidatedTextures() {
   // Clean up any invalidated textures.
   invalidated_textures_mutex_.lock();
-  std::vector<Texture*>& invalidated_textures = *invalidated_textures_;
+  std::unordered_set<Texture*>& invalidated_textures = *invalidated_textures_;
   if (invalidated_textures_ == &invalidated_textures_sets_[0]) {
     invalidated_textures_ = &invalidated_textures_sets_[1];
   } else {
     invalidated_textures_ = &invalidated_textures_sets_[0];
   }
   invalidated_textures_mutex_.unlock();
+
+  // Append all invalidated textures to a deletion queue. They will be deleted
+  // when all command buffers using them have finished executing.
   if (!invalidated_textures.empty()) {
     for (auto it = invalidated_textures.begin();
          it != invalidated_textures.end(); ++it) {
diff --git a/src/xenia/gpu/vulkan/texture_cache.h b/src/xenia/gpu/vulkan/texture_cache.h
index 488924bb4..af1e10447 100644
--- a/src/xenia/gpu/vulkan/texture_cache.h
+++ b/src/xenia/gpu/vulkan/texture_cache.h
@@ -11,6 +11,7 @@
 #define XENIA_GPU_VULKAN_TEXTURE_CACHE_H_
 
 #include <unordered_map>
+#include <unordered_set>
 
 #include "xenia/gpu/register_file.h"
 #include "xenia/gpu/sampler_info.h"
@@ -203,8 +204,8 @@ class TextureCache {
   std::list<Texture*> pending_delete_textures_;
 
   std::mutex invalidated_textures_mutex_;
-  std::vector<Texture*>* invalidated_textures_;
-  std::vector<Texture*> invalidated_textures_sets_[2];
+  std::unordered_set<Texture*>* invalidated_textures_;
+  std::unordered_set<Texture*> invalidated_textures_sets_[2];
 
   struct UpdateSetInfo {
     // Bitmap of all 32 fetch constants and whether they have been setup yet.