diff --git a/Source/Core/VideoCommon/NativeVertexFormat.h b/Source/Core/VideoCommon/NativeVertexFormat.h
index 8dd53c77a6..d1912e1cbf 100644
--- a/Source/Core/VideoCommon/NativeVertexFormat.h
+++ b/Source/Core/VideoCommon/NativeVertexFormat.h
@@ -76,6 +76,15 @@ struct PortableVertexDeclaration
 	AttributeFormat colors[2];
 	AttributeFormat texcoords[8];
 	AttributeFormat posmtx;
+
+	inline bool operator<(const PortableVertexDeclaration& b) const
+	{
+		return memcmp(this, &b, sizeof(PortableVertexDeclaration)) < 0;
+	}
+	inline bool operator==(const PortableVertexDeclaration& b) const
+	{
+		return memcmp(this, &b, sizeof(PortableVertexDeclaration)) == 0;
+	}
 };
 
 // The implementation of this class is specific for GL/DX, so NativeVertexFormat.cpp
diff --git a/Source/Core/VideoCommon/VertexLoader.cpp b/Source/Core/VideoCommon/VertexLoader.cpp
index 83f9aaa081..bdf58899e4 100644
--- a/Source/Core/VideoCommon/VertexLoader.cpp
+++ b/Source/Core/VideoCommon/VertexLoader.cpp
@@ -463,7 +463,6 @@ VertexLoader::VertexLoader(const TVtxDesc &vtx_desc, const VAT &vtx_attr)
 	m_compiledCode = nullptr;
 	m_numLoadedVertices = 0;
 	m_VertexSize = 0;
-	m_NativeFmt = nullptr;
 	loop_counter = 0;
 	VertexLoader_Normal::Init();
 	VertexLoader_Position::Init();
@@ -488,7 +487,6 @@ VertexLoader::~VertexLoader()
 	#ifdef USE_VERTEX_LOADER_JIT
 	FreeCodeSpace();
 	#endif
-	delete m_NativeFmt;
 }
 
 void VertexLoader::CompileVertexTranslator()
@@ -773,9 +771,7 @@ void VertexLoader::CompileVertexTranslator()
 	ABI_PopAllCalleeSavedRegsAndAdjustStack();
 	RET();
 #endif
-	m_NativeFmt = g_vertex_manager->CreateNativeVertexFormat();
-	m_NativeFmt->m_components = components;
-	m_NativeFmt->Initialize(vtx_decl);
+	m_NativeFmt = VertexLoaderManager::GetNativeVertexFormat(vtx_decl, components);
 }
 
 void VertexLoader::WriteCall(TPipelineFunction func)
@@ -825,9 +821,6 @@ void VertexLoader::SetupRunVertices(int vtx_attr_group, int primitive, int const
 	// Flush if our vertex format is different from the currently set.
 	if (g_nativeVertexFmt != nullptr && g_nativeVertexFmt != m_NativeFmt)
 	{
-		// We really must flush here. It's possible that the native representations
-		// of the two vtx formats are the same, but we have no way to easily check that
-		// now.
 		VertexManager::Flush();
 		// Also move the Set() here?
 	}
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.cpp b/Source/Core/VideoCommon/VertexLoaderManager.cpp
index b39577dd26..dd1196a48e 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderManager.cpp
@@ -3,6 +3,7 @@
 // Refer to the license.txt file included.
 
 #include <algorithm>
+#include <memory>
 #include <unordered_map>
 #include <vector>
 
@@ -33,11 +34,13 @@ struct hash<VertexLoaderUID>
 }
 
 typedef std::unordered_map<VertexLoaderUID, VertexLoader*> VertexLoaderMap;
+typedef std::map<PortableVertexDeclaration, std::unique_ptr<NativeVertexFormat>> NativeVertexLoaderMap;
 
 namespace VertexLoaderManager
 {
 
 static VertexLoaderMap g_VertexLoaderMap;
+static NativeVertexLoaderMap s_native_vertex_map;
 // TODO - change into array of pointers. Keep a map of all seen so far.
 
 void Init()
@@ -55,6 +58,7 @@ void Shutdown()
 		delete p.second;
 	}
 	g_VertexLoaderMap.clear();
+	s_native_vertex_map.clear();
 }
 
 namespace
@@ -131,6 +135,19 @@ int GetVertexSize(int vtx_attr_group)
 	return RefreshLoader(vtx_attr_group)->GetVertexSize();
 }
 
+NativeVertexFormat* GetNativeVertexFormat(const PortableVertexDeclaration& format, u32 components)
+{
+	auto& native = s_native_vertex_map[format];
+	if (!native)
+	{
+		auto raw_pointer = g_vertex_manager->CreateNativeVertexFormat();
+		native = std::unique_ptr<NativeVertexFormat>(raw_pointer);
+		native->m_components = components;
+		native->Initialize(format);
+	}
+	return native.get();
+}
+
 }  // namespace
 
 void LoadCPReg(u32 sub_cmd, u32 value)
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.h b/Source/Core/VideoCommon/VertexLoaderManager.h
index 7e6c46818a..909701bf6d 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.h
+++ b/Source/Core/VideoCommon/VertexLoaderManager.h
@@ -7,6 +7,7 @@
 #include <string>
 
 #include "Common/Common.h"
+#include "VideoCommon/NativeVertexFormat.h"
 
 namespace VertexLoaderManager
 {
@@ -20,6 +21,8 @@ namespace VertexLoaderManager
 
 	// For debugging
 	void AppendListToString(std::string *dest);
+
+	NativeVertexFormat* GetNativeVertexFormat(const PortableVertexDeclaration& format, u32 components);
 };
 
 void RecomputeCachedArraybases();