diff --git a/Source/Core/VideoCommon/VertexLoader.cpp b/Source/Core/VideoCommon/VertexLoader.cpp
index c2d3c1bd29..89965f8710 100644
--- a/Source/Core/VideoCommon/VertexLoader.cpp
+++ b/Source/Core/VideoCommon/VertexLoader.cpp
@@ -149,9 +149,16 @@ void VertexLoader::CompileVertexTranslator()
         VertexLoader_Color::GetFunction(m_VtxDesc.low.Color[i], m_VtxAttr.GetColorFormat(i));
 
     if (pFunc != nullptr)
+    {
       WriteCall(pFunc);
+    }
     else
+    {
       ASSERT(m_VtxDesc.low.Color[i] == VertexComponentFormat::NotPresent);
+      // Keep colIndex in sync if color 0 is absent but color 1 is present
+      if (i == 0 && m_VtxDesc.low.Color[1] != VertexComponentFormat::NotPresent)
+        WriteCall(VertexLoader_Color::GetDummyFunction());
+    }
 
     if (m_VtxDesc.low.Color[i] != VertexComponentFormat::NotPresent)
     {
diff --git a/Source/Core/VideoCommon/VertexLoader_Color.cpp b/Source/Core/VideoCommon/VertexLoader_Color.cpp
index fab4177fca..87cd19063a 100644
--- a/Source/Core/VideoCommon/VertexLoader_Color.cpp
+++ b/Source/Core/VideoCommon/VertexLoader_Color.cpp
@@ -16,6 +16,11 @@
 
 namespace
 {
+void Color_Read_Dummy(VertexLoader* loader)
+{
+  loader->m_colIndex++;
+}
+
 constexpr u32 alpha_mask = 0xFF000000;
 
 void SetCol(VertexLoader* loader, u32 val)
@@ -201,3 +206,8 @@ TPipelineFunction VertexLoader_Color::GetFunction(VertexComponentFormat type, Co
   }
   return s_table_read_color[type][format];
 }
+
+TPipelineFunction VertexLoader_Color::GetDummyFunction()
+{
+  return Color_Read_Dummy;
+}
diff --git a/Source/Core/VideoCommon/VertexLoader_Color.h b/Source/Core/VideoCommon/VertexLoader_Color.h
index 0e66bfbf86..da56d61212 100644
--- a/Source/Core/VideoCommon/VertexLoader_Color.h
+++ b/Source/Core/VideoCommon/VertexLoader_Color.h
@@ -25,6 +25,9 @@ public:
 
   static TPipelineFunction GetFunction(VertexComponentFormat type, ColorFormat format);
 
+  // It is important to synchronize colIndex, or else the wrong color array will be used
+  static TPipelineFunction GetDummyFunction();
+
 private:
   template <typename T, auto last_member>
   using EnumMap = typename Common::EnumMap<T, last_member>;