diff --git a/Source/Core/Core/DolphinAnalytics.cpp b/Source/Core/Core/DolphinAnalytics.cpp
index ab5ea2c772..42eb7febe1 100644
--- a/Source/Core/Core/DolphinAnalytics.cpp
+++ b/Source/Core/Core/DolphinAnalytics.cpp
@@ -136,7 +136,7 @@ void DolphinAnalytics::ReportGameStart()
 }
 
 // Keep in sync with enum class GameQuirk definition.
-constexpr std::array<const char*, 27> GAME_QUIRKS_NAMES{
+constexpr std::array<const char*, 28> GAME_QUIRKS_NAMES{
     "icache-matters",
     "directly-reads-wiimote-input",
     "uses-DVDLowStopLaser",
@@ -164,6 +164,7 @@ constexpr std::array<const char*, 27> GAME_QUIRKS_NAMES{
     "mismatched-gpu-colors-between-cp-and-xf",
     "mismatched-gpu-normals-between-cp-and-xf",
     "mismatched-gpu-tex-coords-between-cp-and-xf",
+    "mismatched-gpu-matrix-indices-between-cp-and-xf",
 };
 static_assert(GAME_QUIRKS_NAMES.size() == static_cast<u32>(GameQuirk::COUNT),
               "Game quirks names and enum definition are out of sync.");
diff --git a/Source/Core/Core/DolphinAnalytics.h b/Source/Core/Core/DolphinAnalytics.h
index 6e8acb8a8f..c708ad3a13 100644
--- a/Source/Core/Core/DolphinAnalytics.h
+++ b/Source/Core/Core/DolphinAnalytics.h
@@ -87,6 +87,10 @@ enum class GameQuirk
   MISMATCHED_GPU_COLORS_BETWEEN_CP_AND_XF,
   MISMATCHED_GPU_NORMALS_BETWEEN_CP_AND_XF,
   MISMATCHED_GPU_TEX_COORDS_BETWEEN_CP_AND_XF,
+  // Both CP and XF have normally-identical matrix index information.  We currently always
+  // use the CP one in the hardware renderers and the XF one in the software renderer,
+  // but testing is needed to find out which of these is actually used for what.
+  MISMATCHED_GPU_MATRIX_INDICES_BETWEEN_CP_AND_XF,
 
   COUNT,
 };
diff --git a/Source/Core/VideoBackends/Software/SWVertexLoader.cpp b/Source/Core/VideoBackends/Software/SWVertexLoader.cpp
index 4b9825132e..aaf91d9891 100644
--- a/Source/Core/VideoBackends/Software/SWVertexLoader.cpp
+++ b/Source/Core/VideoBackends/Software/SWVertexLoader.cpp
@@ -106,20 +106,6 @@ void SWVertexLoader::DrawCurrentBatch(u32 base_index, u32 num_indices, u32 base_
 
 void SWVertexLoader::SetFormat()
 {
-  // matrix index from xf regs or cp memory?
-  if (xfmem.MatrixIndexA.PosNormalMtxIdx != g_main_cp_state.matrix_index_a.PosNormalMtxIdx ||
-      xfmem.MatrixIndexA.Tex0MtxIdx != g_main_cp_state.matrix_index_a.Tex0MtxIdx ||
-      xfmem.MatrixIndexA.Tex1MtxIdx != g_main_cp_state.matrix_index_a.Tex1MtxIdx ||
-      xfmem.MatrixIndexA.Tex2MtxIdx != g_main_cp_state.matrix_index_a.Tex2MtxIdx ||
-      xfmem.MatrixIndexA.Tex3MtxIdx != g_main_cp_state.matrix_index_a.Tex3MtxIdx ||
-      xfmem.MatrixIndexB.Tex4MtxIdx != g_main_cp_state.matrix_index_b.Tex4MtxIdx ||
-      xfmem.MatrixIndexB.Tex5MtxIdx != g_main_cp_state.matrix_index_b.Tex5MtxIdx ||
-      xfmem.MatrixIndexB.Tex6MtxIdx != g_main_cp_state.matrix_index_b.Tex6MtxIdx ||
-      xfmem.MatrixIndexB.Tex7MtxIdx != g_main_cp_state.matrix_index_b.Tex7MtxIdx)
-  {
-    ERROR_LOG_FMT(VIDEO, "Matrix indices don't match");
-  }
-
   m_vertex.posMtx = xfmem.MatrixIndexA.PosNormalMtxIdx;
   m_vertex.texMtx[0] = xfmem.MatrixIndexA.Tex0MtxIdx;
   m_vertex.texMtx[1] = xfmem.MatrixIndexA.Tex1MtxIdx;
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.cpp b/Source/Core/VideoCommon/VertexLoaderManager.cpp
index eea5183a81..f3fc4d9e7e 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderManager.cpp
@@ -321,6 +321,18 @@ static void CheckCPConfiguration(int vtx_attr_group)
     // Don't bail out, though; we can still render something successfully
     // (real hardware seems to hang in this case, though)
   }
+
+  if (g_main_cp_state.matrix_index_a.Hex != xfmem.MatrixIndexA.Hex ||
+      g_main_cp_state.matrix_index_b.Hex != xfmem.MatrixIndexB.Hex)
+  {
+    PanicAlertFmt("Mismatched matrix index configuration between CP and XF stages - "
+                  "index A: {:08x}/{:08x}, index B {:08x}/{:08x}. "
+                  "Please report on the issue tracker.",
+                  g_main_cp_state.matrix_index_a.Hex, xfmem.MatrixIndexA.Hex,
+                  g_main_cp_state.matrix_index_b.Hex, xfmem.MatrixIndexB.Hex);
+    DolphinAnalytics::Instance().ReportGameQuirk(
+        GameQuirk::MISMATCHED_GPU_MATRIX_INDICES_BETWEEN_CP_AND_XF);
+  }
 }
 
 int RunVertices(int vtx_attr_group, OpcodeDecoder::Primitive primitive, int count, DataReader src,