diff --git a/Source/Core/VideoCommon/VertexLoader.cpp b/Source/Core/VideoCommon/VertexLoader.cpp
index e1410b49f2..1db8deec0c 100644
--- a/Source/Core/VideoCommon/VertexLoader.cpp
+++ b/Source/Core/VideoCommon/VertexLoader.cpp
@@ -23,7 +23,7 @@ static void PosMtx_ReadDirect_UByte(VertexLoader* loader)
 {
   u32 posmtx = DataRead<u8>() & 0x3f;
   if (loader->m_counter < 3)
-    VertexLoaderManager::position_matrix_index[loader->m_counter] = posmtx;
+    VertexLoaderManager::position_matrix_index[loader->m_counter + 1] = posmtx;
   DataWrite<u32>(posmtx);
   PRIM_LOG("posmtx: %d, ", posmtx);
 }
diff --git a/Source/Core/VideoCommon/VertexLoaderARM64.cpp b/Source/Core/VideoCommon/VertexLoaderARM64.cpp
index cf0fd576f4..87ac4c6722 100644
--- a/Source/Core/VideoCommon/VertexLoaderARM64.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderARM64.cpp
@@ -194,7 +194,7 @@ int VertexLoaderARM64::ReadVertex(u64 attribute, int format, int count_in, int c
   {
     CMP(count_reg, 3);
     FixupBranch dont_store = B(CC_GT);
-    MOVI2R(EncodeRegTo64(scratch2_reg), (u64)VertexLoaderManager::position_cache);
+    MOVP2R(EncodeRegTo64(scratch2_reg), VertexLoaderManager::position_cache);
     ADD(EncodeRegTo64(scratch1_reg), EncodeRegTo64(scratch2_reg), EncodeRegTo64(count_reg),
         ArithOption(EncodeRegTo64(count_reg), ST_LSL, 4));
     m_float_emit.STUR(write_size, coords, EncodeRegTo64(scratch1_reg), -16);
@@ -392,11 +392,11 @@ void VertexLoaderARM64::GenerateVertexLoader()
     MOV(skipped_reg, WZR);
   MOV(saved_count, count_reg);
 
-  MOVI2R(stride_reg, (u64)&g_main_cp_state.array_strides);
-  MOVI2R(arraybase_reg, (u64)&VertexLoaderManager::cached_arraybases);
+  MOVP2R(stride_reg, g_main_cp_state.array_strides);
+  MOVP2R(arraybase_reg, VertexLoaderManager::cached_arraybases);
 
   if (need_scale)
-    MOVI2R(scale_reg, (u64)&scale_factors);
+    MOVP2R(scale_reg, scale_factors);
 
   const u8* loop_start = GetCodePtr();
 
@@ -409,8 +409,7 @@ void VertexLoaderARM64::GenerateVertexLoader()
     // Z-Freeze
     CMP(count_reg, 3);
     FixupBranch dont_store = B(CC_GT);
-    MOVI2R(EncodeRegTo64(scratch2_reg),
-           (u64)VertexLoaderManager::position_matrix_index - sizeof(u32));
+    MOVP2R(EncodeRegTo64(scratch2_reg), VertexLoaderManager::position_matrix_index);
     STR(INDEX_UNSIGNED, scratch1_reg, EncodeRegTo64(scratch2_reg), 0);
     SetJumpTarget(dont_store);
 
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.cpp b/Source/Core/VideoCommon/VertexLoaderManager.cpp
index e9c4533244..6579802445 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderManager.cpp
@@ -28,7 +28,10 @@
 namespace VertexLoaderManager
 {
 float position_cache[3][4];
-u32 position_matrix_index[3];
+
+// The counter added to the address of the array is 1, 2, or 3, but never zero.
+// So only index 1 - 3 are used.
+u32 position_matrix_index[4];
 
 static NativeVertexFormatMap s_native_vertex_map;
 static NativeVertexFormat* s_current_vtx_fmt;
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.h b/Source/Core/VideoCommon/VertexLoaderManager.h
index 2b273bb80f..f8a4235341 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.h
+++ b/Source/Core/VideoCommon/VertexLoaderManager.h
@@ -42,7 +42,7 @@ void UpdateVertexArrayPointers();
 // Position cache for zfreeze (3 vertices, 4 floats each to allow SIMD overwrite).
 // These arrays are in reverse order.
 extern float position_cache[3][4];
-extern u32 position_matrix_index[3];
+extern u32 position_matrix_index[4];
 
 // VB_HAS_X. Bitmask telling what vertex components are present.
 extern u32 g_current_components;
diff --git a/Source/Core/VideoCommon/VertexLoaderX64.cpp b/Source/Core/VideoCommon/VertexLoaderX64.cpp
index 4aa6802e41..606a3ebae4 100644
--- a/Source/Core/VideoCommon/VertexLoaderX64.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderX64.cpp
@@ -416,7 +416,7 @@ void VertexLoaderX64::GenerateVertexLoader()
     // zfreeze
     CMP(32, R(count_reg), Imm8(3));
     FixupBranch dont_store = J_CC(CC_A);
-    MOV(32, MPIC(VertexLoaderManager::position_matrix_index - 1, count_reg, SCALE_4), R(scratch1));
+    MOV(32, MPIC(VertexLoaderManager::position_matrix_index, count_reg, SCALE_4), R(scratch1));
     SetJumpTarget(dont_store);
 
     m_native_components |= VB_HAS_POSMTXIDX;
diff --git a/Source/Core/VideoCommon/VertexManagerBase.cpp b/Source/Core/VideoCommon/VertexManagerBase.cpp
index 8ac365a735..ab4dced0e9 100644
--- a/Source/Core/VideoCommon/VertexManagerBase.cpp
+++ b/Source/Core/VideoCommon/VertexManagerBase.cpp
@@ -304,7 +304,7 @@ void VertexManagerBase::CalculateZSlope(NativeVertexFormat* format)
   {
     // If this vertex format has per-vertex position matrix IDs, look it up.
     if (vert_decl.posmtx.enable)
-      mtxIdx = VertexLoaderManager::position_matrix_index[2 - i];
+      mtxIdx = VertexLoaderManager::position_matrix_index[3 - i];
 
     if (vert_decl.position.components == 2)
       VertexLoaderManager::position_cache[2 - i][2] = 0;