diff --git a/pcsx2/GS/GSBlock.cpp b/pcsx2/GS/GSBlock.cpp
index b3bd16aa9a..c4e735a6d6 100644
--- a/pcsx2/GS/GSBlock.cpp
+++ b/pcsx2/GS/GSBlock.cpp
@@ -16,6 +16,8 @@
 #include "PrecompiledHeader.h"
 #include "GSBlock.h"
 
+MULTI_ISA_UNSHARED_IMPL;
+
 CONSTINIT const GSVector4i GSBlock::m_r16mask(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
 CONSTINIT const GSVector4i GSBlock::m_r8mask(0, 4, 2, 6, 8, 12, 10, 14, 1, 5, 3, 7, 9, 13, 11, 15);
 CONSTINIT const GSVector4i GSBlock::m_r4mask(0, 8, 4, 12, 1, 9, 5, 13, 2, 10, 6, 14, 3, 11, 7, 15);
diff --git a/pcsx2/GS/GSBlock.h b/pcsx2/GS/GSBlock.h
index eb475184bd..d70d47e2b1 100644
--- a/pcsx2/GS/GSBlock.h
+++ b/pcsx2/GS/GSBlock.h
@@ -18,6 +18,9 @@
 #include "GSRegs.h"
 #include "GSTables.h"
 #include "GSVector.h"
+#include "MultiISA.h"
+
+MULTI_ISA_UNSHARED_START
 
 class GSBlock
 {
@@ -1940,3 +1943,5 @@ public:
 
 	// TODO: ReadAndExpandBlock4HH_16
 };
+
+MULTI_ISA_UNSHARED_END
diff --git a/pcsx2/GS/GSLocalMemory.h b/pcsx2/GS/GSLocalMemory.h
index a6e8e4fea5..c8d07dc82c 100644
--- a/pcsx2/GS/GSLocalMemory.h
+++ b/pcsx2/GS/GSLocalMemory.h
@@ -17,7 +17,6 @@
 
 #include "GSTables.h"
 #include "GSVector.h"
-#include "GSBlock.h"
 #include "GSClut.h"
 #include "MultiISA.h"
 #include <array>
diff --git a/pcsx2/GS/GSLocalMemoryMultiISA.cpp b/pcsx2/GS/GSLocalMemoryMultiISA.cpp
index 5cba6b585f..67a0b3e2d0 100644
--- a/pcsx2/GS/GSLocalMemoryMultiISA.cpp
+++ b/pcsx2/GS/GSLocalMemoryMultiISA.cpp
@@ -16,6 +16,7 @@
 #include "PrecompiledHeader.h"
 #include "GSLocalMemory.h"
 #include "GS.h"
+#include "GSBlock.h"
 #include "GSExtra.h"
 
 class CURRENT_ISA::GSLocalMemoryFunctions
diff --git a/pcsx2/GS/GSVector.cpp b/pcsx2/GS/GSVector.cpp
index deb00ac09d..dab402d2b1 100644
--- a/pcsx2/GS/GSVector.cpp
+++ b/pcsx2/GS/GSVector.cpp
@@ -71,8 +71,6 @@ CONSTINIT const GSVector4 GSVector4::m_xc1e00000000fffff = cxpr64(0xc1e00000000f
 CONSTINIT const GSVector4 GSVector4::m_max = cxpr(FLT_MAX);
 CONSTINIT const GSVector4 GSVector4::m_min = cxpr(FLT_MIN);
 
-#if _M_SSE >= 0x500
-
 CONSTINIT const GSVector8 GSVector8::m_half = cxpr(0.5f);
 CONSTINIT const GSVector8 GSVector8::m_one = cxpr(1.0f);
 CONSTINIT const GSVector8 GSVector8::m_x7fffffff = cxpr(0x7fffffff);
@@ -83,9 +81,6 @@ CONSTINIT const GSVector8 GSVector8::m_xc1e00000000fffff = cxpr64(0xc1e00000000f
 CONSTINIT const GSVector8 GSVector8::m_max = cxpr(FLT_MAX);
 CONSTINIT const GSVector8 GSVector8::m_min = cxpr(FLT_MAX);
 
-#endif
-
-#if _M_SSE >= 0x501
 CONSTINIT const GSVector8i GSVector8i::m_xff[33] =
 {
 	cxpr(0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000),
@@ -159,7 +154,6 @@ CONSTINIT const GSVector8i GSVector8i::m_x0f[33] =
 	cxpr(0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x000f0f0f),
 	cxpr(0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f),
 };
-#endif
 
 GSVector4i GSVector4i::fit(int arx, int ary) const
 {
diff --git a/pcsx2/GS/GSVector8.h b/pcsx2/GS/GSVector8.h
index 1f60768f39..c0b87b3c1d 100644
--- a/pcsx2/GS/GSVector8.h
+++ b/pcsx2/GS/GSVector8.h
@@ -15,8 +15,6 @@
 
 #include <cassert>
 
-#if _M_SSE >= 0x500
-
 class alignas(32) GSVector8
 {
 	struct cxpr_init_tag {};
@@ -53,7 +51,9 @@ public:
 		u16 U16[16];
 		u32 U32[8];
 		u64 U64[4];
+#if _M_SSE >= 0x500
 		__m256 m;
+#endif
 		__m128 m0, m1;
 	};
 
@@ -104,6 +104,8 @@ public:
 		return GSVector8(cxpr_init, x, x, x, x);
 	}
 
+#if _M_SSE >= 0x500
+
 	__forceinline GSVector8(float x0, float y0, float z0, float w0, float x1, float y1, float z1, float w1)
 	{
 		m = _mm256_set_ps(w1, z1, y1, x1, w0, z0, y0, x0);
@@ -948,6 +950,5 @@ public:
 	// TODO: v.(x0|y0|z0|w0|x1|y1|z1|w1) // broadcast element
 
 #endif
-};
-
 #endif
+};
diff --git a/pcsx2/GS/GSVector8i.h b/pcsx2/GS/GSVector8i.h
index 3456ef889d..8bff2eef42 100644
--- a/pcsx2/GS/GSVector8i.h
+++ b/pcsx2/GS/GSVector8i.h
@@ -15,8 +15,6 @@
 
 #include <cassert>
 
-#if _M_SSE >= 0x501
-
 class alignas(32) GSVector8i
 {
 	static const GSVector8i m_xff[33];
@@ -45,7 +43,9 @@ public:
 		u16 U16[16];
 		u32 U32[8];
 		u64 U64[4];
+#if _M_SSE >= 0x501
 		__m256i m;
+#endif
 		__m128i m0, m1;
 	};
 
@@ -61,6 +61,8 @@ public:
 		return GSVector8i(cxpr_init, x, x, x, x, x, x, x, x);
 	}
 
+#if _M_SSE >= 0x501
+
 	__forceinline explicit GSVector8i(const GSVector8& v, bool truncate = true);
 
 	__forceinline static GSVector8i cast(const GSVector8& v);
@@ -1887,6 +1889,6 @@ public:
 
 	__forceinline static GSVector8i xff(int n) { return m_xff[n]; }
 	__forceinline static GSVector8i x0f(int n) { return m_x0f[n]; }
+#endif
 };
 
-#endif
diff --git a/tests/ctest/GS/swizzle_test_main.cpp b/tests/ctest/GS/swizzle_test_main.cpp
index 9057aa49b4..fccd85334c 100644
--- a/tests/ctest/GS/swizzle_test_main.cpp
+++ b/tests/ctest/GS/swizzle_test_main.cpp
@@ -16,9 +16,12 @@
 #include "PrecompiledHeader.h"
 #include "GSBlock.h"
 #include "GSClut.h"
+#include "MultiISA.h"
 #include <gtest/gtest.h>
 #include <string.h>
 
+using namespace CURRENT_ISA;
+
 static void swizzle(const u8* table, u8* dst, const u8* src, int bpp, bool deswizzle)
 {
 	int pxbytes = bpp / 8;