diff --git a/common/Pcsx2Defs.h b/common/Pcsx2Defs.h
index 372c153906..ac82bf1594 100644
--- a/common/Pcsx2Defs.h
+++ b/common/Pcsx2Defs.h
@@ -24,14 +24,9 @@
 
 #include "Pcsx2Types.h"
 #include <cstddef>
-#include <cassert>
 
 #include "common/emitter/x86_intrin.h"
 
-// The C++ standard doesn't allow `offsetof` to be used on non-constant values (e.g. `offsetof(class, field[i])`)
-// Use this in those situations
-#define OFFSETOF(a, b) (reinterpret_cast<size_t>(&(static_cast<a*>(0)->b)))
-
 // --------------------------------------------------------------------------------------
 // Dev / Debug conditionals - Consts for using if() statements instead of uglier #ifdef.
 // --------------------------------------------------------------------------------------
diff --git a/pcsx2/GS/Renderers/SW/GSDrawScanlineCodeGenerator.all.cpp b/pcsx2/GS/Renderers/SW/GSDrawScanlineCodeGenerator.all.cpp
index 6aa4ab5a1b..c8d3b79330 100644
--- a/pcsx2/GS/Renderers/SW/GSDrawScanlineCodeGenerator.all.cpp
+++ b/pcsx2/GS/Renderers/SW/GSDrawScanlineCodeGenerator.all.cpp
@@ -1,5 +1,5 @@
 /*  PCSX2 - PS2 Emulator for PCs
- *  Copyright (C) 2002-2021 PCSX2 Dev Team
+ *  Copyright (C) 2002-2023 PCSX2 Dev Team
  *
  *  PCSX2 is free software: you can redistribute it and/or modify it under the terms
  *  of the GNU Lesser General Public License as published by the Free Software Found-
@@ -19,12 +19,18 @@
 #include "GSVertexSW.h"
 #include "common/Perf.h"
 
+#include <cstddef>
+
 MULTI_ISA_UNSHARED_IMPL;
 using namespace Xbyak;
 
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Winvalid-offsetof" // We know what we're doing.
+#endif
+
 #define _rip_const(cptr) ptr[rip + ((char*)(cptr))]
-#define _rip_local(field) ptr[_m_local + OFFSETOF(GSScanlineLocalData, field)]
-#define _rip_global(field) ptr[_m_local__gd + OFFSETOF(GSScanlineGlobalData, field)]
+#define _rip_local(field) ptr[_m_local + offsetof(GSScanlineLocalData, field)]
+#define _rip_global(field) ptr[_m_local__gd + offsetof(GSScanlineGlobalData, field)]
 
 /// On AVX, does a v-prefixed separate destination operation
 /// On SSE, moves src1 into dst using movdqa, then does the operation
diff --git a/pcsx2/GS/Renderers/SW/GSSetupPrimCodeGenerator.all.cpp b/pcsx2/GS/Renderers/SW/GSSetupPrimCodeGenerator.all.cpp
index b165a93f2e..e29886a975 100644
--- a/pcsx2/GS/Renderers/SW/GSSetupPrimCodeGenerator.all.cpp
+++ b/pcsx2/GS/Renderers/SW/GSSetupPrimCodeGenerator.all.cpp
@@ -1,5 +1,5 @@
 /*  PCSX2 - PS2 Emulator for PCs
- *  Copyright (C) 2002-2021 PCSX2 Dev Team
+ *  Copyright (C) 2002-2023 PCSX2 Dev Team
  *
  *  PCSX2 is free software: you can redistribute it and/or modify it under the terms
  *  of the GNU Lesser General Public License as published by the Free Software Found-
@@ -18,10 +18,12 @@
 #include "GSVertexSW.h"
 #include "common/Perf.h"
 
+#include <cstddef>
+
 MULTI_ISA_UNSHARED_IMPL;
 using namespace Xbyak;
 
-#define _rip_local(field) (ptr[_m_local + OFFSETOF(GSScanlineLocalData, field)])
+#define _rip_local(field) (ptr[_m_local + offsetof(GSScanlineLocalData, field)])
 
 #define _64_m_local _64_t0