diff --git a/pcsx2-qt/EmuThread.cpp b/pcsx2-qt/EmuThread.cpp
index c131ce4ec8..f4cf66c45f 100644
--- a/pcsx2-qt/EmuThread.cpp
+++ b/pcsx2-qt/EmuThread.cpp
@@ -217,7 +217,7 @@ void EmuThread::saveStateToSlot(qint32 slot)
 
 void EmuThread::run()
 {
-	PerformanceMetrics::SetCPUThreadTimer(Common::ThreadCPUTimer::GetForCallingThread());
+	PerformanceMetrics::SetCPUThread(Threading::ThreadHandle::GetForCallingThread());
 	m_event_loop = new QEventLoop();
 	m_started_semaphore.release();
 
@@ -244,7 +244,7 @@ void EmuThread::run()
 	destroyBackgroundControllerPollTimer();
 	InputManager::CloseSources();
 	VMManager::Internal::ReleaseMemory();
-	PerformanceMetrics::SetCPUThreadTimer(Common::ThreadCPUTimer());
+	PerformanceMetrics::SetCPUThread(Threading::ThreadHandle());
 	moveToThread(m_ui_thread);
 	deleteLater();
 }
diff --git a/pcsx2/GS/Renderers/SW/GSRasterizer.cpp b/pcsx2/GS/Renderers/SW/GSRasterizer.cpp
index e3c5c2cacd..2ccc59b2a9 100644
--- a/pcsx2/GS/Renderers/SW/GSRasterizer.cpp
+++ b/pcsx2/GS/Renderers/SW/GSRasterizer.cpp
@@ -1204,7 +1204,7 @@ GSRasterizerList::~GSRasterizerList()
 void GSRasterizerList::OnWorkerStartup(int i)
 {
 	Threading::SetNameOfCurrentThread(StringUtil::StdStringFromFormat("GS-SW-%d", i).c_str());
-	PerformanceMetrics::SetGSSWThreadTimer(i, Common::ThreadCPUTimer::GetForCallingThread());
+	PerformanceMetrics::SetGSSWThread(i, Threading::ThreadHandle::GetForCallingThread());
 }
 
 void GSRasterizerList::OnWorkerShutdown(int i)
diff --git a/pcsx2/PerformanceMetrics.cpp b/pcsx2/PerformanceMetrics.cpp
index 8006b32b9d..1ff4eb7fb3 100644
--- a/pcsx2/PerformanceMetrics.cpp
+++ b/pcsx2/PerformanceMetrics.cpp
@@ -18,6 +18,9 @@
 #include <chrono>
 #include <vector>
 
+#include "common/Timer.h"
+#include "common/Threading.h"
+
 #include "PerformanceMetrics.h"
 #include "System.h"
 #include "System/SysThreads.h"
@@ -50,7 +53,8 @@ static PerformanceMetrics::InternalFPSMethod s_internal_fps_method = Performance
 static u32 s_gs_framebuffer_blits_since_last_update = 0;
 static u32 s_gs_privileged_register_writes_since_last_update = 0;
 
-static Common::ThreadCPUTimer s_cpu_thread_timer;
+static Threading::ThreadHandle s_cpu_thread_handle;
+static u64 s_last_cpu_time = 0;
 static u64 s_last_gs_time = 0;
 static u64 s_last_vu_time = 0;
 static u64 s_last_ticks = 0;
@@ -64,7 +68,8 @@ static float s_vu_thread_time = 0.0f;
 
 struct GSSWThreadStats
 {
-	Common::ThreadCPUTimer timer;
+	Threading::ThreadHandle handle;
+	u64 last_cpu_time = 0;
 	double usage = 0.0;
 	double time = 0.0;
 };
@@ -112,10 +117,13 @@ void PerformanceMetrics::Reset()
 	s_last_update_time.Reset();
 	s_last_frame_time.Reset();
 
-	s_cpu_thread_timer.Reset();
+	s_last_cpu_time = s_cpu_thread_handle.GetCPUTime();
 	s_last_gs_time = GetMTGS().GetCpuTime();
 	s_last_vu_time = THREAD_VU1 ? vu1Thread.GetCpuTime() : 0;
 	s_last_ticks = GetCPUTicks();
+
+	for (GSSWThreadStats& stat : s_gs_sw_threads)
+		stat.last_cpu_time = stat.handle.GetCPUTime();
 }
 
 void PerformanceMetrics::Update(bool gs_register_write, bool fb_blit)
@@ -164,22 +172,9 @@ void PerformanceMetrics::Update(bool gs_register_write, bool fb_blit)
 	s_gs_privileged_register_writes_since_last_update = 0;
 	s_gs_framebuffer_blits_since_last_update = 0;
 
-	s_cpu_thread_timer.GetUsageInMillisecondsAndReset(ticks_diff, &s_cpu_thread_time, &s_cpu_thread_usage);
-	s_cpu_thread_time /= static_cast<double>(s_frames_since_last_update);
-
-	for (GSSWThreadStats& thread : s_gs_sw_threads)
-	{
-		thread.timer.GetUsageInMillisecondsAndReset(ticks_diff, &thread.time, &thread.usage);
-		thread.time /= static_cast<double>(s_frames_since_last_update);
-	}
-
-	const u64 gs_time = GetMTGS().GetCpuTime();
-	const u64 vu_time = THREAD_VU1 ? vu1Thread.GetCpuTime() : 0;
 	const u64 ticks = GetCPUTicks();
-
-	const u64 gs_delta = gs_time - s_last_gs_time;
-	const u64 vu_delta = vu_time - s_last_vu_time;
 	const u64 ticks_delta = ticks - s_last_ticks;
+	s_last_ticks = ticks;
 
 	const double pct_divider =
 		100.0 * (1.0 / ((static_cast<double>(ticks_delta) * static_cast<double>(GetThreadTicksPerSecond())) /
@@ -187,14 +182,32 @@ void PerformanceMetrics::Update(bool gs_register_write, bool fb_blit)
 	const double time_divider = 1000.0 * (1.0 / static_cast<double>(GetThreadTicksPerSecond())) *
 								(1.0 / static_cast<double>(s_frames_since_last_update));
 
+	const u64 cpu_time = s_cpu_thread_handle.GetCPUTime();
+	const u64 gs_time = GetMTGS().GetCpuTime();
+	const u64 vu_time = THREAD_VU1 ? vu1Thread.GetCpuTime() : 0;
+
+	const u64 cpu_delta = cpu_time - s_last_cpu_time;
+	const u64 gs_delta = gs_time - s_last_gs_time;
+	const u64 vu_delta = vu_time - s_last_vu_time;
+	s_last_cpu_time = cpu_time;
+	s_last_gs_time = gs_time;
+	s_last_vu_time = vu_time;
+
+	s_cpu_thread_usage = static_cast<double>(cpu_delta) * pct_divider;
 	s_gs_thread_usage = static_cast<double>(gs_delta) * pct_divider;
 	s_vu_thread_usage = static_cast<double>(vu_delta) * pct_divider;
+	s_cpu_thread_time = static_cast<double>(cpu_delta) * time_divider;
 	s_gs_thread_time = static_cast<double>(gs_delta) * time_divider;
 	s_vu_thread_time = static_cast<double>(vu_delta) * time_divider;
 
-	s_last_gs_time = gs_time;
-	s_last_vu_time = vu_time;
-	s_last_ticks = ticks;
+	for (GSSWThreadStats& thread : s_gs_sw_threads)
+	{
+		const u64 time = thread.handle.GetCPUTime();
+		const u64 delta = time - thread.last_cpu_time;
+		thread.last_cpu_time = time;
+		thread.usage = static_cast<double>(delta) * pct_divider;
+		thread.time = static_cast<double>(delta) * time_divider;
+	}
 
 	s_frames_since_last_update = 0;
 	s_presents_since_last_update = 0;
@@ -210,9 +223,10 @@ void PerformanceMetrics::OnGPUPresent(float gpu_time)
 	s_presents_since_last_update++;
 }
 
-void PerformanceMetrics::SetCPUThreadTimer(Common::ThreadCPUTimer timer)
+void PerformanceMetrics::SetCPUThread(Threading::ThreadHandle thread)
 {
-	s_cpu_thread_timer = std::move(timer);
+	s_last_cpu_time = thread ? thread.GetCPUTime() : 0;
+	s_cpu_thread_handle = std::move(thread);
 }
 
 void PerformanceMetrics::SetGSSWThreadCount(u32 count)
@@ -221,9 +235,10 @@ void PerformanceMetrics::SetGSSWThreadCount(u32 count)
 	s_gs_sw_threads.resize(count);
 }
 
-void PerformanceMetrics::SetGSSWThreadTimer(u32 index, Common::ThreadCPUTimer timer)
+void PerformanceMetrics::SetGSSWThread(u32 index, Threading::ThreadHandle thread)
 {
-	s_gs_sw_threads[index].timer = std::move(timer);
+	s_gs_sw_threads[index].last_cpu_time = thread ? thread.GetCPUTime() : 0;
+	s_gs_sw_threads[index].handle = std::move(thread);
 }
 
 void PerformanceMetrics::SetVerticalFrequency(float rate)
diff --git a/pcsx2/PerformanceMetrics.h b/pcsx2/PerformanceMetrics.h
index 9795800d60..4b4d575166 100644
--- a/pcsx2/PerformanceMetrics.h
+++ b/pcsx2/PerformanceMetrics.h
@@ -14,7 +14,7 @@
  */
 
 #pragma once
-#include "common/Timer.h"
+#include "common/Threading.h"
 
 namespace PerformanceMetrics
 {
@@ -31,11 +31,11 @@ namespace PerformanceMetrics
 	void OnGPUPresent(float gpu_time);
 
 	/// Sets the EE thread for CPU usage calculations.
-	void SetCPUThreadTimer(Common::ThreadCPUTimer timer);
+	void SetCPUThread(Threading::ThreadHandle thread);
 
 	/// Sets timers for GS software threads.
 	void SetGSSWThreadCount(u32 count);
-	void SetGSSWThreadTimer(u32 index, Common::ThreadCPUTimer timer);
+	void SetGSSWThread(u32 index, Threading::ThreadHandle thread);
 
 	/// Sets the vertical frequency, used in speed calculations.
 	void SetVerticalFrequency(float rate);
diff --git a/pcsx2/System/SysCoreThread.cpp b/pcsx2/System/SysCoreThread.cpp
index 769d8d488b..f2f0d7960b 100644
--- a/pcsx2/System/SysCoreThread.cpp
+++ b/pcsx2/System/SysCoreThread.cpp
@@ -341,12 +341,12 @@ void SysCoreThread::TearDownSystems(SystemsMask systemsToTearDown)
 	if (systemsToTearDown & System_SPU2) SPU2close();
 	if (systemsToTearDown & System_MCD) FileMcd_EmuClose();
 
-	PerformanceMetrics::SetCPUThreadTimer(Common::ThreadCPUTimer());
+	PerformanceMetrics::SetCPUThread(Threading::ThreadHandle());
 }
 
 void SysCoreThread::OnResumeInThread(SystemsMask systemsToReinstate)
 {
-	PerformanceMetrics::SetCPUThreadTimer(Common::ThreadCPUTimer::GetForCallingThread());
+	PerformanceMetrics::SetCPUThread(Threading::ThreadHandle::GetForCallingThread());
 	PerformanceMetrics::Reset();
 
 	GetMTGS().WaitForOpen();