diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index 07a2740fc..f6cc9dcc8 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -118,6 +118,7 @@ add_library(core
   spu.h
   system.cpp
   system.h
+  system_private.h
   timers.cpp
   timers.h
   timing_event.cpp
diff --git a/src/core/core.vcxproj b/src/core/core.vcxproj
index 8b12fec78..f3798eb86 100644
--- a/src/core/core.vcxproj
+++ b/src/core/core.vcxproj
@@ -170,6 +170,7 @@
     <ClInclude Include="sio.h" />
     <ClInclude Include="spu.h" />
     <ClInclude Include="system.h" />
+    <ClInclude Include="system_private.h" />
     <ClInclude Include="timers.h" />
     <ClInclude Include="timing_event.h" />
     <ClInclude Include="types.h" />
diff --git a/src/core/core.vcxproj.filters b/src/core/core.vcxproj.filters
index b0d8a5369..5a8d278a2 100644
--- a/src/core/core.vcxproj.filters
+++ b/src/core/core.vcxproj.filters
@@ -148,6 +148,7 @@
     <ClInclude Include="gpu_dump.h" />
     <ClInclude Include="cdrom_subq_replacement.h" />
     <ClInclude Include="performance_counters.h" />
+    <ClInclude Include="system_private.h" />
   </ItemGroup>
   <ItemGroup>
     <None Include="gpu_sw_rasterizer.inl" />
diff --git a/src/core/fullscreen_ui.cpp b/src/core/fullscreen_ui.cpp
index 166065156..83bd31239 100644
--- a/src/core/fullscreen_ui.cpp
+++ b/src/core/fullscreen_ui.cpp
@@ -11,6 +11,7 @@
 #include "host.h"
 #include "settings.h"
 #include "system.h"
+#include "system_private.h"
 
 #include "scmversion/scmversion.h"
 
diff --git a/src/core/game_list.h b/src/core/game_list.h
index e596439cf..1d87635b2 100644
--- a/src/core/game_list.h
+++ b/src/core/game_list.h
@@ -16,7 +16,6 @@
 #include <span>
 #include <string>
 
-class ByteStream;
 class ProgressCallback;
 
 struct SystemBootParameters;
diff --git a/src/core/gpu.cpp b/src/core/gpu.cpp
index 405a6ce20..dcea1824e 100644
--- a/src/core/gpu.cpp
+++ b/src/core/gpu.cpp
@@ -11,6 +11,7 @@
 #include "performance_counters.h"
 #include "settings.h"
 #include "system.h"
+#include "system_private.h"
 #include "timers.h"
 #include "timing_event.h"
 
@@ -1063,6 +1064,7 @@ void GPU::CRTCTickEvent(TickCount ticks)
         // TODO: move present in here I guess
         FlushRender();
         UpdateDisplay();
+        System::IncrementFrameNumber();
         frame_done = true;
 
         // switch fields early. this is needed so we draw to the correct one.
@@ -3105,6 +3107,7 @@ void GPU::ProcessGPUDumpPacket(GPUDump::PacketType type, const std::span<const u
 
       FlushRender();
       UpdateDisplay();
+      System::IncrementFrameNumber();
       System::FrameDone();
     }
     break;
diff --git a/src/core/performance_counters.cpp b/src/core/performance_counters.cpp
index 44059c886..5a6ad2a20 100644
--- a/src/core/performance_counters.cpp
+++ b/src/core/performance_counters.cpp
@@ -4,6 +4,7 @@
 #include "performance_counters.h"
 #include "gpu.h"
 #include "system.h"
+#include "system_private.h"
 
 #include "util/media_capture.h"
 
@@ -148,7 +149,7 @@ void PerformanceCounters::Reset()
 
   s_state.last_frame_number = System::GetFrameNumber();
   s_state.last_internal_frame_number = System::GetInternalFrameNumber();
-  s_state.last_cpu_time = System::Internal::GetCPUThreadHandle().GetCPUTime();
+  s_state.last_cpu_time = System::GetCPUThreadHandle().GetCPUTime();
   if (const Threading::Thread* sw_thread = g_gpu->GetSWThread(); sw_thread)
     s_state.last_sw_time = sw_thread->GetCPUTime();
   else
@@ -202,7 +203,7 @@ void PerformanceCounters::Update(u32 frame_number, u32 internal_frame_number)
   s_state.speed = (s_state.vps / System::GetVideoFrameRate()) * 100.0f;
 
   const Threading::Thread* sw_thread = g_gpu->GetSWThread();
-  const u64 cpu_time = System::Internal::GetCPUThreadHandle().GetCPUTime();
+  const u64 cpu_time = System::GetCPUThreadHandle().GetCPUTime();
   const u64 sw_time = sw_thread ? sw_thread->GetCPUTime() : 0;
   const u64 cpu_delta = cpu_time - s_state.last_cpu_time;
   const u64 sw_delta = sw_time - s_state.last_sw_time;
diff --git a/src/core/settings.cpp b/src/core/settings.cpp
index 0144056e5..ec06a2712 100644
--- a/src/core/settings.cpp
+++ b/src/core/settings.cpp
@@ -2311,7 +2311,6 @@ static const char* s_log_filters[] = {
   "AutoUpdaterDialog",
   "BIOS",
   "Bus",
-  "ByteStream",
   "CDImage",
   "CDImageBin",
   "CDImageCHD",
diff --git a/src/core/system.cpp b/src/core/system.cpp
index 6740df10d..e115dc6e0 100644
--- a/src/core/system.cpp
+++ b/src/core/system.cpp
@@ -33,6 +33,7 @@
 #include "save_state_version.h"
 #include "sio.h"
 #include "spu.h"
+#include "system_private.h"
 #include "timers.h"
 
 #include "scmversion/scmversion.h"
@@ -113,7 +114,6 @@ SystemBootParameters::~SystemBootParameters() = default;
 
 namespace System {
 
-/// Memory save states - only for internal use.
 namespace {
 
 struct SaveStateBuffer
@@ -127,14 +127,6 @@ struct SaveStateBuffer
   DynamicHeapArray<u8> state_data;
   size_t state_size;
 };
-struct MemorySaveState
-{
-  std::unique_ptr<GPUTexture> vram_texture;
-  DynamicHeapArray<u8> state_data;
-#ifdef PROFILE_MEMORY_SAVE_STATES
-  size_t state_size;
-#endif
-};
 
 } // namespace
 
@@ -197,15 +189,10 @@ static void UpdatePerGameMemoryCards();
 static std::unique_ptr<MemoryCard> GetMemoryCardForSlot(u32 slot, MemoryCardType type);
 static void UpdateMultitaps();
 
-/// Returns the maximum size of a save state, considering the current configuration.
-static size_t GetMaxSaveStateSize();
-
 static std::string GetMediaPathFromSaveState(const char* path);
 static bool SaveUndoLoadState();
 static void UpdateMemorySaveStateSettings();
 static bool LoadRewindState(u32 skip_saves = 0, bool consume_state = true);
-static bool SaveMemoryState(MemorySaveState* mss);
-static bool LoadMemoryState(const MemorySaveState& mss);
 static bool LoadStateFromBuffer(const SaveStateBuffer& buffer, Error* error, bool update_display);
 static bool LoadStateBufferFromFile(SaveStateBuffer* buffer, std::FILE* fp, Error* error, bool read_title,
                                     bool read_media_path, bool read_screenshot, bool read_data);
@@ -216,7 +203,6 @@ static bool SaveStateBufferToFile(const SaveStateBuffer& buffer, std::FILE* fp,
                                   SaveStateCompressionMode compression_mode);
 static u32 CompressAndWriteStateData(std::FILE* fp, std::span<const u8> src, SaveStateCompressionMode method,
                                      u32* header_type, Error* error);
-static bool DoState(StateWrapper& sw, GPUTexture** host_texture, bool update_display, bool is_memory_state);
 
 static bool IsExecutionInterrupted();
 static void CheckForAndExitExecution();
@@ -337,7 +323,7 @@ static TinyString GetTimestampStringForFileName()
   return TinyString::from_format("{:%Y-%m-%d-%H-%M-%S}", fmt::localtime(std::time(nullptr)));
 }
 
-bool System::Internal::PerformEarlyHardwareChecks(Error* error)
+bool System::PerformEarlyHardwareChecks(Error* error)
 {
   // This shouldn't fail... if it does, just hope for the best.
   cpuinfo_initialize();
@@ -457,7 +443,7 @@ void System::LogStartupInformation()
 #endif
 }
 
-bool System::Internal::ProcessStartup(Error* error)
+bool System::ProcessStartup(Error* error)
 {
   Common::Timer timer;
 
@@ -482,13 +468,13 @@ bool System::Internal::ProcessStartup(Error* error)
   return true;
 }
 
-void System::Internal::ProcessShutdown()
+void System::ProcessShutdown()
 {
   Bus::ReleaseMemory();
   CPU::CodeCache::ProcessShutdown();
 }
 
-bool System::Internal::CPUThreadInitialize(Error* error)
+bool System::CPUThreadInitialize(Error* error)
 {
   Threading::SetNameOfCurrentThread("CPU Thread");
 
@@ -520,7 +506,7 @@ bool System::Internal::CPUThreadInitialize(Error* error)
   return true;
 }
 
-void System::Internal::CPUThreadShutdown()
+void System::CPUThreadShutdown()
 {
 #ifdef ENABLE_DISCORD_PRESENCE
   ShutdownDiscordPresence();
@@ -535,12 +521,12 @@ void System::Internal::CPUThreadShutdown()
 #endif
 }
 
-const Threading::ThreadHandle& System::Internal::GetCPUThreadHandle()
+const Threading::ThreadHandle& System::GetCPUThreadHandle()
 {
   return s_cpu_thread_handle;
 }
 
-void System::Internal::IdlePollUpdate()
+void System::IdlePollUpdate()
 {
   InputManager::PollSources();
 
@@ -2048,8 +2034,6 @@ void System::Execute()
 
 void System::FrameDone()
 {
-  s_frame_number++;
-
   // Vertex buffer is shared, need to flush what we have.
   g_gpu->FlushRender();
 
@@ -2330,6 +2314,11 @@ void System::SingleStepCPU()
     PauseSystem(false);
 }
 
+void System::IncrementFrameNumber()
+{
+  s_frame_number++;
+}
+
 void System::IncrementInternalFrameNumber()
 {
   if (IsFastForwardingBoot()) [[unlikely]]
@@ -4886,7 +4875,7 @@ void System::DoRewind()
 
   InvalidateDisplay();
   Host::PumpMessagesOnCPUThread();
-  Internal::IdlePollUpdate();
+  IdlePollUpdate();
 
   Throttle(Common::Timer::GetCurrentValue());
 }
diff --git a/src/core/system.h b/src/core/system.h
index 781ac8bae..1e064b2a1 100644
--- a/src/core/system.h
+++ b/src/core/system.h
@@ -217,8 +217,6 @@ void UpdateOverclock();
 GlobalTicks GetGlobalTickCounter();
 u32 GetFrameNumber();
 u32 GetInternalFrameNumber();
-void IncrementInternalFrameNumber();
-void FrameDone();
 
 const std::string& GetDiscPath();
 const std::string& GetGameSerial();
@@ -437,72 +435,11 @@ void ReleaseSocketMultiplexer();
 /// Called when rich presence changes.
 void UpdateRichPresence(bool update_session_time);
 
-namespace Internal {
-/// Performs mandatory hardware checks.
-bool PerformEarlyHardwareChecks(Error* error);
-
-/// Called on process startup, as early as possible.
-bool ProcessStartup(Error* error);
-
-/// Called on process shutdown.
-void ProcessShutdown();
-
-/// Called on CPU thread initialization.
-bool CPUThreadInitialize(Error* error);
-
-/// Called on CPU thread shutdown.
-void CPUThreadShutdown();
-
-/// Returns a handle to the CPU thread.
-const Threading::ThreadHandle& GetCPUThreadHandle();
-
-/// Polls input, updates subsystems which are present while paused/inactive.
-void IdlePollUpdate();
-} // namespace Internal
-
 } // namespace System
 
 namespace Host {
-/// Called with the settings lock held, when system settings are being loaded (should load input sources, etc).
-void LoadSettings(const SettingsInterface& si, std::unique_lock<std::mutex>& lock);
-
-/// Called after settings are updated.
-void CheckForSettingsChanges(const Settings& old_settings);
-
-/// Called when the VM is starting initialization, but has not been completed yet.
-void OnSystemStarting();
-
-/// Called when the VM is created.
-void OnSystemStarted();
-
-/// Called when the VM is shut down or destroyed.
-void OnSystemDestroyed();
-
-/// Called when the VM is paused.
-void OnSystemPaused();
-
-/// Called when the VM is resumed after being paused.
-void OnSystemResumed();
-
-/// Called when the pause state changes, or fullscreen UI opens.
-void OnIdleStateChanged();
-
-/// Called when performance metrics are updated, approximately once a second.
-void OnPerformanceCountersUpdated();
-
-/// Provided by the host; called when the running executable changes.
-void OnGameChanged(const std::string& disc_path, const std::string& game_serial, const std::string& game_name);
-
-/// Called when media capture starts/stops.
-void OnMediaCaptureStarted();
-void OnMediaCaptureStopped();
-
-/// Provided by the host; called once per frame at guest vsync.
-void PumpMessagesOnCPUThread();
-
-/// Requests a specific display window size.
-void RequestResizeHostDisplay(s32 width, s32 height);
 
 /// Requests shut down of the current virtual machine.
 void RequestSystemShutdown(bool allow_confirm, bool save_state);
+
 } // namespace Host
diff --git a/src/core/system_private.h b/src/core/system_private.h
new file mode 100644
index 000000000..7ba41e226
--- /dev/null
+++ b/src/core/system_private.h
@@ -0,0 +1,95 @@
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-License-Identifier: CC-BY-NC-ND-4.0
+
+#pragma once
+
+#include "system.h"
+
+namespace System {
+
+/// Memory save states - only for internal use.
+struct MemorySaveState
+{
+  std::unique_ptr<GPUTexture> vram_texture;
+  DynamicHeapArray<u8> state_data;
+  size_t state_size;
+};
+
+bool SaveMemoryState(MemorySaveState* mss);
+bool LoadMemoryState(const MemorySaveState& mss);
+
+/// Returns the maximum size of a save state, considering the current configuration.
+size_t GetMaxSaveStateSize();
+
+bool DoState(StateWrapper& sw, GPUTexture** host_texture, bool update_display, bool is_memory_state);
+
+void IncrementFrameNumber();
+void IncrementInternalFrameNumber();
+void FrameDone();
+
+/// Performs mandatory hardware checks.
+bool PerformEarlyHardwareChecks(Error* error);
+
+/// Called on process startup, as early as possible.
+bool ProcessStartup(Error* error);
+
+/// Called on process shutdown.
+void ProcessShutdown();
+
+/// Called on CPU thread initialization.
+bool CPUThreadInitialize(Error* error);
+
+/// Called on CPU thread shutdown.
+void CPUThreadShutdown();
+
+/// Returns a handle to the CPU thread.
+const Threading::ThreadHandle& GetCPUThreadHandle();
+
+/// Polls input, updates subsystems which are present while paused/inactive.
+void IdlePollUpdate();
+
+} // namespace System
+
+namespace Host {
+
+/// Called with the settings lock held, when system settings are being loaded (should load input sources, etc).
+void LoadSettings(const SettingsInterface& si, std::unique_lock<std::mutex>& lock);
+
+/// Called after settings are updated.
+void CheckForSettingsChanges(const Settings& old_settings);
+
+/// Called when the VM is starting initialization, but has not been completed yet.
+void OnSystemStarting();
+
+/// Called when the VM is created.
+void OnSystemStarted();
+
+/// Called when the VM is shut down or destroyed.
+void OnSystemDestroyed();
+
+/// Called when the VM is paused.
+void OnSystemPaused();
+
+/// Called when the VM is resumed after being paused.
+void OnSystemResumed();
+
+/// Called when the pause state changes, or fullscreen UI opens.
+void OnIdleStateChanged();
+
+/// Called when performance metrics are updated, approximately once a second.
+void OnPerformanceCountersUpdated();
+
+/// Provided by the host; called when the running executable changes.
+void OnGameChanged(const std::string& disc_path, const std::string& game_serial, const std::string& game_name);
+
+/// Called when media capture starts/stops.
+void OnMediaCaptureStarted();
+void OnMediaCaptureStopped();
+
+/// Provided by the host; called once per frame at guest vsync.
+void PumpMessagesOnCPUThread();
+
+/// Requests a specific display window size.
+void RequestResizeHostDisplay(s32 width, s32 height);
+
+} // namespace Host
diff --git a/src/duckstation-qt/qthost.cpp b/src/duckstation-qt/qthost.cpp
index 8b755f25f..76719b7e3 100644
--- a/src/duckstation-qt/qthost.cpp
+++ b/src/duckstation-qt/qthost.cpp
@@ -26,6 +26,7 @@
 #include "core/performance_counters.h"
 #include "core/spu.h"
 #include "core/system.h"
+#include "core/system_private.h"
 
 #include "common/assert.h"
 #include "common/crash_handler.h"
@@ -162,7 +163,7 @@ void QtHost::RegisterTypes()
 bool QtHost::PerformEarlyHardwareChecks()
 {
   Error error;
-  const bool okay = System::Internal::PerformEarlyHardwareChecks(&error);
+  const bool okay = System::PerformEarlyHardwareChecks(&error);
   if (okay && !error.IsValid()) [[likely]]
     return true;
 
@@ -189,7 +190,7 @@ bool QtHost::EarlyProcessStartup()
 #endif
 
   Error error;
-  if (System::Internal::ProcessStartup(&error)) [[likely]]
+  if (System::ProcessStartup(&error)) [[likely]]
     return true;
 
   QMessageBox::critical(nullptr, QStringLiteral("Process Startup Failed"),
@@ -1775,7 +1776,7 @@ void EmuThread::processAuxiliaryRenderWindowInputEvent(void* userdata, quint32 e
 
 void EmuThread::doBackgroundControllerPoll()
 {
-  System::Internal::IdlePollUpdate();
+  System::IdlePollUpdate();
 }
 
 void EmuThread::createBackgroundControllerPollTimer()
@@ -1850,7 +1851,7 @@ void EmuThread::run()
   // input source setup must happen on emu thread
   {
     Error startup_error;
-    if (!System::Internal::CPUThreadInitialize(&startup_error))
+    if (!System::CPUThreadInitialize(&startup_error))
     {
       moveToThread(m_ui_thread);
       Host::ReportFatalError("Fatal Startup Error", startup_error.GetDescription());
@@ -1880,7 +1881,7 @@ void EmuThread::run()
       }
 
       m_event_loop->processEvents(QEventLoop::AllEvents);
-      System::Internal::IdlePollUpdate();
+      System::IdlePollUpdate();
       if (g_gpu_device && g_gpu_device->HasMainSwapChain())
       {
         System::PresentDisplay(false, 0);
@@ -1894,7 +1895,7 @@ void EmuThread::run()
     System::ShutdownSystem(false);
 
   destroyBackgroundControllerPollTimer();
-  System::Internal::CPUThreadShutdown();
+  System::CPUThreadShutdown();
 
   // move back to UI thread
   moveToThread(m_ui_thread);
@@ -2724,7 +2725,7 @@ shutdown_and_exit:
   // Ensure log is flushed.
   Log::SetFileOutputParams(false, nullptr);
 
-  System::Internal::ProcessShutdown();
+  System::ProcessShutdown();
 
   return result;
 }
diff --git a/src/duckstation-qt/qthost.h b/src/duckstation-qt/qthost.h
index 2d96ab5bf..b01c60275 100644
--- a/src/duckstation-qt/qthost.h
+++ b/src/duckstation-qt/qthost.h
@@ -32,8 +32,6 @@
 #include <utility>
 #include <vector>
 
-class ByteStream;
-
 class QActionGroup;
 class QEventLoop;
 class QMenu;
diff --git a/src/duckstation-regtest/regtest_host.cpp b/src/duckstation-regtest/regtest_host.cpp
index 05108c0d7..baf1e5a66 100644
--- a/src/duckstation-regtest/regtest_host.cpp
+++ b/src/duckstation-regtest/regtest_host.cpp
@@ -8,6 +8,7 @@
 #include "core/gpu.h"
 #include "core/host.h"
 #include "core/system.h"
+#include "core/system_private.h"
 
 #include "scmversion/scmversion.h"
 
@@ -738,8 +739,7 @@ std::string RegTestHost::GetFrameDumpFilename(u32 frame)
 int main(int argc, char* argv[])
 {
   Error startup_error;
-  if (!System::Internal::PerformEarlyHardwareChecks(&startup_error) ||
-      !System::Internal::ProcessStartup(&startup_error))
+  if (!System::PerformEarlyHardwareChecks(&startup_error) || !System::ProcessStartup(&startup_error))
   {
     ERROR_LOG("CPUThreadInitialize() failed: {}", startup_error.GetDescription());
     return EXIT_FAILURE;
@@ -763,7 +763,7 @@ int main(int argc, char* argv[])
   if (!RegTestHost::SetNewDataRoot(autoboot->filename))
     return EXIT_FAILURE;
 
-  if (!System::Internal::CPUThreadInitialize(&startup_error))
+  if (!System::CPUThreadInitialize(&startup_error))
   {
     ERROR_LOG("CPUThreadInitialize() failed: {}", startup_error.GetDescription());
     return EXIT_FAILURE;
@@ -810,7 +810,7 @@ int main(int argc, char* argv[])
   result = 0;
 
 cleanup:
-  System::Internal::CPUThreadShutdown();
-  System::Internal::ProcessShutdown();
+  System::CPUThreadShutdown();
+  System::ProcessShutdown();
   return result;
 }