diff --git a/Source/Android/jni/MainAndroid.cpp b/Source/Android/jni/MainAndroid.cpp
index f150c2c9e9..77c373a481 100644
--- a/Source/Android/jni/MainAndroid.cpp
+++ b/Source/Android/jni/MainAndroid.cpp
@@ -521,6 +521,7 @@ JNIEXPORT void JNICALL Java_org_dolphinemu_dolphinemu_NativeLibrary_Initialize(J
   Common::AndroidSetReportHandler(&ReportSend);
   DolphinAnalytics::AndroidSetGetValFunc(&GetAnalyticValue);
   UICommon::Init();
+  GCAdapter::Init();
 }
 
 JNIEXPORT void JNICALL Java_org_dolphinemu_dolphinemu_NativeLibrary_ReportStartToAnalytics(JNIEnv*,
diff --git a/Source/Core/DolphinNoGUI/MainNoGUI.cpp b/Source/Core/DolphinNoGUI/MainNoGUI.cpp
index 185bc447d9..cec4788f1c 100644
--- a/Source/Core/DolphinNoGUI/MainNoGUI.cpp
+++ b/Source/Core/DolphinNoGUI/MainNoGUI.cpp
@@ -30,6 +30,8 @@
 #endif
 #include "UICommon/UICommon.h"
 
+#include "InputCommon/GCAdapter.h"
+
 #include "VideoCommon/RenderBase.h"
 #include "VideoCommon/VideoBackendBase.h"
 
@@ -226,6 +228,7 @@ int main(int argc, char* argv[])
 
   UICommon::SetUserDirectory(user_directory);
   UICommon::Init();
+  GCAdapter::Init();
 
   s_platform = GetPlatform(options);
   if (!s_platform || !s_platform->Init())
diff --git a/Source/Core/DolphinQt/MainWindow.cpp b/Source/Core/DolphinQt/MainWindow.cpp
index 9c7d4d37ae..3be8f9d6a0 100644
--- a/Source/Core/DolphinQt/MainWindow.cpp
+++ b/Source/Core/DolphinQt/MainWindow.cpp
@@ -113,6 +113,7 @@
 #include "DolphinQt/WiiUpdate.h"
 
 #include "InputCommon/ControllerInterface/ControllerInterface.h"
+#include "InputCommon/GCAdapter.h"
 
 #include "UICommon/DiscordPresence.h"
 #include "UICommon/GameFile.h"
@@ -328,6 +329,7 @@ void MainWindow::InitControllers()
              "No default device has been added in time. EmulatedController(s) defaulting adds"
              " input mappings made for a specific default device depending on the platform");
   }
+  GCAdapter::Init();
   Pad::Initialize();
   Pad::InitializeGBA();
   Keyboard::Initialize();
diff --git a/Source/Core/InputCommon/GCAdapter.cpp b/Source/Core/InputCommon/GCAdapter.cpp
index e03cf99f82..973d1a1d5f 100644
--- a/Source/Core/InputCommon/GCAdapter.cpp
+++ b/Source/Core/InputCommon/GCAdapter.cpp
@@ -75,7 +75,7 @@ static bool s_libusb_hotplug_enabled = false;
 static libusb_hotplug_callback_handle s_hotplug_handle;
 #endif
 
-static LibusbUtils::Context s_libusb_context;
+static std::unique_ptr<LibusbUtils::Context> s_libusb_context;
 
 static u8 s_endpoint_in = 0;
 static u8 s_endpoint_out = 0;
@@ -172,7 +172,7 @@ static void ScanThreadFunc()
   if (s_libusb_hotplug_enabled)
   {
     if (libusb_hotplug_register_callback(
-            s_libusb_context,
+            *s_libusb_context,
             (libusb_hotplug_event)(LIBUSB_HOTPLUG_EVENT_DEVICE_ARRIVED |
                                    LIBUSB_HOTPLUG_EVENT_DEVICE_LEFT),
             LIBUSB_HOTPLUG_ENUMERATE, 0x057e, 0x0337, LIBUSB_HOTPLUG_MATCH_ANY, HotplugCallback,
@@ -218,6 +218,8 @@ void Init()
   if (s_handle != nullptr)
     return;
 
+  s_libusb_context = std::make_unique<LibusbUtils::Context>();
+
   if (Core::GetState() != Core::State::Uninitialized && Core::GetState() != Core::State::Starting)
   {
     if ((CoreTiming::GetTicks() - s_last_init) < SystemTimers::GetTicksPerSecond())
@@ -240,7 +242,7 @@ void StartScanThread()
 {
   if (s_adapter_detect_thread_running.IsSet())
     return;
-  if (!s_libusb_context.IsValid())
+  if (!s_libusb_context->IsValid())
     return;
   s_adapter_detect_thread_running.Set(true);
   s_adapter_detect_thread = std::thread(ScanThreadFunc);
@@ -266,7 +268,7 @@ static void Setup()
   s_controller_type.fill(ControllerTypes::CONTROLLER_NONE);
   s_controller_rumble.fill(0);
 
-  s_libusb_context.GetDeviceList([](libusb_device* device) {
+  s_libusb_context->GetDeviceList([](libusb_device* device) {
     if (CheckDeviceAccess(device))
     {
       // Only connect to a single adapter in case the user has multiple connected
@@ -397,9 +399,10 @@ void Shutdown()
 {
   StopScanThread();
 #if defined(LIBUSB_API_VERSION) && LIBUSB_API_VERSION >= 0x01000102
-  if (s_libusb_context.IsValid() && s_libusb_hotplug_enabled)
-    libusb_hotplug_deregister_callback(s_libusb_context, s_hotplug_handle);
+  if (s_libusb_context->IsValid() && s_libusb_hotplug_enabled)
+    libusb_hotplug_deregister_callback(*s_libusb_context, s_hotplug_handle);
 #endif
+  s_libusb_context.reset();
   Reset();
 
   s_status = NO_ADAPTER_DETECTED;
diff --git a/Source/Core/UICommon/UICommon.cpp b/Source/Core/UICommon/UICommon.cpp
index 4d043d371c..cb857fd3c7 100644
--- a/Source/Core/UICommon/UICommon.cpp
+++ b/Source/Core/UICommon/UICommon.cpp
@@ -108,7 +108,6 @@ void Init()
   SConfig::Init();
   Discord::Init();
   Common::Log::LogManager::Init();
-  GCAdapter::Init();
   VideoBackendBase::ActivateBackend(Config::Get(Config::MAIN_GFX_BACKEND));
 
   Common::SetEnableAlert(Config::Get(Config::MAIN_USE_PANIC_HANDLERS));