diff --git a/Source/Core/Common/CMakeLists.txt b/Source/Core/Common/CMakeLists.txt
index c894a297c3..4e1f21b55f 100644
--- a/Source/Core/Common/CMakeLists.txt
+++ b/Source/Core/Common/CMakeLists.txt
@@ -40,11 +40,6 @@ set(SRCS
   Logging/LogManager.cpp
 )
 
-if(LIBUSB_FOUND)
-  set(LIBS ${LIBS} ${LIBUSB_LIBRARIES})
-  set(SRCS ${SRCS} LibusbContext.cpp)
-endif(LIBUSB_FOUND)
-
 if(ANDROID)
   set(SRCS ${SRCS}
     Logging/ConsoleListenerDroid.cpp)
diff --git a/Source/Core/Common/Common.vcxproj b/Source/Core/Common/Common.vcxproj
index 928c8c8196..13256bdce7 100644
--- a/Source/Core/Common/Common.vcxproj
+++ b/Source/Core/Common/Common.vcxproj
@@ -118,7 +118,6 @@
     <ClInclude Include="Hash.h" />
     <ClInclude Include="IniFile.h" />
     <ClInclude Include="JitRegister.h" />
-    <ClInclude Include="LibusbContext.h" />
     <ClInclude Include="LinearDiskCache.h" />
     <ClInclude Include="MathUtil.h" />
     <ClInclude Include="MD5.h" />
@@ -170,9 +169,6 @@
     <ClCompile Include="Hash.cpp" />
     <ClCompile Include="IniFile.cpp" />
     <ClCompile Include="JitRegister.cpp" />
-    <ClCompile Include="LibusbContext.cpp">
-      <DisableSpecificWarnings>4200;%(DisableSpecificWarnings)</DisableSpecificWarnings>
-    </ClCompile>
     <ClCompile Include="Logging\ConsoleListenerWin.cpp" />
     <ClCompile Include="MathUtil.cpp" />
     <ClCompile Include="MD5.cpp" />
diff --git a/Source/Core/Common/Common.vcxproj.filters b/Source/Core/Common/Common.vcxproj.filters
index b6dd7d27e0..cb6fba5ad5 100644
--- a/Source/Core/Common/Common.vcxproj.filters
+++ b/Source/Core/Common/Common.vcxproj.filters
@@ -49,7 +49,6 @@
     <ClInclude Include="FPURoundMode.h" />
     <ClInclude Include="Hash.h" />
     <ClInclude Include="IniFile.h" />
-    <ClInclude Include="LibusbContext.h" />
     <ClInclude Include="LinearDiskCache.h" />
     <ClInclude Include="MathUtil.h" />
     <ClInclude Include="MemArena.h" />
@@ -251,7 +250,6 @@
     <ClCompile Include="FileUtil.cpp" />
     <ClCompile Include="Hash.cpp" />
     <ClCompile Include="IniFile.cpp" />
-    <ClCompile Include="LibusbContext.cpp" />
     <ClCompile Include="MathUtil.cpp" />
     <ClCompile Include="MemArena.cpp" />
     <ClCompile Include="MemoryUtil.cpp" />
diff --git a/Source/Core/Common/LibusbContext.cpp b/Source/Core/Common/LibusbContext.cpp
deleted file mode 100644
index 1eb729c4a0..0000000000
--- a/Source/Core/Common/LibusbContext.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2017 Dolphin Emulator Project
-// Licensed under GPLv2+
-// Refer to the license.txt file included.
-
-#include <libusb.h>
-#include <memory>
-#include <mutex>
-
-#include "Common/LibusbContext.h"
-#include "Common/MsgHandler.h"
-#include "Common/StringUtil.h"
-
-namespace LibusbContext
-{
-static std::shared_ptr<libusb_context> s_libusb_context;
-static std::once_flag s_context_initialized;
-
-static libusb_context* Create()
-{
-  libusb_context* context;
-  const int ret = libusb_init(&context);
-  if (ret < LIBUSB_SUCCESS)
-  {
-    bool is_windows = false;
-#ifdef _WIN32
-    is_windows = true;
-#endif
-    const std::string reason =
-        is_windows && ret == LIBUSB_ERROR_NOT_FOUND ?
-            GetStringT("Failed to initialize libusb because usbdk is not installed.") :
-            StringFromFormat(GetStringT("Failed to initialize libusb (%s).").c_str(),
-                             libusb_error_name(ret));
-    PanicAlertT("%s\nSome USB features will not work.", reason.c_str());
-    return nullptr;
-  }
-  return context;
-}
-
-std::shared_ptr<libusb_context> Get()
-{
-  std::call_once(s_context_initialized, []() {
-    s_libusb_context.reset(Create(), [](auto* context) {
-      if (context != nullptr)
-        libusb_exit(context);
-    });
-  });
-  return s_libusb_context;
-}
-}
diff --git a/Source/Core/Common/LibusbContext.h b/Source/Core/Common/LibusbContext.h
deleted file mode 100644
index 99af12ba4c..0000000000
--- a/Source/Core/Common/LibusbContext.h
+++ /dev/null
@@ -1,15 +0,0 @@
-// Copyright 2017 Dolphin Emulator Project
-// Licensed under GPLv2+
-// Refer to the license.txt file included.
-
-#include <memory>
-
-struct libusb_context;
-
-namespace LibusbContext
-{
-// libusb on Windows is limited to only a single context. Trying to open more
-// than one can cause issues with device enumerations.
-// libusb is thread-safe so this context can be safely used from different threads.
-std::shared_ptr<libusb_context> Get();
-}
diff --git a/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp b/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp
index e76becfc96..cba2a3f394 100644
--- a/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp
+++ b/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp
@@ -15,7 +15,6 @@
 
 #include "Common/Assert.h"
 #include "Common/ChunkFile.h"
-#include "Common/LibusbContext.h"
 #include "Common/Logging/Log.h"
 #include "Common/MsgHandler.h"
 #include "Common/Network.h"
@@ -60,7 +59,11 @@ namespace Device
 BluetoothReal::BluetoothReal(u32 device_id, const std::string& device_name)
     : BluetoothBase(device_id, device_name)
 {
-  m_libusb_context = LibusbContext::Get();
+  const int ret = libusb_init(&m_libusb_context);
+  if (ret < 0)
+  {
+    PanicAlertT("Couldn't initialise libusb for Bluetooth passthrough: %s", libusb_error_name(ret));
+  }
   LoadLinkKeys();
 }
 
@@ -77,6 +80,7 @@ BluetoothReal::~BluetoothReal()
     libusb_unref_device(m_device);
   }
 
+  libusb_exit(m_libusb_context);
   SaveLinkKeys();
 }
 
@@ -86,15 +90,21 @@ ReturnCode BluetoothReal::Open(const OpenRequest& request)
     return IPC_EACCES;
 
   libusb_device** list;
-  const ssize_t cnt = libusb_get_device_list(m_libusb_context.get(), &list);
-  _dbg_assert_msg_(IOS, cnt > 0, "Couldn't get device list");
+  const ssize_t cnt = libusb_get_device_list(m_libusb_context, &list);
+  if (cnt < 0)
+  {
+    ERROR_LOG(IOS_WIIMOTE, "Couldn't get device list: %s",
+              libusb_error_name(static_cast<int>(cnt)));
+    return IPC_ENOENT;
+  }
+
   for (ssize_t i = 0; i < cnt; ++i)
   {
     libusb_device* device = list[i];
     libusb_device_descriptor device_descriptor;
     libusb_config_descriptor* config_descriptor;
     libusb_get_device_descriptor(device, &device_descriptor);
-    const int ret = libusb_get_active_config_descriptor(device, &config_descriptor);
+    const int ret = libusb_get_config_descriptor(device, 0, &config_descriptor);
     if (ret != 0)
     {
       ERROR_LOG(IOS_WIIMOTE, "Failed to get config descriptor for device %04x:%04x: %s",
@@ -601,7 +611,7 @@ void BluetoothReal::TransferThread()
   Common::SetCurrentThreadName("BT USB Thread");
   while (m_thread_running.IsSet())
   {
-    libusb_handle_events_completed(m_libusb_context.get(), nullptr);
+    libusb_handle_events_completed(m_libusb_context, nullptr);
   }
 }
 
diff --git a/Source/Core/Core/IOS/USB/Bluetooth/BTReal.h b/Source/Core/Core/IOS/USB/Bluetooth/BTReal.h
index ba98483c7c..dc1679925a 100644
--- a/Source/Core/Core/IOS/USB/Bluetooth/BTReal.h
+++ b/Source/Core/Core/IOS/USB/Bluetooth/BTReal.h
@@ -76,7 +76,7 @@ private:
 
   libusb_device* m_device = nullptr;
   libusb_device_handle* m_handle = nullptr;
-  std::shared_ptr<libusb_context> m_libusb_context;
+  libusb_context* m_libusb_context = nullptr;
 
   Common::Flag m_thread_running;
   std::thread m_thread;
diff --git a/Source/Core/Core/IOS/USB/Host.cpp b/Source/Core/Core/IOS/USB/Host.cpp
index 226c107eb0..0ad6bc9cce 100644
--- a/Source/Core/Core/IOS/USB/Host.cpp
+++ b/Source/Core/Core/IOS/USB/Host.cpp
@@ -13,7 +13,6 @@
 #include "Common/Assert.h"
 #include "Common/ChunkFile.h"
 #include "Common/CommonTypes.h"
-#include "Common/LibusbContext.h"
 #include "Common/Logging/Log.h"
 #include "Common/Thread.h"
 #include "Core/ConfigManager.h"
@@ -30,6 +29,18 @@ namespace Device
 {
 USBHost::USBHost(u32 device_id, const std::string& device_name) : Device(device_id, device_name)
 {
+#ifdef __LIBUSB__
+  const int ret = libusb_init(&m_libusb_context);
+  _dbg_assert_msg_(IOS_USB, ret == 0, "Failed to init libusb for USB passthrough.");
+#endif
+}
+
+USBHost::~USBHost()
+{
+#ifdef __LIBUSB__
+  if (m_libusb_context)
+    libusb_exit(m_libusb_context);
+#endif
 }
 
 ReturnCode USBHost::Open(const OpenRequest& request)
@@ -116,11 +127,10 @@ bool USBHost::AddNewDevices(std::set<u64>& new_devices, DeviceChangeHooks& hooks
   if (SConfig::GetInstance().m_usb_passthrough_devices.empty())
     return true;
 
-  auto libusb_context = LibusbContext::Get();
-  if (libusb_context)
+  if (m_libusb_context)
   {
     libusb_device** list;
-    const ssize_t count = libusb_get_device_list(libusb_context.get(), &list);
+    const ssize_t count = libusb_get_device_list(m_libusb_context, &list);
     if (count < 0)
     {
       WARN_LOG(IOS_USB, "Failed to get device list: %s",
@@ -135,17 +145,25 @@ bool USBHost::AddNewDevices(std::set<u64>& new_devices, DeviceChangeHooks& hooks
       libusb_get_device_descriptor(device, &descriptor);
       if (!SConfig::GetInstance().IsUSBDeviceWhitelisted(
               {descriptor.idVendor, descriptor.idProduct}))
+      {
+        libusb_unref_device(device);
         continue;
+      }
 
       auto usb_device = std::make_unique<USB::LibusbDevice>(device, descriptor);
       if (!ShouldAddDevice(*usb_device))
+      {
+        libusb_unref_device(device);
         continue;
+      }
       const u64 id = usb_device->GetId();
       new_devices.insert(id);
       if (AddDevice(std::move(usb_device)) || always_add_hooks)
         hooks.emplace(GetDeviceById(id), ChangeEvent::Inserted);
+      else
+        libusb_unref_device(device);
     }
-    libusb_free_device_list(list, 1);
+    libusb_free_device_list(list, 0);
   }
 #endif
   return true;
@@ -200,12 +218,11 @@ void USBHost::StartThreads()
   }
 
 #ifdef __LIBUSB__
-  if (!m_event_thread_running.IsSet())
+  if (!m_event_thread_running.IsSet() && m_libusb_context)
   {
     m_event_thread_running.Set();
     m_event_thread = std::thread([this] {
       Common::SetCurrentThreadName("USB Passthrough Thread");
-      std::shared_ptr<libusb_context> context;
       while (m_event_thread_running.IsSet())
       {
         if (SConfig::GetInstance().m_usb_passthrough_devices.empty())
@@ -214,15 +231,8 @@ void USBHost::StartThreads()
           continue;
         }
 
-        if (!context)
-          context = LibusbContext::Get();
-
-        // If we failed to get a libusb context, stop the thread.
-        if (!context)
-          return;
-
         static timeval tv = {0, 50000};
-        libusb_handle_events_timeout_completed(context.get(), &tv, nullptr);
+        libusb_handle_events_timeout_completed(m_libusb_context, &tv, nullptr);
       }
     });
   }
diff --git a/Source/Core/Core/IOS/USB/Host.h b/Source/Core/Core/IOS/USB/Host.h
index 27d94d805f..3bdcb2445b 100644
--- a/Source/Core/Core/IOS/USB/Host.h
+++ b/Source/Core/Core/IOS/USB/Host.h
@@ -34,7 +34,7 @@ class USBHost : public Device
 {
 public:
   USBHost(u32 device_id, const std::string& device_name);
-  virtual ~USBHost() = default;
+  virtual ~USBHost();
 
   ReturnCode Open(const OpenRequest& request) override;
 
@@ -71,6 +71,8 @@ private:
   void DispatchHooks(const DeviceChangeHooks& hooks);
 
 #ifdef __LIBUSB__
+  libusb_context* m_libusb_context = nullptr;
+
   // Event thread for libusb
   Common::Flag m_event_thread_running;
   std::thread m_event_thread;
diff --git a/Source/Core/InputCommon/GCAdapter.cpp b/Source/Core/InputCommon/GCAdapter.cpp
index 1b251b3f5e..5916ab600f 100644
--- a/Source/Core/InputCommon/GCAdapter.cpp
+++ b/Source/Core/InputCommon/GCAdapter.cpp
@@ -4,11 +4,9 @@
 
 #include <algorithm>
 #include <libusb.h>
-#include <memory>
 #include <mutex>
 
 #include "Common/Flag.h"
-#include "Common/LibusbContext.h"
 #include "Common/Logging/Log.h"
 #include "Common/Thread.h"
 #include "Core/ConfigManager.h"
@@ -52,7 +50,7 @@ static Common::Flag s_adapter_detect_thread_running;
 static std::function<void(void)> s_detect_callback;
 
 static bool s_libusb_driver_not_supported = false;
-static std::shared_ptr<libusb_context> s_libusb_context;
+static libusb_context* s_libusb_context;
 #if defined(__FreeBSD__) && __FreeBSD__ >= 11
 static bool s_libusb_hotplug_enabled = true;
 #else
@@ -118,8 +116,8 @@ static void ScanThreadFunc()
   if (s_libusb_hotplug_enabled)
   {
     if (libusb_hotplug_register_callback(
-            s_libusb_context.get(), (libusb_hotplug_event)(LIBUSB_HOTPLUG_EVENT_DEVICE_ARRIVED |
-                                                           LIBUSB_HOTPLUG_EVENT_DEVICE_LEFT),
+            s_libusb_context, (libusb_hotplug_event)(LIBUSB_HOTPLUG_EVENT_DEVICE_ARRIVED |
+                                                     LIBUSB_HOTPLUG_EVENT_DEVICE_LEFT),
             LIBUSB_HOTPLUG_ENUMERATE, 0x057e, 0x0337, LIBUSB_HOTPLUG_MATCH_ANY, HotplugCallback,
             nullptr, &s_hotplug_handle) != LIBUSB_SUCCESS)
       s_libusb_hotplug_enabled = false;
@@ -133,7 +131,7 @@ static void ScanThreadFunc()
     if (s_libusb_hotplug_enabled)
     {
       static timeval tv = {0, 500000};
-      libusb_handle_events_timeout(s_libusb_context.get(), &tv);
+      libusb_handle_events_timeout(s_libusb_context, &tv);
     }
     else
     {
@@ -179,9 +177,12 @@ void StartScanThread()
   if (s_adapter_detect_thread_running.IsSet())
     return;
 
-  s_libusb_context = LibusbContext::Get();
-  if (!s_libusb_context)
+  const int ret = libusb_init(&s_libusb_context);
+  if (ret < 0)
+  {
+    ERROR_LOG(SERIALINTERFACE, "libusb_init failed with error: %d", ret);
     return;
+  }
   s_adapter_detect_thread_running.Set(true);
   s_adapter_detect_thread = std::thread(ScanThreadFunc);
 }
@@ -197,7 +198,7 @@ void StopScanThread()
 static void Setup()
 {
   libusb_device** list;
-  ssize_t cnt = libusb_get_device_list(s_libusb_context.get(), &list);
+  ssize_t cnt = libusb_get_device_list(s_libusb_context, &list);
 
   for (int i = 0; i < SerialInterface::MAX_SI_CHANNELS; i++)
   {
@@ -329,11 +330,15 @@ void Shutdown()
   StopScanThread();
 #if defined(LIBUSB_API_VERSION) && LIBUSB_API_VERSION >= 0x01000102
   if (s_libusb_context && s_libusb_hotplug_enabled)
-    libusb_hotplug_deregister_callback(s_libusb_context.get(), s_hotplug_handle);
+    libusb_hotplug_deregister_callback(s_libusb_context, s_hotplug_handle);
 #endif
   Reset();
 
-  s_libusb_context.reset();
+  if (s_libusb_context)
+  {
+    libusb_exit(s_libusb_context);
+    s_libusb_context = nullptr;
+  }
   s_libusb_driver_not_supported = false;
 }
 
diff --git a/Source/Core/UICommon/USBUtils.cpp b/Source/Core/UICommon/USBUtils.cpp
index 4fbabda856..001e1dbbb1 100644
--- a/Source/Core/UICommon/USBUtils.cpp
+++ b/Source/Core/UICommon/USBUtils.cpp
@@ -2,23 +2,14 @@
 // Licensed under GPLv2+
 // Refer to the license.txt file included.
 
-#include <memory>
-#include <mutex>
-
 #ifdef __LIBUSB__
 #include <libusb.h>
-#include "Common/LibusbContext.h"
 #endif
 
 #include "Common/CommonTypes.h"
 #include "Common/StringUtil.h"
 #include "UICommon/USBUtils.h"
 
-#ifdef __LIBUSB__
-static std::once_flag s_tried_libusb_init;
-static std::shared_ptr<libusb_context> s_libusb_context;
-#endif
-
 // Because opening and getting the device name from devices is slow, especially on Windows
 // with usbdk, we cannot do that for every single device. We should however still show
 // device names for known Wii peripherals.
@@ -43,12 +34,12 @@ std::map<std::pair<u16, u16>, std::string> GetInsertedDevices()
   std::map<std::pair<u16, u16>, std::string> devices;
 
 #ifdef __LIBUSB__
-  std::call_once(s_tried_libusb_init, []() { s_libusb_context = LibusbContext::Get(); });
-  if (!s_libusb_context)
+  libusb_context* context = nullptr;
+  if (libusb_init(&context) < 0)
     return devices;
 
   libusb_device** list;
-  const ssize_t cnt = libusb_get_device_list(s_libusb_context.get(), &list);
+  const ssize_t cnt = libusb_get_device_list(context, &list);
   for (ssize_t i = 0; i < cnt; ++i)
   {
     libusb_device_descriptor descr;
@@ -57,6 +48,7 @@ std::map<std::pair<u16, u16>, std::string> GetInsertedDevices()
     devices[vid_pid] = GetDeviceName(vid_pid);
   }
   libusb_free_device_list(list, 1);
+  libusb_exit(context);
 #endif
 
   return devices;