diff --git a/src/core/controller.cpp b/src/core/controller.cpp
index 0f087615d..9bf210f1a 100644
--- a/src/core/controller.cpp
+++ b/src/core/controller.cpp
@@ -23,7 +23,7 @@ bool Controller::Transfer(const u8 data_in, u8* data_out)
 
 void Controller::SetButtonState(s32 button_code, bool pressed) {}
 
-std::shared_ptr<Controller> Controller::Create(std::string_view type_name)
+std::unique_ptr<Controller> Controller::Create(std::string_view type_name)
 {
   if (type_name == "DigitalController")
     return DigitalController::Create();
diff --git a/src/core/controller.h b/src/core/controller.h
index c47f007e7..ffe92f8b9 100644
--- a/src/core/controller.h
+++ b/src/core/controller.h
@@ -25,7 +25,7 @@ public:
   virtual void SetButtonState(s32 button_code, bool pressed);
 
   /// Creates a new controller of the specified type.
-  static std::shared_ptr<Controller> Create(std::string_view type_name);
+  static std::unique_ptr<Controller> Create(std::string_view type_name);
 
   /// Gets the integer code for a button in the specified controller type.
   static std::optional<s32> GetButtonCodeByName(std::string_view type_name, std::string_view button_name);
diff --git a/src/core/digital_controller.cpp b/src/core/digital_controller.cpp
index 480951193..2c8820762 100644
--- a/src/core/digital_controller.cpp
+++ b/src/core/digital_controller.cpp
@@ -76,9 +76,9 @@ bool DigitalController::Transfer(const u8 data_in, u8* data_out)
   }
 }
 
-std::shared_ptr<DigitalController> DigitalController::Create()
+std::unique_ptr<DigitalController> DigitalController::Create()
 {
-  return std::make_shared<DigitalController>();
+  return std::make_unique<DigitalController>();
 }
 
 std::optional<s32> DigitalController::GetButtonCodeByName(std::string_view button_name)
diff --git a/src/core/digital_controller.h b/src/core/digital_controller.h
index 412208d28..2c2089ef7 100644
--- a/src/core/digital_controller.h
+++ b/src/core/digital_controller.h
@@ -31,7 +31,7 @@ public:
   DigitalController();
   ~DigitalController() override;
 
-  static std::shared_ptr<DigitalController> Create();
+  static std::unique_ptr<DigitalController> Create();
   static std::optional<s32> GetButtonCodeByName(std::string_view button_name);
 
   void SetButtonState(Button button, bool pressed);
diff --git a/src/core/host_interface.cpp b/src/core/host_interface.cpp
index c58811eff..df427e5c0 100644
--- a/src/core/host_interface.cpp
+++ b/src/core/host_interface.cpp
@@ -66,7 +66,6 @@ bool HostInterface::BootSystem(const char* filename, const char* state_filename)
     return false;
 
   m_paused = m_settings.start_paused;
-  ConnectControllers();
   UpdateSpeedLimiterState();
 
   if (state_filename && !LoadState(state_filename))
@@ -208,8 +207,6 @@ std::optional<std::vector<u8>> HostInterface::GetBIOSImage(ConsoleRegion region)
   return BIOS::LoadImageFromFile(m_settings.bios_path);
 }
 
-void HostInterface::ConnectControllers() {}
-
 void HostInterface::Throttle()
 {
   // Allow variance of up to 40ms either way.
diff --git a/src/core/host_interface.h b/src/core/host_interface.h
index 131d16bc2..14391fd34 100644
--- a/src/core/host_interface.h
+++ b/src/core/host_interface.h
@@ -60,9 +60,6 @@ protected:
     float duration;
   };
 
-  /// Connects controllers. TODO: Clean this up later...
-  virtual void ConnectControllers();
-
   /// Throttles the system, i.e. sleeps until it's time to execute the next frame.
   void Throttle();
 
diff --git a/src/core/memory_card.cpp b/src/core/memory_card.cpp
index 964dde4a4..378d3ee45 100644
--- a/src/core/memory_card.cpp
+++ b/src/core/memory_card.cpp
@@ -230,16 +230,16 @@ bool MemoryCard::Transfer(const u8 data_in, u8* data_out)
   return ack;
 }
 
-std::shared_ptr<MemoryCard> MemoryCard::Create(System* system)
+std::unique_ptr<MemoryCard> MemoryCard::Create(System* system)
 {
-  auto mc = std::make_shared<MemoryCard>(system);
+  std::unique_ptr<MemoryCard> mc = std::make_unique<MemoryCard>(system);
   mc->Format();
   return mc;
 }
 
-std::shared_ptr<MemoryCard> MemoryCard::Open(System* system, std::string_view filename)
+std::unique_ptr<MemoryCard> MemoryCard::Open(System* system, std::string_view filename)
 {
-  auto mc = std::make_shared<MemoryCard>(system);
+  std::unique_ptr<MemoryCard> mc = std::make_unique<MemoryCard>(system);
   mc->m_filename = filename;
   if (!mc->LoadFromFile())
   {
diff --git a/src/core/memory_card.h b/src/core/memory_card.h
index 3e029a87e..df8246bcb 100644
--- a/src/core/memory_card.h
+++ b/src/core/memory_card.h
@@ -21,8 +21,8 @@ public:
   MemoryCard(System* system);
   ~MemoryCard();
 
-  static std::shared_ptr<MemoryCard> Create(System* system);
-  static std::shared_ptr<MemoryCard> Open(System* system, std::string_view filename);
+  static std::unique_ptr<MemoryCard> Create(System* system);
+  static std::unique_ptr<MemoryCard> Open(System* system, std::string_view filename);
 
   void Reset();
   bool DoState(StateWrapper& sw);
diff --git a/src/core/pad.cpp b/src/core/pad.cpp
index 4ebc06257..ba9bf9ea7 100644
--- a/src/core/pad.cpp
+++ b/src/core/pad.cpp
@@ -95,6 +95,16 @@ bool Pad::DoState(StateWrapper& sw)
   return !sw.HasError();
 }
 
+void Pad::SetController(u32 slot, std::unique_ptr<Controller> dev)
+{
+  m_controllers[slot] = std::move(dev);
+}
+
+void Pad::SetMemoryCard(u32 slot, std::unique_ptr<MemoryCard> dev)
+{
+  m_memory_cards[slot] = std::move(dev);
+}
+
 u32 Pad::ReadRegister(u32 offset)
 {
   switch (offset)
@@ -289,8 +299,8 @@ void Pad::DoTransfer()
 {
   Log_DebugPrintf("Transferring slot %d", m_JOY_CTRL.SLOT.GetValue());
 
-  const std::shared_ptr<Controller>& controller = m_controllers[m_JOY_CTRL.SLOT];
-  const std::shared_ptr<MemoryCard>& memory_card = m_memory_cards[m_JOY_CTRL.SLOT];
+  Controller* const controller = m_controllers[m_JOY_CTRL.SLOT].get();
+  MemoryCard* const memory_card = m_memory_cards[m_JOY_CTRL.SLOT].get();
 
   // set rx?
   m_JOY_CTRL.RXEN = true;
diff --git a/src/core/pad.h b/src/core/pad.h
index 24a0c2b51..69ef2bde6 100644
--- a/src/core/pad.h
+++ b/src/core/pad.h
@@ -22,11 +22,11 @@ public:
   void Reset();
   bool DoState(StateWrapper& sw);
 
-  Controller* GetController(u32 slot) { return m_controllers[slot].get(); }
-  void SetController(u32 slot, std::shared_ptr<Controller> dev) { m_controllers[slot] = dev; }
+  Controller* GetController(u32 slot) const { return m_controllers[slot].get(); }
+  void SetController(u32 slot, std::unique_ptr<Controller> dev);
 
   MemoryCard* GetMemoryCard(u32 slot) { return m_memory_cards[slot].get(); }
-  void SetMemoryCard(u32 slot, std::shared_ptr<MemoryCard> dev) { m_memory_cards[slot] = dev; }
+  void SetMemoryCard(u32 slot, std::unique_ptr<MemoryCard> dev);
 
   u32 ReadRegister(u32 offset);
   void WriteRegister(u32 offset, u32 value);
@@ -106,8 +106,8 @@ private:
   System* m_system = nullptr;
   InterruptController* m_interrupt_controller = nullptr;
 
-  std::array<std::shared_ptr<Controller>, NUM_SLOTS> m_controllers;
-  std::array<std::shared_ptr<MemoryCard>, NUM_SLOTS> m_memory_cards;
+  std::array<std::unique_ptr<Controller>, NUM_SLOTS> m_controllers;
+  std::array<std::unique_ptr<MemoryCard>, NUM_SLOTS> m_memory_cards;
 
   State m_state = State::Idle;
   TickCount m_ticks_remaining = 0;
diff --git a/src/core/system.cpp b/src/core/system.cpp
index 1e2617a81..a4ee45029 100644
--- a/src/core/system.cpp
+++ b/src/core/system.cpp
@@ -146,6 +146,7 @@ bool System::Boot(const char* filename)
 
   // Component setup.
   InitializeComponents();
+  UpdateControllers();
   UpdateMemoryCards();
   Reset();
 
@@ -451,9 +452,21 @@ void System::StallCPU(TickCount ticks)
   m_cpu->AddPendingTicks(ticks);
 }
 
-void System::SetController(u32 slot, std::shared_ptr<Controller> dev)
+Controller* System::GetController(u32 slot) const
 {
-  m_pad->SetController(slot, std::move(dev));
+  return m_pad->GetController(slot);
+}
+
+void System::UpdateControllers()
+{
+  m_pad->SetController(0, nullptr);
+  m_pad->SetController(1, nullptr);
+
+  {
+    std::unique_ptr<Controller> controller = Controller::Create("DigitalController");
+    if (controller)
+      m_pad->SetController(0, std::move(controller));
+  }
 }
 
 void System::UpdateMemoryCards()
@@ -464,14 +477,14 @@ void System::UpdateMemoryCards()
   const Settings& settings = m_host_interface->GetSettings();
   if (!settings.memory_card_a_path.empty())
   {
-    std::shared_ptr<MemoryCard> card = MemoryCard::Open(this, settings.memory_card_a_path);
+    std::unique_ptr<MemoryCard> card = MemoryCard::Open(this, settings.memory_card_a_path);
     if (card)
       m_pad->SetMemoryCard(0, std::move(card));
   }
 
   if (!settings.memory_card_b_path.empty())
   {
-    std::shared_ptr<MemoryCard> card = MemoryCard::Open(this, settings.memory_card_b_path);
+    std::unique_ptr<MemoryCard> card = MemoryCard::Open(this, settings.memory_card_b_path);
     if (card)
       m_pad->SetMemoryCard(1, std::move(card));
   }
diff --git a/src/core/system.h b/src/core/system.h
index 9c8b79179..da2b8e84d 100644
--- a/src/core/system.h
+++ b/src/core/system.h
@@ -79,7 +79,9 @@ public:
   // Adds ticks to the global tick counter, simulating the CPU being stalled.
   void StallCPU(TickCount ticks);
 
-  void SetController(u32 slot, std::shared_ptr<Controller> dev);
+  // Access controllers for simulating input.
+  Controller* GetController(u32 slot) const;
+  void UpdateControllers();
   void UpdateMemoryCards();
   void UpdateCPUExecutionMode();
 
diff --git a/src/duckstation/sdl_host_interface.cpp b/src/duckstation/sdl_host_interface.cpp
index 73d7ab7ae..d019793cf 100644
--- a/src/duckstation/sdl_host_interface.cpp
+++ b/src/duckstation/sdl_host_interface.cpp
@@ -160,12 +160,6 @@ void SDLHostInterface::SaveSettings()
   m_settings.Save(m_settings_filename.c_str());
 }
 
-void SDLHostInterface::ConnectControllers()
-{
-  m_controller = DigitalController::Create();
-  m_system->SetController(0, m_controller);
-}
-
 void SDLHostInterface::QueueSwitchGPURenderer()
 {
   SDL_Event ev = {};
@@ -489,8 +483,9 @@ void SDLHostInterface::HandleSDLEvent(const SDL_Event* event)
 
     case SDL_CONTROLLERAXISMOTION:
     {
-      if (m_controller)
-        HandleSDLControllerAxisEventForController(event, m_controller.get());
+      DigitalController* controller = static_cast<DigitalController*>(m_system->GetController(0));
+      if (controller)
+        HandleSDLControllerAxisEventForController(event, controller);
     }
     break;
 
@@ -503,8 +498,9 @@ void SDLHostInterface::HandleSDLEvent(const SDL_Event* event)
         m_focus_main_menu_bar = true;
       }
 
-      if (m_controller)
-        HandleSDLControllerButtonEventForController(event, m_controller.get());
+      DigitalController* controller = static_cast<DigitalController*>(m_system->GetController(0));
+      if (controller)
+        HandleSDLControllerButtonEventForController(event, controller);
     }
     break;
 
@@ -520,7 +516,8 @@ void SDLHostInterface::HandleSDLEvent(const SDL_Event* event)
 void SDLHostInterface::HandleSDLKeyEvent(const SDL_Event* event)
 {
   const bool repeat = event->key.repeat != 0;
-  if (!repeat && m_controller && HandleSDLKeyEventForController(event, m_controller.get()))
+  DigitalController* controller = static_cast<DigitalController*>(m_system->GetController(0));
+  if (!repeat && controller && HandleSDLKeyEventForController(event, controller))
     return;
 
   const bool pressed = (event->type == SDL_KEYDOWN);
diff --git a/src/duckstation/sdl_host_interface.h b/src/duckstation/sdl_host_interface.h
index 03b41fcb0..6c61e34f0 100644
--- a/src/duckstation/sdl_host_interface.h
+++ b/src/duckstation/sdl_host_interface.h
@@ -13,8 +13,6 @@
 #include <mutex>
 
 class System;
-class DigitalController;
-class MemoryCard;
 class AudioStream;
 
 class SDLHostInterface final : public HostInterface
@@ -33,9 +31,6 @@ public:
 
   void Run();
 
-protected:
-  void ConnectControllers() override;
-
 private:
   static constexpr u32 NUM_QUICK_SAVE_STATES = 10;
   static constexpr char RESUME_SAVESTATE_FILENAME[] = "savestate_resume.bin";
@@ -98,8 +93,6 @@ private:
 
   std::map<int, SDL_GameController*> m_sdl_controllers;
 
-  std::shared_ptr<DigitalController> m_controller;
-
   u32 m_switch_gpu_renderer_event_id = 0;
 
   bool m_quit_request = false;