diff --git a/Source/Core/Core/Core.cpp b/Source/Core/Core/Core.cpp
index 2ca4942921..30d64b6d38 100644
--- a/Source/Core/Core/Core.cpp
+++ b/Source/Core/Core/Core.cpp
@@ -361,8 +361,13 @@ void EmuThread()
 		return;
 	}
 
-	Keyboard::Initialize(s_window_handle);
-	Pad::Initialize(s_window_handle);
+	bool init_controllers = false;
+	if (!g_controller_interface.IsInit())
+	{
+		Pad::Initialize(s_window_handle);
+		Keyboard::Initialize(s_window_handle);
+		init_controllers = true;
+	}
 
 	// Load and Init Wiimotes - only if we are booting in Wii mode
 	if (core_parameter.bWii)
@@ -479,10 +484,13 @@ void EmuThread()
 	HW::Shutdown();
 	INFO_LOG(CONSOLE, "%s", StopMessage(false, "HW shutdown").c_str());
 
-	Wiimote::Shutdown();
-
-	Keyboard::Shutdown();
-	Pad::Shutdown();
+	if (init_controllers)
+	{
+		Wiimote::Shutdown();
+		Keyboard::Shutdown();
+		Pad::Shutdown();
+		init_controllers = false;
+	}
 
 	g_video_backend->Shutdown();
 	AudioCommon::ShutdownSoundStream();
diff --git a/Source/Core/Core/HW/GCKeyboard.cpp b/Source/Core/Core/HW/GCKeyboard.cpp
index df08522887..1e57e8e802 100644
--- a/Source/Core/Core/HW/GCKeyboard.cpp
+++ b/Source/Core/Core/HW/GCKeyboard.cpp
@@ -35,8 +35,9 @@ void Shutdown()
 // if plugin isn't initialized, init and load config
 void Initialize(void* const hwnd)
 {
-	for (unsigned int i=0; i<4; ++i)
-		s_config.controllers.push_back(new GCKeyboard(i));
+	if (s_config.controllers.empty())
+		for (unsigned int i = 0; i < 4; ++i)
+			s_config.controllers.push_back(new GCKeyboard(i));
 
 	g_controller_interface.Initialize(hwnd);
 
diff --git a/Source/Core/Core/HW/GCPad.cpp b/Source/Core/Core/HW/GCPad.cpp
index 7b3802d268..34ae9ea618 100644
--- a/Source/Core/Core/HW/GCPad.cpp
+++ b/Source/Core/Core/HW/GCPad.cpp
@@ -36,8 +36,9 @@ void Shutdown()
 void Initialize(void* const hwnd)
 {
 	// add 4 gcpads
-	for (unsigned int i=0; i<4; ++i)
-		s_config.controllers.push_back(new GCPad(i));
+	if (s_config.controllers.empty())
+		for (unsigned int i = 0; i < 4; ++i)
+			s_config.controllers.push_back(new GCPad(i));
 
 	g_controller_interface.Initialize(hwnd);
 
diff --git a/Source/Core/Core/HW/Wiimote.cpp b/Source/Core/Core/HW/Wiimote.cpp
index e7636b3429..357b3b1fea 100644
--- a/Source/Core/Core/HW/Wiimote.cpp
+++ b/Source/Core/Core/HW/Wiimote.cpp
@@ -40,8 +40,9 @@ void Shutdown()
 void Initialize(void* const hwnd, bool wait)
 {
 	// add 4 Wiimotes
-	for (unsigned int i = WIIMOTE_CHAN_0; i<MAX_BBMOTES; ++i)
-		s_config.controllers.push_back(new WiimoteEmu::Wiimote(i));
+	if (s_config.controllers.empty())
+		for (unsigned int i = WIIMOTE_CHAN_0; i < MAX_BBMOTES; ++i)
+			s_config.controllers.push_back(new WiimoteEmu::Wiimote(i));
 
 	g_controller_interface.Initialize(hwnd);
 
diff --git a/Source/Core/Core/HotkeyManager.cpp b/Source/Core/Core/HotkeyManager.cpp
index 431ff1696f..e677c96cef 100644
--- a/Source/Core/Core/HotkeyManager.cpp
+++ b/Source/Core/Core/HotkeyManager.cpp
@@ -48,7 +48,7 @@ const std::string hotkey_labels[] =
 	_trans("Change Disc"),
 	_trans("Refresh List"),
 
-	_trans("Play/Pause"),
+	_trans("Toggle Pause"),
 	_trans("Stop"),
 	_trans("Reset"),
 	_trans("Frame Advance"),
@@ -205,7 +205,8 @@ bool IsPressed(int Id, bool held)
 
 void Initialize(void* const hwnd)
 {
-	s_config.controllers.push_back(new HotkeyManager());
+	if (s_config.controllers.empty())
+		s_config.controllers.push_back(new HotkeyManager());
 
 	g_controller_interface.Initialize(hwnd);
 
diff --git a/Source/Core/DolphinWX/Frame.cpp b/Source/Core/DolphinWX/Frame.cpp
index 83ad8f8afc..2446cb4cd3 100644
--- a/Source/Core/DolphinWX/Frame.cpp
+++ b/Source/Core/DolphinWX/Frame.cpp
@@ -50,6 +50,9 @@
 #include "Core/Movie.h"
 #include "Core/State.h"
 #include "Core/HW/DVDInterface.h"
+#include "Core/HW/GCKeyboard.h"
+#include "Core/HW/GCPad.h"
+#include "Core/HW/Wiimote.h"
 
 #include "DolphinWX/Frame.h"
 #include "DolphinWX/GameListCtrl.h"
@@ -345,15 +348,21 @@ END_EVENT_TABLE()
 // Creation and close, quit functions
 
 
-bool CFrame::InitHotkeys()
+bool CFrame::InitControllers()
 {
 	if (!g_controller_interface.IsInit())
 	{
 #if defined(HAVE_X11) && HAVE_X11
 		Window win = X11Utils::XWindowFromHandle(GetHandle());
 		HotkeyManagerEmu::Initialize(reinterpret_cast<void*>(win));
+		Pad::Initialize(reinterpret_cast<void*>(win));
+		Keyboard::Initialize(reinterpret_cast<void*>(win));
+		Wiimote::Initialize(reinterpret_cast<void*>(win));
 #else
 		HotkeyManagerEmu::Initialize(reinterpret_cast<void*>(GetHandle()));
+		Pad::Initialize(reinterpret_cast<void*>(GetHandle()));
+		Keyboard::Initialize(reinterpret_cast<void*>(GetHandle()));
+		Wiimote::Initialize(reinterpret_cast<void*>(GetHandle()));
 #endif
 		return true;
 	}
@@ -497,7 +506,7 @@ CFrame::CFrame(wxFrame* parent,
 		g_pCodeWindow->UpdateButtonStates();
 
 	// check if game is running
-	m_bHotkeysInit = InitHotkeys();
+	m_bHotkeysInit = InitControllers();
 
 	m_poll_hotkey_timer = new wxTimer(this);
 	Bind(wxEVT_TIMER, &CFrame::PollHotkeys, this);
@@ -510,7 +519,11 @@ CFrame::~CFrame()
 
 	if (m_bHotkeysInit)
 	{
+		Wiimote::Shutdown();
+		Keyboard::Shutdown();
+		Pad::Shutdown();
 		HotkeyManagerEmu::Shutdown();
+		m_bHotkeysInit = false;
 	}
 
 	drives.clear();
@@ -938,6 +951,9 @@ void CFrame::OnGameListCtrl_ItemActivated(wxListEvent& WXUNUSED (event))
 
 static bool IsHotkey(wxKeyEvent &event, int Id, bool keyUp = false)
 {
+	if (Core::GetState() == Core::CORE_UNINITIALIZED)
+		return false;
+
 	// Input event hotkey
 	if (event.GetKeyCode() == WXK_NONE)
 	{
@@ -1261,7 +1277,7 @@ void CFrame::PollHotkeys(wxTimerEvent& event)
 {
 	if (Core::GetState() == Core::CORE_UNINITIALIZED || Core::GetState() == Core::CORE_PAUSE)
 	{
-		InitHotkeys();
+		m_bHotkeysInit = InitControllers();
 		g_controller_interface.UpdateInput();
 	}
 
diff --git a/Source/Core/DolphinWX/Frame.h b/Source/Core/DolphinWX/Frame.h
index a636f843d2..bcb51447ad 100644
--- a/Source/Core/DolphinWX/Frame.h
+++ b/Source/Core/DolphinWX/Frame.h
@@ -346,7 +346,7 @@ private:
 	void PollHotkeys(wxTimerEvent&);
 	void ParseHotkeys(wxKeyEvent &event);
 
-	bool InitHotkeys();
+	bool InitControllers();
 
 	// Event table
 	DECLARE_EVENT_TABLE();