diff --git a/Source/Core/DolphinQt2/Config/Mapping/MappingButton.cpp b/Source/Core/DolphinQt2/Config/Mapping/MappingButton.cpp
index 5e2aa189fa..b63073cfa6 100644
--- a/Source/Core/DolphinQt2/Config/Mapping/MappingButton.cpp
+++ b/Source/Core/DolphinQt2/Config/Mapping/MappingButton.cpp
@@ -4,6 +4,8 @@
 
 #include <thread>
 
+#include <QApplication>
+#include <QFontMetrics>
 #include <QMouseEvent>
 #include <QRegExp>
 #include <QString>
@@ -27,6 +29,7 @@
 #include "InputCommon/ControllerInterface/Device.h"
 
 constexpr int SLIDER_TICK_COUNT = 100;
+constexpr int VERTICAL_PADDING = 2;
 
 static QString EscapeAmpersand(QString&& string)
 {
@@ -42,6 +45,17 @@ MappingButton::MappingButton(MappingWidget* widget, ControlReference* ref, bool
     : ElidedButton(EscapeAmpersand(QString::fromStdString(ref->GetExpression()))), m_parent(widget),
       m_reference(ref)
 {
+  // Force all mapping buttons to use stay at a minimal height
+  int height = QFontMetrics(qApp->font()).height() + 2 * VERTICAL_PADDING;
+
+  setMinimumHeight(height);
+  setMaximumHeight(height);
+
+  // Make sure that long entries don't throw our layout out of whack
+  setMaximumWidth(115);
+
+  setSizePolicy(QSizePolicy::Fixed, QSizePolicy::Fixed);
+
   Connect();
   setToolTip(
       tr("Left-click to detect input.\nMiddle-click to clear.\nRight-click for more options."));
@@ -76,10 +90,6 @@ MappingButton::MappingButton(MappingWidget* widget, ControlReference* ref, bool
   });
 
   m_timer->start(1000 / 30);
-
-  setMaximumHeight(24);
-  setMaximumWidth(200);
-  setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Fixed);
 }
 
 void MappingButton::Connect()