diff --git a/driver.c b/driver.c
index 88281248a2..7d0df5bed0 100644
--- a/driver.c
+++ b/driver.c
@@ -718,10 +718,19 @@ void init_video_input(void)
          rarch_fail(1, "init_video_input()");
       }
    }
+
+   driver.overlay = input_overlay_new(NULL);
 }
 
 void uninit_video_input(void)
 {
+   if (driver.overlay)
+   {
+      input_overlay_free(driver.overlay);
+      driver.overlay = NULL;
+      driver.overlay_state = 0;
+   }
+
    if (driver.input_data != driver.video_data && driver.input)
       input_free_func();
 
diff --git a/driver.h b/driver.h
index dffafd6f69..199311bb2c 100644
--- a/driver.h
+++ b/driver.h
@@ -24,6 +24,7 @@
 #include <stdint.h>
 #include "msvc/msvc_compat.h"
 #include "gfx/scaler/scaler.h"
+#include "input/overlay.h"
 
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -277,6 +278,9 @@ typedef struct driver
 
    struct scaler_ctx scaler;
    void *scaler_out;
+
+   input_overlay_t *overlay;
+   uint64_t overlay_state;
 } driver_t;
 
 void init_drivers(void);
diff --git a/driver_funcs.h b/driver_funcs.h
index 41f9651fa5..ce5def2432 100644
--- a/driver_funcs.h
+++ b/driver_funcs.h
@@ -56,6 +56,7 @@ static inline bool input_key_pressed_func(int key)
       return false;
 
    bool ret = driver.input->key_pressed(driver.input_data, key);
+   ret |= driver.overlay_state & (UINT64_C(1) << key);
 #ifdef HAVE_COMMAND
    if (!ret && driver.command)
       ret = rarch_cmd_get(driver.command, key);
diff --git a/input/x11_input.c b/input/x11_input.c
index d296501f6f..052d2f4e48 100644
--- a/input/x11_input.c
+++ b/input/x11_input.c
@@ -14,7 +14,6 @@
  */
 
 #include "input_common.h"
-#include "overlay.h"
 
 #include "../driver.h"
 
@@ -31,10 +30,6 @@ typedef struct x11_input
 {
    const rarch_joypad_driver_t *joypad;
 
-   bool ol_defer;
-   input_overlay_t *ol;
-   uint64_t ol_state;
-
    Display *display;
    Window win;
 
@@ -63,11 +58,6 @@ static void *x_input_init(void)
    x11->joypad = input_joypad_init_first();
    input_init_keyboard_lut(rarch_key_map_x11);
 
-   if (driver.video_data) // Video driver isn't initialized yet, init later.
-      x11->ol = input_overlay_new(NULL);
-   else
-      x11->ol_defer = true;
-
    return x11;
 }
 
@@ -97,8 +87,7 @@ static bool x_bind_button_pressed(void *data, int key)
 {
    x11_input_t *x11 = (x11_input_t*)data;
    return x_is_pressed(x11, g_settings.input.binds[0], key) ||
-      input_joypad_pressed(x11->joypad, 0, &g_settings.input.binds[0][key]) ||
-      (x11->ol_state & (UINT64_C(1) << key));
+      input_joypad_pressed(x11->joypad, 0, &g_settings.input.binds[0][key]);
 }
 
 static int16_t x_mouse_state(x11_input_t *x11, unsigned id)
@@ -176,8 +165,7 @@ static int16_t x_input_state(void *data, const struct retro_keybind **binds, uns
    {
       case RETRO_DEVICE_JOYPAD:
          return x_is_pressed(x11, binds[port], id) ||
-            input_joypad_pressed(x11->joypad, port, &binds[port][id]) ||
-            ((port == 0) && (x11->ol_state & (UINT64_C(1) << id)));
+            input_joypad_pressed(x11->joypad, port, &binds[port][id]);
 
       case RETRO_DEVICE_KEYBOARD:
          return x_key_pressed(x11, id);
@@ -206,27 +194,9 @@ static void x_input_free(void *data)
    if (x11->joypad)
       x11->joypad->destroy();
 
-   if (x11->ol)
-      input_overlay_free(x11->ol);
-
    free(data);
 }
 
-static void x_input_poll_overlay(x11_input_t *x11)
-{
-   if (!x11->ol)
-      return;
-
-   if (x11->mouse_l)
-   {
-      int16_t norm_x = 0, norm_y = 0;
-      bool valid = input_translate_coord_viewport(x11->mouse_x, x11->mouse_y, &norm_x, &norm_y);
-      x11->ol_state = valid ? input_overlay_poll(x11->ol, norm_x, norm_y) : 0;
-   }
-   else
-      x11->ol_state = 0;
-}
-
 static void x_input_poll_mouse(x11_input_t *x11)
 {
    Window root_win, child_win;
@@ -248,20 +218,12 @@ static void x_input_poll_mouse(x11_input_t *x11)
    x11->mouse_l = mask & Button1Mask; 
    x11->mouse_m = mask & Button2Mask; 
    x11->mouse_r = mask & Button3Mask; 
-
-   x_input_poll_overlay(x11);
 }
 
 static void x_input_poll(void *data)
 {
    x11_input_t *x11 = (x11_input_t*)data;
 
-   if (x11->ol_defer)
-   {
-      x11->ol = input_overlay_new(NULL);
-      x11->ol_defer = false;
-   }
-
    if (video_focus_func())
       XQueryKeymap(x11->display, x11->state);
    else
diff --git a/retroarch.c b/retroarch.c
index b2323dcec4..57a43011e3 100644
--- a/retroarch.c
+++ b/retroarch.c
@@ -469,9 +469,29 @@ size_t audio_sample_batch(const int16_t *data, size_t frames)
    return frames;
 }
 
+static inline void input_poll_overlay(void)
+{
+   bool pressed = input_input_state_func(NULL, 0,
+         RETRO_DEVICE_POINTER, 0, RETRO_DEVICE_ID_POINTER_PRESSED);
+
+   driver.overlay_state = 0;
+   if (!pressed)
+      return;
+
+   int16_t x = input_input_state_func(NULL, 0,
+         RETRO_DEVICE_POINTER, 0, RETRO_DEVICE_ID_POINTER_X);
+   int16_t y = input_input_state_func(NULL, 0,
+         RETRO_DEVICE_POINTER, 0, RETRO_DEVICE_ID_POINTER_Y);
+
+   driver.overlay_state = input_overlay_poll(driver.overlay, x, y);
+}
+
 static void input_poll(void)
 {
    input_poll_func();
+
+   if (driver.overlay) // Poll overlay state
+      input_poll_overlay();
 }
 
 // Turbo scheme: If turbo button is held, all buttons pressed except for D-pad will go into
@@ -520,6 +540,9 @@ static int16_t input_state(unsigned port, unsigned device, unsigned index, unsig
    if (id < RARCH_FIRST_META_KEY || device == RETRO_DEVICE_KEYBOARD)
       res = input_input_state_func(binds, port, device, index, id);
 
+   if (device == RETRO_DEVICE_JOYPAD && port == 0)
+      res |= driver.overlay_state & (UINT64_C(1) << id) ? 1 : 0;
+
    // Don't allow turbo for D-pad.
    if (device == RETRO_DEVICE_JOYPAD && (id < RETRO_DEVICE_ID_JOYPAD_UP || id > RETRO_DEVICE_ID_JOYPAD_RIGHT))
       res = input_apply_turbo(port, id, res);