[UI] Android ImGui touch and mouse input

android/android_studio_project/app/src/main/java/jp/xenia/emulator/WindowedAppActivity.java

@@ -1,8 +1,10 @@
 package jp.xenia.emulator;
 
+import android.annotation.SuppressLint;
 import android.app.Activity;
 import android.content.res.AssetManager;
 import android.os.Bundle;
+import android.view.MotionEvent;
 import android.view.Surface;
 import android.view.SurfaceHolder;
 import android.view.View;
@@ -16,10 +18,7 @@ public abstract class WindowedAppActivity extends Activity {
         System.loadLibrary("xenia-app");
     }
 
-    private final WindowSurfaceOnLayoutChangeListener mWindowSurfaceOnLayoutChangeListener =
+    private final WindowSurfaceListener mWindowSurfaceListener = new WindowSurfaceListener();
-            new WindowSurfaceOnLayoutChangeListener();
-    private final WindowSurfaceHolderCallback mWindowSurfaceHolderCallback =
-            new WindowSurfaceHolderCallback();
 
     // May be 0 while destroying (mainly while the superclass is).
     private long mAppContext = 0;
@@ -35,6 +34,8 @@ public abstract class WindowedAppActivity extends Activity {
     private native void onWindowSurfaceLayoutChange(
             long appContext, int left, int top, int right, int bottom);
 
+    private native boolean onWindowSurfaceMotionEvent(long appContext, MotionEvent event);
+
     private native void onWindowSurfaceChanged(long appContext, Surface windowSurface);
 
     private native void paintWindow(long appContext, boolean forcePaint);
@@ -48,8 +49,10 @@ public abstract class WindowedAppActivity extends Activity {
 
         // Detach from the old surface.
         if (mWindowSurfaceView != null) {
-            mWindowSurfaceView.getHolder().removeCallback(mWindowSurfaceHolderCallback);
+            mWindowSurfaceView.getHolder().removeCallback(mWindowSurfaceListener);
-            mWindowSurfaceView.removeOnLayoutChangeListener(mWindowSurfaceOnLayoutChangeListener);
+            mWindowSurfaceView.setOnTouchListener(null);
+            mWindowSurfaceView.setOnGenericMotionListener(null);
+            mWindowSurfaceView.removeOnLayoutChangeListener(mWindowSurfaceListener);
             mWindowSurfaceView = null;
             if (mAppContext != 0) {
                 onWindowSurfaceChanged(mAppContext, null);
@@ -61,12 +64,12 @@ public abstract class WindowedAppActivity extends Activity {
         }
 
         mWindowSurfaceView = windowSurfaceView;
-        // The native window code assumes that, when the surface exists, it covers the entire
-        // window.
         // FIXME(Triang3l): This doesn't work if the layout has already been performed.
-        mWindowSurfaceView.addOnLayoutChangeListener(mWindowSurfaceOnLayoutChangeListener);
+        mWindowSurfaceView.addOnLayoutChangeListener(mWindowSurfaceListener);
+        mWindowSurfaceView.setOnGenericMotionListener(mWindowSurfaceListener);
+        mWindowSurfaceView.setOnTouchListener(mWindowSurfaceListener);
         final SurfaceHolder windowSurfaceHolder = mWindowSurfaceView.getHolder();
-        windowSurfaceHolder.addCallback(mWindowSurfaceHolderCallback);
+        windowSurfaceHolder.addCallback(mWindowSurfaceListener);
         // If setting after the creation of the surface.
         if (mAppContext != 0) {
             final Surface windowSurface = windowSurfaceHolder.getSurface();
@@ -115,7 +118,11 @@ public abstract class WindowedAppActivity extends Activity {
         super.onDestroy();
     }
 
-    private class WindowSurfaceOnLayoutChangeListener implements View.OnLayoutChangeListener {
+    private class WindowSurfaceListener implements
+            View.OnGenericMotionListener,
+            View.OnLayoutChangeListener,
+            View.OnTouchListener,
+            SurfaceHolder.Callback2 {
         @Override
         public void onLayoutChange(
                 final View v, final int left, final int top, final int right, final int bottom,
@@ -124,9 +131,24 @@ public abstract class WindowedAppActivity extends Activity {
                 onWindowSurfaceLayoutChange(mAppContext, left, top, right, bottom);
             }
         }
+
+        @Override
+        public boolean onGenericMotion(View v, MotionEvent event) {
+            if (mAppContext == 0) {
+                return false;
+            }
+            return onWindowSurfaceMotionEvent(mAppContext, event);
+        }
+
+        @SuppressLint("ClickableViewAccessibility")
+        @Override
+        public boolean onTouch(View v, MotionEvent event) {
+            if (mAppContext == 0) {
+                return false;
+            }
+            return onWindowSurfaceMotionEvent(mAppContext, event);
         }
 
-    private class WindowSurfaceHolderCallback implements SurfaceHolder.Callback2 {
         @Override
         public void surfaceCreated(final SurfaceHolder holder) {
             if (mAppContext == 0) {

src/xenia/ui/imgui_drawer.cc

@@ -43,7 +43,8 @@ ImGuiDrawer::~ImGuiDrawer() {
     window_->RemoveInputListener(this);
     if (internal_state_) {
       ImGui::SetCurrentContext(internal_state_);
-      if (ImGui::IsAnyMouseDown()) {
+      if (touch_pointer_id_ == TouchEvent::kPointerIDNone &&
+          ImGui::IsAnyMouseDown()) {
         window_->ReleaseMouse();
       }
     }
@@ -213,6 +214,9 @@ void ImGuiDrawer::Initialize() {
 
   frame_time_tick_frequency_ = double(Clock::QueryHostTickFrequency());
   last_frame_time_ticks_ = Clock::QueryHostTickCount();
+
+  touch_pointer_id_ = TouchEvent::kPointerIDNone;
+  reset_mouse_position_after_next_frame_ = false;
 }
 
 void ImGuiDrawer::SetupFontTexture() {
@@ -310,6 +314,11 @@ void ImGuiDrawer::Draw(UIDrawContext& ui_draw_context) {
     RenderDrawLists(draw_data, ui_draw_context);
   }
 
+  if (reset_mouse_position_after_next_frame_) {
+    reset_mouse_position_after_next_frame_ = false;
+    io.MousePos = ImVec2(-FLT_MAX, -FLT_MAX);
+  }
+
   if (dialogs_.empty()) {
     // All dialogs have removed themselves during the draw, detach.
     presenter_->RemoveUIDrawerFromUIThread(this);
@@ -382,7 +391,7 @@ void ImGuiDrawer::OnKeyChar(KeyEvent& e) {
 }
 
 void ImGuiDrawer::OnMouseDown(MouseEvent& e) {
-  UpdateMousePosition(e);
+  SwitchToPhysicalMouseAndUpdateMousePosition(e);
   auto& io = GetIO();
   int button = -1;
   switch (e.button()) {
@@ -409,10 +418,12 @@ void ImGuiDrawer::OnMouseDown(MouseEvent& e) {
   }
 }
 
-void ImGuiDrawer::OnMouseMove(MouseEvent& e) { UpdateMousePosition(e); }
+void ImGuiDrawer::OnMouseMove(MouseEvent& e) {
+  SwitchToPhysicalMouseAndUpdateMousePosition(e);
+}
 
 void ImGuiDrawer::OnMouseUp(MouseEvent& e) {
-  UpdateMousePosition(e);
+  SwitchToPhysicalMouseAndUpdateMousePosition(e);
   auto& io = GetIO();
   int button = -1;
   switch (e.button()) {
@@ -440,14 +451,48 @@ void ImGuiDrawer::OnMouseUp(MouseEvent& e) {
 }
 
 void ImGuiDrawer::OnMouseWheel(MouseEvent& e) {
-  UpdateMousePosition(e);
+  SwitchToPhysicalMouseAndUpdateMousePosition(e);
   auto& io = GetIO();
   io.MouseWheel += float(e.scroll_y()) / float(MouseEvent::kScrollPerDetent);
 }
 
+void ImGuiDrawer::OnTouchEvent(TouchEvent& e) {
+  auto& io = GetIO();
+  TouchEvent::Action action = e.action();
+  uint32_t pointer_id = e.pointer_id();
+  if (action == TouchEvent::Action::kDown) {
+    // The latest pointer needs to be controlling the ImGui mouse.
+    if (touch_pointer_id_ == TouchEvent::kPointerIDNone) {
+      // Switching from the mouse to touch input.
+      if (ImGui::IsAnyMouseDown()) {
+        std::memset(io.MouseDown, 0, sizeof(io.MouseDown));
+        window_->ReleaseMouse();
+      }
+    }
+    touch_pointer_id_ = pointer_id;
+  } else {
+    if (pointer_id != touch_pointer_id_) {
+      return;
+    }
+  }
+  UpdateMousePosition(e.x(), e.y());
+  if (action == TouchEvent::Action::kUp ||
+      action == TouchEvent::Action::kCancel) {
+    io.MouseDown[0] = false;
+    touch_pointer_id_ = TouchEvent::kPointerIDNone;
+    // Make sure that after a touch, the ImGui mouse isn't hovering over
+    // anything.
+    reset_mouse_position_after_next_frame_ = true;
+  } else {
+    io.MouseDown[0] = true;
+    reset_mouse_position_after_next_frame_ = false;
+  }
+}
+
 void ImGuiDrawer::ClearInput() {
   auto& io = GetIO();
-  if (ImGui::IsAnyMouseDown()) {
+  if (touch_pointer_id_ == TouchEvent::kPointerIDNone &&
+      ImGui::IsAnyMouseDown()) {
     window_->ReleaseMouse();
   }
   io.MousePos = ImVec2(-FLT_MAX, -FLT_MAX);
@@ -460,6 +505,8 @@ void ImGuiDrawer::ClearInput() {
   io.KeySuper = false;
   std::memset(io.KeysDown, 0, sizeof(io.KeysDown));
   io.ClearInputCharacters();
+  touch_pointer_id_ = TouchEvent::kPointerIDNone;
+  reset_mouse_position_after_next_frame_ = false;
 }
 
 void ImGuiDrawer::OnKey(KeyEvent& e, bool is_down) {
@@ -487,12 +534,27 @@ void ImGuiDrawer::OnKey(KeyEvent& e, bool is_down) {
   }
 }
 
-void ImGuiDrawer::UpdateMousePosition(const MouseEvent& e) {
+void ImGuiDrawer::UpdateMousePosition(float x, float y) {
   auto& io = GetIO();
   float physical_to_logical =
       float(window_->GetMediumDpi()) / float(window_->GetDpi());
-  io.MousePos.x = e.x() * physical_to_logical;
+  io.MousePos.x = x * physical_to_logical;
-  io.MousePos.y = e.y() * physical_to_logical;
+  io.MousePos.y = y * physical_to_logical;
+}
+
+void ImGuiDrawer::SwitchToPhysicalMouseAndUpdateMousePosition(
+    const MouseEvent& e) {
+  if (touch_pointer_id_ != TouchEvent::kPointerIDNone) {
+    touch_pointer_id_ = TouchEvent::kPointerIDNone;
+    auto& io = GetIO();
+    std::memset(io.MouseDown, 0, sizeof(io.MouseDown));
+    // Nothing needs to be done regarding CaptureMouse and ReleaseMouse - all
+    // buttons as well as mouse capture have been released when switching to
+    // touch input, the mouse is never captured during touch input, and now
+    // resetting to no buttons down (therefore not capturing).
+  }
+  reset_mouse_position_after_next_frame_ = false;
+  UpdateMousePosition(float(e.x()), float(e.y()));
 }
 
 }  // namespace ui

src/xenia/ui/imgui_drawer.h

@@ -59,6 +59,7 @@ class ImGuiDrawer : public WindowInputListener, public UIDrawer {
   void OnMouseMove(MouseEvent& e) override;
   void OnMouseUp(MouseEvent& e) override;
   void OnMouseWheel(MouseEvent& e) override;
+  void OnTouchEvent(TouchEvent& e) override;
   // For now, no need for OnDpiChanged because redrawing is done continuously.
 
  private:
@@ -70,7 +71,8 @@ class ImGuiDrawer : public WindowInputListener, public UIDrawer {
 
   void ClearInput();
   void OnKey(KeyEvent& e, bool is_down);
-  void UpdateMousePosition(const MouseEvent& e);
+  void UpdateMousePosition(float x, float y);
+  void SwitchToPhysicalMouseAndUpdateMousePosition(const MouseEvent& e);
 
   Window* window_;
   size_t z_order_;
@@ -92,6 +94,16 @@ class ImGuiDrawer : public WindowInputListener, public UIDrawer {
   // detaching the presenter.
   std::unique_ptr<ImmediateTexture> font_texture_;
 
+  // If there's an active pointer, the ImGui mouse is controlled by this touch.
+  // If it's TouchEvent::kPointerIDNone, the ImGui mouse is controlled by the
+  // mouse.
+  uint32_t touch_pointer_id_ = TouchEvent::kPointerIDNone;
+  // Whether after the next frame (since the mouse up event needs to be handled
+  // with the correct mouse position still), the ImGui mouse position should be
+  // reset (for instance, after releasing a touch), so it's not hovering over
+  // anything.
+  bool reset_mouse_position_after_next_frame_ = false;
+
   double frame_time_tick_frequency_;
   uint64_t last_frame_time_ticks_;
 };

src/xenia/ui/ui_event.h

@@ -10,6 +10,7 @@
 #ifndef XENIA_UI_UI_EVENT_H_
 #define XENIA_UI_UI_EVENT_H_
 
+#include <cstdint>
 #include <filesystem>
 
 #include "xenia/ui/virtual_key.h"
@@ -156,6 +157,46 @@ class MouseEvent : public UIEvent {
   int32_t scroll_y_ = 0;
 };
 
+class TouchEvent : public UIEvent {
+ public:
+  enum class Action {
+    kDown,
+    kUp,
+    // Should be treated as an up event, but without performing the usual action
+    // for releasing.
+    kCancel,
+    kMove,
+  };
+
+  // Can be used by event listeners as the value for when there's no current
+  // pointer, for example.
+  static constexpr uint32_t kPointerIDNone = UINT32_MAX;
+
+  explicit TouchEvent(Window* target, uint32_t pointer_id, Action action,
+                      float x, float y)
+      : UIEvent(target),
+        pointer_id_(pointer_id),
+        action_(action),
+        x_(x),
+        y_(y) {}
+
+  bool is_handled() const { return handled_; }
+  void set_handled(bool value) { handled_ = value; }
+
+  uint32_t pointer_id() { return pointer_id_; }
+  Action action() const { return action_; }
+  // Can be outside the boundaries of the surface.
+  float x() const { return x_; }
+  float y() const { return y_; }
+
+ private:
+  bool handled_ = false;
+  uint32_t pointer_id_;
+  Action action_;
+  float x_;
+  float y_;
+};
+
 }  // namespace ui
 }  // namespace xe
 

src/xenia/ui/window.cc

@@ -645,6 +645,19 @@ void Window::OnMouseWheel(MouseEvent& e,
   }
 }
 
+void Window::OnTouchEvent(TouchEvent& e,
+                          WindowDestructionReceiver& destruction_receiver) {
+  PropagateEventThroughInputListeners(
+      [&e](auto listener) {
+        listener->OnTouchEvent(e);
+        return e.is_handled();
+      },
+      destruction_receiver);
+  if (destruction_receiver.IsWindowDestroyed()) {
+    return;
+  }
+}
+
 void Window::SendEventToListeners(
     std::function<void(WindowListener*)> fn,
     WindowDestructionReceiver& destruction_receiver) {

src/xenia/ui/window.h

@@ -597,6 +597,9 @@ class Window {
   void OnMouseWheel(MouseEvent& e,
                     WindowDestructionReceiver& destruction_receiver);
 
+  void OnTouchEvent(TouchEvent& e,
+                    WindowDestructionReceiver& destruction_receiver);
+
  private:
   struct ListenerIterationContext {
     explicit ListenerIterationContext(ListenerIterationContext* outer_context,

src/xenia/ui/window_android.cc

@@ -9,10 +9,16 @@
 
 #include "xenia/ui/window_android.h"
 
+#include <android/input.h>
+#include <jni.h>
+#include <algorithm>
+#include <cstdint>
 #include <memory>
 #include <utility>
 
 #include "xenia/base/assert.h"
+#include "xenia/base/logging.h"
+#include "xenia/base/math.h"
 #include "xenia/ui/surface_android.h"
 #include "xenia/ui/windowed_app_context_android.h"
 
@@ -55,6 +61,181 @@ void AndroidWindow::OnActivitySurfaceLayoutChange() {
   }
 }
 
+bool AndroidWindow::OnActivitySurfaceMotionEvent(jobject event) {
+  auto& android_app_context =
+      static_cast<const AndroidWindowedAppContext&>(app_context());
+  JNIEnv* jni_env = android_app_context.ui_thread_jni_env();
+  const AndroidWindowedAppContext::JniIDs& jni_ids =
+      android_app_context.jni_ids();
+
+  int32_t source =
+      jni_env->CallIntMethod(event, jni_ids.motion_event_get_source);
+
+  switch (source) {
+    case AINPUT_SOURCE_TOUCHSCREEN: {
+      // Returning true for all touch events regardless of whether they have
+      // been handled to keep receiving touch events for the pointers in the
+      // event (if returning false for a down event, no more events will be sent
+      // for the pointers in it), and also because there are multiple pointers
+      // in a single event, and different handler invocations may result in
+      // different is_handled.
+      WindowDestructionReceiver destruction_receiver(this);
+      int32_t action_and_pointer_index =
+          jni_env->CallIntMethod(event, jni_ids.motion_event_get_action);
+      int32_t action = action_and_pointer_index & AMOTION_EVENT_ACTION_MASK;
+      // For pointer ACTION_POINTER_DOWN and ACTION_POINTER_UP.
+      int32_t pointer_index = (action_and_pointer_index &
+                               AMOTION_EVENT_ACTION_POINTER_INDEX_MASK) >>
+                              AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT;
+      if (action == AMOTION_EVENT_ACTION_POINTER_DOWN ||
+          action == AMOTION_EVENT_ACTION_POINTER_UP) {
+        int32_t touch_pointer_index =
+            (action_and_pointer_index &
+             AMOTION_EVENT_ACTION_POINTER_INDEX_MASK) >>
+            AMOTION_EVENT_ACTION_POINTER_INDEX_SHIFT;
+        TouchEvent e(
+            this,
+            jni_env->CallIntMethod(event, jni_ids.motion_event_get_pointer_id,
+                                   touch_pointer_index),
+            (action == AMOTION_EVENT_ACTION_POINTER_DOWN)
+                ? TouchEvent::Action::kDown
+                : TouchEvent::Action::kUp,
+            jni_env->CallFloatMethod(event, jni_ids.motion_event_get_x,
+                                     pointer_index),
+            jni_env->CallFloatMethod(event, jni_ids.motion_event_get_y,
+                                     pointer_index));
+        OnTouchEvent(e, destruction_receiver);
+        if (destruction_receiver.IsWindowDestroyed()) {
+          return true;
+        }
+      } else {
+        TouchEvent::Action touch_event_action;
+        switch (action) {
+          case AMOTION_EVENT_ACTION_DOWN:
+            touch_event_action = TouchEvent::Action::kDown;
+            break;
+          case AMOTION_EVENT_ACTION_UP:
+            touch_event_action = TouchEvent::Action::kUp;
+            break;
+          case AMOTION_EVENT_ACTION_MOVE:
+            touch_event_action = TouchEvent::Action::kMove;
+            break;
+          case AMOTION_EVENT_ACTION_CANCEL:
+            touch_event_action = TouchEvent::Action::kCancel;
+            break;
+          default:
+            return true;
+        }
+        int32_t touch_pointer_count = jni_env->CallIntMethod(
+            event, jni_ids.motion_event_get_pointer_count);
+        for (int32_t i = 0; i < touch_pointer_count; ++i) {
+          TouchEvent e(
+              this,
+              jni_env->CallIntMethod(event, jni_ids.motion_event_get_pointer_id,
+                                     i),
+              touch_event_action,
+              jni_env->CallFloatMethod(event, jni_ids.motion_event_get_x, i),
+              jni_env->CallFloatMethod(event, jni_ids.motion_event_get_y, i));
+          OnTouchEvent(e, destruction_receiver);
+          if (destruction_receiver.IsWindowDestroyed()) {
+            return true;
+          }
+        }
+      }
+      return true;
+    } break;
+
+    case AINPUT_SOURCE_MOUSE: {
+      WindowDestructionReceiver destruction_receiver(this);
+      // X and Y can be outside the View (have negative coordinates, or beyond
+      // the size of the element), and not only for ACTION_HOVER_EXIT (it's
+      // predeced by ACTION_HOVER_MOVE, at least on Android API level 30, also
+      // with out-of-bounds coordinates), when moving the mouse outside the
+      // View, or when starting moving the mouse when the pointer was previously
+      // outside the View in some cases.
+      int32_t mouse_x = int32_t(
+          std::min(float(GetActualPhysicalWidth()),
+                   std::max(0.0f, jni_env->CallFloatMethod(
+                                      event, jni_ids.motion_event_get_x, 0))) +
+          0.5f);
+      int32_t mouse_y = int32_t(
+          std::min(float(GetActualPhysicalHeight()),
+                   std::max(0.0f, jni_env->CallFloatMethod(
+                                      event, jni_ids.motion_event_get_y, 0))) +
+          0.5f);
+      static const MouseEvent::Button kMouseEventButtons[] = {
+          MouseEvent::Button::kLeft,   MouseEvent::Button::kRight,
+          MouseEvent::Button::kMiddle, MouseEvent::Button::kX1,
+          MouseEvent::Button::kX2,
+      };
+      static constexpr uint32_t kUsedMouseButtonMask =
+          (UINT32_C(1) << xe::countof(kMouseEventButtons)) - 1;
+      uint32_t new_mouse_button_state = uint32_t(
+          jni_env->CallIntMethod(event, jni_ids.motion_event_get_button_state));
+      // OnMouseUp.
+      uint32_t mouse_buttons_remaining =
+          mouse_button_state_ & ~new_mouse_button_state & kUsedMouseButtonMask;
+      uint32_t mouse_button_index;
+      while (
+          xe::bit_scan_forward(mouse_buttons_remaining, &mouse_button_index)) {
+        mouse_buttons_remaining &= ~(UINT32_C(1) << mouse_button_index);
+        MouseEvent e(this, kMouseEventButtons[mouse_button_index], mouse_x,
+                     mouse_y);
+        OnMouseUp(e, destruction_receiver);
+        if (destruction_receiver.IsWindowDestroyed()) {
+          return true;
+        }
+      }
+      // Generic OnMouseMove regardless of the action since any event can
+      // provide new coordinates.
+      {
+        MouseEvent e(this, MouseEvent::Button::kNone, mouse_x, mouse_y);
+        OnMouseMove(e, destruction_receiver);
+        if (destruction_receiver.IsWindowDestroyed()) {
+          return true;
+        }
+      }
+      // OnMouseWheel.
+      // The axis value may be outside -1...1 if multiple scrolls have occurred
+      // quickly.
+      int32_t scroll_x = int32_t(
+          jni_env->CallFloatMethod(event, jni_ids.motion_event_get_axis_value,
+                                   AMOTION_EVENT_AXIS_HSCROLL, 0) *
+          float(MouseEvent::kScrollPerDetent));
+      int32_t scroll_y = int32_t(
+          jni_env->CallFloatMethod(event, jni_ids.motion_event_get_axis_value,
+                                   AMOTION_EVENT_AXIS_VSCROLL, 0) *
+          float(MouseEvent::kScrollPerDetent));
+      if (scroll_x || scroll_y) {
+        MouseEvent e(this, MouseEvent::Button::kNone, mouse_x, mouse_y,
+                     scroll_x, scroll_y);
+        OnMouseWheel(e, destruction_receiver);
+        if (destruction_receiver.IsWindowDestroyed()) {
+          return true;
+        }
+      }
+      // OnMouseDown.
+      mouse_buttons_remaining =
+          new_mouse_button_state & ~mouse_button_state_ & kUsedMouseButtonMask;
+      while (
+          xe::bit_scan_forward(mouse_buttons_remaining, &mouse_button_index)) {
+        mouse_buttons_remaining &= ~(UINT32_C(1) << mouse_button_index);
+        MouseEvent e(this, kMouseEventButtons[mouse_button_index], mouse_x,
+                     mouse_y);
+        OnMouseDown(e, destruction_receiver);
+        if (destruction_receiver.IsWindowDestroyed()) {
+          return true;
+        }
+      }
+      // Update the button state for state differences.
+      mouse_button_state_ = new_mouse_button_state;
+      return true;
+    } break;
+  }
+
+  return false;
+}
+
 uint32_t AndroidWindow::GetLatestDpiImpl() const {
   auto& android_app_context =
       static_cast<const AndroidWindowedAppContext&>(app_context());
@@ -62,6 +243,9 @@ uint32_t AndroidWindow::GetLatestDpiImpl() const {
 }
 
 bool AndroidWindow::OpenImpl() {
+  // Reset the input.
+  mouse_button_state_ = 0;
+
   // The window is a proxy between the main activity and Xenia, so there can be
   // only one open window for an activity.
   auto& android_app_context =

src/xenia/ui/window_android.h

@@ -10,6 +10,8 @@
 #ifndef XENIA_UI_WINDOW_ANDROID_H_
 #define XENIA_UI_WINDOW_ANDROID_H_
 
+#include <jni.h>
+
 #include "xenia/ui/menu_item.h"
 #include "xenia/ui/window.h"
 
@@ -30,6 +32,7 @@ class AndroidWindow : public Window {
   uint32_t GetMediumDpi() const override { return 160; }
 
   void OnActivitySurfaceLayoutChange();
+  bool OnActivitySurfaceMotionEvent(jobject event);
   void OnActivitySurfaceChanged() { OnSurfaceChanged(true); }
   void PaintActivitySurface(bool force_paint) { OnPaint(force_paint); }
 
@@ -42,6 +45,9 @@ class AndroidWindow : public Window {
   std::unique_ptr<Surface> CreateSurfaceImpl(
       Surface::TypeFlags allowed_types) override;
   void RequestPaintImpl() override;
+
+ private:
+  uint32_t mouse_button_state_ = 0;
 };
 
 // Dummy for the menu item - menus are controlled by the layout.

src/xenia/ui/window_listener.h

@@ -48,6 +48,8 @@ class WindowInputListener {
   virtual void OnMouseMove(MouseEvent& e) {}
   virtual void OnMouseUp(MouseEvent& e) {}
   virtual void OnMouseWheel(MouseEvent& e) {}
+
+  virtual void OnTouchEvent(TouchEvent& e) {}
 };
 
 }  // namespace ui

src/xenia/ui/windowed_app_context_android.cc

@@ -20,6 +20,7 @@
 #include <unistd.h>
 #include <array>
 #include <cstdint>
+#include <cstring>
 
 #include "xenia/base/assert.h"
 #include "xenia/base/logging.h"
@@ -130,6 +131,12 @@ void AndroidWindowedAppContext::JniActivityOnWindowSurfaceLayoutChange(
   }
 }
 
+bool AndroidWindowedAppContext::JniActivityOnWindowSurfaceMotionEvent(
+    jobject event) {
+  return activity_window_ &&
+         activity_window_->OnActivitySurfaceMotionEvent(event);
+}
+
 void AndroidWindowedAppContext::JniActivityOnWindowSurfaceChanged(
     jobject window_surface_object) {
   // Detach from the old surface.
@@ -250,6 +257,62 @@ bool AndroidWindowedAppContext::Initialize(JNIEnv* ui_thread_jni_env,
   android_base_initialized_ = true;
   ui_thread_jni_env_->DeleteLocalRef(application_context_init_ref);
 
+  // Initialize common windowed app JNI IDs.
+  {
+    jclass motion_event_class_local_ref =
+        ui_thread_jni_env_->FindClass("android/view/MotionEvent");
+    if (!motion_event_class_local_ref) {
+      XELOGE(
+          "AndroidWindowedAppContext: Failed to find the motion event class");
+      Shutdown();
+      return false;
+    }
+    motion_event_class_ =
+        reinterpret_cast<jclass>(ui_thread_jni_env_->NewGlobalRef(
+            reinterpret_cast<jobject>(motion_event_class_local_ref)));
+    ui_thread_jni_env_->DeleteLocalRef(
+        reinterpret_cast<jobject>(motion_event_class_local_ref));
+  }
+  if (!motion_event_class_) {
+    XELOGE(
+        "AndroidWindowedAppContext: Failed to create a global reference to the "
+        "motion event class");
+    Shutdown();
+    return false;
+  }
+  bool motion_event_ids_obtained = true;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_action = ui_thread_jni_env_->GetMethodID(
+           motion_event_class_, "getAction", "()I")) != nullptr;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_axis_value = ui_thread_jni_env_->GetMethodID(
+           motion_event_class_, "getAxisValue", "(II)F")) != nullptr;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_button_state = ui_thread_jni_env_->GetMethodID(
+           motion_event_class_, "getButtonState", "()I")) != nullptr;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_pointer_id = ui_thread_jni_env_->GetMethodID(
+           motion_event_class_, "getPointerId", "(I)I")) != nullptr;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_pointer_count =
+           ui_thread_jni_env_->GetMethodID(
+               motion_event_class_, "getPointerCount", "()I")) != nullptr;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_source = ui_thread_jni_env_->GetMethodID(
+           motion_event_class_, "getSource", "()I")) != nullptr;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_x = ui_thread_jni_env_->GetMethodID(
+           motion_event_class_, "getX", "(I)F")) != nullptr;
+  motion_event_ids_obtained &=
+      (jni_ids_.motion_event_get_y = ui_thread_jni_env_->GetMethodID(
+           motion_event_class_, "getY", "(I)F")) != nullptr;
+  if (!motion_event_ids_obtained) {
+    XELOGE(
+        "AndroidWindowedAppContext: Failed to get the motion event class IDs");
+    Shutdown();
+    return false;
+  }
+
   // Initialize interfacing with the WindowedAppActivity.
   activity_ = ui_thread_jni_env_->NewGlobalRef(activity);
   if (!activity_) {
@@ -343,6 +406,12 @@ void AndroidWindowedAppContext::Shutdown() {
     activity_ = nullptr;
   }
 
+  std::memset(&jni_ids_, 0, sizeof(jni_ids_));
+  if (motion_event_class_) {
+    ui_thread_jni_env_->DeleteGlobalRef(motion_event_class_);
+    motion_event_class_ = nullptr;
+  }
+
   if (android_base_initialized_) {
     xe::ShutdownAndroidAppFromMainThread();
     android_base_initialized_ = false;
@@ -495,6 +564,14 @@ Java_jp_xenia_emulator_WindowedAppActivity_onWindowSurfaceLayoutChange(
       ->JniActivityOnWindowSurfaceLayoutChange(left, top, right, bottom);
 }
 
+JNIEXPORT jboolean JNICALL
+Java_jp_xenia_emulator_WindowedAppActivity_onWindowSurfaceMotionEvent(
+    JNIEnv* jni_env, jobject activity, jlong app_context_ptr, jobject event) {
+  return jboolean(
+      reinterpret_cast<xe::ui::AndroidWindowedAppContext*>(app_context_ptr)
+          ->JniActivityOnWindowSurfaceMotionEvent(event));
+}
+
 JNIEXPORT void JNICALL
 Java_jp_xenia_emulator_WindowedAppActivity_onWindowSurfaceChanged(
     JNIEnv* jni_env, jobject activity, jlong app_context_ptr,

src/xenia/ui/windowed_app_context_android.h

@@ -28,17 +28,35 @@ class WindowedApp;
 
 class AndroidWindowedAppContext final : public WindowedAppContext {
  public:
+  // Precached JNI references and IDs that may be used for windowed app purposes
+  // by external code.
+  struct JniIDs {
+    // android.view.MotionEvent.
+    jmethodID motion_event_get_action;
+    jmethodID motion_event_get_axis_value;
+    jmethodID motion_event_get_button_state;
+    jmethodID motion_event_get_pointer_count;
+    jmethodID motion_event_get_pointer_id;
+    jmethodID motion_event_get_source;
+    jmethodID motion_event_get_x;
+    jmethodID motion_event_get_y;
+  };
+
   WindowedApp* app() const { return app_.get(); }
 
   void NotifyUILoopOfPendingFunctions() override;
 
   void PlatformQuitFromUIThread() override;
 
+  JNIEnv* ui_thread_jni_env() const { return ui_thread_jni_env_; }
+
   uint32_t GetPixelDensity() const {
     return configuration_ ? uint32_t(AConfiguration_getDensity(configuration_))
                           : 160;
   }
 
+  const JniIDs& jni_ids() const { return jni_ids_; }
+
   int32_t window_surface_layout_left() const {
     return window_surface_layout_left_;
   }
@@ -70,6 +88,7 @@ class AndroidWindowedAppContext final : public WindowedAppContext {
   void JniActivityOnDestroy();
   void JniActivityOnWindowSurfaceLayoutChange(jint left, jint top, jint right,
                                               jint bottom);
+  bool JniActivityOnWindowSurfaceMotionEvent(jobject event);
   void JniActivityOnWindowSurfaceChanged(jobject window_surface_object);
   void JniActivityPaintWindow(bool force_paint);
 
@@ -117,6 +136,9 @@ class AndroidWindowedAppContext final : public WindowedAppContext {
 
   bool android_base_initialized_ = false;
 
+  jclass motion_event_class_ = nullptr;
+  JniIDs jni_ids_ = {};
+
   jobject activity_ = nullptr;
   jmethodID activity_method_finish_ = nullptr;
   jmethodID activity_method_post_invalidate_window_surface_ = nullptr;