From d19396718ee0c0ba7fb578f595a6553c0458da59 Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 12 Oct 2024 09:23:31 +0200 Subject: [PATCH] Inject display-related events to virtual display Mouse and touch events must be sent to the virtual display id (used for mirroring), other events (like key events) must be sent to the original display id. Fixes #4598 Fixes #5137 PR #5370 Co-authored-by: nightmare --- .../genymobile/scrcpy/control/Controller.java | 74 ++++++++++++------- .../scrcpy/video/ScreenCapture.java | 11 ++- .../scrcpy/video/VirtualDisplayListener.java | 2 +- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/control/Controller.java b/server/src/main/java/com/genymobile/scrcpy/control/Controller.java index ac870f27..5175ed5e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/control/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/control/Controller.java @@ -17,7 +17,6 @@ import android.content.Intent; import android.os.Build; import android.os.SystemClock; import android.view.InputDevice; -import android.view.InputEvent; import android.view.KeyCharacterMap; import android.view.KeyEvent; import android.view.MotionEvent; @@ -31,6 +30,28 @@ import java.util.concurrent.atomic.AtomicReference; public class Controller implements AsyncProcessor, VirtualDisplayListener { + /* + * For event injection, there are two display ids: + * - the displayId passed to the constructor (which comes from --display-id passed by the client, 0 for the main display); + * - the virtualDisplayId used for mirroring, notified by the capture instance via the VirtualDisplayListener interface. + * + * (In case the ScreenCapture uses the "SurfaceControl API", then both ids are equals, but this is an implementation detail.) + * + * In order to make events work correctly in all cases: + * - virtualDisplayId must be used for events relative to the display (mouse and touch events with coordinates); + * - displayId must be used for other events (like key events). + */ + + private static final class DisplayData { + private final int virtualDisplayId; + private final PositionMapper positionMapper; + + private DisplayData(int virtualDisplayId, PositionMapper positionMapper) { + this.virtualDisplayId = virtualDisplayId; + this.positionMapper = positionMapper; + } + } + private static final int DEFAULT_DEVICE_ID = 0; // control_msg.h values of the pointerId field in inject_touch_event message @@ -54,7 +75,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { private final AtomicBoolean isSettingClipboard = new AtomicBoolean(); - private final AtomicReference positionMapper = new AtomicReference<>(); + private final AtomicReference displayData = new AtomicReference<>(); private long lastTouchDown; private final PointersState pointersState = new PointersState(); @@ -102,8 +123,9 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { } @Override - public void onNewVirtualDisplay(PositionMapper positionMapper) { - this.positionMapper.set(positionMapper); + public void onNewVirtualDisplay(int virtualDisplayId, PositionMapper positionMapper) { + DisplayData data = new DisplayData(virtualDisplayId, positionMapper); + this.displayData.set(data); } private UhidManager getUhidManager() { @@ -284,7 +306,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { return false; } for (KeyEvent event : events) { - if (!injectEvent(event, Device.INJECT_MODE_ASYNC)) { + if (!Device.injectEvent(event, displayId, Device.INJECT_MODE_ASYNC)) { return false; } } @@ -306,7 +328,12 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { private boolean injectTouch(int action, long pointerId, Position position, float pressure, int actionButton, int buttons) { long now = SystemClock.uptimeMillis(); - Point point = getPhysicalPoint(position); + // it hides the field on purpose, to read it with atomic access + @SuppressWarnings("checkstyle:HiddenField") + DisplayData displayData = this.displayData.get(); + assert displayData != null : "Cannot receive a touch event without a display"; + + Point point = displayData.positionMapper.map(position); if (point == null) { Ln.w("Ignore touch event, it was generated for a different device size"); return false; @@ -365,7 +392,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { // First button pressed: ACTION_DOWN MotionEvent downEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_DOWN, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0); - if (!injectEvent(downEvent, Device.INJECT_MODE_ASYNC)) { + if (!Device.injectEvent(downEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) { return false; } } @@ -376,7 +403,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { if (!InputManager.setActionButton(pressEvent, actionButton)) { return false; } - if (!injectEvent(pressEvent, Device.INJECT_MODE_ASYNC)) { + if (!Device.injectEvent(pressEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) { return false; } @@ -390,7 +417,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { if (!InputManager.setActionButton(releaseEvent, actionButton)) { return false; } - if (!injectEvent(releaseEvent, Device.INJECT_MODE_ASYNC)) { + if (!Device.injectEvent(releaseEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) { return false; } @@ -398,7 +425,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { // Last button released: ACTION_UP MotionEvent upEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_UP, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0); - if (!injectEvent(upEvent, Device.INJECT_MODE_ASYNC)) { + if (!Device.injectEvent(upEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) { return false; } } @@ -409,12 +436,18 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { MotionEvent event = MotionEvent.obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0); - return injectEvent(event, Device.INJECT_MODE_ASYNC); + return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC); } private boolean injectScroll(Position position, float hScroll, float vScroll, int buttons) { long now = SystemClock.uptimeMillis(); - Point point = getPhysicalPoint(position); + + // it hides the field on purpose, to read it with atomic access + @SuppressWarnings("checkstyle:HiddenField") + DisplayData displayData = this.displayData.get(); + assert displayData != null : "Cannot receive a scroll event without a display"; + + Point point = displayData.positionMapper.map(position); if (point == null) { Ln.w("Ignore scroll event, it was generated for a different device size"); return false; @@ -431,18 +464,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { MotionEvent event = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, InputDevice.SOURCE_MOUSE, 0); - return injectEvent(event, Device.INJECT_MODE_ASYNC); - } - - private Point getPhysicalPoint(Position position) { - // it hides the field on purpose, to read it with atomic access - @SuppressWarnings("checkstyle:HiddenField") - PositionMapper positionMapper = this.positionMapper.get(); - if (positionMapper == null) { - return null; - } - - return positionMapper.map(position); + return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC); } /** @@ -520,10 +542,6 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener { ServiceManager.getActivityManager().startActivity(intent); } - private boolean injectEvent(InputEvent event, int injectMode) { - return Device.injectEvent(event, displayId, injectMode); - } - private boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState, int injectMode) { return Device.injectKeyEvent(action, keyCode, repeat, metaState, displayId, injectMode); } diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index 95faaf39..7e516909 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -106,10 +106,16 @@ public class ScreenCapture extends SurfaceCapture { virtualDisplay = null; } + int virtualDisplayId; + PositionMapper positionMapper; try { Size videoSize = screenInfo.getVideoSize(); virtualDisplay = ServiceManager.getDisplayManager() .createVirtualDisplay("scrcpy", videoSize.getWidth(), videoSize.getHeight(), displayId, surface); + virtualDisplayId = virtualDisplay.getDisplay().getDisplayId(); + Rect contentRect = new Rect(0, 0, videoSize.getWidth(), videoSize.getHeight()); + // The position are relative to the virtual display, not the original display + positionMapper = new PositionMapper(videoSize, contentRect, 0); Ln.d("Display: using DisplayManager API"); } catch (Exception displayManagerException) { try { @@ -123,6 +129,8 @@ public class ScreenCapture extends SurfaceCapture { int layerStack = displayInfo.getLayerStack(); setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); + virtualDisplayId = displayId; + positionMapper = PositionMapper.from(screenInfo); Ln.d("Display: using SurfaceControl API"); } catch (Exception surfaceControlException) { Ln.e("Could not create display using DisplayManager", displayManagerException); @@ -132,8 +140,7 @@ public class ScreenCapture extends SurfaceCapture { } if (vdListener != null) { - PositionMapper positionMapper = PositionMapper.from(screenInfo); - vdListener.onNewVirtualDisplay(positionMapper); + vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper); } } diff --git a/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java b/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java index d978361e..c079265e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java @@ -3,5 +3,5 @@ package com.genymobile.scrcpy.video; import com.genymobile.scrcpy.control.PositionMapper; public interface VirtualDisplayListener { - void onNewVirtualDisplay(PositionMapper positionMapper); + void onNewVirtualDisplay(int displayId, PositionMapper positionMapper); }