drgreen/scrcpy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inject display-related events to virtual display

Browse Source 
Mouse and touch events must be sent to the virtual display id (used for
mirroring), other events (like key events) must be sent to the original
display id.

Fixes #4598 <https://github.com/Genymobile/scrcpy/issues/4598>
Fixes #5137 <https://github.com/Genymobile/scrcpy/issues/5137>
PR #5370 <https://github.com/Genymobile/scrcpy/pull/5370>

Co-authored-by: nightmare <mengyanshou@gmail.com>
This commit is contained in:
Romain Vimont 2024-10-12 09:23:31 +02:00
parent 7024d38199
commit d19396718e
3 changed files with 56 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
server/src/main/java/com/genymobile/scrcpy/control/Controller.java
View File

@ -17,7 +17,6 @@ import android.content.Intent;
import android.os.Build; import android.os.Build;
import android.os.SystemClock; import android.os.SystemClock;
import android.view.InputDevice; import android.view.InputDevice;
import android.view.InputEvent;
import android.view.KeyCharacterMap; import android.view.KeyCharacterMap;
import android.view.KeyEvent; import android.view.KeyEvent;
import android.view.MotionEvent; import android.view.MotionEvent;
@ -31,6 +30,28 @@ import java.util.concurrent.atomic.AtomicReference;
public class Controller implements AsyncProcessor, VirtualDisplayListener { public class Controller implements AsyncProcessor, VirtualDisplayListener {
/*
* For event injection, there are two display ids:
* - the displayId passed to the constructor (which comes from --display-id passed by the client, 0 for the main display);
* - the virtualDisplayId used for mirroring, notified by the capture instance via the VirtualDisplayListener interface.
*
* (In case the ScreenCapture uses the "SurfaceControl API", then both ids are equals, but this is an implementation detail.)
*
* In order to make events work correctly in all cases:
* - virtualDisplayId must be used for events relative to the display (mouse and touch events with coordinates);
* - displayId must be used for other events (like key events).
*/
private static final class DisplayData {
private final int virtualDisplayId;
private final PositionMapper positionMapper;
private DisplayData(int virtualDisplayId, PositionMapper positionMapper) {
this.virtualDisplayId = virtualDisplayId;
this.positionMapper = positionMapper;
}
}
private static final int DEFAULT_DEVICE_ID = 0; private static final int DEFAULT_DEVICE_ID = 0;
// control_msg.h values of the pointerId field in inject_touch_event message // control_msg.h values of the pointerId field in inject_touch_event message
@ -54,7 +75,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
private final AtomicBoolean isSettingClipboard = new AtomicBoolean(); private final AtomicBoolean isSettingClipboard = new AtomicBoolean();
private final AtomicReference<PositionMapper> positionMapper = new AtomicReference<>(); private final AtomicReference<DisplayData> displayData = new AtomicReference<>();
private long lastTouchDown; private long lastTouchDown;
private final PointersState pointersState = new PointersState(); private final PointersState pointersState = new PointersState();
@ -102,8 +123,9 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
} }
@Override @Override
public void onNewVirtualDisplay(PositionMapper positionMapper) { public void onNewVirtualDisplay(int virtualDisplayId, PositionMapper positionMapper) {
this.positionMapper.set(positionMapper); DisplayData data = new DisplayData(virtualDisplayId, positionMapper);
this.displayData.set(data);
} }
private UhidManager getUhidManager() { private UhidManager getUhidManager() {
@ -284,7 +306,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
return false; return false;
} }
for (KeyEvent event : events) { for (KeyEvent event : events) {
if (!injectEvent(event, Device.INJECT_MODE_ASYNC)) { if (!Device.injectEvent(event, displayId, Device.INJECT_MODE_ASYNC)) {
return false; return false;
} }
} }
@ -306,7 +328,12 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
private boolean injectTouch(int action, long pointerId, Position position, float pressure, int actionButton, int buttons) { private boolean injectTouch(int action, long pointerId, Position position, float pressure, int actionButton, int buttons) {
long now = SystemClock.uptimeMillis(); long now = SystemClock.uptimeMillis();
Point point = getPhysicalPoint(position); // it hides the field on purpose, to read it with atomic access
@SuppressWarnings("checkstyle:HiddenField")
DisplayData displayData = this.displayData.get();
assert displayData != null : "Cannot receive a touch event without a display";
Point point = displayData.positionMapper.map(position);
if (point == null) { if (point == null) {
Ln.w("Ignore touch event, it was generated for a different device size"); Ln.w("Ignore touch event, it was generated for a different device size");
return false; return false;
@ -365,7 +392,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
// First button pressed: ACTION_DOWN // First button pressed: ACTION_DOWN
MotionEvent downEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_DOWN, pointerCount, pointerProperties, MotionEvent downEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_DOWN, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0); pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!injectEvent(downEvent, Device.INJECT_MODE_ASYNC)) { if (!Device.injectEvent(downEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false; return false;
} }
} }
@ -376,7 +403,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
if (!InputManager.setActionButton(pressEvent, actionButton)) { if (!InputManager.setActionButton(pressEvent, actionButton)) {
return false; return false;
} }
if (!injectEvent(pressEvent, Device.INJECT_MODE_ASYNC)) { if (!Device.injectEvent(pressEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false; return false;
} }
@ -390,7 +417,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
if (!InputManager.setActionButton(releaseEvent, actionButton)) { if (!InputManager.setActionButton(releaseEvent, actionButton)) {
return false; return false;
} }
if (!injectEvent(releaseEvent, Device.INJECT_MODE_ASYNC)) { if (!Device.injectEvent(releaseEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false; return false;
} }
@ -398,7 +425,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
// Last button released: ACTION_UP // Last button released: ACTION_UP
MotionEvent upEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_UP, pointerCount, pointerProperties, MotionEvent upEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_UP, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0); pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!injectEvent(upEvent, Device.INJECT_MODE_ASYNC)) { if (!Device.injectEvent(upEvent, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC)) {
return false; return false;
} }
} }
@ -409,12 +436,18 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
MotionEvent event = MotionEvent.obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, MotionEvent event = MotionEvent.obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f,
DEFAULT_DEVICE_ID, 0, source, 0); DEFAULT_DEVICE_ID, 0, source, 0);
return injectEvent(event, Device.INJECT_MODE_ASYNC); return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC);
} }
private boolean injectScroll(Position position, float hScroll, float vScroll, int buttons) { private boolean injectScroll(Position position, float hScroll, float vScroll, int buttons) {
long now = SystemClock.uptimeMillis(); long now = SystemClock.uptimeMillis();
Point point = getPhysicalPoint(position);
// it hides the field on purpose, to read it with atomic access
@SuppressWarnings("checkstyle:HiddenField")
DisplayData displayData = this.displayData.get();
assert displayData != null : "Cannot receive a scroll event without a display";
Point point = displayData.positionMapper.map(position);
if (point == null) { if (point == null) {
Ln.w("Ignore scroll event, it was generated for a different device size"); Ln.w("Ignore scroll event, it was generated for a different device size");
return false; return false;
@ -431,18 +464,7 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
MotionEvent event = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, MotionEvent event = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, buttons, 1f, 1f,
DEFAULT_DEVICE_ID, 0, InputDevice.SOURCE_MOUSE, 0); DEFAULT_DEVICE_ID, 0, InputDevice.SOURCE_MOUSE, 0);
return injectEvent(event, Device.INJECT_MODE_ASYNC); return Device.injectEvent(event, displayData.virtualDisplayId, Device.INJECT_MODE_ASYNC);
}
private Point getPhysicalPoint(Position position) {
// it hides the field on purpose, to read it with atomic access
@SuppressWarnings("checkstyle:HiddenField")
PositionMapper positionMapper = this.positionMapper.get();
if (positionMapper == null) {
return null;
}
return positionMapper.map(position);
} }
/** /**
@ -520,10 +542,6 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
ServiceManager.getActivityManager().startActivity(intent); ServiceManager.getActivityManager().startActivity(intent);
} }
private boolean injectEvent(InputEvent event, int injectMode) {
return Device.injectEvent(event, displayId, injectMode);
}
private boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState, int injectMode) { private boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState, int injectMode) {
return Device.injectKeyEvent(action, keyCode, repeat, metaState, displayId, injectMode); return Device.injectKeyEvent(action, keyCode, repeat, metaState, displayId, injectMode);
} }

11
server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java
View File

@ -106,10 +106,16 @@ public class ScreenCapture extends SurfaceCapture {
virtualDisplay = null; virtualDisplay = null;
} }
int virtualDisplayId;
PositionMapper positionMapper;
try { try {
Size videoSize = screenInfo.getVideoSize(); Size videoSize = screenInfo.getVideoSize();
virtualDisplay = ServiceManager.getDisplayManager() virtualDisplay = ServiceManager.getDisplayManager()
.createVirtualDisplay("scrcpy", videoSize.getWidth(), videoSize.getHeight(), displayId, surface); .createVirtualDisplay("scrcpy", videoSize.getWidth(), videoSize.getHeight(), displayId, surface);
virtualDisplayId = virtualDisplay.getDisplay().getDisplayId();
Rect contentRect = new Rect(0, 0, videoSize.getWidth(), videoSize.getHeight());
// The position are relative to the virtual display, not the original display
positionMapper = new PositionMapper(videoSize, contentRect, 0);
Ln.d("Display: using DisplayManager API"); Ln.d("Display: using DisplayManager API");
} catch (Exception displayManagerException) { } catch (Exception displayManagerException) {
try { try {
@ -123,6 +129,8 @@ public class ScreenCapture extends SurfaceCapture {
int layerStack = displayInfo.getLayerStack(); int layerStack = displayInfo.getLayerStack();
setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack); setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
virtualDisplayId = displayId;
positionMapper = PositionMapper.from(screenInfo);
Ln.d("Display: using SurfaceControl API"); Ln.d("Display: using SurfaceControl API");
} catch (Exception surfaceControlException) { } catch (Exception surfaceControlException) {
Ln.e("Could not create display using DisplayManager", displayManagerException); Ln.e("Could not create display using DisplayManager", displayManagerException);
@ -132,8 +140,7 @@ public class ScreenCapture extends SurfaceCapture {
} }
if (vdListener != null) { if (vdListener != null) {
PositionMapper positionMapper = PositionMapper.from(screenInfo); vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper);
vdListener.onNewVirtualDisplay(positionMapper);
} }
} }

2
server/src/main/java/com/genymobile/scrcpy/video/VirtualDisplayListener.java
View File

@ -3,5 +3,5 @@ package com.genymobile.scrcpy.video;
import com.genymobile.scrcpy.control.PositionMapper; import com.genymobile.scrcpy.control.PositionMapper;
public interface VirtualDisplayListener { public interface VirtualDisplayListener {
void onNewVirtualDisplay(PositionMapper positionMapper); void onNewVirtualDisplay(int displayId, PositionMapper positionMapper);
} }