Compare commits

...

16 Commits

Author SHA1 Message Date
e0d87bd903 AudioEncoder WIP 2023-02-02 10:25:07 +01:00
97014889c3 Capture device audio WIP
Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>
2023-02-01 23:06:01 +01:00
05462a5467 Move Workarounds call
Workarounds are not specific to the screen encoder.

Co-authored-by: Simon Chan <1330321+yume-chan@users.noreply.github.com>
2023-02-01 23:05:25 +01:00
05fa3a3358 Use FakeContext for Application instance
This will expose the correct package name and UID to the application
context.
2023-02-01 22:54:12 +01:00
4e115065a2 Use shell package name for workarounds
For consistency.
2023-02-01 22:54:12 +01:00
5174dfaad9 Use PACKAGE_NAME from FakeContext
Remove duplicated constant.
2023-02-01 22:54:12 +01:00
6048aa7378 Use AttributionSource from FakeContext
FakeContext already provides an AttributeSource instance.

Co-authored-by: Simon Chan <1330321+yume-chan@users.noreply.github.com>
2023-02-01 22:54:12 +01:00
2871ebe31a Add a fake Android Context
Since scrcpy-server is not an Android application (it's a java
executable), it has no Context.

Some features will require a Context instance to get the package name
and the UID. Add a FakeContext for this purpose.

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>
2023-02-01 22:54:12 +01:00
2989566c55 Use Process.ROOT_UID
Replace ServiceManager.USER_ID by existing constant Process.ROOT_UID.
2023-02-01 22:54:12 +01:00
653387acdb Compute relative PTS on the client-side
The PTS received by MediaCodec are expressed relative to an arbitrary
clock origin. We consider the PTS of the first frame to be 0, and the
PTS of every other frame is relative to this first PTS (note that the
PTS is only used for recording, it is ignored for mirroring).

For simplicity, this relative PTS was computed on the server-side.

To prepare support for multiple stream (video and audio), send the
packet with its original PTS, and handle the PTS offset on the
client-side (by the recorder).

Since we can't know in advance which stream will produce the first
packet with the lowest PTS (a packet received later on one stream may
have a PTS lower than a packet received earlier on another stream),
computing the PTS on the server-side would require unnecessary waiting.
2023-02-01 22:54:12 +01:00
d7f3683a1b Do not expose controller threads
The way the controller execute its events asynchronously is an
implementation detail.
2023-02-01 22:54:10 +01:00
9b286ec8a7 Inject additional ACTION_BUTTON_* events for mouse
On mouse click events:
 - the first button pressed must first generate ACTION_DOWN;
 - all button pressed (including the first one) must generate
   ACTION_BUTTON_PRESS;
 - all button released (including the last one) must generate
   ACTION_BUTTON_RELEASE;
 - the last button released must in addition generate ACTION_UP.

Otherwise, Chrome does not work properly.

Fixes #3635 <https://github.com/Genymobile/scrcpy/issues/3635>

Signed-off-by: Romain Vimont <rom@rom1v.com>
2023-01-30 20:57:54 +01:00
8c5c55f9e1 Fix mouse pointer state update
If the pointer is a mouse, the pointer is UP only when no buttons are
pressed (not when a button is released, because there might be other
buttons still pressed).

Refs #3635 <https://github.com/Genymobile/scrcpy/issues/3635>

Signed-off-by: Romain Vimont <rom@rom1v.com>
2023-01-30 20:57:54 +01:00
0afef0c634 Forward action button to device
On click event, only the whole buttons state was passed to the device.
In addition, on ACTION_DOWN and ACTION_UP, pass the button associated to
the action.

Refs #3635 <https://github.com/Genymobile/scrcpy/issues/3635>

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>
2023-01-30 20:57:54 +01:00
07806ba915 Retry on spurious error
MediaCodec may fail spuriously, typically when stopping an encoding and
starting a new one immediately (for example on device rotation).

In that case, retry a few times, in many cases it should work.

Refs #3693 <https://github.com/Genymobile/scrcpy/issues/3693>
2023-01-30 20:55:51 +01:00
a52053421a Extract retry handling
Move the code to downscale and retry on error out of the catch-block.

Refs 26b4104844
2023-01-29 14:49:36 +01:00
22 changed files with 529 additions and 137 deletions

View File

@ -117,8 +117,9 @@ sc_control_msg_serialize(const struct sc_control_msg *msg, unsigned char *buf) {
uint16_t pressure =
sc_float_to_u16fp(msg->inject_touch_event.pressure);
sc_write16be(&buf[22], pressure);
sc_write32be(&buf[24], msg->inject_touch_event.buttons);
return 28;
sc_write32be(&buf[24], msg->inject_touch_event.action_button);
sc_write32be(&buf[28], msg->inject_touch_event.buttons);
return 32;
case SC_CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT:
write_position(&buf[1], &msg->inject_scroll_event.position);
int16_t hscroll =
@ -179,22 +180,25 @@ sc_control_msg_log(const struct sc_control_msg *msg) {
if (pointer_name) {
// string pointer id
LOG_CMSG("touch [id=%s] %-4s position=%" PRIi32 ",%" PRIi32
" pressure=%f buttons=%06lx",
" pressure=%f action_button=%06lx buttons=%06lx",
pointer_name,
MOTIONEVENT_ACTION_LABEL(action),
msg->inject_touch_event.position.point.x,
msg->inject_touch_event.position.point.y,
msg->inject_touch_event.pressure,
(long) msg->inject_touch_event.action_button,
(long) msg->inject_touch_event.buttons);
} else {
// numeric pointer id
LOG_CMSG("touch [id=%" PRIu64_ "] %-4s position=%" PRIi32 ",%"
PRIi32 " pressure=%f buttons=%06lx",
PRIi32 " pressure=%f action_button=%06lx"
" buttons=%06lx",
id,
MOTIONEVENT_ACTION_LABEL(action),
msg->inject_touch_event.position.point.x,
msg->inject_touch_event.position.point.y,
msg->inject_touch_event.pressure,
(long) msg->inject_touch_event.action_button,
(long) msg->inject_touch_event.buttons);
}
break;

View File

@ -65,6 +65,7 @@ struct sc_control_msg {
} inject_text;
struct {
enum android_motionevent_action action;
enum android_motionevent_buttons action_button;
enum android_motionevent_buttons buttons;
uint64_t pointer_id;
struct sc_position position;

View File

@ -339,6 +339,7 @@ simulate_virtual_finger(struct sc_input_manager *im,
im->forward_all_clicks ? POINTER_ID_VIRTUAL_MOUSE
: POINTER_ID_VIRTUAL_FINGER;
msg.inject_touch_event.pressure = up ? 0.0f : 1.0f;
msg.inject_touch_event.action_button = 0;
msg.inject_touch_event.buttons = 0;
if (!sc_controller_push_msg(im->controller, &msg)) {

View File

@ -93,6 +93,7 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp,
.pointer_id = event->pointer_id,
.position = event->position,
.pressure = event->action == SC_ACTION_DOWN ? 1.f : 0.f,
.action_button = convert_mouse_buttons(event->button),
.buttons = convert_mouse_buttons(event->buttons_state),
},
};

View File

@ -11,6 +11,8 @@
/** Downcast packet_sink to recorder */
#define DOWNCAST(SINK) container_of(SINK, struct sc_recorder, packet_sink)
#define SC_PTS_ORIGIN_NONE UINT64_C(-1)
static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us
static const AVOutputFormat *
@ -128,6 +130,8 @@ sc_recorder_write(struct sc_recorder *recorder, AVPacket *packet) {
return true;
}
LOGI("==== %" PRIu64, packet->pts);
sc_recorder_rescale_packet(recorder, packet);
return av_write_frame(recorder->ctx, packet) >= 0;
}
@ -169,6 +173,18 @@ run_recorder(void *data) {
sc_mutex_unlock(&recorder->mutex);
if (recorder->pts_origin == SC_PTS_ORIGIN_NONE
&& rec->packet->pts != AV_NOPTS_VALUE) {
// First PTS received
recorder->pts_origin = rec->packet->pts;
}
if (rec->packet->pts != AV_NOPTS_VALUE) {
// Set PTS relatve to the origin
rec->packet->pts -= recorder->pts_origin;
rec->packet->dts = rec->packet->pts;
}
// recorder->previous is only written from this thread, no need to lock
struct sc_record_packet *previous = recorder->previous;
recorder->previous = rec;
@ -243,6 +259,7 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) {
recorder->failed = false;
recorder->header_written = false;
recorder->previous = NULL;
recorder->pts_origin = SC_PTS_ORIGIN_NONE;
const char *format_name = sc_recorder_get_format_name(recorder->format);
assert(format_name);

View File

@ -28,6 +28,8 @@ struct sc_recorder {
struct sc_size declared_frame_size;
bool header_written;
uint64_t pts_origin;
sc_thread thread;
sc_mutex mutex;
sc_cond queue_cond;

View File

@ -90,13 +90,14 @@ static void test_serialize_inject_touch_event(void) {
},
},
.pressure = 1.0f,
.action_button = AMOTION_EVENT_BUTTON_PRIMARY,
.buttons = AMOTION_EVENT_BUTTON_PRIMARY,
},
};
unsigned char buf[SC_CONTROL_MSG_MAX_SIZE];
size_t size = sc_control_msg_serialize(&msg, buf);
assert(size == 28);
assert(size == 32);
const unsigned char expected[] = {
SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT,
@ -105,7 +106,8 @@ static void test_serialize_inject_touch_event(void) {
0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xc8, // 100 200
0x04, 0x38, 0x07, 0x80, // 1080 1920
0xff, 0xff, // pressure
0x00, 0x00, 0x00, 0x01 // AMOTION_EVENT_BUTTON_PRIMARY
0x00, 0x00, 0x00, 0x01, // AMOTION_EVENT_BUTTON_PRIMARY (action button)
0x00, 0x00, 0x00, 0x01, // AMOTION_EVENT_BUTTON_PRIMARY (buttons)
};
assert(!memcmp(buf, expected, sizeof(expected)));
}

View File

@ -0,0 +1,196 @@
package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.ServiceManager;
import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.content.ComponentName;
import android.content.Intent;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.AudioTimestamp;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaRecorder;
import android.os.Build;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.SystemClock;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
public final class AudioEncoder {
private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS;
private static final int SAMPLE_RATE = 48000;
private static final int CHANNELS = 2;
private static final int BIT_RATE = 128000;
private static int BUFFER_MS = 15; // milliseconds
private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000;
private AudioRecord recorder;
private MediaCodec mediaCodec;
private HandlerThread thread;
private final AtomicBoolean interrupted = new AtomicBoolean();
private final Semaphore endSemaphore = new Semaphore(0); // blocks until encoding is ended
private static AudioFormat createAudioFormat() {
AudioFormat.Builder builder = new AudioFormat.Builder();
builder.setEncoding(AudioFormat.ENCODING_PCM_16BIT);
builder.setSampleRate(SAMPLE_RATE);
builder.setChannelMask(CHANNELS == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO);
return builder.build();
}
@TargetApi(Build.VERSION_CODES.M)
@SuppressLint({"WrongConstant", "MissingPermission"})
private static AudioRecord createAudioRecord() {
AudioRecord.Builder builder = new AudioRecord.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
// On older APIs, Workarounds.fillAppInfo() must be called beforehand
builder.setContext(FakeContext.get());
}
builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
builder.setAudioFormat(createAudioFormat());
builder.setBufferSizeInBytes(1024 * 1024);
return builder.build();
}
private static MediaFormat createFormat() {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, MIMETYPE);
format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
return format;
}
@TargetApi(Build.VERSION_CODES.M)
public void start() throws IOException {
mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException
recorder = createAudioRecord();
MediaFormat format = createFormat();
mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
recorder.startRecording();
thread = new HandlerThread("AudioEncoder");
thread.start();
class AudioEncoderCallbacks extends MediaCodec.Callback {
private final AudioTimestamp timestamp = new AudioTimestamp();
private long nextPts;
private boolean eofSignaled;
private boolean ended;
private void notifyEnded() {
assert(!ended);
ended = true;
endSemaphore.release();
}
@TargetApi(Build.VERSION_CODES.N)
@Override
public void onInputBufferAvailable(MediaCodec codec, int index) {
if (eofSignaled) {
return;
}
ByteBuffer inputBuffer = codec.getInputBuffer(index);
int r = recorder.read(inputBuffer, BUFFER_SIZE);
long pts;
int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
if (ret == AudioRecord.SUCCESS) {
pts = timestamp.nanoTime / 1000;
} else {
if (nextPts == 0) {
Ln.w("Could not get any audio timestamp");
}
// compute from previous timestamp and packet size
pts = nextPts;
}
long durationMs = r * 1000 / CHANNELS / SAMPLE_RATE;
nextPts = pts + durationMs;
int flags = 0;
if (interrupted.get()) {
flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM;
eofSignaled = true;
}
codec.queueInputBuffer(index, 0, r, pts, flags);
}
@Override
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) {
if (ended) {
return;
}
ByteBuffer codecBuffer = codec.getOutputBuffer(index);
try {
boolean isConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0;
long pts = bufferInfo.presentationTimeUs;
Ln.i("Audio packet: pts=" + pts + " " + codecBuffer.remaining() + " bytes");
} finally {
codec.releaseOutputBuffer(index, false);
}
boolean eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
if (eof) {
notifyEnded();
}
}
@Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Ln.e("MediaCodec error", e);
if (!ended) {
notifyEnded();
}
}
@Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
// ignore
}
}
mediaCodec.setCallback(new AudioEncoderCallbacks(), new Handler(thread.getLooper()));
mediaCodec.start();
}
private void waitEnded() {
try {
endSemaphore.acquire();
} catch (InterruptedException e) {
// ignore
}
}
public void stop() {
Ln.i("==== STOP");
if (thread != null) {
interrupted.set(true);
waitEnded();
thread.interrupt();
thread = null;
mediaCodec.stop();
mediaCodec.release();
recorder.stop();
Ln.i("==== STOPPED");
}
}
}

View File

@ -29,6 +29,7 @@ public final class ControlMessage {
private int metaState; // KeyEvent.META_*
private int action; // KeyEvent.ACTION_* or MotionEvent.ACTION_* or POWER_MODE_*
private int keycode; // KeyEvent.KEYCODE_*
private int actionButton; // MotionEvent.BUTTON_*
private int buttons; // MotionEvent.BUTTON_*
private long pointerId;
private float pressure;
@ -60,13 +61,15 @@ public final class ControlMessage {
return msg;
}
public static ControlMessage createInjectTouchEvent(int action, long pointerId, Position position, float pressure, int buttons) {
public static ControlMessage createInjectTouchEvent(int action, long pointerId, Position position, float pressure, int actionButton,
int buttons) {
ControlMessage msg = new ControlMessage();
msg.type = TYPE_INJECT_TOUCH_EVENT;
msg.action = action;
msg.pointerId = pointerId;
msg.pressure = pressure;
msg.position = position;
msg.actionButton = actionButton;
msg.buttons = buttons;
return msg;
}
@ -140,6 +143,10 @@ public final class ControlMessage {
return keycode;
}
public int getActionButton() {
return actionButton;
}
public int getButtons() {
return buttons;
}

View File

@ -9,7 +9,7 @@ import java.nio.charset.StandardCharsets;
public class ControlMessageReader {
static final int INJECT_KEYCODE_PAYLOAD_LENGTH = 13;
static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 27;
static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 31;
static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
static final int BACK_OR_SCREEN_ON_LENGTH = 1;
static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
@ -140,8 +140,9 @@ public class ControlMessageReader {
long pointerId = buffer.getLong();
Position position = readPosition(buffer);
float pressure = Binary.u16FixedPointToFloat(buffer.getShort());
int actionButton = buffer.getInt();
int buttons = buffer.getInt();
return ControlMessage.createInjectTouchEvent(action, pointerId, position, pressure, buttons);
return ControlMessage.createInjectTouchEvent(action, pointerId, position, pressure, actionButton, buttons);
}
private ControlMessage parseInjectScrollEvent() {

View File

@ -1,5 +1,7 @@
package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.InputManager;
import android.os.Build;
import android.os.SystemClock;
import android.view.InputDevice;
@ -22,6 +24,8 @@ public class Controller {
private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor();
private Thread thread;
private final Device device;
private final DesktopConnection connection;
private final DeviceMessageSender sender;
@ -60,7 +64,7 @@ public class Controller {
}
}
public void control() throws IOException {
private void control() throws IOException {
// on start, power on the device
if (powerOn && !Device.isScreenOn()) {
device.pressReleaseKeycode(KeyEvent.KEYCODE_POWER, Device.INJECT_MODE_ASYNC);
@ -80,6 +84,27 @@ public class Controller {
}
}
public void start() {
thread = new Thread(() -> {
try {
control();
} catch (IOException e) {
// this is expected on close
Ln.d("Controller stopped");
}
});
thread.start();
sender.start();
}
public void stop() {
if (thread != null) {
thread.interrupt();
thread = null;
}
sender.stop();
}
public DeviceMessageSender getSender() {
return sender;
}
@ -99,7 +124,7 @@ public class Controller {
break;
case ControlMessage.TYPE_INJECT_TOUCH_EVENT:
if (device.supportsInputEvents()) {
injectTouch(msg.getAction(), msg.getPointerId(), msg.getPosition(), msg.getPressure(), msg.getButtons());
injectTouch(msg.getAction(), msg.getPointerId(), msg.getPosition(), msg.getPressure(), msg.getActionButton(), msg.getButtons());
}
break;
case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
@ -179,7 +204,7 @@ public class Controller {
return successCount;
}
private boolean injectTouch(int action, long pointerId, Position position, float pressure, int buttons) {
private boolean injectTouch(int action, long pointerId, Position position, float pressure, int actionButton, int buttons) {
long now = SystemClock.uptimeMillis();
Point point = device.getPhysicalPoint(position);
@ -196,22 +221,23 @@ public class Controller {
Pointer pointer = pointersState.get(pointerIndex);
pointer.setPoint(point);
pointer.setPressure(pressure);
pointer.setUp(action == MotionEvent.ACTION_UP);
int source;
int pointerCount = pointersState.update(pointerProperties, pointerCoords);
if (pointerId == POINTER_ID_MOUSE || pointerId == POINTER_ID_VIRTUAL_MOUSE) {
// real mouse event (forced by the client when --forward-on-click)
pointerProperties[pointerIndex].toolType = MotionEvent.TOOL_TYPE_MOUSE;
source = InputDevice.SOURCE_MOUSE;
pointer.setUp(buttons == 0);
} else {
// POINTER_ID_GENERIC_FINGER, POINTER_ID_VIRTUAL_FINGER or real touch from device
pointerProperties[pointerIndex].toolType = MotionEvent.TOOL_TYPE_FINGER;
source = InputDevice.SOURCE_TOUCHSCREEN;
// Buttons must not be set for touch events
buttons = 0;
pointer.setUp(action == MotionEvent.ACTION_UP);
}
int pointerCount = pointersState.update(pointerProperties, pointerCoords);
if (pointerCount == 1) {
if (action == MotionEvent.ACTION_DOWN) {
lastTouchDown = now;
@ -225,6 +251,62 @@ public class Controller {
}
}
/* If the input device is a mouse (on API >= 23):
* - the first button pressed must first generate ACTION_DOWN;
* - all button pressed (including the first one) must generate ACTION_BUTTON_PRESS;
* - all button released (including the last one) must generate ACTION_BUTTON_RELEASE;
* - the last button released must in addition generate ACTION_UP.
*
* Otherwise, Chrome does not work properly: <https://github.com/Genymobile/scrcpy/issues/3635>
*/
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && source == InputDevice.SOURCE_MOUSE) {
if (action == MotionEvent.ACTION_DOWN) {
if (actionButton == buttons) {
// First button pressed: ACTION_DOWN
MotionEvent downEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_DOWN, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!device.injectEvent(downEvent, Device.INJECT_MODE_ASYNC)) {
return false;
}
}
// Any button pressed: ACTION_BUTTON_PRESS
MotionEvent pressEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_BUTTON_PRESS, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!InputManager.setActionButton(pressEvent, actionButton)) {
return false;
}
if (!device.injectEvent(pressEvent, Device.INJECT_MODE_ASYNC)) {
return false;
}
return true;
}
if (action == MotionEvent.ACTION_UP) {
// Any button released: ACTION_BUTTON_RELEASE
MotionEvent releaseEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_BUTTON_RELEASE, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!InputManager.setActionButton(releaseEvent, actionButton)) {
return false;
}
if (!device.injectEvent(releaseEvent, Device.INJECT_MODE_ASYNC)) {
return false;
}
if (buttons == 0) {
// Last button released: ACTION_UP
MotionEvent upEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_UP, pointerCount, pointerProperties,
pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
if (!device.injectEvent(upEvent, Device.INJECT_MODE_ASYNC)) {
return false;
}
}
return true;
}
}
MotionEvent event = MotionEvent
.obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source,
0);

View File

@ -6,6 +6,8 @@ public final class DeviceMessageSender {
private final DesktopConnection connection;
private Thread thread;
private String clipboardText;
private long ack;
@ -24,7 +26,7 @@ public final class DeviceMessageSender {
notify();
}
public void loop() throws IOException, InterruptedException {
private void loop() throws IOException, InterruptedException {
while (!Thread.currentThread().isInterrupted()) {
String text;
long sequence;
@ -49,4 +51,22 @@ public final class DeviceMessageSender {
}
}
}
public void start() {
thread = new Thread(() -> {
try {
loop();
} catch (IOException | InterruptedException e) {
// this is expected on close
Ln.d("Device message sender stopped");
}
});
thread.start();
}
public void stop() {
if (thread != null) {
thread.interrupt();
thread = null;
}
}
}

View File

@ -0,0 +1,40 @@
package com.genymobile.scrcpy;
import android.annotation.TargetApi;
import android.content.AttributionSource;
import android.content.ContextWrapper;
import android.os.Build;
import android.os.Process;
public final class FakeContext extends ContextWrapper {
public static final String PACKAGE_NAME = "com.android.shell";
private static final FakeContext INSTANCE = new FakeContext();
public static FakeContext get() {
return INSTANCE;
}
private FakeContext() {
super(null);
}
@Override
public String getPackageName() {
return PACKAGE_NAME;
}
@Override
public String getOpPackageName() {
return PACKAGE_NAME;
}
@TargetApi(Build.VERSION_CODES.S)
@Override
public AttributionSource getAttributionSource() {
AttributionSource.Builder builder = new AttributionSource.Builder(Process.SHELL_UID);
builder.setPackageName(PACKAGE_NAME);
return builder.build();
}
}

View File

@ -9,6 +9,7 @@ import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.os.Build;
import android.os.IBinder;
import android.os.SystemClock;
import android.view.Surface;
import java.io.FileDescriptor;
@ -27,6 +28,7 @@ public class ScreenEncoder implements Device.RotationListener {
// Keep the values in descending order
private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800};
private static final int MAX_CONSECUTIVE_ERRORS = 3;
private static final long PACKET_FLAG_CONFIG = 1L << 63;
private static final long PACKET_FLAG_KEY_FRAME = 1L << 62;
@ -40,9 +42,9 @@ public class ScreenEncoder implements Device.RotationListener {
private final int maxFps;
private final boolean sendFrameMeta;
private final boolean downsizeOnError;
private long ptsOrigin;
private boolean firstFrameSent;
private int consecutiveErrors;
public ScreenEncoder(boolean sendFrameMeta, int bitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
boolean downsizeOnError) {
@ -64,17 +66,6 @@ public class ScreenEncoder implements Device.RotationListener {
}
public void streamScreen(Device device, FileDescriptor fd) throws IOException {
Workarounds.prepareMainLooper();
if (Build.BRAND.equalsIgnoreCase("meizu")) {
// <https://github.com/Genymobile/scrcpy/issues/240>
// <https://github.com/Genymobile/scrcpy/issues/2656>
Workarounds.fillAppInfo();
}
internalStreamScreen(device, fd);
}
private void internalStreamScreen(Device device, FileDescriptor fd) throws IOException {
MediaCodec codec = createCodec(encoderName);
MediaFormat format = createFormat(bitRate, maxFps, codecOptions);
IBinder display = createDisplay();
@ -108,20 +99,10 @@ public class ScreenEncoder implements Device.RotationListener {
codec.stop();
} catch (IllegalStateException | IllegalArgumentException e) {
Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
if (!downsizeOnError || firstFrameSent) {
// Fail immediately
if (!prepareRetry(device, screenInfo)) {
throw e;
}
int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
if (newMaxSize == 0) {
// Definitively fail
throw e;
}
// Retry with a smaller device size
Ln.i("Retrying with -m" + newMaxSize + "...");
device.setMaxSize(newMaxSize);
Ln.i("Retrying...");
alive = true;
} finally {
codec.reset();
@ -137,6 +118,39 @@ public class ScreenEncoder implements Device.RotationListener {
}
}
private boolean prepareRetry(Device device, ScreenInfo screenInfo) {
if (firstFrameSent) {
++consecutiveErrors;
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
// Definitively fail
return false;
}
// Wait a bit to increase the probability that retrying will fix the problem
SystemClock.sleep(50);
return true;
}
if (!downsizeOnError) {
// Must fail immediately
return false;
}
// Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising)
int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
Ln.i("newMaxSize = " + newMaxSize);
if (newMaxSize == 0) {
// Must definitively fail
return false;
}
// Retry with a smaller device size
Ln.i("Retrying with -m" + newMaxSize + "...");
device.setMaxSize(newMaxSize);
return true;
}
private static int chooseMaxSizeFallback(Size failedSize) {
int currentMaxSize = Math.max(failedSize.getWidth(), failedSize.getHeight());
for (int value : MAX_SIZE_FALLBACK) {
@ -172,6 +186,7 @@ public class ScreenEncoder implements Device.RotationListener {
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
// If this is not a config packet, then it contains a frame
firstFrameSent = true;
consecutiveErrors = 0;
}
}
} finally {
@ -191,10 +206,7 @@ public class ScreenEncoder implements Device.RotationListener {
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
pts = PACKET_FLAG_CONFIG; // non-media data packet
} else {
if (ptsOrigin == 0) {
ptsOrigin = bufferInfo.presentationTimeUs;
}
pts = bufferInfo.presentationTimeUs - ptsOrigin;
pts = bufferInfo.presentationTimeUs;
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) {
pts |= PACKET_FLAG_KEY_FRAME;
}

View File

@ -4,6 +4,7 @@ import android.graphics.Rect;
import android.media.MediaCodecInfo;
import android.os.BatteryManager;
import android.os.Build;
import android.provider.MediaStore;
import java.io.IOException;
import java.util.List;
@ -69,8 +70,24 @@ public final class Server {
int uid = options.getUid();
boolean tunnelForward = options.isTunnelForward();
boolean control = options.getControl();
boolean audio = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R; // TODO option
boolean sendDummyByte = options.getSendDummyByte();
Workarounds.prepareMainLooper();
// <https://github.com/Genymobile/scrcpy/issues/240>
// <https://github.com/Genymobile/scrcpy/issues/2656>
boolean mustFillAppInfo = Build.BRAND.equalsIgnoreCase("meizu");
// Before Android 11, audio is not supported.
// Since Android 12, we can properly set a context on the AudioRecord.
// Only on Android 11 we must fill app info for the AudioRecord to work.
mustFillAppInfo |= audio && Build.VERSION.SDK_INT == Build.VERSION_CODES.R;
if (mustFillAppInfo) {
Workarounds.fillAppInfo();
}
try (DesktopConnection connection = DesktopConnection.open(uid, tunnelForward, control, sendDummyByte)) {
if (options.getSendDeviceMeta()) {
Size videoSize = device.getScreenInfo().getVideoSize();
@ -79,16 +96,19 @@ public final class Server {
ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate(), options.getMaxFps(), codecOptions,
options.getEncoderName(), options.getDownsizeOnError());
Thread controllerThread = null;
Thread deviceMessageSenderThread = null;
Controller controller = null;
if (control) {
final Controller controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
controller = new Controller(device, connection, options.getClipboardAutosync(), options.getPowerOn());
controller.start();
// asynchronous
controllerThread = startController(controller);
deviceMessageSenderThread = startDeviceMessageSender(controller.getSender());
final Controller controllerRef = controller;
device.setClipboardListener(text -> controllerRef.getSender().pushClipboardText(text));
}
device.setClipboardListener(text -> controller.getSender().pushClipboardText(text));
AudioEncoder audioEncoder = null;
if (audio) {
audioEncoder = new AudioEncoder();
audioEncoder.start();
}
try {
@ -99,11 +119,11 @@ public final class Server {
Ln.d("Screen streaming stopped");
} finally {
initThread.interrupt();
if (controllerThread != null) {
controllerThread.interrupt();
if (controller != null) {
controller.stop();
}
if (deviceMessageSenderThread != null) {
deviceMessageSenderThread.interrupt();
if (audioEncoder != null) {
audioEncoder.stop();
}
}
}
@ -115,32 +135,6 @@ public final class Server {
return thread;
}
private static Thread startController(final Controller controller) {
Thread thread = new Thread(() -> {
try {
controller.control();
} catch (IOException e) {
// this is expected on close
Ln.d("Controller stopped");
}
});
thread.start();
return thread;
}
private static Thread startDeviceMessageSender(final DeviceMessageSender sender) {
Thread thread = new Thread(() -> {
try {
sender.loop();
} catch (IOException | InterruptedException e) {
// this is expected on close
Ln.d("Device message sender stopped");
}
});
thread.start();
return thread;
}
private static Options createOptions(String... args) {
if (args.length < 1) {
throw new IllegalArgumentException("Missing client version");

View File

@ -3,13 +3,12 @@ package com.genymobile.scrcpy;
import android.annotation.SuppressLint;
import android.app.Application;
import android.app.Instrumentation;
import android.content.Context;
import android.content.ContextWrapper;
import android.content.pm.ApplicationInfo;
import android.os.Looper;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
public final class Workarounds {
private Workarounds() {
@ -50,7 +49,7 @@ public final class Workarounds {
Object appBindData = appBindDataConstructor.newInstance();
ApplicationInfo applicationInfo = new ApplicationInfo();
applicationInfo.packageName = "com.genymobile.scrcpy";
applicationInfo.packageName = FakeContext.PACKAGE_NAME;
// appBindData.appInfo = applicationInfo;
Field appInfoField = appBindDataClass.getDeclaredField("appInfo");
@ -62,11 +61,10 @@ public final class Workarounds {
mBoundApplicationField.setAccessible(true);
mBoundApplicationField.set(activityThread, appBindData);
// Context ctx = activityThread.getSystemContext();
Method getSystemContextMethod = activityThreadClass.getDeclaredMethod("getSystemContext");
Context ctx = (Context) getSystemContextMethod.invoke(activityThread);
Application app = Instrumentation.newApplication(Application.class, ctx);
Application app = Application.class.newInstance();
Field baseField = ContextWrapper.class.getDeclaredField("mBase");
baseField.setAccessible(true);
baseField.set(app, FakeContext.get());
// activityThread.mInitialApplication = app;
Field mInitialApplicationField = activityThreadClass.getDeclaredField("mInitialApplication");

View File

@ -5,6 +5,7 @@ import com.genymobile.scrcpy.Ln;
import android.os.Binder;
import android.os.IBinder;
import android.os.IInterface;
import android.os.Process;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
@ -48,10 +49,10 @@ public class ActivityManager {
Object[] args;
if (getContentProviderExternalMethodNewVersion) {
// new version
args = new Object[]{name, ServiceManager.USER_ID, token, null};
args = new Object[]{name, Process.ROOT_UID, token, null};
} else {
// old version
args = new Object[]{name, ServiceManager.USER_ID, token};
args = new Object[]{name, Process.ROOT_UID, token};
}
// ContentProviderHolder providerHolder = getContentProviderExternal(...);
Object providerHolder = method.invoke(manager, args);

View File

@ -1,11 +1,13 @@
package com.genymobile.scrcpy.wrappers;
import com.genymobile.scrcpy.FakeContext;
import com.genymobile.scrcpy.Ln;
import android.content.ClipData;
import android.content.IOnPrimaryClipChangedListener;
import android.os.Build;
import android.os.IInterface;
import android.os.Process;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -58,22 +60,22 @@ public class ClipboardManager {
private static ClipData getPrimaryClip(Method method, boolean alternativeMethod, IInterface manager)
throws InvocationTargetException, IllegalAccessException {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME);
return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME);
}
if (alternativeMethod) {
return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID);
return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, null, Process.ROOT_UID);
}
return (ClipData) method.invoke(manager, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID);
return (ClipData) method.invoke(manager, FakeContext.PACKAGE_NAME, Process.ROOT_UID);
}
private static void setPrimaryClip(Method method, boolean alternativeMethod, IInterface manager, ClipData clipData)
throws InvocationTargetException, IllegalAccessException {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME);
method.invoke(manager, clipData, FakeContext.PACKAGE_NAME);
} else if (alternativeMethod) {
method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID);
method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, null, Process.ROOT_UID);
} else {
method.invoke(manager, clipData, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID);
method.invoke(manager, clipData, FakeContext.PACKAGE_NAME, Process.ROOT_UID);
}
}
@ -106,11 +108,11 @@ public class ClipboardManager {
private static void addPrimaryClipChangedListener(Method method, boolean alternativeMethod, IInterface manager,
IOnPrimaryClipChangedListener listener) throws InvocationTargetException, IllegalAccessException {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
method.invoke(manager, listener, ServiceManager.PACKAGE_NAME);
method.invoke(manager, listener, FakeContext.PACKAGE_NAME);
} else if (alternativeMethod) {
method.invoke(manager, listener, ServiceManager.PACKAGE_NAME, null, ServiceManager.USER_ID);
method.invoke(manager, listener, FakeContext.PACKAGE_NAME, null, Process.ROOT_UID);
} else {
method.invoke(manager, listener, ServiceManager.PACKAGE_NAME, ServiceManager.USER_ID);
method.invoke(manager, listener, FakeContext.PACKAGE_NAME, Process.ROOT_UID);
}
}

View File

@ -1,11 +1,15 @@
package com.genymobile.scrcpy.wrappers;
import com.genymobile.scrcpy.FakeContext;
import com.genymobile.scrcpy.Ln;
import com.genymobile.scrcpy.SettingsException;
import android.annotation.SuppressLint;
import android.content.AttributionSource;
import android.os.Build;
import android.os.Bundle;
import android.os.IBinder;
import android.os.Process;
import java.io.Closeable;
import java.lang.reflect.InvocationTargetException;
@ -51,11 +55,10 @@ public class ContentProvider implements Closeable {
@SuppressLint("PrivateApi")
private Method getCallMethod() throws NoSuchMethodException {
if (callMethod == null) {
try {
Class<?> attributionSourceClass = Class.forName("android.content.AttributionSource");
callMethod = provider.getClass().getMethod("call", attributionSourceClass, String.class, String.class, String.class, Bundle.class);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
callMethod = provider.getClass().getMethod("call", AttributionSource.class, String.class, String.class, String.class, Bundle.class);
callMethodVersion = 0;
} catch (NoSuchMethodException | ClassNotFoundException e0) {
} else {
// old versions
try {
callMethod = provider.getClass()
@ -75,40 +78,29 @@ public class ContentProvider implements Closeable {
return callMethod;
}
@SuppressLint("PrivateApi")
private Object getAttributionSource()
throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
if (attributionSource == null) {
Class<?> cl = Class.forName("android.content.AttributionSource$Builder");
Object builder = cl.getConstructor(int.class).newInstance(ServiceManager.USER_ID);
cl.getDeclaredMethod("setPackageName", String.class).invoke(builder, ServiceManager.PACKAGE_NAME);
attributionSource = cl.getDeclaredMethod("build").invoke(builder);
}
return attributionSource;
}
private Bundle call(String callMethod, String arg, Bundle extras)
throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException {
throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
try {
Method method = getCallMethod();
Object[] args;
switch (callMethodVersion) {
case 0:
args = new Object[]{getAttributionSource(), "settings", callMethod, arg, extras};
break;
case 1:
args = new Object[]{ServiceManager.PACKAGE_NAME, null, "settings", callMethod, arg, extras};
break;
case 2:
args = new Object[]{ServiceManager.PACKAGE_NAME, "settings", callMethod, arg, extras};
break;
default:
args = new Object[]{ServiceManager.PACKAGE_NAME, callMethod, arg, extras};
break;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && callMethodVersion == 0) {
args = new Object[]{FakeContext.get().getAttributionSource(), "settings", callMethod, arg, extras};
} else {
switch (callMethodVersion) {
case 1:
args = new Object[]{FakeContext.PACKAGE_NAME, null, "settings", callMethod, arg, extras};
break;
case 2:
args = new Object[]{FakeContext.PACKAGE_NAME, "settings", callMethod, arg, extras};
break;
default:
args = new Object[]{FakeContext.PACKAGE_NAME, callMethod, arg, extras};
break;
}
}
return (Bundle) method.invoke(provider, args);
} catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException | ClassNotFoundException | InstantiationException e) {
} catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
Ln.e("Could not invoke method", e);
throw e;
}
@ -147,7 +139,7 @@ public class ContentProvider implements Closeable {
public String getValue(String table, String key) throws SettingsException {
String method = getGetMethod(table);
Bundle arg = new Bundle();
arg.putInt(CALL_METHOD_USER_KEY, ServiceManager.USER_ID);
arg.putInt(CALL_METHOD_USER_KEY, Process.ROOT_UID);
try {
Bundle bundle = call(method, key, arg);
if (bundle == null) {
@ -163,7 +155,7 @@ public class ContentProvider implements Closeable {
public void putValue(String table, String key, String value) throws SettingsException {
String method = getPutMethod(table);
Bundle arg = new Bundle();
arg.putInt(CALL_METHOD_USER_KEY, ServiceManager.USER_ID);
arg.putInt(CALL_METHOD_USER_KEY, Process.ROOT_UID);
arg.putString(NAME_VALUE_TABLE_VALUE, value);
try {
call(method, key, arg);

View File

@ -3,6 +3,7 @@ package com.genymobile.scrcpy.wrappers;
import com.genymobile.scrcpy.Ln;
import android.view.InputEvent;
import android.view.MotionEvent;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -17,6 +18,7 @@ public final class InputManager {
private Method injectInputEventMethod;
private static Method setDisplayIdMethod;
private static Method setActionButtonMethod;
public InputManager(android.hardware.input.InputManager manager) {
this.manager = manager;
@ -56,4 +58,22 @@ public final class InputManager {
return false;
}
}
private static Method getSetActionButtonMethod() throws NoSuchMethodException {
if (setActionButtonMethod == null) {
setActionButtonMethod = MotionEvent.class.getMethod("setActionButton", int.class);
}
return setActionButtonMethod;
}
public static boolean setActionButton(MotionEvent motionEvent, int actionButton) {
try {
Method method = getSetActionButtonMethod();
method.invoke(motionEvent, actionButton);
return true;
} catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
Ln.e("Cannot set action button on MotionEvent", e);
return false;
}
}
}

View File

@ -10,9 +10,6 @@ import java.lang.reflect.Method;
@SuppressLint("PrivateApi,DiscouragedPrivateApi")
public final class ServiceManager {
public static final String PACKAGE_NAME = "com.android.shell";
public static final int USER_ID = 0;
private static final Method GET_SERVICE_METHOD;
static {
try {

View File

@ -94,7 +94,8 @@ public class ControlMessageReaderTest {
dos.writeShort(1080);
dos.writeShort(1920);
dos.writeShort(0xffff); // pressure
dos.writeInt(MotionEvent.BUTTON_PRIMARY);
dos.writeInt(MotionEvent.BUTTON_PRIMARY); // action button
dos.writeInt(MotionEvent.BUTTON_PRIMARY); // buttons
byte[] packet = bos.toByteArray();
@ -112,6 +113,7 @@ public class ControlMessageReaderTest {
Assert.assertEquals(1080, event.getPosition().getScreenSize().getWidth());
Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight());
Assert.assertEquals(1f, event.getPressure(), 0f); // must be exact
Assert.assertEquals(MotionEvent.BUTTON_PRIMARY, event.getActionButton());
Assert.assertEquals(MotionEvent.BUTTON_PRIMARY, event.getButtons());
}