Compute relative PTS on the client-side

The PTS received by MediaCodec are expressed relative to an arbitrary
clock origin. We consider the PTS of the first frame to be 0, and the
PTS of every other frame is relative to this first PTS (note that the
PTS is only used for recording, it is ignored for mirroring).

For simplicity, this relative PTS was computed on the server-side.

To prepare support for multiple stream (video and audio), send the
packet with its original PTS, and handle the PTS offset on the
client-side (by the recorder).

Since we can't know in advance which stream will produce the first
packet with the lowest PTS (a packet received later on one stream may
have a PTS lower than a packet received earlier on another stream),
computing the PTS on the server-side would require unnecessary waiting.

app/src/control_msg.c

@@ -117,8 +117,9 @@ sc_control_msg_serialize(const struct sc_control_msg *msg, unsigned char *buf) {
             uint16_t pressure =
                 sc_float_to_u16fp(msg->inject_touch_event.pressure);
             sc_write16be(&buf[22], pressure);
-            sc_write32be(&buf[24], msg->inject_touch_event.buttons);
-            return 28;
+            sc_write32be(&buf[24], msg->inject_touch_event.action_button);
+            sc_write32be(&buf[28], msg->inject_touch_event.buttons);
+            return 32;
         case SC_CONTROL_MSG_TYPE_INJECT_SCROLL_EVENT:
             write_position(&buf[1], &msg->inject_scroll_event.position);
             int16_t hscroll =
@@ -179,22 +180,25 @@ sc_control_msg_log(const struct sc_control_msg *msg) {
             if (pointer_name) {
                 // string pointer id
                 LOG_CMSG("touch [id=%s] %-4s position=%" PRIi32 ",%" PRIi32
-                             " pressure=%f buttons=%06lx",
+                             " pressure=%f action_button=%06lx buttons=%06lx",
                          pointer_name,
                          MOTIONEVENT_ACTION_LABEL(action),
                          msg->inject_touch_event.position.point.x,
                          msg->inject_touch_event.position.point.y,
                          msg->inject_touch_event.pressure,
+                         (long) msg->inject_touch_event.action_button,
                          (long) msg->inject_touch_event.buttons);
             } else {
                 // numeric pointer id
                 LOG_CMSG("touch [id=%" PRIu64_ "] %-4s position=%" PRIi32 ",%"
-                             PRIi32 " pressure=%f buttons=%06lx",
+                             PRIi32 " pressure=%f action_button=%06lx"
+                             " buttons=%06lx",
                          id,
                          MOTIONEVENT_ACTION_LABEL(action),
                          msg->inject_touch_event.position.point.x,
                          msg->inject_touch_event.position.point.y,
                          msg->inject_touch_event.pressure,
+                         (long) msg->inject_touch_event.action_button,
                          (long) msg->inject_touch_event.buttons);
             }
             break;

app/src/control_msg.h

@@ -65,6 +65,7 @@ struct sc_control_msg {
         } inject_text;
         struct {
             enum android_motionevent_action action;
+            enum android_motionevent_buttons action_button;
             enum android_motionevent_buttons buttons;
             uint64_t pointer_id;
             struct sc_position position;

app/src/input_manager.c

@@ -339,6 +339,7 @@ simulate_virtual_finger(struct sc_input_manager *im,
         im->forward_all_clicks ? POINTER_ID_VIRTUAL_MOUSE
                                : POINTER_ID_VIRTUAL_FINGER;
     msg.inject_touch_event.pressure = up ? 0.0f : 1.0f;
+    msg.inject_touch_event.action_button = 0;
     msg.inject_touch_event.buttons = 0;
 
     if (!sc_controller_push_msg(im->controller, &msg)) {

app/src/mouse_inject.c

@@ -93,6 +93,7 @@ sc_mouse_processor_process_mouse_click(struct sc_mouse_processor *mp,
             .pointer_id = event->pointer_id,
             .position = event->position,
             .pressure = event->action == SC_ACTION_DOWN ? 1.f : 0.f,
+            .action_button = convert_mouse_buttons(event->button),
             .buttons = convert_mouse_buttons(event->buttons_state),
         },
     };

app/src/recorder.c

@@ -11,6 +11,8 @@
 /** Downcast packet_sink to recorder */
 #define DOWNCAST(SINK) container_of(SINK, struct sc_recorder, packet_sink)
 
+#define SC_PTS_ORIGIN_NONE UINT64_C(-1)
+
 static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us
 
 static const AVOutputFormat *
@@ -128,6 +130,8 @@ sc_recorder_write(struct sc_recorder *recorder, AVPacket *packet) {
         return true;
     }
 
+    LOGI("==== %" PRIu64, packet->pts);
+
     sc_recorder_rescale_packet(recorder, packet);
     return av_write_frame(recorder->ctx, packet) >= 0;
 }
@@ -169,6 +173,18 @@ run_recorder(void *data) {
 
         sc_mutex_unlock(&recorder->mutex);
 
+        if (recorder->pts_origin == SC_PTS_ORIGIN_NONE
+                && rec->packet->pts != AV_NOPTS_VALUE) {
+            // First PTS received
+            recorder->pts_origin = rec->packet->pts;
+        }
+
+        if (rec->packet->pts != AV_NOPTS_VALUE) {
+            // Set PTS relatve to the origin
+            rec->packet->pts -= recorder->pts_origin;
+            rec->packet->dts = rec->packet->pts;
+        }
+
         // recorder->previous is only written from this thread, no need to lock
         struct sc_record_packet *previous = recorder->previous;
         recorder->previous = rec;
@@ -243,6 +259,7 @@ sc_recorder_open(struct sc_recorder *recorder, const AVCodec *input_codec) {
     recorder->failed = false;
     recorder->header_written = false;
     recorder->previous = NULL;
+    recorder->pts_origin = SC_PTS_ORIGIN_NONE;
 
     const char *format_name = sc_recorder_get_format_name(recorder->format);
     assert(format_name);

app/src/recorder.h

@@ -28,6 +28,8 @@ struct sc_recorder {
     struct sc_size declared_frame_size;
     bool header_written;
 
+    uint64_t pts_origin;
+
     sc_thread thread;
     sc_mutex mutex;
     sc_cond queue_cond;

app/tests/test_control_msg_serialize.c

@@ -90,13 +90,14 @@ static void test_serialize_inject_touch_event(void) {
                 },
             },
             .pressure = 1.0f,
+            .action_button = AMOTION_EVENT_BUTTON_PRIMARY,
             .buttons = AMOTION_EVENT_BUTTON_PRIMARY,
         },
     };
 
     unsigned char buf[SC_CONTROL_MSG_MAX_SIZE];
     size_t size = sc_control_msg_serialize(&msg, buf);
-    assert(size == 28);
+    assert(size == 32);
 
     const unsigned char expected[] = {
         SC_CONTROL_MSG_TYPE_INJECT_TOUCH_EVENT,
@@ -105,7 +106,8 @@ static void test_serialize_inject_touch_event(void) {
         0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xc8, // 100 200
         0x04, 0x38, 0x07, 0x80, // 1080 1920
         0xff, 0xff, // pressure
-        0x00, 0x00, 0x00, 0x01 // AMOTION_EVENT_BUTTON_PRIMARY
+        0x00, 0x00, 0x00, 0x01, // AMOTION_EVENT_BUTTON_PRIMARY (action button)
+        0x00, 0x00, 0x00, 0x01, // AMOTION_EVENT_BUTTON_PRIMARY (buttons)
     };
     assert(!memcmp(buf, expected, sizeof(expected)));
 }

server/src/main/java/com/genymobile/scrcpy/ControlMessage.java

@@ -29,6 +29,7 @@ public final class ControlMessage {
     private int metaState; // KeyEvent.META_*
     private int action; // KeyEvent.ACTION_* or MotionEvent.ACTION_* or POWER_MODE_*
     private int keycode; // KeyEvent.KEYCODE_*
+    private int actionButton; // MotionEvent.BUTTON_*
     private int buttons; // MotionEvent.BUTTON_*
     private long pointerId;
     private float pressure;
@@ -60,13 +61,15 @@ public final class ControlMessage {
         return msg;
     }
 
-    public static ControlMessage createInjectTouchEvent(int action, long pointerId, Position position, float pressure, int buttons) {
+    public static ControlMessage createInjectTouchEvent(int action, long pointerId, Position position, float pressure, int actionButton,
+            int buttons) {
         ControlMessage msg = new ControlMessage();
         msg.type = TYPE_INJECT_TOUCH_EVENT;
         msg.action = action;
         msg.pointerId = pointerId;
         msg.pressure = pressure;
         msg.position = position;
+        msg.actionButton = actionButton;
         msg.buttons = buttons;
         return msg;
     }
@@ -140,6 +143,10 @@ public final class ControlMessage {
         return keycode;
     }
 
+    public int getActionButton() {
+        return actionButton;
+    }
+
     public int getButtons() {
         return buttons;
     }

server/src/main/java/com/genymobile/scrcpy/ControlMessageReader.java

@@ -9,7 +9,7 @@ import java.nio.charset.StandardCharsets;
 public class ControlMessageReader {
 
     static final int INJECT_KEYCODE_PAYLOAD_LENGTH = 13;
-    static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 27;
+    static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 31;
     static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
     static final int BACK_OR_SCREEN_ON_LENGTH = 1;
     static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
@@ -140,8 +140,9 @@ public class ControlMessageReader {
         long pointerId = buffer.getLong();
         Position position = readPosition(buffer);
         float pressure = Binary.u16FixedPointToFloat(buffer.getShort());
+        int actionButton = buffer.getInt();
         int buttons = buffer.getInt();
-        return ControlMessage.createInjectTouchEvent(action, pointerId, position, pressure, buttons);
+        return ControlMessage.createInjectTouchEvent(action, pointerId, position, pressure, actionButton, buttons);
     }
 
     private ControlMessage parseInjectScrollEvent() {

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -1,5 +1,7 @@
 package com.genymobile.scrcpy;
 
+import com.genymobile.scrcpy.wrappers.InputManager;
+
 import android.os.Build;
 import android.os.SystemClock;
 import android.view.InputDevice;
@@ -99,7 +101,7 @@ public class Controller {
                 break;
             case ControlMessage.TYPE_INJECT_TOUCH_EVENT:
                 if (device.supportsInputEvents()) {
-                    injectTouch(msg.getAction(), msg.getPointerId(), msg.getPosition(), msg.getPressure(), msg.getButtons());
+                    injectTouch(msg.getAction(), msg.getPointerId(), msg.getPosition(), msg.getPressure(), msg.getActionButton(), msg.getButtons());
                 }
                 break;
             case ControlMessage.TYPE_INJECT_SCROLL_EVENT:
@@ -179,7 +181,7 @@ public class Controller {
         return successCount;
     }
 
-    private boolean injectTouch(int action, long pointerId, Position position, float pressure, int buttons) {
+    private boolean injectTouch(int action, long pointerId, Position position, float pressure, int actionButton, int buttons) {
         long now = SystemClock.uptimeMillis();
 
         Point point = device.getPhysicalPoint(position);
@@ -196,22 +198,23 @@ public class Controller {
         Pointer pointer = pointersState.get(pointerIndex);
         pointer.setPoint(point);
         pointer.setPressure(pressure);
-        pointer.setUp(action == MotionEvent.ACTION_UP);
 
         int source;
-        int pointerCount = pointersState.update(pointerProperties, pointerCoords);
         if (pointerId == POINTER_ID_MOUSE || pointerId == POINTER_ID_VIRTUAL_MOUSE) {
             // real mouse event (forced by the client when --forward-on-click)
             pointerProperties[pointerIndex].toolType = MotionEvent.TOOL_TYPE_MOUSE;
             source = InputDevice.SOURCE_MOUSE;
+            pointer.setUp(buttons == 0);
         } else {
             // POINTER_ID_GENERIC_FINGER, POINTER_ID_VIRTUAL_FINGER or real touch from device
             pointerProperties[pointerIndex].toolType = MotionEvent.TOOL_TYPE_FINGER;
             source = InputDevice.SOURCE_TOUCHSCREEN;
             // Buttons must not be set for touch events
             buttons = 0;
+            pointer.setUp(action == MotionEvent.ACTION_UP);
         }
 
+        int pointerCount = pointersState.update(pointerProperties, pointerCoords);
         if (pointerCount == 1) {
             if (action == MotionEvent.ACTION_DOWN) {
                 lastTouchDown = now;
@@ -225,6 +228,62 @@ public class Controller {
             }
         }
 
+        /* If the input device is a mouse (on API >= 23):
+         *   - the first button pressed must first generate ACTION_DOWN;
+         *   - all button pressed (including the first one) must generate ACTION_BUTTON_PRESS;
+         *   - all button released (including the last one) must generate ACTION_BUTTON_RELEASE;
+         *   - the last button released must in addition generate ACTION_UP.
+         *
+         * Otherwise, Chrome does not work properly: <https://github.com/Genymobile/scrcpy/issues/3635>
+         */
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && source == InputDevice.SOURCE_MOUSE) {
+            if (action == MotionEvent.ACTION_DOWN) {
+                if (actionButton == buttons) {
+                    // First button pressed: ACTION_DOWN
+                    MotionEvent downEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_DOWN, pointerCount, pointerProperties,
+                            pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
+                    if (!device.injectEvent(downEvent, Device.INJECT_MODE_ASYNC)) {
+                        return false;
+                    }
+                }
+
+                // Any button pressed: ACTION_BUTTON_PRESS
+                MotionEvent pressEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_BUTTON_PRESS, pointerCount, pointerProperties,
+                        pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
+                if (!InputManager.setActionButton(pressEvent, actionButton)) {
+                    return false;
+                }
+                if (!device.injectEvent(pressEvent, Device.INJECT_MODE_ASYNC)) {
+                    return false;
+                }
+
+                return true;
+            }
+
+            if (action == MotionEvent.ACTION_UP) {
+                // Any button released: ACTION_BUTTON_RELEASE
+                MotionEvent releaseEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_BUTTON_RELEASE, pointerCount, pointerProperties,
+                        pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
+                if (!InputManager.setActionButton(releaseEvent, actionButton)) {
+                    return false;
+                }
+                if (!device.injectEvent(releaseEvent, Device.INJECT_MODE_ASYNC)) {
+                    return false;
+                }
+
+                if (buttons == 0) {
+                    // Last button released: ACTION_UP
+                    MotionEvent upEvent = MotionEvent.obtain(lastTouchDown, now, MotionEvent.ACTION_UP, pointerCount, pointerProperties,
+                            pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source, 0);
+                    if (!device.injectEvent(upEvent, Device.INJECT_MODE_ASYNC)) {
+                        return false;
+                    }
+                }
+
+                return true;
+            }
+        }
+
         MotionEvent event = MotionEvent
                 .obtain(lastTouchDown, now, action, pointerCount, pointerProperties, pointerCoords, 0, buttons, 1f, 1f, DEFAULT_DEVICE_ID, 0, source,
                         0);

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -9,6 +9,7 @@ import android.media.MediaCodecList;
 import android.media.MediaFormat;
 import android.os.Build;
 import android.os.IBinder;
+import android.os.SystemClock;
 import android.view.Surface;
 
 import java.io.FileDescriptor;
@@ -27,6 +28,7 @@ public class ScreenEncoder implements Device.RotationListener {
 
     // Keep the values in descending order
     private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800};
+    private static final int MAX_CONSECUTIVE_ERRORS = 3;
 
     private static final long PACKET_FLAG_CONFIG = 1L << 63;
     private static final long PACKET_FLAG_KEY_FRAME = 1L << 62;
@@ -40,9 +42,9 @@ public class ScreenEncoder implements Device.RotationListener {
     private final int maxFps;
     private final boolean sendFrameMeta;
     private final boolean downsizeOnError;
-    private long ptsOrigin;
 
     private boolean firstFrameSent;
+    private int consecutiveErrors;
 
     public ScreenEncoder(boolean sendFrameMeta, int bitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
             boolean downsizeOnError) {
@@ -108,20 +110,10 @@ public class ScreenEncoder implements Device.RotationListener {
                     codec.stop();
                 } catch (IllegalStateException | IllegalArgumentException e) {
                     Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
-                    if (!downsizeOnError || firstFrameSent) {
-                        // Fail immediately
+                    if (!prepareRetry(device, screenInfo)) {
                         throw e;
                     }
-
-                    int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
-                    if (newMaxSize == 0) {
-                        // Definitively fail
-                        throw e;
-                    }
-
-                    // Retry with a smaller device size
-                    Ln.i("Retrying with -m" + newMaxSize + "...");
-                    device.setMaxSize(newMaxSize);
+                    Ln.i("Retrying...");
                     alive = true;
                 } finally {
                     codec.reset();
@@ -137,6 +129,39 @@ public class ScreenEncoder implements Device.RotationListener {
         }
     }
 
+    private boolean prepareRetry(Device device, ScreenInfo screenInfo) {
+        if (firstFrameSent) {
+            ++consecutiveErrors;
+            if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
+                // Definitively fail
+                return false;
+            }
+
+            // Wait a bit to increase the probability that retrying will fix the problem
+            SystemClock.sleep(50);
+            return true;
+        }
+
+        if (!downsizeOnError) {
+            // Must fail immediately
+            return false;
+        }
+
+        // Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising)
+
+        int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
+        Ln.i("newMaxSize = " + newMaxSize);
+        if (newMaxSize == 0) {
+            // Must definitively fail
+            return false;
+        }
+
+        // Retry with a smaller device size
+        Ln.i("Retrying with -m" + newMaxSize + "...");
+        device.setMaxSize(newMaxSize);
+        return true;
+    }
+
     private static int chooseMaxSizeFallback(Size failedSize) {
         int currentMaxSize = Math.max(failedSize.getWidth(), failedSize.getHeight());
         for (int value : MAX_SIZE_FALLBACK) {
@@ -172,6 +197,7 @@ public class ScreenEncoder implements Device.RotationListener {
                     if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
                         // If this is not a config packet, then it contains a frame
                         firstFrameSent = true;
+                        consecutiveErrors = 0;
                     }
                 }
             } finally {
@@ -191,10 +217,7 @@ public class ScreenEncoder implements Device.RotationListener {
         if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
             pts = PACKET_FLAG_CONFIG; // non-media data packet
         } else {
-            if (ptsOrigin == 0) {
-                ptsOrigin = bufferInfo.presentationTimeUs;
-            }
-            pts = bufferInfo.presentationTimeUs - ptsOrigin;
+            pts = bufferInfo.presentationTimeUs;
             if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) {
                 pts |= PACKET_FLAG_KEY_FRAME;
             }

server/src/main/java/com/genymobile/scrcpy/wrappers/InputManager.java

@@ -3,6 +3,7 @@ package com.genymobile.scrcpy.wrappers;
 import com.genymobile.scrcpy.Ln;
 
 import android.view.InputEvent;
+import android.view.MotionEvent;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -17,6 +18,7 @@ public final class InputManager {
     private Method injectInputEventMethod;
 
     private static Method setDisplayIdMethod;
+    private static Method setActionButtonMethod;
 
     public InputManager(android.hardware.input.InputManager manager) {
         this.manager = manager;
@@ -56,4 +58,22 @@ public final class InputManager {
             return false;
         }
     }
+
+    private static Method getSetActionButtonMethod() throws NoSuchMethodException {
+        if (setActionButtonMethod == null) {
+            setActionButtonMethod = MotionEvent.class.getMethod("setActionButton", int.class);
+        }
+        return setActionButtonMethod;
+    }
+
+    public static boolean setActionButton(MotionEvent motionEvent, int actionButton) {
+        try {
+            Method method = getSetActionButtonMethod();
+            method.invoke(motionEvent, actionButton);
+            return true;
+        } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException e) {
+            Ln.e("Cannot set action button on MotionEvent", e);
+            return false;
+        }
+    }
 }

server/src/test/java/com/genymobile/scrcpy/ControlMessageReaderTest.java

@@ -94,7 +94,8 @@ public class ControlMessageReaderTest {
         dos.writeShort(1080);
         dos.writeShort(1920);
         dos.writeShort(0xffff); // pressure
-        dos.writeInt(MotionEvent.BUTTON_PRIMARY);
+        dos.writeInt(MotionEvent.BUTTON_PRIMARY); // action button
+        dos.writeInt(MotionEvent.BUTTON_PRIMARY); // buttons
 
         byte[] packet = bos.toByteArray();
 
@@ -112,6 +113,7 @@ public class ControlMessageReaderTest {
         Assert.assertEquals(1080, event.getPosition().getScreenSize().getWidth());
         Assert.assertEquals(1920, event.getPosition().getScreenSize().getHeight());
         Assert.assertEquals(1f, event.getPressure(), 0f); // must be exact
+        Assert.assertEquals(MotionEvent.BUTTON_PRIMARY, event.getActionButton());
         Assert.assertEquals(MotionEvent.BUTTON_PRIMARY, event.getButtons());
     }