Introduce --keyboard and --mouse

Until now, there was two modes for keyboard and mouse: - event injection using the Android system API (default) - HID/AOA over USB For this reason, the options were exposed as simple flags: - -K or --hid-keyboard to enable physical keyboard simulation (AOA) - -M or --hid-mouse to enable physical mouse simulation (AOA) Replace them by explicit --keyboard and --mouse options, with 3 possible values: - disabled - sdk (default) - aoa This will allow to add a new mode (uhid). Co-authored-by: Romain Vimont <rom@rom1v.com> Signed-off-by: Romain Vimont <rom@rom1v.com>
2023-11-28 17:17:35 +08:00 · 2023-11-28 17:17:35 +08:00 · 266644560d
commit 266644560d
parent e572e841a8
8 changed files with 213 additions and 78 deletions
--- a/app/data/bash-completion/scrcpy
+++ b/app/data/bash-completion/scrcpy
@ -27,8 +27,8 @@ _scrcpy() {
        --force-adb-forward
        --forward-all-clicks
        -h --help
+        --keyboard
        --kill-adb-on-close
-        -K --hid-keyboard
        --legacy-paste
        --list-camera-sizes
        --list-cameras
@ -39,6 +39,7 @@ _scrcpy() {
        -m --max-size=
        -M --hid-mouse
        --max-fps=
+        --mouse=
        -n --no-control
        -N --no-playback
        --no-audio
@ -115,6 +116,14 @@ _scrcpy() {
            COMPREPLY=($(compgen -W 'front back external' -- "$cur"))
            return
            ;;
+        --keyboard)
+            COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur"))
+            return
+            ;;
+        --mouse)
+            COMPREPLY=($(compgen -W 'disabled sdk aoa' -- "$cur"))
+            return
+            ;;
        --orientation|--display-orientation)
            COMPREPLY=($(compgen -W '0 90 180 270 flip0 flip90 flip180 flip270' -- "$cur"))
            return
--- a/app/data/zsh-completion/_scrcpy
+++ b/app/data/zsh-completion/_scrcpy
@ -34,8 +34,8 @@ arguments=(
    '--force-adb-forward[Do not attempt to use \"adb reverse\" to connect to the device]'
    '--forward-all-clicks[Forward clicks to device]'
    {-h,--help}'[Print the help]'
+    '--keyboard[Set the keyboard input mode]:mode:(disabled sdk aoa)'
    '--kill-adb-on-close[Kill adb when scrcpy terminates]'
-    {-K,--hid-keyboard}'[Simulate a physical keyboard by using HID over AOAv2]'
    '--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]'
    '--list-camera-sizes[List the valid camera capture sizes]'
    '--list-cameras[List cameras available on the device]'
@ -45,6 +45,7 @@ arguments=(
    {-m,--max-size=}'[Limit both the width and height of the video to value]'
    {-M,--hid-mouse}'[Simulate a physical mouse by using HID over AOAv2]'
    '--max-fps=[Limit the frame rate of screen capture]'
+    '--mouse[Set the mouse input mode]:mode:(disabled sdk aoa)'
    {-n,--no-control}'[Disable device control \(mirror the device in read only\)]'
    {-N,--no-playback}'[Disable video and audio playback]'
    '--no-audio[Disable audio forwarding]'
--- a/app/scrcpy.1
+++ b/app/scrcpy.1
@ -172,24 +172,26 @@ By default, right-click triggers BACK (or POWER on) and middle-click triggers HO
 Print this help.

 .TP
-.B \-\-kill\-adb\-on\-close
-Kill adb when scrcpy terminates.
+.BI "\-\-keyboard " mode
+Select how to send keyboard inputs to the device.

-.TP
-.B \-K, \-\-hid\-keyboard
-Simulate a physical keyboard by using HID over AOAv2.
+Possible values are "disabled", "sdk" and "aoa":

-This provides a better experience for IME users, and allows to generate non-ASCII characters, contrary to the default injection method.
+ - "disabled" does not send keyboard inputs to the device.
+ - "sdk" uses the Android system API to deliver keyboard events to applications.
+ - "aoa" simulates a physical keyboard using the AOAv2 protocol. It may only work over USB.

-It may only work over USB.
-
-The keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly:
+For "aoa", the keyboard layout must be configured (once and for all) on the device, via Settings -> System -> Languages and input -> Physical keyboard. This settings page can be started directly:

    adb shell am start -a android.settings.HARD_KEYBOARD_SETTINGS

-However, the option is only available when the HID keyboard is enabled (or a physical keyboard is connected).
+This option is only available when the HID keyboard is enabled (or a physical keyboard is connected).

-Also see \fB\-\-hid\-mouse\fR.
+Also see \fB\-\-mouse\fR.
+
+.TP
+.B \-\-kill\-adb\-on\-close
+Kill adb when scrcpy terminates.

 .TP
 .B \-\-legacy\-paste
@ -230,20 +232,25 @@ Limit both the width and height of the video to \fIvalue\fR. The other dimension
 Default is 0 (unlimited).

 .TP
-.B \-M, \-\-hid\-mouse
-Simulate a physical mouse by using HID over AOAv2.
+.BI "\-\-max\-fps " value
+Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions).

-In this mode, the computer mouse is captured to control the device directly (relative mouse mode).
+.TP
+.BI "\-\-mouse " mode
+Select how to send mouse inputs to the device.
+
+Possible values are "disabled", "sdk" and "aoa":
+
+ - "disabled" does not send mouse inputs to the device.
+ - "sdk" uses the Android system API to deliver mouse events to applications.
+ - "aoa" simulates a physical mouse using the AOAv2 protocol. It may only work over USB.
+
+In "aoa" mode, the computer mouse is captured to control the device directly (relative mouse mode).

 LAlt, LSuper or RSuper toggle the capture mode, to give control of the mouse back to the computer.

-It may only work over USB.
+Also see \fB\-\-keyboard\fR.

-Also see \fB\-\-hid\-keyboard\fR.
-
-.TP
-.BI "\-\-max\-fps " value
-Limit the framerate of screen capture (officially supported since Android 10, but may work on earlier versions).

 .TP
 .B \-n, \-\-no\-control
--- a/app/src/cli.c
+++ b/app/src/cli.c
@ -93,6 +93,8 @@ enum {
    OPT_DISPLAY_ORIENTATION,
    OPT_RECORD_ORIENTATION,
    OPT_ORIENTATION,
+    OPT_KEYBOARD,
+    OPT_MOUSE,
 };

 struct sc_option {
@ -358,27 +360,35 @@ static const struct sc_option options[] = {
        .longopt = "help",
        .text = "Print this help.",
    },
+    {
+        .longopt_id = OPT_KEYBOARD,
+        .longopt = "keyboard",
+        .argdesc = "mode",
+        .text = "Select how to send keyboard inputs to the device.\n"
+                "Possible values are \"disabled\", \"sdk\" and \"aoa\".\n"
+                "\"disabled\" does not send keyboard inputs to the device.\n"
+                "\"sdk\" uses the Android system API to deliver keyboard\n"
+                "events to applications.\n"
+                "\"aoa\" simulates a physical keyboard using the AOAv2\n"
+                "protocol. It may only work over USB.\n"
+                "For \"aoa\", the keyboard layout must be configured (once and "
+                "for all) on the device, via Settings -> System -> Languages "
+                "and input -> Physical keyboard. This settings page can be "
+                "started directly: `adb shell am start -a "
+                "android.settings.HARD_KEYBOARD_SETTINGS`.\n"
+                "This option is only available when the HID keyboard is "
+                "enabled (or a physical keyboard is connected).\n"
+                "Also see --mouse.",
+    },
    {
        .longopt_id = OPT_KILL_ADB_ON_CLOSE,
        .longopt = "kill-adb-on-close",
        .text = "Kill adb when scrcpy terminates.",
    },
    {
+        // deprecated
        .shortopt = 'K',
        .longopt = "hid-keyboard",
-        .text = "Simulate a physical keyboard by using HID over AOAv2.\n"
-                "It provides a better experience for IME users, and allows to "
-                "generate non-ASCII characters, contrary to the default "
-                "injection method.\n"
-                "It may only work over USB.\n"
-                "The keyboard layout must be configured (once and for all) on "
-                "the device, via Settings -> System -> Languages and input -> "
-                "Physical keyboard. This settings page can be started "
-                "directly: `adb shell am start -a "
-                "android.settings.HARD_KEYBOARD_SETTINGS`.\n"
-                "However, the option is only available when the HID keyboard "
-                "is enabled (or a physical keyboard is connected).\n"
-                "Also see --hid-mouse.",
    },
    {
        .longopt_id = OPT_LEGACY_PASTE,
@ -432,15 +442,9 @@ static const struct sc_option options[] = {
                "Default is 0 (unlimited).",
    },
    {
+        // deprecated
        .shortopt = 'M',
        .longopt = "hid-mouse",
-        .text = "Simulate a physical mouse by using HID over AOAv2.\n"
-                "In this mode, the computer mouse is captured to control the "
-                "device directly (relative mouse mode).\n"
-                "LAlt, LSuper or RSuper toggle the capture mode, to give "
-                "control of the mouse back to the computer.\n"
-                "It may only work over USB.\n"
-                "Also see --hid-keyboard.",
    },
    {
        .longopt_id = OPT_MAX_FPS,
@ -449,6 +453,23 @@ static const struct sc_option options[] = {
        .text = "Limit the frame rate of screen capture (officially supported "
                "since Android 10, but may work on earlier versions).",
    },
+    {
+        .longopt_id = OPT_MOUSE,
+        .longopt = "mouse",
+        .argdesc = "mode",
+        .text = "Select how to send mouse inputs to the device.\n"
+                "Possible values are \"disabled\", \"sdk\" and \"aoa\".\n"
+                "\"disabled\" does not send mouse inputs to the device.\n"
+                "\"sdk\" uses the Android system API to deliver mouse\n"
+                "events to applications.\n"
+                "\"aoa\" simulates a physical mouse using the AOAv2 protocol\n"
+                "It may only work over USB.\n"
+                "In \"aoa\" mode, the computer mouse is captured to control "
+                "the device directly (relative mouse mode).\n"
+                "LAlt, LSuper or RSuper toggle the capture mode, to give "
+                "control of the mouse back to the computer.\n"
+                "Also see --keyboard.",
+    },
    {
        .shortopt = 'n',
        .longopt = "no-control",
@ -543,10 +564,10 @@ static const struct sc_option options[] = {
                "mirroring is disabled.\n"
                "LAlt, LSuper or RSuper toggle the mouse capture mode, to give "
                "control of the mouse back to the computer.\n"
-                "If any of --hid-keyboard or --hid-mouse is set, only enable "
-                "keyboard or mouse respectively, otherwise enable both.\n"
+                "Keyboard and mouse may be disabled separately using\n"
+                "--keyboard=disabled and --mouse=disabled.\n"
                "It may only work over USB.\n"
-                "See --hid-keyboard and --hid-mouse.",
+                "See --keyboard and --mouse.",
    },
    {
        .shortopt = 'p',
@ -1902,6 +1923,58 @@ parse_camera_fps(const char *s, uint16_t *camera_fps) {
    return true;
 }

+static bool
+parse_keyboard(const char *optarg, enum sc_keyboard_input_mode *mode) {
+    if (!strcmp(optarg, "disabled")) {
+        *mode = SC_KEYBOARD_INPUT_MODE_DISABLED;
+        return true;
+    }
+
+    if (!strcmp(optarg, "sdk")) {
+        *mode = SC_KEYBOARD_INPUT_MODE_SDK;
+        return true;
+    }
+
+    if (!strcmp(optarg, "aoa")) {
+#ifdef HAVE_USB
+        *mode = SC_KEYBOARD_INPUT_MODE_AOA;
+        return true;
+#else
+        LOGE("--keyboard=aoa is disabled.");
+        return false;
+#endif
+    }
+
+    LOGE("Unsupported keyboard: %s (expected disabled, sdk or aoa)", optarg);
+    return false;
+}
+
+static bool
+parse_mouse(const char *optarg, enum sc_mouse_input_mode *mode) {
+    if (!strcmp(optarg, "disabled")) {
+        *mode = SC_MOUSE_INPUT_MODE_DISABLED;
+        return true;
+    }
+
+    if (!strcmp(optarg, "sdk")) {
+        *mode = SC_MOUSE_INPUT_MODE_SDK;
+        return true;
+    }
+
+    if (!strcmp(optarg, "aoa")) {
+#ifdef HAVE_USB
+        *mode = SC_MOUSE_INPUT_MODE_AOA;
+        return true;
+#else
+        LOGE("--mouse=aoa is disabled.");
+        return false;
+#endif
+    }
+
+    LOGE("Unsupported mouse: %s (expected disabled, sdk or aoa)", optarg);
+    return false;
+}
+
 static bool
 parse_time_limit(const char *s, sc_tick *tick) {
    long value;
@ -1991,12 +2064,19 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                break;
            case 'K':
 #ifdef HAVE_USB
-                opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
+                LOGW("-K/--hid-keyboard is deprecated, use --keyboard=aoa "
+                     "instead.");
+                opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AOA;
                break;
 #else
                LOGE("HID over AOA (-K/--hid-keyboard) is disabled.");
                return false;
 #endif
+            case OPT_KEYBOARD:
+                if (!parse_keyboard(optarg, &opts->keyboard_input_mode)) {
+                    return false;
+                }
+                break;
            case OPT_MAX_FPS:
                if (!parse_max_fps(optarg, &opts->max_fps)) {
                    return false;
@ -2009,12 +2089,18 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                break;
            case 'M':
 #ifdef HAVE_USB
-                opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_HID;
+                LOGW("-M/--hid-mouse is deprecated, use --mouse=aoa instead.");
+                opts->mouse_input_mode = SC_MOUSE_INPUT_MODE_AOA;
                break;
 #else
                LOGE("HID over AOA (-M/--hid-mouse) is disabled.");
                return false;
 #endif
+            case OPT_MOUSE:
+                if (!parse_mouse(optarg, &opts->mouse_input_mode)) {
+                    return false;
+                }
+                break;
            case OPT_LOCK_VIDEO_ORIENTATION:
                if (!parse_lock_video_orientation(optarg,
                        &opts->lock_video_orientation)) {
@ -2461,6 +2547,31 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
    }
 #endif

+    if (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AUTO) {
+        opts->keyboard_input_mode = otg ? SC_KEYBOARD_INPUT_MODE_AOA
+                                        : SC_KEYBOARD_INPUT_MODE_SDK;
+    }
+    if (opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_AUTO) {
+        opts->mouse_input_mode = otg ? SC_MOUSE_INPUT_MODE_AOA
+                                     : SC_MOUSE_INPUT_MODE_SDK;
+    }
+
+    if (otg) {
+        enum sc_keyboard_input_mode kmode = opts->keyboard_input_mode;
+        if (kmode != SC_KEYBOARD_INPUT_MODE_AOA
+                && kmode != SC_KEYBOARD_INPUT_MODE_DISABLED) {
+            LOGE("In OTG mode, --keyboard only supports aoa or disabled.");
+            return false;
+        }
+
+        enum sc_mouse_input_mode mmode = opts->mouse_input_mode;
+        if (mmode != SC_MOUSE_INPUT_MODE_AOA
+                && mmode != SC_MOUSE_INPUT_MODE_DISABLED) {
+            LOGE("In OTG mode, --mouse only supports aoa or disabled.");
+            return false;
+        }
+    }
+
    if ((opts->tunnel_host || opts->tunnel_port) && !opts->force_adb_forward) {
        LOGI("Tunnel host/port is set, "
             "--force-adb-forward automatically enabled.");
@ -2621,12 +2732,12 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
    }

 # ifdef _WIN32
-    if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID
-                || opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID)) {
+    if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA
+                || opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA)) {
        LOGE("On Windows, it is not possible to open a USB device already open "
             "by another process (like adb).");
-        LOGE("Therefore, -K/--hid-keyboard and -M/--hid-mouse may only work in "
-             "OTG mode (--otg).");
+        LOGE("Therefore, --keyboard=aoa and --mouse=aoa may only work in OTG"
+             "mode (--otg).");
        return false;
    }
 # endif
--- a/app/src/options.c
+++ b/app/src/options.c
@ -21,8 +21,8 @@ const struct scrcpy_options scrcpy_options_default = {
    .video_source = SC_VIDEO_SOURCE_DISPLAY,
    .audio_source = SC_AUDIO_SOURCE_AUTO,
    .record_format = SC_RECORD_FORMAT_AUTO,
-    .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT,
-    .mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT,
+    .keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AUTO,
+    .mouse_input_mode = SC_MOUSE_INPUT_MODE_AUTO,
    .camera_facing = SC_CAMERA_FACING_ANY,
    .port_range = {
        .first = DEFAULT_LOCAL_PORT_RANGE_FIRST,
--- a/app/src/options.h
+++ b/app/src/options.h
@ -140,13 +140,17 @@ enum sc_lock_video_orientation {
 };

 enum sc_keyboard_input_mode {
-    SC_KEYBOARD_INPUT_MODE_INJECT,
-    SC_KEYBOARD_INPUT_MODE_HID,
+    SC_KEYBOARD_INPUT_MODE_AUTO,
+    SC_KEYBOARD_INPUT_MODE_DISABLED,
+    SC_KEYBOARD_INPUT_MODE_SDK,
+    SC_KEYBOARD_INPUT_MODE_AOA,
 };

 enum sc_mouse_input_mode {
-    SC_MOUSE_INPUT_MODE_INJECT,
-    SC_MOUSE_INPUT_MODE_HID,
+    SC_MOUSE_INPUT_MODE_AUTO,
+    SC_MOUSE_INPUT_MODE_DISABLED,
+    SC_MOUSE_INPUT_MODE_SDK,
+    SC_MOUSE_INPUT_MODE_AOA,
 };

 enum sc_key_inject_mode {
--- a/app/src/scrcpy.c
+++ b/app/src/scrcpy.c
@ -543,11 +543,11 @@ scrcpy(struct scrcpy_options *options) {

    if (options->control) {
 #ifdef HAVE_USB
-        bool use_hid_keyboard =
-            options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID;
-        bool use_hid_mouse =
-            options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID;
-        if (use_hid_keyboard || use_hid_mouse) {
+        bool use_aoa_keyboard =
+            options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA;
+        bool use_aoa_mouse =
+            options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA;
+        if (use_aoa_keyboard || use_aoa_mouse) {
            bool ok = sc_acksync_init(&s->acksync);
            if (!ok) {
                goto end;
@ -590,7 +590,7 @@ scrcpy(struct scrcpy_options *options) {
                goto aoa_hid_end;
            }

-            if (use_hid_keyboard) {
+            if (use_aoa_keyboard) {
                if (sc_hid_keyboard_init(&s->keyboard_hid, &s->aoa)) {
                    hid_keyboard_initialized = true;
                    kp = &s->keyboard_hid.key_processor;
@ -599,7 +599,7 @@ scrcpy(struct scrcpy_options *options) {
                }
            }

-            if (use_hid_mouse) {
+            if (use_aoa_mouse) {
                if (sc_hid_mouse_init(&s->mouse_hid, &s->aoa)) {
                    hid_mouse_initialized = true;
                    mp = &s->mouse_hid.mouse_processor;
@ -634,25 +634,23 @@ aoa_hid_end:
                }
            }

-            if (use_hid_keyboard && !hid_keyboard_initialized) {
-                LOGE("Fallback to default keyboard injection method "
-                     "(-K/--hid-keyboard ignored)");
-                options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_INJECT;
+            if (use_aoa_keyboard && !hid_keyboard_initialized) {
+                LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)");
+                options->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_SDK;
            }

-            if (use_hid_mouse && !hid_mouse_initialized) {
-                LOGE("Fallback to default mouse injection method "
-                     "(-M/--hid-mouse ignored)");
-                options->mouse_input_mode = SC_MOUSE_INPUT_MODE_INJECT;
+            if (use_aoa_mouse && !hid_mouse_initialized) {
+                LOGE("Fallback to --keyboard=sdk (--keyboard=aoa ignored)");
+                options->mouse_input_mode = SC_MOUSE_INPUT_MODE_SDK;
            }
        }
 #else
-        assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_HID);
-        assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_HID);
+        assert(options->keyboard_input_mode != SC_KEYBOARD_INPUT_MODE_AOA);
+        assert(options->mouse_input_mode != SC_MOUSE_INPUT_MODE_AOA);
 #endif

        // keyboard_input_mode may have been reset if HID mode failed
-        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_INJECT) {
+        if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_SDK) {
            sc_keyboard_inject_init(&s->keyboard_inject, &s->controller,
                                    options->key_inject_mode,
                                    options->forward_key_repeat);
@ -660,7 +658,7 @@ aoa_hid_end:
        }

        // mouse_input_mode may have been reset if HID mode failed
-        if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_INJECT) {
+        if (options->mouse_input_mode == SC_MOUSE_INPUT_MODE_SDK) {
            sc_mouse_inject_init(&s->mouse_inject, &s->controller);
            mp = &s->mouse_inject.mouse_processor;
        }
--- a/app/src/usb/scrcpy_otg.c
+++ b/app/src/usb/scrcpy_otg.c
@ -117,10 +117,15 @@ scrcpy_otg(struct scrcpy_options *options) {
    }
    aoa_initialized = true;

+    assert(options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA
+        || options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_DISABLED);
+    assert(options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA
+        || options->mouse_input_mode == SC_MOUSE_INPUT_MODE_DISABLED);
+
    bool enable_keyboard =
-        options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID;
+        options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA;
    bool enable_mouse =
-        options->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID;
+        options->mouse_input_mode == SC_MOUSE_INPUT_MODE_AOA;

    // If neither --hid-keyboard or --hid-mouse is passed, enable both
    if (!enable_keyboard && !enable_mouse) {