Handle UHID output

Use UHID output reports to synchronize CapsLock and VerrNum states.

Co-authored-by: Romain Vimont <rom@rom1v.com>
Signed-off-by: Romain Vimont <rom@rom1v.com>

app/meson.build

@@ -36,6 +36,7 @@ src = [
     'src/trait/frame_source.c',
     'src/trait/packet_source.c',
     'src/uhid/keyboard_uhid.c',
+    'src/uhid/uhid_output.c',
     'src/util/acksync.c',
     'src/util/audiobuf.c',
     'src/util/average.c',

app/src/controller.c

@@ -43,9 +43,11 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket) {
 }
 
 void
-sc_controller_set_acksync(struct sc_controller *controller,
+sc_controller_configure(struct sc_controller *controller,
-                          struct sc_acksync *acksync) {
+                        struct sc_acksync *acksync,
+                        struct sc_uhid_devices *uhid_devices) {
     controller->receiver.acksync = acksync;
+    controller->receiver.uhid_devices = uhid_devices;
 }
 
 void

app/src/controller.h

@@ -28,8 +28,9 @@ bool
 sc_controller_init(struct sc_controller *controller, sc_socket control_socket);
 
 void
-sc_controller_set_acksync(struct sc_controller *controller,
+sc_controller_configure(struct sc_controller *controller,
-                          struct sc_acksync *acksync);
+                        struct sc_acksync *acksync,
+                        struct sc_uhid_devices *uhid_devices);
 
 void
 sc_controller_destroy(struct sc_controller *controller);

app/src/device_msg.c

@@ -46,6 +46,31 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
             msg->ack_clipboard.sequence = sequence;
             return 9;
         }
+        case DEVICE_MSG_TYPE_UHID_OUTPUT: {
+            if (len < 5) {
+                // at least id + size
+                return 0; // not available
+            }
+            uint16_t id = sc_read16be(&buf[1]);
+            size_t size = sc_read16be(&buf[3]);
+            if (size < len - 5) {
+                return 0; // not available
+            }
+            uint8_t *data = malloc(size);
+            if (!data) {
+                LOG_OOM();
+                return -1;
+            }
+            if (size) {
+                memcpy(data, &buf[5], size);
+            }
+
+            msg->uhid_output.id = id;
+            msg->uhid_output.size = size;
+            msg->uhid_output.data = data;
+
+            return 5 + size;
+        }
         default:
             LOGW("Unknown device message type: %d", (int) msg->type);
             return -1; // error, we cannot recover
@@ -54,7 +79,15 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
 
 void
 sc_device_msg_destroy(struct sc_device_msg *msg) {
-    if (msg->type == DEVICE_MSG_TYPE_CLIPBOARD) {
+    switch (msg->type) {
-        free(msg->clipboard.text);
+        case DEVICE_MSG_TYPE_CLIPBOARD:
+            free(msg->clipboard.text);
+            break;
+        case DEVICE_MSG_TYPE_UHID_OUTPUT:
+            free(msg->uhid_output.data);
+            break;
+        default:
+            // nothing to do
+            break;
     }
 }

app/src/device_msg.h

@@ -14,6 +14,7 @@
 enum sc_device_msg_type {
     DEVICE_MSG_TYPE_CLIPBOARD,
     DEVICE_MSG_TYPE_ACK_CLIPBOARD,
+    DEVICE_MSG_TYPE_UHID_OUTPUT,
 };
 
 struct sc_device_msg {
@@ -25,6 +26,11 @@ struct sc_device_msg {
         struct {
             uint64_t sequence;
         } ack_clipboard;
+        struct {
+            uint16_t id;
+            uint16_t size;
+            uint8_t *data; // owned, to be freed by free()
+        } uhid_output;
     };
 };
 

app/src/receiver.c

@@ -6,6 +6,7 @@
 
 #include "device_msg.h"
 #include "util/log.h"
+#include "util/str.h"
 
 bool
 sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
@@ -16,6 +17,7 @@ sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
 
     receiver->control_socket = control_socket;
     receiver->acksync = NULL;
+    receiver->uhid_devices = NULL;
 
     return true;
 }
@@ -47,6 +49,31 @@ process_msg(struct sc_receiver *receiver, struct sc_device_msg *msg) {
                  msg->ack_clipboard.sequence);
             sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
             break;
+        case DEVICE_MSG_TYPE_UHID_OUTPUT:
+            if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
+                char *hex = sc_str_to_hex_string(msg->uhid_output.data,
+                                                 msg->uhid_output.size);
+                if (hex) {
+                    LOGV("UHID output [%" PRIu16 "] %s",
+                         msg->uhid_output.id, hex);
+                    free(hex);
+                } else {
+                    LOGV("UHID output [%" PRIu16 "] size=%" PRIu16,
+                         msg->uhid_output.id, msg->uhid_output.size);
+                }
+            }
+            assert(receiver->uhid_devices);
+            struct sc_uhid_receiver *uhid_receiver =
+                sc_uhid_devices_get_receiver(receiver->uhid_devices,
+                                             msg->uhid_output.id);
+            if (uhid_receiver) {
+                uhid_receiver->ops->process_output(uhid_receiver,
+                                                   msg->uhid_output.data,
+                                                   msg->uhid_output.size);
+            } else {
+                LOGW("No UHID receiver for id %" PRIu16, msg->uhid_output.id);
+            }
+            break;
     }
 }
 

app/src/receiver.h

@@ -5,6 +5,7 @@
 
 #include <stdbool.h>
 
+#include "uhid/uhid_output.h"
 #include "util/acksync.h"
 #include "util/net.h"
 #include "util/thread.h"
@@ -17,6 +18,7 @@ struct sc_receiver {
     sc_mutex mutex;
 
     struct sc_acksync *acksync;
+    struct sc_uhid_devices *uhid_devices;
 };
 
 bool

app/src/scrcpy.c

@@ -62,6 +62,7 @@ struct scrcpy {
     struct sc_aoa aoa;
     // sequence/ack helper to synchronize clipboard and Ctrl+v via HID
     struct sc_acksync acksync;
+    struct sc_uhid_devices uhid_devices;
 #endif
     union {
         struct sc_keyboard_sdk keyboard_sdk;
@@ -342,6 +343,7 @@ scrcpy(struct scrcpy_options *options) {
     bool timeout_started = false;
 
     struct sc_acksync *acksync = NULL;
+    struct sc_uhid_devices *uhid_devices = NULL;
 
     uint32_t scid = scrcpy_generate_scid();
 
@@ -665,10 +667,12 @@ aoa_hid_end:
                                  options->forward_key_repeat);
             kp = &s->keyboard_sdk.key_processor;
         } else if (options->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_UHID) {
-            bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller);
+            bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller,
+                                            &s->uhid_devices);
             if (!ok) {
                 goto end;
             }
+            uhid_devices = &s->uhid_devices;
             kp = &s->keyboard_uhid.key_processor;
         }
 
@@ -678,7 +682,7 @@ aoa_hid_end:
             mp = &s->mouse_sdk.mouse_processor;
         }
 
-        sc_controller_set_acksync(&s->controller, acksync);
+        sc_controller_configure(&s->controller, acksync, uhid_devices);
 
         if (!sc_controller_start(&s->controller)) {
             goto end;

app/src/uhid/keyboard_uhid.c

@@ -5,8 +5,52 @@
 /** Downcast key processor to keyboard_uhid */
 #define DOWNCAST(KP) container_of(KP, struct sc_keyboard_uhid, key_processor)
 
+/** Downcast uhid_receiver to keyboard_uhid */
+#define DOWNCAST_RECEIVER(UR) \
+    container_of(UR, struct sc_keyboard_uhid, uhid_receiver)
+
 #define UHID_KEYBOARD_ID 1
 
+static void
+sc_keyboard_uhid_send_input(struct sc_keyboard_uhid *kb,
+                            const struct sc_hid_event *event) {
+    struct sc_control_msg msg;
+    msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
+    msg.uhid_input.id = UHID_KEYBOARD_ID;
+
+    assert(event->size <= SC_HID_MAX_SIZE);
+    memcpy(msg.uhid_input.data, event->data, event->size);
+    msg.uhid_input.size = event->size;
+
+    if (!sc_controller_push_msg(kb->controller, &msg)) {
+        LOGE("Could not send UHID_INPUT message (key)");
+    }
+}
+
+static void
+sc_keyboard_uhid_synchronize_mod(struct sc_keyboard_uhid *kb) {
+    SDL_Keymod sdl_mod = SDL_GetModState();
+    uint16_t mod = sc_mods_state_from_sdl(sdl_mod) & (SC_MOD_CAPS | SC_MOD_NUM);
+
+    uint16_t device_mod =
+        atomic_load_explicit(&kb->device_mod, memory_order_relaxed);
+    uint16_t diff = mod ^ device_mod;
+
+    if (diff) {
+        // Inherently racy (the HID output reports arrive asynchronously in
+        // response to key presses), but will re-synchronize on next key press
+        // or HID output anyway
+        atomic_store_explicit(&kb->device_mod, mod, memory_order_relaxed);
+
+        struct sc_hid_event hid_event;
+        sc_hid_keyboard_event_from_mods(&hid_event, diff);
+
+        LOGV("HID keyboard state synchronized");
+
+        sc_keyboard_uhid_send_input(kb, &hid_event);
+    }
+}
+
 static void
 sc_key_processor_process_key(struct sc_key_processor *kp,
                              const struct sc_key_event *event,
@@ -25,26 +69,55 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
 
     // Not all keys are supported, just ignore unsupported keys
     if (sc_hid_keyboard_event_from_key(&kb->hid, &hid_event, event)) {
-        struct sc_control_msg msg;
+        if (event->scancode == SC_SCANCODE_CAPSLOCK) {
-        msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
+            atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_CAPS,
-        msg.uhid_input.id = UHID_KEYBOARD_ID;
+                                      memory_order_relaxed);
-
+        } else if (event->scancode == SC_SCANCODE_NUMLOCK) {
-        assert(hid_event.size <= SC_HID_MAX_SIZE);
+            atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_NUM,
-        memcpy(msg.uhid_input.data, hid_event.data, hid_event.size);
+                                      memory_order_relaxed);
-        msg.uhid_input.size = hid_event.size;
+        } else {
-
+            // Synchronize modifiers (only if the scancode itself does not
-        if (!sc_controller_push_msg(kb->controller, &msg)) {
+            // change the modifiers)
-            LOGE("Could not send UHID_INPUT message (key)");
+            sc_keyboard_uhid_synchronize_mod(kb);
         }
+        sc_keyboard_uhid_send_input(kb, &hid_event);
     }
 }
 
+static unsigned
+sc_keyboard_uhid_to_sc_mod(uint8_t hid_led) {
+    // <https://www.usb.org/sites/default/files/documents/hut1_12v2.pdf>
+    // (chapter 11: LED page)
+    unsigned mod = 0;
+    if (hid_led & 0x01) {
+        mod |= SC_MOD_NUM;
+    }
+    if (hid_led & 0x02) {
+        mod |= SC_MOD_CAPS;
+    }
+    return mod;
+}
+
+static void
+sc_uhid_receiver_process_output(struct sc_uhid_receiver *receiver,
+                                const uint8_t *data, size_t len) {
+    // Called from the thread receiving device messages
+    assert(len);
+    struct sc_keyboard_uhid *kb = DOWNCAST_RECEIVER(receiver);
+
+    uint8_t hid_led = data[0];
+    uint16_t device_mod = sc_keyboard_uhid_to_sc_mod(hid_led);
+    atomic_store_explicit(&kb->device_mod, device_mod, memory_order_relaxed);
+}
+
 bool
 sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
-                      struct sc_controller *controller) {
+                      struct sc_controller *controller,
+                      struct sc_uhid_devices *uhid_devices) {
     sc_hid_keyboard_init(&kb->hid);
 
     kb->controller = controller;
+    atomic_init(&kb->device_mod, 0);
 
     static const struct sc_key_processor_ops ops = {
         .process_key = sc_key_processor_process_key,
@@ -58,6 +131,14 @@ sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
     kb->key_processor.async_paste = false;
     kb->key_processor.ops = &ops;
 
+    static const struct sc_uhid_receiver_ops uhid_receiver_ops = {
+        .process_output = sc_uhid_receiver_process_output,
+    };
+
+    kb->uhid_receiver.id = UHID_KEYBOARD_ID;
+    kb->uhid_receiver.ops = &uhid_receiver_ops;
+    sc_uhid_devices_add_receiver(uhid_devices, &kb->uhid_receiver);
+
     struct sc_control_msg msg;
     msg.type = SC_CONTROL_MSG_TYPE_UHID_CREATE;
     msg.uhid_create.id = UHID_KEYBOARD_ID;

app/src/uhid/keyboard_uhid.h

@@ -7,17 +7,21 @@
 
 #include "controller.h"
 #include "hid/hid_keyboard.h"
+#include "uhid/uhid_output.h"
 #include "trait/key_processor.h"
 
 struct sc_keyboard_uhid {
     struct sc_key_processor key_processor; // key processor trait
+    struct sc_uhid_receiver uhid_receiver;
 
     struct sc_hid_keyboard hid;
     struct sc_controller *controller;
+    atomic_uint_least16_t device_mod;
 };
 
 bool
 sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
-                      struct sc_controller *controller);
+                      struct sc_controller *controller,
+                      struct sc_uhid_devices *uhid_devices);
 
 #endif

app/tests/test_device_msg_deserialize.c

@@ -61,6 +61,28 @@ static void test_deserialize_ack_set_clipboard(void) {
     assert(msg.ack_clipboard.sequence == UINT64_C(0x0102030405060708));
 }
 
+static void test_deserialize_uhid_output(void) {
+    const uint8_t input[] = {
+        DEVICE_MSG_TYPE_UHID_OUTPUT,
+        0, 42, // id
+        0, 5, // size
+        0x01, 0x02, 0x03, 0x04, 0x05, // data
+    };
+
+    struct sc_device_msg msg;
+    ssize_t r = sc_device_msg_deserialize(input, sizeof(input), &msg);
+    assert(r == 10);
+
+    assert(msg.type == DEVICE_MSG_TYPE_UHID_OUTPUT);
+    assert(msg.uhid_output.id == 42);
+    assert(msg.uhid_output.size == 5);
+
+    uint8_t expected[] = {1, 2, 3, 4, 5};
+    assert(!memcmp(msg.uhid_output.data, expected, sizeof(expected)));
+
+    sc_device_msg_destroy(&msg);
+}
+
 int main(int argc, char *argv[]) {
     (void) argc;
     (void) argv;
@@ -68,5 +90,6 @@ int main(int argc, char *argv[]) {
     test_deserialize_clipboard();
     test_deserialize_clipboard_big();
     test_deserialize_ack_set_clipboard();
+    test_deserialize_uhid_output();
     return 0;
 }

server/src/main/java/com/genymobile/scrcpy/Controller.java

@@ -52,7 +52,7 @@ public class Controller implements AsyncProcessor {
         this.powerOn = powerOn;
         initPointers();
         sender = new DeviceMessageSender(controlChannel);
-        uhidManager = new UhidManager();
+        uhidManager = new UhidManager(sender);
     }
 
     private void initPointers() {

server/src/main/java/com/genymobile/scrcpy/DeviceMessage.java

@@ -4,10 +4,13 @@ public final class DeviceMessage {
 
     public static final int TYPE_CLIPBOARD = 0;
     public static final int TYPE_ACK_CLIPBOARD = 1;
+    public static final int TYPE_UHID_OUTPUT = 2;
 
     private int type;
     private String text;
     private long sequence;
+    private int id;
+    private byte[] data;
 
     private DeviceMessage() {
     }
@@ -26,6 +29,14 @@ public final class DeviceMessage {
         return event;
     }
 
+    public static DeviceMessage createUhidOutput(int id, byte[] data) {
+        DeviceMessage event = new DeviceMessage();
+        event.type = TYPE_UHID_OUTPUT;
+        event.id = id;
+        event.data = data;
+        return event;
+    }
+
     public int getType() {
         return type;
     }
@@ -37,4 +48,12 @@ public final class DeviceMessage {
     public long getSequence() {
         return sequence;
     }
+
+    public int getId() {
+        return id;
+    }
+
+    public byte[] getData() {
+        return data;
+    }
 }

server/src/main/java/com/genymobile/scrcpy/DeviceMessageWriter.java

@@ -29,6 +29,13 @@ public class DeviceMessageWriter {
                 buffer.putLong(msg.getSequence());
                 output.write(rawBuffer, 0, buffer.position());
                 break;
+            case DeviceMessage.TYPE_UHID_OUTPUT:
+                buffer.putShort((short) msg.getId());
+                byte[] data = msg.getData();
+                buffer.putShort((short) data.length);
+                buffer.put(data);
+                output.write(rawBuffer, 0, buffer.position());
+                break;
             default:
                 Ln.w("Unknown device message: " + msg.getType());
                 break;

server/src/main/java/com/genymobile/scrcpy/UhidManager.java

@@ -1,5 +1,8 @@
 package com.genymobile.scrcpy;
 
+import android.os.Build;
+import android.os.HandlerThread;
+import android.os.MessageQueue;
 import android.system.ErrnoException;
 import android.system.Os;
 import android.system.OsConstants;
@@ -7,6 +10,7 @@ import android.util.ArrayMap;
 
 import java.io.FileDescriptor;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.charset.StandardCharsets;
@@ -14,13 +18,31 @@ import java.nio.charset.StandardCharsets;
 public final class UhidManager {
 
     // Linux: include/uapi/linux/uhid.h
+    private static final int UHID_OUTPUT = 6;
     private static final int UHID_CREATE2 = 11;
     private static final int UHID_INPUT2 = 12;
 
     // Linux: include/uapi/linux/input.h
     private static final short BUS_VIRTUAL = 0x06;
 
+    private static final int SIZE_OF_UHID_EVENT = 4380; // sizeof(struct uhid_event)
+
     private final ArrayMap<Integer, FileDescriptor> fds = new ArrayMap<>();
+    private final ByteBuffer buffer = ByteBuffer.allocate(SIZE_OF_UHID_EVENT).order(ByteOrder.nativeOrder());
+
+    private final DeviceMessageSender sender;
+    private final HandlerThread thread = new HandlerThread("UHidManager");
+    private final MessageQueue queue;
+
+    public UhidManager(DeviceMessageSender sender) {
+        this.sender = sender;
+        thread.start();
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+            queue = thread.getLooper().getQueue();
+        } else {
+            queue = null;
+        }
+    }
 
     public void open(int id, byte[] reportDesc) throws IOException {
         try {
@@ -34,6 +56,8 @@ public final class UhidManager {
 
                 byte[] req = buildUhidCreate2Req(reportDesc);
                 Os.write(fd, req, 0, req.length);
+
+                registerUhidListener(id, fd);
             } catch (Exception e) {
                 close(fd);
                 throw e;
@@ -43,6 +67,62 @@ public final class UhidManager {
         }
     }
 
+    private void registerUhidListener(int id, FileDescriptor fd) {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+            queue.addOnFileDescriptorEventListener(fd, MessageQueue.OnFileDescriptorEventListener.EVENT_INPUT, (fd2, events) -> {
+                try {
+                    buffer.clear();
+                    int r = Os.read(fd2, buffer);
+                    buffer.flip();
+                    if (r > 0) {
+                        int type = buffer.getInt();
+                        if (type == UHID_OUTPUT) {
+                            byte[] data = extractHidOutputData(buffer);
+                            if (data != null) {
+                                DeviceMessage msg = DeviceMessage.createUhidOutput(id, data);
+                                sender.send(msg);
+                            }
+                        }
+                    }
+                } catch (ErrnoException | InterruptedIOException e) {
+                    Ln.e("Failed to read UHID output", e);
+                    return 0;
+                }
+                return events;
+            });
+        }
+    }
+
+    private static byte[] extractHidOutputData(ByteBuffer buffer) {
+        /*
+         * #define UHID_DATA_MAX 4096
+         * struct uhid_event {
+         *     uint32_t type;
+         *     union {
+         *         // ...
+         *         struct uhid_output_req {
+         *             __u8 data[UHID_DATA_MAX];
+         *             __u16 size;
+         *             __u8 rtype;
+         *         };
+         *     };
+         * } __attribute__((__packed__));
+         */
+
+        if (buffer.remaining() < 4099) {
+            Ln.w("Incomplete HID output");
+            return null;
+        }
+        int size = buffer.getShort(buffer.position() + 4096) & 0xFFFF;
+        if (size > 4096) {
+            Ln.w("Incorrect HID output size: " + size);
+            return null;
+        }
+        byte[] data = new byte[size];
+        buffer.get(data);
+        return data;
+    }
+
     public void write(int id, byte[] data) throws IOException {
         FileDescriptor fd = fds.get(id);
         if (fd == null) {

server/src/test/java/com/genymobile/scrcpy/DeviceMessageWriterTest.java

@@ -52,4 +52,27 @@ public class DeviceMessageWriterTest {
 
         Assert.assertArrayEquals(expected, actual);
     }
+
+    @Test
+    public void testSerializeUhidOutput() throws IOException {
+        DeviceMessageWriter writer = new DeviceMessageWriter();
+
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        DataOutputStream dos = new DataOutputStream(bos);
+        dos.writeByte(DeviceMessage.TYPE_UHID_OUTPUT);
+        dos.writeShort(42); // id
+        byte[] data = {1, 2, 3, 4, 5};
+        dos.writeShort(data.length);
+        dos.write(data);
+
+        byte[] expected = bos.toByteArray();
+
+        DeviceMessage msg = DeviceMessage.createUhidOutput(42, data);
+        bos = new ByteArrayOutputStream();
+        writer.writeTo(msg, bos);
+
+        byte[] actual = bos.toByteArray();
+
+        Assert.assertArrayEquals(expected, actual);
+    }
 }