Handle UHID output

Use UHID output reports to synchronize CapsLock and VerrNum states.

Co-authored-by: Romain Vimont <[email protected]>
Signed-off-by: Romain Vimont <[email protected]>

app/meson.build

@@ -36,6 +36,7 @@ src = [
     'src/trait/frame_source.c',
     'src/trait/packet_source.c',
     'src/uhid/keyboard_uhid.c',
+    'src/uhid/uhid_output.c',
     'src/util/acksync.c',
     'src/util/audiobuf.c',
     'src/util/average.c',

app/src/controller.c

@@ -43,9 +43,11 @@ sc_controller_init(struct sc_controller *controller, sc_socket control_socket) {
 }
 
 void
-sc_controller_set_acksync(struct sc_controller *controller,
-                          struct sc_acksync *acksync) {
+sc_controller_configure(struct sc_controller *controller,
+                        struct sc_acksync *acksync,
+                        struct sc_uhid_devices *uhid_devices) {
     controller->receiver.acksync = acksync;
+    controller->receiver.uhid_devices = uhid_devices;
 }
 
 void

app/src/controller.h

@@ -28,8 +28,9 @@ bool
 sc_controller_init(struct sc_controller *controller, sc_socket control_socket);
 
 void
-sc_controller_set_acksync(struct sc_controller *controller,
-                          struct sc_acksync *acksync);
+sc_controller_configure(struct sc_controller *controller,
+                        struct sc_acksync *acksync,
+                        struct sc_uhid_devices *uhid_devices);
 
 void
 sc_controller_destroy(struct sc_controller *controller);

app/src/device_msg.c

@@ -46,6 +46,31 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
             msg->ack_clipboard.sequence = sequence;
             return 9;
         }
+        case DEVICE_MSG_TYPE_UHID_OUTPUT: {
+            if (len < 5) {
+                // at least id + size
+                return 0; // not available
+            }
+            uint16_t id = sc_read16be(&buf[1]);
+            size_t size = sc_read16be(&buf[3]);
+            if (size < len - 5) {
+                return 0; // not available
+            }
+            uint8_t *data = malloc(size);
+            if (!data) {
+                LOG_OOM();
+                return -1;
+            }
+            if (size) {
+                memcpy(data, &buf[5], size);
+            }
+
+            msg->uhid_output.id = id;
+            msg->uhid_output.size = size;
+            msg->uhid_output.data = data;
+
+            return 5 + size;
+        }
         default:
             LOGW("Unknown device message type: %d", (int) msg->type);
             return -1; // error, we cannot recover
@@ -54,7 +79,15 @@ sc_device_msg_deserialize(const uint8_t *buf, size_t len,
 
 void
 sc_device_msg_destroy(struct sc_device_msg *msg) {
-    if (msg->type == DEVICE_MSG_TYPE_CLIPBOARD) {
-        free(msg->clipboard.text);
+    switch (msg->type) {
+        case DEVICE_MSG_TYPE_CLIPBOARD:
+            free(msg->clipboard.text);
+            break;
+        case DEVICE_MSG_TYPE_UHID_OUTPUT:
+            free(msg->uhid_output.data);
+            break;
+        default:
+            // nothing to do
+            break;
     }
 }

app/src/device_msg.h

@@ -14,6 +14,7 @@
 enum sc_device_msg_type {
     DEVICE_MSG_TYPE_CLIPBOARD,
     DEVICE_MSG_TYPE_ACK_CLIPBOARD,
+    DEVICE_MSG_TYPE_UHID_OUTPUT,
 };
 
 struct sc_device_msg {
@@ -25,6 +26,11 @@ struct sc_device_msg {
         struct {
             uint64_t sequence;
         } ack_clipboard;
+        struct {
+            uint16_t id;
+            uint16_t size;
+            uint8_t *data; // owned, to be freed by free()
+        } uhid_output;
     };
 };
 

app/src/receiver.c

@@ -1,11 +1,13 @@
 #include "receiver.h"
 
 #include <assert.h>
+#include <inttypes.h>
 #include <stdint.h>
 #include <SDL2/SDL_clipboard.h>
 
 #include "device_msg.h"
 #include "util/log.h"
+#include "util/str.h"
 
 bool
 sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
@@ -16,6 +18,7 @@ sc_receiver_init(struct sc_receiver *receiver, sc_socket control_socket) {
 
     receiver->control_socket = control_socket;
     receiver->acksync = NULL;
+    receiver->uhid_devices = NULL;
 
     return true;
 }
@@ -47,6 +50,31 @@ process_msg(struct sc_receiver *receiver, struct sc_device_msg *msg) {
                  msg->ack_clipboard.sequence);
             sc_acksync_ack(receiver->acksync, msg->ack_clipboard.sequence);
             break;
+        case DEVICE_MSG_TYPE_UHID_OUTPUT:
+            if (sc_get_log_level() <= SC_LOG_LEVEL_VERBOSE) {
+                char *hex = sc_str_to_hex_string(msg->uhid_output.data,
+                                                 msg->uhid_output.size);
+                if (hex) {
+                    LOGV("UHID output [%" PRIu16 "] %s",
+                         msg->uhid_output.id, hex);
+                    free(hex);
+                } else {
+                    LOGV("UHID output [%" PRIu16 "] size=%" PRIu16,
+                         msg->uhid_output.id, msg->uhid_output.size);
+                }
+            }
+            assert(receiver->uhid_devices);
+            struct sc_uhid_receiver *uhid_receiver =
+                sc_uhid_devices_get_receiver(receiver->uhid_devices,
+                                             msg->uhid_output.id);
+            if (uhid_receiver) {
+                uhid_receiver->ops->process_output(uhid_receiver,
+                                                   msg->uhid_output.data,
+                                                   msg->uhid_output.size);
+            } else {
+                LOGW("No UHID receiver for id %" PRIu16, msg->uhid_output.id);
+            }
+            break;
     }
 }
 

app/src/receiver.h

@@ -5,6 +5,7 @@
 
 #include <stdbool.h>
 
+#include "uhid/uhid_output.h"
 #include "util/acksync.h"
 #include "util/net.h"
 #include "util/thread.h"
@@ -17,6 +18,7 @@ struct sc_receiver {
     sc_mutex mutex;
 
     struct sc_acksync *acksync;
+    struct sc_uhid_devices *uhid_devices;
 };
 
 bool

app/src/scrcpy.c

@@ -62,6 +62,7 @@ struct scrcpy {
     struct sc_aoa aoa;
     // sequence/ack helper to synchronize clipboard and Ctrl+v via HID
     struct sc_acksync acksync;
+    struct sc_uhid_devices uhid_devices;
 #endif
     union {
         struct sc_keyboard_sdk keyboard_sdk;
@@ -342,6 +343,7 @@ scrcpy(struct scrcpy_options *options) {
     bool timeout_started = false;
 
     struct sc_acksync *acksync = NULL;
+    struct sc_uhid_devices *uhid_devices = NULL;
 
     uint32_t scid = scrcpy_generate_scid();
 
@@ -666,10 +668,12 @@ scrcpy(struct scrcpy_options *options) {
             kp = &s->keyboard_sdk.key_processor;
         } else if (options->keyboard_input_mode
                 == SC_KEYBOARD_INPUT_MODE_UHID) {
-            bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller);
+            bool ok = sc_keyboard_uhid_init(&s->keyboard_uhid, &s->controller,
+                                            &s->uhid_devices);
             if (!ok) {
                 goto end;
             }
+            uhid_devices = &s->uhid_devices;
             kp = &s->keyboard_uhid.key_processor;
         }
 
@@ -679,7 +683,7 @@ scrcpy(struct scrcpy_options *options) {
             mp = &s->mouse_sdk.mouse_processor;
         }
 
-        sc_controller_set_acksync(&s->controller, acksync);
+        sc_controller_configure(&s->controller, acksync, uhid_devices);
 
         if (!sc_controller_start(&s->controller)) {
             goto end;

app/src/uhid/keyboard_uhid.c

@@ -5,8 +5,52 @@
 /** Downcast key processor to keyboard_uhid */
 #define DOWNCAST(KP) container_of(KP, struct sc_keyboard_uhid, key_processor)
 
+/** Downcast uhid_receiver to keyboard_uhid */
+#define DOWNCAST_RECEIVER(UR) \
+    container_of(UR, struct sc_keyboard_uhid, uhid_receiver)
+
 #define UHID_KEYBOARD_ID 1
 
+static void
+sc_keyboard_uhid_send_input(struct sc_keyboard_uhid *kb,
+                            const struct sc_hid_event *event) {
+    struct sc_control_msg msg;
+    msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
+    msg.uhid_input.id = UHID_KEYBOARD_ID;
+
+    assert(event->size <= SC_HID_MAX_SIZE);
+    memcpy(msg.uhid_input.data, event->data, event->size);
+    msg.uhid_input.size = event->size;
+
+    if (!sc_controller_push_msg(kb->controller, &msg)) {
+        LOGE("Could not send UHID_INPUT message (key)");
+    }
+}
+
+static void
+sc_keyboard_uhid_synchronize_mod(struct sc_keyboard_uhid *kb) {
+    SDL_Keymod sdl_mod = SDL_GetModState();
+    uint16_t mod = sc_mods_state_from_sdl(sdl_mod) & (SC_MOD_CAPS | SC_MOD_NUM);
+
+    uint16_t device_mod =
+        atomic_load_explicit(&kb->device_mod, memory_order_relaxed);
+    uint16_t diff = mod ^ device_mod;
+
+    if (diff) {
+        // Inherently racy (the HID output reports arrive asynchronously in
+        // response to key presses), but will re-synchronize on next key press
+        // or HID output anyway
+        atomic_store_explicit(&kb->device_mod, mod, memory_order_relaxed);
+
+        struct sc_hid_event hid_event;
+        sc_hid_keyboard_event_from_mods(&hid_event, diff);
+
+        LOGV("HID keyboard state synchronized");
+
+        sc_keyboard_uhid_send_input(kb, &hid_event);
+    }
+}
+
 static void
 sc_key_processor_process_key(struct sc_key_processor *kp,
                              const struct sc_key_event *event,
@@ -25,26 +69,56 @@ sc_key_processor_process_key(struct sc_key_processor *kp,
 
     // Not all keys are supported, just ignore unsupported keys
     if (sc_hid_keyboard_event_from_key(&kb->hid, &hid_event, event)) {
-        struct sc_control_msg msg;
-        msg.type = SC_CONTROL_MSG_TYPE_UHID_INPUT;
-        msg.uhid_input.id = UHID_KEYBOARD_ID;
-
-        assert(hid_event.size <= SC_HID_MAX_SIZE);
-        memcpy(msg.uhid_input.data, hid_event.data, hid_event.size);
-        msg.uhid_input.size = hid_event.size;
-
-        if (!sc_controller_push_msg(kb->controller, &msg)) {
-            LOGE("Could not send UHID_INPUT message (key)");
+        if (event->scancode == SC_SCANCODE_CAPSLOCK) {
+            atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_CAPS,
+                                      memory_order_relaxed);
+        } else if (event->scancode == SC_SCANCODE_NUMLOCK) {
+            atomic_fetch_xor_explicit(&kb->device_mod, SC_MOD_NUM,
+                                      memory_order_relaxed);
+        } else {
+            // Synchronize modifiers (only if the scancode itself does not
+            // change the modifiers)
+            sc_keyboard_uhid_synchronize_mod(kb);
         }
+        sc_keyboard_uhid_send_input(kb, &hid_event);
+    }
+}
+
+static unsigned
+sc_keyboard_uhid_to_sc_mod(uint8_t hid_led) {
+    // <https://www.usb.org/sites/default/files/documents/hut1_12v2.pdf>
+    // (chapter 11: LED page)
+    unsigned mod = 0;
+    if (hid_led & 0x01) {
+        mod |= SC_MOD_NUM;
     }
+    if (hid_led & 0x02) {
+        mod |= SC_MOD_CAPS;
+    }
+    return mod;
+}
+
+static void
+sc_uhid_receiver_process_output(struct sc_uhid_receiver *receiver,
+                                const uint8_t *data, size_t len) {
+    // Called from the thread receiving device messages
+    assert(len);
+    (void) len;
+    struct sc_keyboard_uhid *kb = DOWNCAST_RECEIVER(receiver);
+
+    uint8_t hid_led = data[0];
+    uint16_t device_mod = sc_keyboard_uhid_to_sc_mod(hid_led);
+    atomic_store_explicit(&kb->device_mod, device_mod, memory_order_relaxed);
 }
 
 bool
 sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
-                      struct sc_controller *controller) {
+                      struct sc_controller *controller,
+                      struct sc_uhid_devices *uhid_devices) {
     sc_hid_keyboard_init(&kb->hid);
 
     kb->controller = controller;
+    atomic_init(&kb->device_mod, 0);
 
     static const struct sc_key_processor_ops ops = {
         .process_key = sc_key_processor_process_key,
@@ -58,6 +132,14 @@ sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
     kb->key_processor.async_paste = false;
     kb->key_processor.ops = &ops;
 
+    static const struct sc_uhid_receiver_ops uhid_receiver_ops = {
+        .process_output = sc_uhid_receiver_process_output,
+    };
+
+    kb->uhid_receiver.id = UHID_KEYBOARD_ID;
+    kb->uhid_receiver.ops = &uhid_receiver_ops;
+    sc_uhid_devices_add_receiver(uhid_devices, &kb->uhid_receiver);
+
     struct sc_control_msg msg;
     msg.type = SC_CONTROL_MSG_TYPE_UHID_CREATE;
     msg.uhid_create.id = UHID_KEYBOARD_ID;

app/src/uhid/keyboard_uhid.h

@@ -7,17 +7,21 @@
 
 #include "controller.h"
 #include "hid/hid_keyboard.h"
+#include "uhid/uhid_output.h"
 #include "trait/key_processor.h"
 
 struct sc_keyboard_uhid {
     struct sc_key_processor key_processor; // key processor trait
+    struct sc_uhid_receiver uhid_receiver;
 
     struct sc_hid_keyboard hid;
     struct sc_controller *controller;
+    atomic_uint_least16_t device_mod;
 };
 
 bool
 sc_keyboard_uhid_init(struct sc_keyboard_uhid *kb,
-                      struct sc_controller *controller);
+                      struct sc_controller *controller,
+                      struct sc_uhid_devices *uhid_devices);
 
 #endif