[WIP] UHID mouse/keyboard input

app/data/bash-completion/scrcpy

@@ -28,7 +28,7 @@ _scrcpy() {
         --forward-all-clicks
         -h --help
         --kill-adb-on-close
-        -K --hid-keyboard
+        --keyboard-input-mode=
         --legacy-paste
         --list-camera-sizes
         --list-cameras
@@ -158,6 +158,10 @@ _scrcpy() {
             COMPREPLY=($(compgen -W "$("${ADB:-adb}" devices | awk '$2 == "device" {print $1}')" -- ${cur}))
             return
             ;;
+        --keyboard-input-mode)
+            COMPREPLY=($(compgen -W "disable inject aoa uhid" -- ${cur}))
+            return
+            ;;
         --audio-bit-rate \
         |--audio-buffer \
         |-b|--video-bit-rate \

app/data/zsh-completion/_scrcpy

@@ -35,7 +35,7 @@ arguments=(
     '--forward-all-clicks[Forward clicks to device]'
     {-h,--help}'[Print the help]'
     '--kill-adb-on-close[Kill adb when scrcpy terminates]'
-    {-K,--hid-keyboard}'[Simulate a physical keyboard by using HID over AOAv2]'
+    '--keyboard-input-mode=[Set the keyboard input mode]:mode:(disable inject aoa uhid)'
     '--legacy-paste[Inject computer clipboard text as a sequence of key events on Ctrl+v]'
     '--list-camera-sizes[List the valid camera capture sizes]'
     '--list-cameras[List cameras available on the device]'

app/meson.build

@@ -19,6 +19,10 @@ src = [
     'src/file_pusher.c',
     'src/fps_counter.c',
     'src/frame_buffer.c',
+    'src/hid/hid_event.c',
+    'src/hid/hid_interface.c',
+    'src/hid/hid_keyboard.c',
+    'src/hid/uhid_hid.c',
     'src/input_manager.c',
     'src/keyboard_inject.c',
     'src/mouse_inject.c',
@@ -88,7 +92,6 @@ usb_support = get_option('usb')
 if usb_support
     src += [
         'src/usb/aoa_hid.c',
-        'src/usb/hid_keyboard.c',
         'src/usb/hid_mouse.c',
         'src/usb/scrcpy_otg.c',
         'src/usb/screen_otg.c',

app/src/cli.c

@@ -93,6 +93,7 @@ enum {
     OPT_DISPLAY_ORIENTATION,
     OPT_RECORD_ORIENTATION,
     OPT_ORIENTATION,
+    OPT_KEYBOARD_INPUT_MODE,
 };
 
 struct sc_option {
@@ -366,19 +367,27 @@ static const struct sc_option options[] = {
     {
         .shortopt = 'K',
         .longopt = "hid-keyboard",
-        .text = "Simulate a physical keyboard by using HID over AOAv2.\n"
-                "It provides a better experience for IME users, and allows to "
-                "generate non-ASCII characters, contrary to the default "
-                "injection method.\n"
-                "It may only work over USB.\n"
+    },
+    {
+        .longopt_id = OPT_KEYBOARD_INPUT_MODE,
+        .longopt = "keyboard-input-mode",
+        .argdesc = "value",
+        .text = "Select how to send keyboard inputs to the device.\n"
+                "Possible values are \"disable\", \"inject\", \"aoa\" and \"uhid\".\n"
+                "\"disable\" doesn't send keyboard inputs to device.\n"
+                "\"inject\" uses Android system API to deliver keyboard events to applications.\n"
+                "\"aoa\" simulates a physical HID keyboard using AoAv2 protocol. It only works over USB, but doesn't need USB debugging to be on.\n"
+                "\"uhid\" simulates a physical HID keyboard using Linux's UHID kernel module. It works over both USB and TCP/IP but requires USB debugging.\n"
+                "Simulating physical keyboards (via \"aoa\" or \"uhid\") works on a lower level, thus provides better compatibilities with IMEs, allowing to producing non-ASCII characters.\n"
                 "The keyboard layout must be configured (once and for all) on "
                 "the device, via Settings -> System -> Languages and input -> "
                 "Physical keyboard. This settings page can be started "
                 "directly: `adb shell am start -a "
                 "android.settings.HARD_KEYBOARD_SETTINGS`.\n"
                 "However, the option is only available when the HID keyboard "
                 "is enabled (or a physical keyboard is connected).\n"
-                "Also see --hid-mouse.",
+                "Default is \"inject\"."
+                "Also see --otg, --mouse-input-mode and --gamepad-input-mode.",
     },
     {
         .longopt_id = OPT_LEGACY_PASTE,
@@ -440,7 +449,7 @@ static const struct sc_option options[] = {
                 "LAlt, LSuper or RSuper toggle the capture mode, to give "
                 "control of the mouse back to the computer.\n"
                 "It may only work over USB.\n"
-                "Also see --hid-keyboard.",
+                "Also see --keyboard-input-mode.",
     },
     {
         .longopt_id = OPT_MAX_FPS,
@@ -543,10 +552,9 @@ static const struct sc_option options[] = {
                 "mirroring is disabled.\n"
                 "LAlt, LSuper or RSuper toggle the mouse capture mode, to give "
                 "control of the mouse back to the computer.\n"
-                "If any of --hid-keyboard or --hid-mouse is set, only enable "
-                "keyboard or mouse respectively, otherwise enable both.\n"
+                "--keyboard-input-mode=disable can be used to disable keyboard separately.\n"
                 "It may only work over USB.\n"
-                "See --hid-keyboard and --hid-mouse.",
+                "See --keyboard-input-mode and --hid-mouse.",
     },
     {
         .shortopt = 'p',
@@ -1898,6 +1906,37 @@ parse_camera_fps(const char *s, uint16_t *camera_fps) {
     return true;
 }
 
+static bool
+parse_keyboard_input_mode(const char *optarg, enum sc_keyboard_input_mode *mode) {
+    if (!strcmp(optarg, "disable")) {
+        *mode = SC_KEYBOARD_INPUT_MODE_DISABLED;
+        return true;
+    }
+
+    if (!strcmp(optarg, "inject")) {
+        *mode = SC_KEYBOARD_INPUT_MODE_INJECT;
+        return true;
+    }
+
+    if (!strcmp(optarg, "aoa")) {
+#ifdef HAVE_USB
+        *mode = SC_KEYBOARD_INPUT_MODE_AOA;
+        return true;
+#else
+        LOGE("--keyboard-input-mode=aoa is disabled.");
+        return false;
+#endif
+    }
+
+    if (!strcmp(optarg, "uhid")) {
+        *mode = SC_KEYBOARD_INPUT_MODE_UHID;
+        return true;
+    }
+
+    LOGE("Unsupported keyboard input mode: %s (expected inject, aoa or uhid)", optarg);
+    return false;
+}
+
 static bool
 parse_time_limit(const char *s, sc_tick *tick) {
     long value;
@@ -1987,12 +2026,18 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                 break;
             case 'K':
 #ifdef HAVE_USB
-                opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_HID;
+                LOGW("-K/--hid-keyboard is deprecated, use --keyboard-input-mode=aoa instead");
+                opts->keyboard_input_mode = SC_KEYBOARD_INPUT_MODE_AOA;
                 break;
 #else
                 LOGE("HID over AOA (-K/--hid-keyboard) is disabled.");
                 return false;
 #endif
+            case OPT_KEYBOARD_INPUT_MODE:
+                if (!parse_keyboard_input_mode(optarg, &opts->keyboard_input_mode)){
+                    return false;
+                }
+                break;
             case OPT_MAX_FPS:
                 if (!parse_max_fps(optarg, &opts->max_fps)) {
                     return false;
@@ -2615,11 +2660,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
     }
 
 # ifdef _WIN32
-    if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_HID
+    if (!otg && (opts->keyboard_input_mode == SC_KEYBOARD_INPUT_MODE_AOA
                 || opts->mouse_input_mode == SC_MOUSE_INPUT_MODE_HID)) {
         LOGE("On Windows, it is not possible to open a USB device already open "
              "by another process (like adb).");
-        LOGE("Therefore, -K/--hid-keyboard and -M/--hid-mouse may only work in "
+        LOGE("Therefore, --keyboard-input-mode=aoa and -M/--hid-mouse may only work in "
              "OTG mode (--otg).");
         return false;
     }

app/src/control_msg.c

@@ -152,6 +152,15 @@ sc_control_msg_serialize(const struct sc_control_msg *msg, unsigned char *buf) {
         case SC_CONTROL_MSG_TYPE_ROTATE_DEVICE:
             // no additional data
             return 1;
+        case SC_CONTROL_MSG_TYPE_UHID_OPEN:
+        case SC_CONTROL_MSG_TYPE_UHID_WRITE:
+            sc_write32be(&buf[1], msg->uhid_open.id);
+            sc_write32be(&buf[5], msg->uhid_open.size);
+            memcpy(&buf[9], msg->uhid_open.data, msg->uhid_open.size);
+            return 9 + msg->uhid_open.size;
+        case SC_CONTROL_MSG_TYPE_UHID_CLOSE:
+            sc_write32be(&buf[1], msg->uhid_close.id);
+            return 5;
         default:
             LOGW("Unknown message type: %u", (unsigned) msg->type);
             return 0;
@@ -242,6 +251,15 @@ sc_control_msg_log(const struct sc_control_msg *msg) {
         case SC_CONTROL_MSG_TYPE_ROTATE_DEVICE:
             LOG_CMSG("rotate device");
             break;
+        case SC_CONTROL_MSG_TYPE_UHID_OPEN:
+            LOG_CMSG("uhid open id=%" PRIu32 ", data size=%" PRIu32, msg->uhid_open.id, msg->uhid_open.size);
+            break;
+        case SC_CONTROL_MSG_TYPE_UHID_WRITE:
+            LOG_CMSG("uhid write id=%" PRIu32 ", data size=%" PRIu32, msg->uhid_write.id, msg->uhid_write.size);
+            break;
+        case SC_CONTROL_MSG_TYPE_UHID_CLOSE:
+            LOG_CMSG("uhid close id=%" PRIu32, msg->uhid_close.id);
+            break;
         default:
             LOG_CMSG("unknown type: %u", (unsigned) msg->type);
             break;
@@ -257,6 +275,10 @@ sc_control_msg_destroy(struct sc_control_msg *msg) {
         case SC_CONTROL_MSG_TYPE_SET_CLIPBOARD:
             free(msg->set_clipboard.text);
             break;
+        case SC_CONTROL_MSG_TYPE_UHID_OPEN:
+        case SC_CONTROL_MSG_TYPE_UHID_WRITE:
+            free(msg->uhid_open.data);
+            break;
         default:
             // do nothing
             break;

app/src/control_msg.h

@@ -37,6 +37,9 @@ enum sc_control_msg_type {
     SC_CONTROL_MSG_TYPE_SET_CLIPBOARD,
     SC_CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE,
     SC_CONTROL_MSG_TYPE_ROTATE_DEVICE,
+    SC_CONTROL_MSG_TYPE_UHID_OPEN,
+    SC_CONTROL_MSG_TYPE_UHID_WRITE,
+    SC_CONTROL_MSG_TYPE_UHID_CLOSE,
 };
 
 enum sc_screen_power_mode {
@@ -92,6 +95,19 @@ struct sc_control_msg {
         struct {
             enum sc_screen_power_mode mode;
         } set_screen_power_mode;
+        struct {
+            int32_t id;
+            uint8_t *data;
+            uint16_t size;
+        } uhid_open;
+        struct {
+            int32_t id;
+            uint8_t *data;
+            uint16_t size;
+        } uhid_write;
+        struct {
+            int32_t id;
+        } uhid_close;
     };
 };
 

app/src/controller.c

@@ -8,15 +8,15 @@
 
 bool
 sc_controller_init(struct sc_controller *controller, sc_socket control_socket,
-                   struct sc_acksync *acksync) {
+                   struct sc_acksync *acksync, struct sc_uhid *uhid) {
     sc_vecdeque_init(&controller->queue);
 
     bool ok = sc_vecdeque_reserve(&controller->queue, SC_CONTROL_MSG_QUEUE_MAX);
     if (!ok) {
         return false;
     }
 
-    ok = sc_receiver_init(&controller->receiver, control_socket, acksync);
+    ok = sc_receiver_init(&controller->receiver, control_socket, acksync, uhid);
     if (!ok) {
         sc_vecdeque_destroy(&controller->queue);
         return false;

app/src/controller.h

@@ -26,7 +26,7 @@ struct sc_controller {
 
 bool
 sc_controller_init(struct sc_controller *controller, sc_socket control_socket,
-                   struct sc_acksync *acksync);
+                   struct sc_acksync *acksync, struct sc_uhid *uhid);
 
 void
 sc_controller_destroy(struct sc_controller *controller);

app/src/device_msg.c

@@ -40,6 +40,25 @@ device_msg_deserialize(const unsigned char *buf, size_t len,
             msg->ack_clipboard.sequence = sequence;
             return 9;
         }
+        case DEVICE_MSG_TYPE_UHID_DATA: {
+            uint32_t id = sc_read32be(&buf[1]);
+            size_t uhid_data_len = sc_read32be(&buf[5]);
+            if (uhid_data_len > len - 9) {
+                return 0; // not available
+            }
+            uint8_t *uhid_data = malloc(uhid_data_len);
+            if (!uhid_data) {
+                LOG_OOM();
+                return -1;
+            }
+            if (uhid_data_len) {
+                memcpy(uhid_data, &buf[9], uhid_data_len);
+            }
+            msg->uhid_data.id = id;
+            msg->uhid_data.data = uhid_data;
+            msg->uhid_data.len = uhid_data_len;
+            return 9 + uhid_data_len;
+        }
         default:
             LOGW("Unknown device message type: %d", (int) msg->type);
             return -1; // error, we cannot recover
@@ -48,7 +67,15 @@ device_msg_deserialize(const unsigned char *buf, size_t len,
 
 void
 device_msg_destroy(struct device_msg *msg) {
-    if (msg->type == DEVICE_MSG_TYPE_CLIPBOARD) {
-        free(msg->clipboard.text);
+    switch (msg->type) {
+        case DEVICE_MSG_TYPE_CLIPBOARD:
+            free(msg->clipboard.text);
+            break;
+        case DEVICE_MSG_TYPE_UHID_DATA:
+            free(msg->uhid_data.data);
+            break;
+        default:
+            // nothing to do
+            break;
     }
 }

app/src/device_msg.h

@@ -14,6 +14,7 @@
 enum device_msg_type {
     DEVICE_MSG_TYPE_CLIPBOARD,
     DEVICE_MSG_TYPE_ACK_CLIPBOARD,
+    DEVICE_MSG_TYPE_UHID_DATA,
 };
 
 struct device_msg {
@@ -25,6 +26,11 @@ struct device_msg {
         struct {
             uint64_t sequence;
         } ack_clipboard;
+        struct {
+            uint32_t id;
+            uint8_t *data; // owned, to be freed by free()
+            uint32_t len;
+        } uhid_data;
     };
 };
 

app/src/hid/hid_event.c

@@ -0,0 +1,18 @@
+#include <stdlib.h>
+
+#include "hid_event.h"
+#include "util/acksync.h"
+
+void
+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
+                  uint8_t *buffer, uint16_t buffer_size) {
+    hid_event->accessory_id = accessory_id;
+    hid_event->buffer = buffer;
+    hid_event->size = buffer_size;
+    hid_event->ack_to_wait = SC_SEQUENCE_INVALID;
+}
+
+void
+sc_hid_event_destroy(struct sc_hid_event *hid_event) {
+    free(hid_event->buffer);
+}

app/src/hid/hid_event.h

@@ -0,0 +1,21 @@
+#ifndef SC_HID_EVENT_H
+#define SC_HID_EVENT_H
+
+#include <stdint.h>
+
+struct sc_hid_event {
+    uint16_t accessory_id;
+    uint8_t *buffer;
+    uint16_t size;
+    uint64_t ack_to_wait;
+};
+
+// Takes ownership of buffer
+void
+sc_hid_event_init(struct sc_hid_event *hid_event, uint16_t accessory_id,
+                  uint8_t *buffer, uint16_t buffer_size);
+
+void
+sc_hid_event_destroy(struct sc_hid_event *hid_event);
+
+#endif