InputService: add basic ASCII keyboard handling

This implements additional keyboard input from VNC viewers in the form of: - ASCII characters, i.e. basic typing - Left and Right arrow handling to select text cursor position - Delete and Backspace handling for editing text - Enter/Return handling on API level >= 30 All display-specific, i.e. provides one keyboard focus per display. Closes #4, finally.
bk138 · May 14, 2024 · 05af5cd · 05af5cd
1 parent 3da1dda
commit 05af5cd
Show file tree

Hide file tree

Showing 3 changed files with 143 additions and 11 deletions.
diff --git a/app/src/main/java/net/christianbeier/droidvnc_ng/InputService.java b/app/src/main/java/net/christianbeier/droidvnc_ng/InputService.java
@@ -21,6 +21,7 @@
 import android.content.Context;
 import android.graphics.Bitmap;
 import android.os.Build;
+import android.os.Bundle;
 import android.os.Handler;
 import android.util.DisplayMetrics;
 import android.util.Log;
@@ -29,6 +30,7 @@
 import android.view.accessibility.AccessibilityEvent;
 import android.view.ViewConfiguration;
 import android.graphics.Path;
+import android.view.accessibility.AccessibilityNodeInfo;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.RequiresApi;
@@ -115,10 +117,39 @@ void setDisplayId(int displayId) {
 	private Handler mMainHandler;
 
 	private final Map<Long, InputContext> mInputContexts = new ConcurrentHashMap<>();
+	/**
+	 * System keyboard input foci, display-specific starting on Android 10, see <a href="https://source.android.com/docs/core/display/multi_display/displays#focus">Android docs</a>
+	 */
+	private final Map<Integer, AccessibilityNodeInfo> mKeyboardFocusNodes = new ConcurrentHashMap<>();
 
 
 	@Override
-	public void onAccessibilityEvent( AccessibilityEvent event ) { }
+	public void onAccessibilityEvent(AccessibilityEvent event) {
+		try {
+			Log.d(TAG, "onAccessibilityEvent: " + event);
+
+			int displayId;
+			if (Build.VERSION.SDK_INT >= 30) {
+				// be display-specific
+				displayId = Objects.requireNonNull(event.getSource()).getWindow().getDisplayId();
+			} else {
+				// assume default display
+				displayId = Display.DEFAULT_DISPLAY;
+			}
+
+			// recycle old node if there
+			AccessibilityNodeInfo previousFocusNode = mKeyboardFocusNodes.get(displayId);
+			if (previousFocusNode != null) {
+				previousFocusNode.recycle();
+			}
+
+			// and put new one
+			mKeyboardFocusNodes.put(displayId, event.getSource());
+
+		} catch (Exception e) {
+			Log.e(TAG, "onAccessibilityEvent: " + Log.getStackTraceString(e));
+		}
+	}
 
 	@Override
 	public void onInterrupt() { }
@@ -271,9 +302,6 @@ public static void onKeyEvent(int down, long keysym, long client) {
 
 		Log.d(TAG, "onKeyEvent: keysym " + keysym + " down " + down + " by client " + client);
 
-		/*
-			Special key handling.
-		 */
 		try {
 			InputContext inputContext = instance.mInputContexts.get(client);
 
@@ -339,6 +367,89 @@ public static void onKeyEvent(int down, long keysym, long client) {
 				instance.performGlobalAction(AccessibilityService.GLOBAL_ACTION_BACK);
 			}
 
+			/*
+				Get current keyboard focus node for input context's display.
+			 */
+			AccessibilityNodeInfo currentFocusNode = instance.mKeyboardFocusNodes.get(inputContext.getDisplayId());
+			// refresh() is important to load the represented view's current text into the node
+			Objects.requireNonNull(currentFocusNode).refresh();
+
+			/*
+			   Left/Right
+			 */
+			if ((keysym == 0xff51 || keysym == 0xff53) && down != 0) {
+				Bundle action = new Bundle();
+				action.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT, AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER);
+				action.putBoolean(AccessibilityNodeInfo.ACTION_ARGUMENT_EXTEND_SELECTION_BOOLEAN, false);
+				if(keysym == 0xff51)
+					Objects.requireNonNull(currentFocusNode).performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY.getId(), action);
+				else
+					Objects.requireNonNull(currentFocusNode).performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_NEXT_AT_MOVEMENT_GRANULARITY.getId(), action);
+			}
+
+			/*
+			    Backspace/Delete
+			    TODO: implement deletions of text selections, right now it's only 1 char at a time
+			 */
+			if ((keysym == 0xff08 || keysym == 0xffff) && down != 0) {
+				CharSequence currentFocusText = Objects.requireNonNull(currentFocusNode).getText();
+				int cursorPos = getCursorPos(currentFocusNode);
+
+				// set new text
+				String newFocusText;
+				if (keysym == 0xff08) {
+					// backspace
+					newFocusText = String.valueOf(currentFocusText.subSequence(0, cursorPos - 1)) + currentFocusText.subSequence(cursorPos, currentFocusText.length());
+				} else {
+					// delete
+					newFocusText = String.valueOf(currentFocusText.subSequence(0, cursorPos)) + currentFocusText.subSequence(cursorPos + 1, currentFocusText.length());
+				}
+				Bundle action = new Bundle();
+				action.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, newFocusText);
+				currentFocusNode.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_TEXT.getId(), action);
+
+				// ACTION_SET_TEXT moves cursor to the end, move cursor back to where it should be
+				setCursorPos(currentFocusNode, keysym == 0xff08 ? cursorPos - 1 : cursorPos);
+			}
+
+			/*
+			    Enter, for API level 30+
+			 */
+			if (keysym == 0xff0d && down != 0) {
+				if (Build.VERSION.SDK_INT >= 30) {
+					Bundle action = new Bundle();
+					Objects.requireNonNull(currentFocusNode).performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_IME_ENTER.getId(), action);
+				}
+			}
+
+			/*
+			    ASCII input
+			 */
+			if (keysym >= 32 && keysym <= 127 && down != 0) {
+				CharSequence currentFocusText = Objects.requireNonNull(currentFocusNode).getText();
+				int cursorPos = getCursorPos(currentFocusNode);
+
+				// set new text
+				String textBeforeCursor = "";
+				try {
+					textBeforeCursor = String.valueOf(currentFocusText.subSequence(0, cursorPos));
+				} catch (IndexOutOfBoundsException ignored) {
+				}
+				String textAfterCursor = "";
+				try {
+					textAfterCursor = String.valueOf(currentFocusText.subSequence(cursorPos, currentFocusText.length()));
+				} catch (IndexOutOfBoundsException ignored) {
+				}
+				String newFocusText = textBeforeCursor + (char) keysym + textAfterCursor;
+
+				Bundle action = new Bundle();
+				action.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, newFocusText);
+				currentFocusNode.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_TEXT.getId(), action);
+
+				// ACTION_SET_TEXT moves cursor to the end, move cursor back to where it should be
+				setCursorPos(currentFocusNode, cursorPos > 0 ? cursorPos + 1 : 1);
+			}
+
 		} catch (Exception e) {
 			// instance probably null
 			Log.e(TAG, "onKeyEvent: failed: " + e);
@@ -541,4 +652,18 @@ private static GestureDescription createSwipe(InputContext inputContext, int x1,
 		swipeBuilder.addStroke( swipeStroke );
 		return swipeBuilder.build();
 	}
+
+	/**
+	 * Returns current cursor position or -1 if no text for node.
+	 */
+	private static int getCursorPos(AccessibilityNodeInfo node) {
+		return node.getTextSelectionEnd();
+	}
+
+	private static void setCursorPos(AccessibilityNodeInfo node, int cursorPos) {
+		Bundle action = new Bundle();
+		action.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, cursorPos);
+		action.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, cursorPos);
+		node.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_SELECTION.getId(), action);
+	}
 }
diff --git a/app/src/main/res/xml-v30/input_service_config.xml b/app/src/main/res/xml-v30/input_service_config.xml
@@ -1,9 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
+<!-- On API 30 and later flagRetrieveInteractiveWindows canRetrieveWindowContent
+     are needed for onAccessibilityEvent() to trigger  -->
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
-    android:accessibilityEventTypes="typeWindowsChanged"
-    android:accessibilityFlags="flagDefault"
-    android:notificationTimeout="50"
+	android:accessibilityEventTypes="typeViewFocused|typeViewClicked|typeViewSelected|typeViewTextChanged|typeViewTextSelectionChanged"
+    android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows"
+    android:notificationTimeout="30"
 	android:description="@string/input_a11y_service_description"
 	android:canPerformGestures="true"
 	android:canTakeScreenshot="true"
+	android:canRetrieveWindowContent="true"
     />
diff --git a/app/src/main/res/xml/input_service_config.xml b/app/src/main/res/xml/input_service_config.xml
@@ -1,8 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
+<!-- accessibilityFeedbackType seems needed pre API 30 so onAccessibilityEvent() triggers,
+ 	 but it seems we don't need flagRetrieveInteractiveWindows -->
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
-    android:accessibilityEventTypes="typeWindowsChanged"
-    android:accessibilityFlags="flagDefault"
-    android:notificationTimeout="50"
+    android:accessibilityEventTypes="typeViewFocused|typeViewClicked|typeViewSelected|typeViewTextChanged|typeViewTextSelectionChanged"
+	android:accessibilityFlags="flagDefault"
+    android:notificationTimeout="30"
 	android:description="@string/input_a11y_service_description"
 	android:canPerformGestures="true"
-    />
+	android:canRetrieveWindowContent="true"
+	android:accessibilityFeedbackType="feedbackVisual"
+	/>