Support role=application web widgets

Background: Within <xxx role=application>...</xxx>, screen readers should not consume DPAD (arrow key) events. Web apps or widgets with role=application have, per the WAI-ARIA spec's contract, their own JavaScript logic for moving focus [1]. [1] w3c/aria#1049, where we discussed this. Problem: TalkBack does not handle role=application so such web apps lose their 4-way (up/down/left/right) navigation. TalkBack only moves forward/backward which breaks authors' pre-defined TV UX. Solution: Whenever accessibility focus (the green rect) goes to some web content with <body role=application> or anywhere within a role=application widget, we don't consume the DPAD events; we let them through. Testing done: From TalkBack's test app, open dpad_a11y.html which has <body role=application>. $ cd testapp $ export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 $ export ANDROID_HOME=~/AndroidSDK $ ./gradlew --debug assemble $ adb install -r app/build/outputs/apk/app-debug.apk $ adb shell am start -n com.android.talkbacktests/.MainActivity Notice: I. Once the WebView gets accessibilty focus, TalkBack won't consume DPAD key events. II. The key events reach the web page's key handlers in JavaScript. III: TalkBack describes HTML elements once they get focused. Signed-off-by: Hugo Holgersson <[email protected]>
google · Jan 27, 2020 · 54d337f · 54d337f
1 parent fa67c12
commit 54d337f
Show file tree

Hide file tree

Showing 6 changed files with 220 additions and 0 deletions.
diff --git a/talkback/src/main/java/controller/TelevisionNavigationController.java b/talkback/src/main/java/controller/TelevisionNavigationController.java
@@ -343,6 +343,15 @@ public boolean processWhenServiceSuspended() {
    */
   private boolean shouldIgnore(AccessibilityNodeInfoCompat node, KeyEvent event) {
     final int keyCode = event.getKeyCode();
+    if (AccessibilityNodeInfoUtils.isWebApplication(node)) {
+      // Web applications and web widgets with role=application have, per the
+      // WAI-ARIA spec's contract, their own JavaScript logic for moving focus.
+      // TalkBack should not consume key events when such an app has accessibility focus.
+      // Debug tip: Forward DPAD events whenever the accessibility cursor is on,
+      // or inside, a WebView: if (WebInterfaceUtils.supportsWebActions(node)) return true;
+      return true;
+    }
+
     if (!mShouldProcessDPadKeyEvent
         && (keyCode == KeyEvent.KEYCODE_DPAD_UP
             || keyCode == KeyEvent.KEYCODE_DPAD_DOWN

diff --git a/testapp/app/src/main/assets/dpad_a11y.html b/testapp/app/src/main/assets/dpad_a11y.html
@@ -0,0 +1,106 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Example TV web app</title>
+    <style>
+      div.outer {display: grid; grid-template-columns: 150px 150px;}
+      div.inner {background: teal; margin: 20px; color: white; font-size: 30px; font-family: Sans-Serif; display: flex; align-items: center; justify-content: center;}
+      div:focus {background: purple;}
+    </style>
+  </head>
+
+<body role="application">
+ <p tabindex="0">This web app allows navigation in 4 directions. Use your arrow keys (DPAD_UP / DPAD_DOWN / DPAD_LEFT / DPAD_RIGHT).</p>
+ <p tabindex="0"><em>role=application</em> makes TalkBack forward DPAD events to Chromium.</p>
+  <div role="grid">
+    <div class="outer" role="row">
+      <div role="gridcell" class="inner" tabindex="0" id="a" aria-label="Astrid">A</div><div role="gridcell" class="inner" tabindex="0" id="c" aria-label="Caroline">C</div>
+    </div>
+    <div class="outer" role="row">
+      <div role="gridcell" class="inner" tabindex="0" id="b" aria-label="Bernie">B</div><div role="gridcell" class="inner" tabindex="0" id="d" aria-label="David">D</div>
+    </div>
+  </div>
+
+  <input type="radio" name="mechanics" id="usecustomjs" checked> <label for="usecustomjs">JavaScript moves web focus (left/right wrap).</label>
+  <input type="radio" name="mechanics" id="usechromium"> <label for="usechromium">Chromium's built-in key handling moves web focus.</label>
+</body>
+
+<script>
+  // These key handlers use preventDefault to stop Chromium's
+  // built-in focus mechanics, called "Spatial Navigation", which
+  // is enabled by default in Android WebViews.
+  function moveFromA(event) {
+    switch (event.code) {
+      case 'ArrowUp':
+        /* go nowhere */ event.preventDefault(); break;
+      case 'ArrowLeft':
+      case 'ArrowRight':
+        c.focus(); event.preventDefault(); break;
+      case 'ArrowDown':
+        b.focus(); event.preventDefault(); break;
+    }
+  }
+
+  function moveFromC(event) {
+    switch (event.code) {
+      case 'ArrowUp':
+        /* go nowhere */ event.preventDefault(); break;
+      case 'ArrowLeft':
+      case 'ArrowRight':
+        a.focus(); event.preventDefault(); break;
+      case 'ArrowDown':
+        d.focus(); event.preventDefault(); break;
+    }
+  }
+
+  function moveFromB(event) {
+    switch (event.code) {
+      case 'ArrowUp':
+        a.focus(); event.preventDefault(); break;
+      case 'ArrowLeft':
+      case 'ArrowRight':
+        d.focus(); event.preventDefault(); break;
+      case 'ArrowDown':
+        usecustomjs.focus(); event.preventDefault(); break;
+    }
+  }
+
+  function moveFromD(event) {
+    switch (event.code) {
+      case 'ArrowUp':
+        c.focus(); event.preventDefault(); break;
+      case 'ArrowLeft':
+      case 'ArrowRight':
+        b.focus(); event.preventDefault(); break;
+      case 'ArrowDown':
+        usechromium.focus(); event.preventDefault(); break;
+      }
+  }
+
+  // Mimic a typical TV app.
+  function enableCustomDpadNavigation() {
+    a.addEventListener('keydown', moveFromA);
+    b.addEventListener('keydown', moveFromB);
+    c.addEventListener('keydown', moveFromC);
+    d.addEventListener('keydown', moveFromD);
+  }
+
+  function disableCustomDpadNavigation() {
+    a.removeEventListener('keydown', moveFromA);
+    b.removeEventListener('keydown', moveFromB);
+    c.removeEventListener('keydown', moveFromC);
+    d.removeEventListener('keydown', moveFromD);
+  }
+
+  function toggleCustomDpad() {
+    if (usecustomjs.checked)
+      enableCustomDpadNavigation();
+    else
+      disableCustomDpadNavigation();
+  }
+  usecustomjs.addEventListener('change', toggleCustomDpad);
+  usechromium.addEventListener('change', toggleCustomDpad);
+
+  enableCustomDpadNavigation();
+  a.focus();
+</script>
diff --git a/testapp/app/src/main/java/com/android/talkbacktests/testsession/WebViewDPADTest.java b/testapp/app/src/main/java/com/android/talkbacktests/testsession/WebViewDPADTest.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.android.talkbacktests.testsession;
+
+import android.content.Context;
+import android.view.KeyEvent;
+import android.view.LayoutInflater;
+import android.view.View;
+import android.view.ViewGroup;
+import android.webkit.WebView;
+
+import com.android.talkbacktests.R;
+
+public class WebViewDPADTest extends BaseTestContent {
+
+    public WebViewDPADTest(Context context, String subtitle, String description) {
+        super(context, subtitle, description);
+    }
+
+    @Override
+    public View getView(final LayoutInflater inflater, ViewGroup container, Context context) {
+        final View view = inflater.inflate(R.layout.test_web_view, container, false);
+        final WebView webView = (WebView) view.findViewById(R.id.webview);
+        webView.getSettings().setJavaScriptEnabled(true);
+        webView.loadUrl("file:///android_asset/dpad_a11y.html");
+        webView.setOnKeyListener(new View.OnKeyListener() {
+            @Override
+            public boolean onKey(View v, int keyCode, KeyEvent event) {
+                if (keyCode == KeyEvent.KEYCODE_BACK) { // Exit the WebView
+                    View parent = (View) v.getParent().getParent().getParent();
+                    View nextButton = parent.findViewById(R.id.next);
+                    // Move Android focus to the native button.
+                    return nextButton != null && nextButton.requestFocus();
+                }
+                return false;
+            }
+        });
+        return view;
+    }
+}
diff --git a/testapp/app/src/main/res/raw/test.json b/testapp/app/src/main/res/raw/test.json
@@ -153,6 +153,17 @@
         }
       ]
     },
+    {
+      "title": "@string/dpad_web_view_session_title",
+      "description": "@string/dpad_web_view_session_description",
+      "content": [
+        {
+          "subtitle": "@string/dpad_web_view_test_subtitle",
+          "description": "@string/dpad_web_view_test_description",
+          "classname": "com.android.talkbacktests.testsession.WebViewDPADTest"
+        }
+      ]
+    },
     {
       "title": "@string/single_line_edit_Field_session_title",
       "description": "@string/single_line_edit_Field_session_description",

diff --git a/testapp/app/src/main/res/values/donottranslate_title_and_description.xml b/testapp/app/src/main/res/values/donottranslate_title_and_description.xml
@@ -63,6 +63,9 @@
     <string name="web_view_session_title">WebView</string>
     <string name="web_view_session_description">View with web content</string>
 
+    <string name="dpad_web_view_session_title">WebView TV app</string>
+    <string name="dpad_web_view_session_description">A TV app where the DPAD moves web focus in four directions.</string>
+
     <string name="single_line_edit_Field_session_title">Single Line Edit Field</string>
     <string name="single_line_edit_Field_session_description">Single-line editable text field</string>
 
@@ -171,6 +174,9 @@
     <string name="web_view_test_subtitle">View with Web Content</string>
     <string name="web_view_test_description">With TalkBack: swipe between web elements, double-tap to activate, swipe up and down to select element type. Note: requires Chrome 50 or above to work.</string>
 
+    <string name="dpad_web_view_test_subtitle">WebView TV app</string>
+    <string name="dpad_web_view_test_description">With TalkBack on Android TV: use the directional keys, the DPAD, to move between the four grid items. This test covers TV web apps that implement custom DPAD navigation, for example, TV apps that display grids of video clips.</string>
+
     <string name="web_view_in_scrollable_view_test_subtitle">WebView Inside ScrollView</string>
     <string name="web_view_in_scrollable_view_test_description">With TalkBack: swipe between web elements until the view scrolls, double-tap to activate. Note: requires Chrome 50 or above to work.</string>
 

diff --git a/utils/src/main/java/AccessibilityNodeInfoUtils.java b/utils/src/main/java/AccessibilityNodeInfoUtils.java
@@ -32,6 +32,7 @@
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityWindowInfo;
 import com.google.android.accessibility.utils.compat.CompatUtils;
+import com.google.android.accessibility.utils.WebInterfaceUtils;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -148,6 +149,39 @@ public boolean accept(AccessibilityNodeInfoCompat node) {
         }
       };
 
+  public static boolean hasApplicationWebRole(AccessibilityNodeInfoCompat node) {
+    return node != null && node.getExtras() != null
+        && node.getExtras().containsKey("AccessibilityNodeInfo.chromeRole")
+        && node.getExtras().get("AccessibilityNodeInfo.chromeRole").equals("application");
+  }
+
+  private static final Filter<AccessibilityNodeInfoCompat> FILTER_IN_WEB_APPLICATION =
+      new Filter<AccessibilityNodeInfoCompat>() {
+        @Override
+        public boolean accept(AccessibilityNodeInfoCompat node) {
+          return hasApplicationWebRole(node);
+        }
+      };
+
+  /**
+   * Returns true if |node| has role=application, i.e. |node| has JavaScript
+   * that handles key events.
+   */
+  public static boolean isWebApplication(AccessibilityNodeInfoCompat node) {
+    // When a WebView-like view (an actual WebView or a browser) has focus:
+    // Check the web content's accessibility tree's first node.
+    // If that node wants raw key event, instead of first "tabbing" the green
+    // rect to it, skip ahead and let the web app directly decide where to go.
+    boolean firstWebNode = WebInterfaceUtils.supportsWebActions(node)
+        && !WebInterfaceUtils.supportsWebActions(node.getParent());
+    boolean firstWebNodeWantsKeyEvents = firstWebNode
+        && node.getChildCount() > 0
+        && hasApplicationWebRole(node.getChild(0));
+
+    return firstWebNodeWantsKeyEvents
+        || AccessibilityNodeInfoUtils.getSelfOrMatchingAncestor(node, FILTER_IN_WEB_APPLICATION) != null;
+  }
+
   private AccessibilityNodeInfoUtils() {
     // This class is not instantiable.
   }