From 27006fe396e0933982c22330282bfb45f4f3b603 Mon Sep 17 00:00:00 2001 From: Callum Moffat Date: Wed, 15 Jun 2022 20:34:51 -0400 Subject: [PATCH] Reland "ChromeOS/Android trackpad gestures" (#34060) * Android trackpad gestures * Try to fix lint problem --- .../android/AndroidTouchProcessor.java | 103 ++++++--- .../android/AndroidTouchProcessorTest.java | 209 ++++++++++++++++++ 2 files changed, 280 insertions(+), 32 deletions(-) create mode 100644 shell/platform/android/test/io/flutter/embedding/android/AndroidTouchProcessorTest.java diff --git a/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java b/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java index 2078bdc161e9f..544828236d2d4 100644 --- a/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java +++ b/shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java @@ -6,9 +6,12 @@ import android.view.MotionEvent; import androidx.annotation.IntDef; import androidx.annotation.NonNull; +import androidx.annotation.VisibleForTesting; import io.flutter.embedding.engine.renderer.FlutterRenderer; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; /** Sends touch information from Android to Flutter in a format that Flutter understands. */ public class AndroidTouchProcessor { @@ -26,7 +29,7 @@ public class AndroidTouchProcessor { PointerChange.PAN_ZOOM_UPDATE, PointerChange.PAN_ZOOM_END }) - private @interface PointerChange { + public @interface PointerChange { int CANCEL = 0; int ADD = 1; int REMOVE = 2; @@ -48,7 +51,7 @@ public class AndroidTouchProcessor { PointerDeviceKind.TRACKPAD, PointerDeviceKind.UNKNOWN }) - private @interface PointerDeviceKind { + public @interface PointerDeviceKind { int TOUCH = 0; int MOUSE = 1; int STYLUS = 2; @@ -59,7 +62,7 @@ public class AndroidTouchProcessor { // Must match the PointerSignalKind enum in pointer.dart. @IntDef({PointerSignalKind.NONE, PointerSignalKind.SCROLL, PointerSignalKind.UNKNOWN}) - private @interface PointerSignalKind { + public @interface PointerSignalKind { int NONE = 0; int SCROLL = 1; int UNKNOWN = 2; @@ -67,7 +70,7 @@ public class AndroidTouchProcessor { // Must match the unpacking code in hooks.dart. private static final int POINTER_DATA_FIELD_COUNT = 35; - private static final int BYTES_PER_FIELD = 8; + @VisibleForTesting static final int BYTES_PER_FIELD = 8; // This value must match the value in framework's platform_view.dart. // This flag indicates whether the original Android pointer events were batched together. @@ -76,12 +79,12 @@ public class AndroidTouchProcessor { @NonNull private final FlutterRenderer renderer; @NonNull private final MotionEventTracker motionEventTracker; - private static final int _POINTER_BUTTON_PRIMARY = 1; - private static final Matrix IDENTITY_TRANSFORM = new Matrix(); private final boolean trackMotionEvents; + private final Map ongoingPans = new HashMap<>(); + /** * Constructs an {@code AndroidTouchProcessor} that will send touch event data to the Flutter * execution context represented by the given {@link FlutterRenderer}. @@ -220,6 +223,28 @@ private void addPointerForIndex( } int pointerKind = getPointerDeviceTypeForToolType(event.getToolType(pointerIndex)); + // We use this in lieu of using event.getRawX and event.getRawY as we wish to support + // earlier versions than API level 29. + float viewToScreenCoords[] = {event.getX(pointerIndex), event.getY(pointerIndex)}; + transformMatrix.mapPoints(viewToScreenCoords); + long buttons; + if (pointerKind == PointerDeviceKind.MOUSE) { + buttons = event.getButtonState() & 0x1F; + if (buttons == 0 + && event.getSource() == InputDevice.SOURCE_MOUSE + && pointerChange == PointerChange.DOWN) { + // Some implementations translate trackpad scrolling into a mouse down-move-up event + // sequence with buttons: 0, such as ARC on a Chromebook. See #11420, a legacy + // implementation that uses the same condition but converts differently. + ongoingPans.put(event.getPointerId(pointerIndex), viewToScreenCoords); + } + } else if (pointerKind == PointerDeviceKind.STYLUS) { + buttons = (event.getButtonState() >> 4) & 0xF; + } else { + buttons = 0; + } + + boolean isTrackpadPan = ongoingPans.containsKey(event.getPointerId(pointerIndex)); int signalKind = event.getActionMasked() == MotionEvent.ACTION_SCROLL @@ -230,39 +255,31 @@ private void addPointerForIndex( packet.putLong(motionEventId); // motionEventId packet.putLong(timeStamp); // time_stamp - packet.putLong(pointerChange); // change - packet.putLong(pointerKind); // kind + if (isTrackpadPan) { + packet.putLong(getPointerChangeForPanZoom(pointerChange)); // change + packet.putLong(PointerDeviceKind.TRACKPAD); // kind + } else { + packet.putLong(pointerChange); // change + packet.putLong(pointerKind); // kind + } packet.putLong(signalKind); // signal_kind packet.putLong(event.getPointerId(pointerIndex)); // device packet.putLong(0); // pointer_identifier, will be generated in pointer_data_packet_converter.cc. - // We use this in lieu of using event.getRawX and event.getRawY as we wish to support - // earlier versions than API level 29. - float viewToScreenCoords[] = {event.getX(pointerIndex), event.getY(pointerIndex)}; - transformMatrix.mapPoints(viewToScreenCoords); - packet.putDouble(viewToScreenCoords[0]); // physical_x - packet.putDouble(viewToScreenCoords[1]); // physical_y + if (isTrackpadPan) { + float[] panStart = ongoingPans.get(event.getPointerId(pointerIndex)); + packet.putDouble(panStart[0]); + packet.putDouble(panStart[1]); + } else { + packet.putDouble(viewToScreenCoords[0]); // physical_x + packet.putDouble(viewToScreenCoords[1]); // physical_y + } packet.putDouble( 0.0); // physical_delta_x, will be generated in pointer_data_packet_converter.cc. packet.putDouble( 0.0); // physical_delta_y, will be generated in pointer_data_packet_converter.cc. - long buttons; - if (pointerKind == PointerDeviceKind.MOUSE) { - buttons = event.getButtonState() & 0x1F; - // TODO(dkwingsmt): Remove this fix after implementing touchpad gestures - // https://github.com/flutter/flutter/issues/23604#issuecomment-524471152 - if (buttons == 0 - && event.getSource() == InputDevice.SOURCE_MOUSE - && (pointerChange == PointerChange.DOWN || pointerChange == PointerChange.MOVE)) { - buttons = _POINTER_BUTTON_PRIMARY; - } - } else if (pointerKind == PointerDeviceKind.STYLUS) { - buttons = (event.getButtonState() >> 4) & 0xF; - } else { - buttons = 0; - } packet.putLong(buttons); // buttons packet.putLong(0); // obscured @@ -317,12 +334,22 @@ private void addPointerForIndex( packet.putDouble(0.0); // scroll_delta_x } - packet.putDouble(0.0); // pan_x - packet.putDouble(0.0); // pan_y + if (isTrackpadPan) { + float[] panStart = ongoingPans.get(event.getPointerId(pointerIndex)); + packet.putDouble(viewToScreenCoords[0] - panStart[0]); + packet.putDouble(viewToScreenCoords[1] - panStart[1]); + } else { + packet.putDouble(0.0); // pan_x + packet.putDouble(0.0); // pan_y + } packet.putDouble(0.0); // pan_delta_x packet.putDouble(0.0); // pan_delta_y packet.putDouble(1.0); // scale packet.putDouble(0.0); // rotation + + if (isTrackpadPan && getPointerChangeForPanZoom(pointerChange) == PointerChange.PAN_ZOOM_END) { + ongoingPans.remove(event.getPointerId(pointerIndex)); + } } @PointerChange @@ -354,7 +381,19 @@ private int getPointerChangeForAction(int maskedAction) { if (maskedAction == MotionEvent.ACTION_SCROLL) { return PointerChange.HOVER; } - return -1; + throw new AssertionError("Unexpected masked action"); + } + + @PointerChange + private int getPointerChangeForPanZoom(int pointerChange) { + if (pointerChange == PointerChange.DOWN) { + return PointerChange.PAN_ZOOM_START; + } else if (pointerChange == PointerChange.MOVE) { + return PointerChange.PAN_ZOOM_UPDATE; + } else if (pointerChange == PointerChange.UP || pointerChange == PointerChange.CANCEL) { + return PointerChange.PAN_ZOOM_END; + } + throw new AssertionError("Unexpected pointer change"); } @PointerDeviceKind diff --git a/shell/platform/android/test/io/flutter/embedding/android/AndroidTouchProcessorTest.java b/shell/platform/android/test/io/flutter/embedding/android/AndroidTouchProcessorTest.java new file mode 100644 index 0000000000000..ae2e31366e986 --- /dev/null +++ b/shell/platform/android/test/io/flutter/embedding/android/AndroidTouchProcessorTest.java @@ -0,0 +1,209 @@ +package io.flutter.embedding.android; + +import static junit.framework.TestCase.assertEquals; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import android.annotation.TargetApi; +import android.view.InputDevice; +import android.view.MotionEvent; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import io.flutter.embedding.engine.renderer.FlutterRenderer; +import java.nio.ByteBuffer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.robolectric.annotation.Config; + +@Config(manifest = Config.NONE) +@RunWith(AndroidJUnit4.class) +@TargetApi(28) +public class AndroidTouchProcessorTest { + @Mock FlutterRenderer mockRenderer; + AndroidTouchProcessor touchProcessor; + @Captor ArgumentCaptor packetCaptor; + @Captor ArgumentCaptor packetSizeCaptor; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + touchProcessor = new AndroidTouchProcessor(mockRenderer, false); + } + + private long readPointerChange(ByteBuffer buffer) { + return buffer.getLong(2 * AndroidTouchProcessor.BYTES_PER_FIELD); + } + + private long readPointerDeviceKind(ByteBuffer buffer) { + return buffer.getLong(3 * AndroidTouchProcessor.BYTES_PER_FIELD); + } + + private long readPointerSignalKind(ByteBuffer buffer) { + return buffer.getLong(4 * AndroidTouchProcessor.BYTES_PER_FIELD); + } + + private double readPointerPhysicalX(ByteBuffer buffer) { + return buffer.getDouble(7 * AndroidTouchProcessor.BYTES_PER_FIELD); + } + + private double readPointerPhysicalY(ByteBuffer buffer) { + return buffer.getDouble(8 * AndroidTouchProcessor.BYTES_PER_FIELD); + } + + private double readPointerPanX(ByteBuffer buffer) { + return buffer.getDouble(29 * AndroidTouchProcessor.BYTES_PER_FIELD); + } + + private double readPointerPanY(ByteBuffer buffer) { + return buffer.getDouble(30 * AndroidTouchProcessor.BYTES_PER_FIELD); + } + + private class MotionEventMocker { + int pointerId; + int source; + int toolType; + + MotionEventMocker(int pointerId, int source, int toolType) { + this.pointerId = pointerId; + this.source = source; + this.toolType = toolType; + } + + MotionEvent mockEvent(int action, float x, float y, int buttonState) { + MotionEvent event = mock(MotionEvent.class); + when(event.getDevice()).thenReturn(null); + when(event.getSource()).thenReturn(source); + when(event.getPointerCount()).thenReturn(1); + when(event.getActionMasked()).thenReturn(action); + when(event.getActionIndex()).thenReturn(0); + when(event.getButtonState()).thenReturn(buttonState); + when(event.getPointerId(0)).thenReturn(pointerId); + when(event.getX(0)).thenReturn(x); + when(event.getY(0)).thenReturn(y); + when(event.getToolType(0)).thenReturn(toolType); + return event; + } + } + + @Test + public void normalTouch() { + MotionEventMocker mocker = + new MotionEventMocker(0, InputDevice.SOURCE_TOUCHSCREEN, MotionEvent.TOOL_TYPE_FINGER); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_DOWN, 0.0f, 0.0f, 0)); + InOrder inOrder = inOrder(mockRenderer); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + ByteBuffer packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.DOWN, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.TOUCH, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(0.0, readPointerPhysicalX(packet)); + assertEquals(0.0, readPointerPhysicalY(packet)); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_MOVE, 10.0f, 5.0f, 0)); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.MOVE, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.TOUCH, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(10.0, readPointerPhysicalX(packet)); + assertEquals(5.0, readPointerPhysicalY(packet)); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_UP, 10.0f, 5.0f, 0)); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.UP, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.TOUCH, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(10.0, readPointerPhysicalX(packet)); + assertEquals(5.0, readPointerPhysicalY(packet)); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void trackpadGesture() { + MotionEventMocker mocker = + new MotionEventMocker(1, InputDevice.SOURCE_MOUSE, MotionEvent.TOOL_TYPE_MOUSE); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_DOWN, 0.0f, 0.0f, 0)); + InOrder inOrder = inOrder(mockRenderer); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + ByteBuffer packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.PAN_ZOOM_START, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.TRACKPAD, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(0.0, readPointerPhysicalX(packet)); + assertEquals(0.0, readPointerPhysicalY(packet)); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_MOVE, 10.0f, 5.0f, 0)); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.PAN_ZOOM_UPDATE, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.TRACKPAD, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(0.0, readPointerPhysicalX(packet)); + assertEquals(0.0, readPointerPhysicalY(packet)); + assertEquals(10.0, readPointerPanX(packet)); + assertEquals(5.0, readPointerPanY(packet)); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_UP, 10.0f, 5.0f, 0)); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.PAN_ZOOM_END, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.TRACKPAD, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(0.0, readPointerPhysicalX(packet)); + assertEquals(0.0, readPointerPhysicalY(packet)); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void mouse() { + MotionEventMocker mocker = + new MotionEventMocker(2, InputDevice.SOURCE_MOUSE, MotionEvent.TOOL_TYPE_MOUSE); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_DOWN, 0.0f, 0.0f, 1)); + InOrder inOrder = inOrder(mockRenderer); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + ByteBuffer packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.DOWN, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.MOUSE, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(0.0, readPointerPhysicalX(packet)); + assertEquals(0.0, readPointerPhysicalY(packet)); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_MOVE, 10.0f, 5.0f, 1)); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.MOVE, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.MOUSE, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(10.0, readPointerPhysicalX(packet)); + assertEquals(5.0, readPointerPhysicalY(packet)); + touchProcessor.onTouchEvent(mocker.mockEvent(MotionEvent.ACTION_UP, 10.0f, 5.0f, 1)); + inOrder + .verify(mockRenderer) + .dispatchPointerDataPacket(packetCaptor.capture(), packetSizeCaptor.capture()); + packet = packetCaptor.getValue(); + assertEquals(AndroidTouchProcessor.PointerChange.UP, readPointerChange(packet)); + assertEquals(AndroidTouchProcessor.PointerDeviceKind.MOUSE, readPointerDeviceKind(packet)); + assertEquals(AndroidTouchProcessor.PointerSignalKind.NONE, readPointerSignalKind(packet)); + assertEquals(10.0, readPointerPhysicalX(packet)); + assertEquals(5.0, readPointerPhysicalY(packet)); + inOrder.verifyNoMoreInteractions(); + } +}