Add CI job for Flutter Android App

.github/cloudbuild/android.yaml

@@ -53,8 +53,7 @@ steps:
 - id: build-apks
   name: gcr.io/$PROJECT_ID/mlperf-mobile-android:latest
   # Image upload usually takes only few seconds.
-  # However, if we generated a new image and the build failed,
-  # it can fail before the build finishes, canceling the upload.
+  # However, if we generated a new image and the build failed, it can cancel the upload.
   # Let's wait for the upload to finish before starting the actual build.
   waitFor: []
   timeout: 10800s # 3 hours
@@ -74,6 +73,8 @@ steps:
     cp bazel-bin/android/androidTest/mlperf_test_app.apk output/artifacts
 - id: instrument-test-on-firebase
   name: gcr.io/cloud-builders/gcloud
+  waitFor:
+  - build-apks
   timeout: 3600s # 1 hour
   entrypoint: bash
   args:
@@ -89,8 +90,8 @@ steps:
 - id: crawler-test-on-firebase
   name: gcr.io/cloud-builders/gcloud
   waitFor:
-  - build-apks # don't wait for the other test to finish, run as soon as APKs are ready
-  timeout: 3600s # 1 hour (we sometimes wait 30+ minutes until the device we need is free)
+  - build-apks
+  timeout: 3600s # 1 hour
   entrypoint: bash
   args:
   - -xc

.github/cloudbuild/flutter-android.yaml

@@ -0,0 +1,123 @@
+substitutions:
+  _IMAGE_NAME: mlperf-mobile-flutter-android
+# Building apks with flutter requires a lot of memory.
+# Builds on standard machines with 4GB of RAM can unexpectedly hang.
+# Also the build is mostly CPU-intensive, so using 8-core machines
+# reduces build time up to 3 times.
+options:
+ machineType: 'E2_HIGHCPU_8'
+
+steps:
+# We need this step to correctly identify dockerfile tag
+- id: fetch-repo-history
+  name: gcr.io/cloud-builders/git
+  timeout: 10s
+  args:
+  - fetch
+  - --unshallow
+# Download DOCKERFILE_COMMIT tag if it exists to skip docker image generation,
+#   or download the :latest tag and use it as a cache,
+#   or skip downloading if :latest doesn't exist yet.
+# This being a separate step helps readability in Cloud Build console.
+- id: cache-old-image
+  name: gcr.io/cloud-builders/docker
+  timeout: 600s # 10 minutes
+  entrypoint: bash
+  args:
+  - -xc
+  - |
+    DOCKERFILE_COMMIT=$$(git log -n 1 --pretty=format:%H -- flutter/android/docker/Dockerfile)
+    docker pull gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT \
+    || docker pull gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest \
+    || true
+# This step overrides the :latest tag of the image, so that we can use it in later steps.
+- id: build-new-image
+  name: gcr.io/cloud-builders/docker
+  timeout: 1800s # 30 minutes
+  entrypoint: bash
+  args:
+  - -xc
+  - |
+    DOCKERFILE_COMMIT=$$(git log -n 1 --pretty=format:%H -- flutter/android/docker/Dockerfile)
+    docker build \
+      -t gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT \
+      -t gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest \
+      --cache-from gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT \
+      --cache-from gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest \
+      flutter/android/docker
+# If the build fails artifacts are not uploaded automatically, so we save them manually before build
+- id: push-new-image
+  name: gcr.io/cloud-builders/docker
+  timeout: 1800s # 30 minutes
+  entrypoint: bash
+  args:
+  - -xc
+  - |
+    DOCKERFILE_COMMIT=$$(git log -n 1 --pretty=format:%H -- flutter/android/docker/Dockerfile)
+    docker push gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT
+    docker push gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest
+- id: build-apks
+  name: gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest
+  # Image upload usually takes only few seconds.
+  # However, if we generated a new image and the build failed, it can cancel the upload.
+  # Let's wait for the upload to finish before starting the actual build.
+  waitFor: []
+  timeout: 10800s # 3 hours
+  entrypoint: bash
+  env:
+  - BAZEL_CACHE_ARG=--remote_cache=https://storage.googleapis.com/$_BAZEL_CACHE_BUCKET --google_default_credentials
+  args:
+  - -xc
+  - |
+    cd flutter || exit $?
+    make ci/android/test_apk || exit $?
+
+    mkdir -p /workspace/output/artifacts || exit $?
+    cp /workspace/flutter/build/app/outputs/flutter-apk/app-release.apk /workspace/output/artifacts/mlperf_app_release.apk || exit $?
+    cp /workspace/flutter/build/app/outputs/apk/debug/app-debug.apk /workspace/output/artifacts/mlperf_app_test_main.apk || exit $?
+    cp /workspace/flutter/build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk /workspace/output/artifacts/mlperf_app_test_helper.apk || exit $?
+- id: instrument-test-on-firebase
+  name: gcr.io/cloud-builders/gcloud
+  waitFor:
+  - build-apks
+  timeout: 3600s # 1 hour
+  entrypoint: bash
+  args:
+  - -xc
+  - |
+    # redfin is Pixel 5e
+    gcloud firebase test android run \
+      --type instrumentation \
+      --app output/artifacts/mlperf_app_test_main.apk \
+      --test output/artifacts/mlperf_app_test_helper.apk \
+      --timeout 45m \
+      --device model=redfin,version=30,locale=en,orientation=portrait
+- id: crawler-test-on-firebase
+  name: gcr.io/cloud-builders/gcloud
+  waitFor:
+  - build-apks
+  timeout: 3600s # 1 hour
+  entrypoint: bash
+  args:
+  - -xc
+  - |
+    # x1q is SM-G981U1 (Samsung Galaxy S20 5G)
+    gcloud firebase test android run \
+      --type robo \
+      --app output/artifacts/mlperf_app_release.apk \
+      --device model=x1q,version=29,locale=en,orientation=portrait \
+      --timeout 90s
+
+# We uploaded both tags earlier, but this option also adds them to the artifacts page of the build
+images:
+- gcr.io/$PROJECT_ID/$_IMAGE_NAME
+
+artifacts:
+  objects:
+    location: gs://$_ARTIFACT_BUCKET/$_ARTIFACT_FOLDER/$COMMIT_SHA-flutter
+    paths:
+    - output/artifacts/mlperf_app_release.apk
+    - output/artifacts/mlperf_app_test_main.apk
+    - output/artifacts/mlperf_app_test_helper.apk
+
+timeout: 18000s # 5 hours

flutter/Makefile

@@ -29,9 +29,7 @@ _bazel_links_arg=--symlink_prefix ${BAZEL_LINKS_DIR} --experimental_no_product_n
 .PHONY: cpp-ios
 cpp-ios:
 	@# NOTE: add `--copt -g` for debug info (but the resulting library would be 0.5 GiB)
-	bazel \
-		${BAZEL_CACHE_FLAG} \
-		build --config=ios_fat64 -c opt //flutter/cpp/flutter:ios_backend_fw_static
+	bazel ${BAZEL_OUTPUT_ROOT_ARG} build --config=ios_fat64 -c opt //flutter/cpp/flutter:ios_backend_fw_static
 
 	rm -rf ${_xcode_fw}
 	cp -a ${_bazel_ios_fw} ${_xcode_fw}
@@ -48,6 +46,33 @@ debug_flags_windows=-c dbg --copt /Od --copt /Z7 --linkopt -debug
 .PHONY: android
 android: backend-bridge-android backends/tflite-android prepare-flutter
 
+.PHONY: android/apk
+android/apk: android
+	flutter clean
+	@# take results from build/app/outputs/flutter-apk/app-release.apk
+	flutter build apk
+
+.PHONY: ci/android/test_apk
+ci/android/test_apk: android/apk
+	@# take results from build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk
+	cd android && ./gradlew app:assembleAndroidTest
+	@# take results from build/app/outputs/apk/debug/app-debug.apk
+	cd android && ./gradlew app:assembleDebug -Ptarget=integration_test/first_test.dart
+
+.PHONY: docker/android/apk
+docker/android/apk: android/docker/image
+	@# if the build fails with java.io.IOException: Input/output error
+	@# remove file android/gradle/wrapper/gradle-wrapper.jar
+	MSYS2_ARG_CONV_EXCL="*" docker run --rm -it --user `id -u`:`id -g` -v $(CURDIR)/..:/mnt/project mlcommons/mlperf_mobile_flutter bash -c "cd /mnt/project/flutter && make prepare-flutter && make android/apk"
+
+.PHONY: android/docker/image
+android/docker/image: build/docker/mlperf_mobile_flutter.stamp
+
+build/docker/mlperf_mobile_flutter.stamp: android/docker/Dockerfile
+	docker image build -t mlcommons/mlperf_mobile_flutter android/docker
+	mkdir -p build/docker
+	touch $@
+
 .PHONY: windows/docker/image
 windows/docker/image:
 	docker build -t mlperf_mobile_flutter:windows-1.0 windows/docker
@@ -135,15 +160,15 @@ windows/flutter-release:
 
 .PHONY: backend-bridge-windows
 backend-bridge-windows:
-	bazel build ${_bazel_links_arg} --config=windows -c opt //flutter/cpp/flutter:backend_bridge.dll
+	bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=windows -c opt //flutter/cpp/flutter:backend_bridge.dll
 	cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/flutter/cpp/flutter/backend_bridge.dll
 	mkdir -p build/win-dlls/
 	rm -f build/win-dlls/backend_bridge.dll
 	cd .. && cp ${BAZEL_LINKS_DIR}bin/flutter/cpp/flutter/backend_bridge.dll flutter/build/win-dlls/backend_bridge.dll
 
 .PHONY: backend-bridge-android
 backend-bridge-android:
-	bazel build ${_bazel_links_arg} --config=android_arm64 -c opt //flutter/cpp/flutter:libbackendbridge.so
+	bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=android_arm64 -c opt //flutter/cpp/flutter:libbackendbridge.so
 	cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/flutter/cpp/flutter/libbackendbridge.so
 	mkdir -p android/app/src/main/jniLibs/arm64-v8a
 	rm -f android/flutter/app/src/main/jniLibs/arm64-v8a/libbackendbridge.so
@@ -160,7 +185,7 @@ backend-bridge-android:
 
 .PHONY: backends/tflite-windows
 backends/tflite-windows:
-	bazel build ${_bazel_links_arg} --config=windows -c opt //mobile_back_tflite:tflitebackenddll
+	bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=windows -c opt //mobile_back_tflite:tflitebackenddll
 	cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.dll
 	mkdir -p build/win-dlls/backends
 	rm -f build/win-dlls/backends/libtflitebackend.dll
@@ -177,7 +202,7 @@ backends/tflite-windows:
 
 .PHONY: backends/tflite-android
 backends/tflite-android:
-	bazel build ${_bazel_links_arg} --config=android_arm64 -c opt //mobile_back_tflite:tflitebackend
+	bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=android_arm64 -c opt //mobile_back_tflite:tflitebackend
 	cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so
 	mkdir -p android/app/src/main/jniLibs/arm64-v8a
 	rm -f android/app/src/main/jniLibs/arm64-v8a/libtflitebackend.so

flutter/android/.gitignore

@@ -13,3 +13,4 @@ key.properties
 **/*.jks
 
 /app/src/main/jniLibs/
+.kotlin

flutter/android/app/build.gradle

@@ -47,6 +47,7 @@ android {
         targetSdkVersion 30
         versionCode flutterVersionCode.toInteger()
         versionName flutterVersionName
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
 
     buildTypes {
@@ -64,4 +65,7 @@ flutter {
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
+    testImplementation 'junit:junit:4.13.2'
+    androidTestImplementation 'androidx.test:runner:1.2.0'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
 }

flutter/android/app/src/androidTest/java/org/mlcommons/android/mlperfbench/MainActivityTest.java

@@ -0,0 +1,13 @@
+package org.mlcommons.android.mlperfbench;
+
+import androidx.test.rule.ActivityTestRule;
+import dev.flutter.plugins.integration_test.FlutterTestRunner;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+@RunWith(FlutterTestRunner.class)
+public class MainActivityTest {
+  @Rule
+  public ActivityTestRule<MainActivity> rule =
+      new ActivityTestRule<>(MainActivity.class, true, false);
+}

flutter/android/docker/Dockerfile

@@ -0,0 +1,62 @@
+FROM ubuntu:20.04
+
+# Without DEBIAN_FRONTEND=noninteractive arg apt-get waits for user input.
+#   Docker desktop shows all previously defined args for each of the commands,
+#   which makes reading layer info harder, so we set this env for for individual commands only.
+# JDK package downloads ~500 MB from slow mirrors, which can take a lot of time,
+#   so a separate layer for it makes image rebuild faster in case we change any other dependencies.
+RUN apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-8-jdk-headless
+RUN apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    apt-transport-https \
+    curl \
+    git \
+    gnupg \
+    make \
+    protobuf-compiler \
+    python3 \
+    python3-pip
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN python3 -m pip install numpy absl-py
+
+# Bazel is not present in standard repositories
+RUN curl -L https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
+    echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
+    apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y bazel=4.2.1
+
+ENV ANDROID_SDK_ROOT=/opt/android
+WORKDIR $ANDROID_SDK_ROOT/cmdline-tools
+# sdkmanager expects to be placed into `$ANDROID_SDK_ROOT/cmdline-tools/tools`
+RUN curl -L https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip | jar x && \
+    mv cmdline-tools tools && \
+    chmod --recursive +x tools/bin
+ENV PATH=$PATH:$ANDROID_SDK_ROOT/cmdline-tools/tools/bin
+
+RUN yes | sdkmanager --licenses >/dev/null
+# Build tools 29.0.2 are required by Flutter 2.5.3
+# Build tools 30 are required by bazel 4.2.1
+RUN yes | sdkmanager \
+    "tools" \
+    "platform-tools" \
+    "build-tools;29.0.2" \
+    "build-tools;30.0.3" \
+    "platforms;android-30"
+# Install NDK in a separate layer to decrease max layer size.
+RUN yes | sdkmanager "ndk;21.4.7075529"
+ENV ANDROID_NDK_ROOT=$ANDROID_SDK_ROOT/ndk/21.4.7075529
+ENV ANDROID_NDK_HOME=$ANDROID_NDK_ROOT
+
+ENV HOME=/image-workdir
+WORKDIR $HOME
+
+ENV PUB_CACHE=$HOME/flutter/.pub-cache
+ENV PATH=$PATH:$HOME/flutter/bin:$PUB_CACHE/bin
+RUN curl -L https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.5.3-stable.tar.xz | tar Jxf -
+RUN dart pub global activate protoc_plugin
+
+ENV GRADLE_USER_HOME=$HOME/.gradle
+ENV ANDROID_SDK_HOME=$HOME/.android
+
+RUN mkdir $ANDROID_SDK_HOME && \
+    keytool -genkey -v -keystore $ANDROID_SDK_HOME/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"
+
+RUN chmod --recursive a=u $HOME

flutter/integration_test/first_test.dart

@@ -1,14 +1,15 @@
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';
-import 'package:path_provider/path_provider.dart';
 import 'package:shared_preferences/shared_preferences.dart';
 import 'dart:io';
 import 'dart:convert';
 
 import 'package:mlcommons_ios_app/main.dart' as app;
 import 'package:mlcommons_ios_app/ui/main_screen.dart';
 import 'package:mlcommons_ios_app/ui/result_screen.dart';
+import 'package:mlcommons_ios_app/benchmark/resource_manager.dart'
+    as resource_manager;
 
 void main() {
   final splashPauseSeconds = 4;
@@ -63,11 +64,13 @@ void main() {
           reason: 'Test results were not found');
 
       final applicationDirectory =
-          (await getApplicationDocumentsDirectory()).path;
+          await resource_manager.ResourceManager.getApplicationDirectory();
       final jsonResultPath = '$applicationDirectory/result.json';
       final file = File(jsonResultPath);
 
-      expect(await file.exists(), true, reason: 'Result.json does not exist');
+      expect(await file.exists(), true,
+          reason:
+              'Result.json does not exist: file $applicationDirectory/result.json is not found');
 
       final jsonResultContent = await file.readAsString();
       final results = jsonDecode(jsonResultContent);

flutter/lib/benchmark/resource_manager.dart

@@ -107,7 +107,7 @@ class ResourceManager {
     return defaultBenchmarksConfiguration;
   }
 
-  Future<void> initSystemPaths() async {
+  static Future<String> getApplicationDirectory() async {
     // applicationDirectory should be visible to user
     Directory? dir;
     if (Platform.isIOS) {
@@ -116,12 +116,18 @@ class ResourceManager {
       dir = await getExternalStorageDirectory();
     } else if (Platform.isWindows) {
       dir = await getDownloadsDirectory();
+    } else {
+      throw 'unsupported platform';
     }
     if (dir != null) {
-      applicationDirectory = dir.path;
+      return dir.path;
     } else {
-      applicationDirectory = (await getApplicationDocumentsDirectory()).path;
+      return (await getApplicationDocumentsDirectory()).path;
     }
+  }
+
+  Future<void> initSystemPaths() async {
+    applicationDirectory = await getApplicationDirectory();
 
     loadedResourcesDir = '$applicationDirectory/loaded_resources';
     externalResourcesDir = '$applicationDirectory/external_resources';