diff --git a/.github/cloudbuild/android.yaml b/.github/cloudbuild/android.yaml index def198fab..ecf6f4752 100644 --- a/.github/cloudbuild/android.yaml +++ b/.github/cloudbuild/android.yaml @@ -53,8 +53,7 @@ steps: - id: build-apks name: gcr.io/$PROJECT_ID/mlperf-mobile-android:latest # Image upload usually takes only few seconds. - # However, if we generated a new image and the build failed, - # it can fail before the build finishes, canceling the upload. + # However, if we generated a new image and the build failed, it can cancel the upload. # Let's wait for the upload to finish before starting the actual build. waitFor: [] timeout: 10800s # 3 hours @@ -74,6 +73,8 @@ steps: cp bazel-bin/android/androidTest/mlperf_test_app.apk output/artifacts - id: instrument-test-on-firebase name: gcr.io/cloud-builders/gcloud + waitFor: + - build-apks timeout: 3600s # 1 hour entrypoint: bash args: @@ -89,8 +90,8 @@ steps: - id: crawler-test-on-firebase name: gcr.io/cloud-builders/gcloud waitFor: - - build-apks # don't wait for the other test to finish, run as soon as APKs are ready - timeout: 3600s # 1 hour (we sometimes wait 30+ minutes until the device we need is free) + - build-apks + timeout: 3600s # 1 hour entrypoint: bash args: - -xc diff --git a/.github/cloudbuild/flutter-android.yaml b/.github/cloudbuild/flutter-android.yaml new file mode 100644 index 000000000..3bdbed244 --- /dev/null +++ b/.github/cloudbuild/flutter-android.yaml @@ -0,0 +1,123 @@ +substitutions: + _IMAGE_NAME: mlperf-mobile-flutter-android +# Building apks with flutter requires a lot of memory. +# Builds on standard machines with 4GB of RAM can unexpectedly hang. +# Also the build is mostly CPU-intensive, so using 8-core machines +# reduces build time up to 3 times. +options: + machineType: 'E2_HIGHCPU_8' + +steps: +# We need this step to correctly identify dockerfile tag +- id: fetch-repo-history + name: gcr.io/cloud-builders/git + timeout: 10s + args: + - fetch + - --unshallow +# Download DOCKERFILE_COMMIT tag if it exists to skip docker image generation, +# or download the :latest tag and use it as a cache, +# or skip downloading if :latest doesn't exist yet. +# This being a separate step helps readability in Cloud Build console. +- id: cache-old-image + name: gcr.io/cloud-builders/docker + timeout: 600s # 10 minutes + entrypoint: bash + args: + - -xc + - | + DOCKERFILE_COMMIT=$$(git log -n 1 --pretty=format:%H -- flutter/android/docker/Dockerfile) + docker pull gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT \ + || docker pull gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest \ + || true +# This step overrides the :latest tag of the image, so that we can use it in later steps. +- id: build-new-image + name: gcr.io/cloud-builders/docker + timeout: 1800s # 30 minutes + entrypoint: bash + args: + - -xc + - | + DOCKERFILE_COMMIT=$$(git log -n 1 --pretty=format:%H -- flutter/android/docker/Dockerfile) + docker build \ + -t gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT \ + -t gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest \ + --cache-from gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT \ + --cache-from gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest \ + flutter/android/docker +# If the build fails artifacts are not uploaded automatically, so we save them manually before build +- id: push-new-image + name: gcr.io/cloud-builders/docker + timeout: 1800s # 30 minutes + entrypoint: bash + args: + - -xc + - | + DOCKERFILE_COMMIT=$$(git log -n 1 --pretty=format:%H -- flutter/android/docker/Dockerfile) + docker push gcr.io/$PROJECT_ID/$_IMAGE_NAME:$$DOCKERFILE_COMMIT + docker push gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest +- id: build-apks + name: gcr.io/$PROJECT_ID/$_IMAGE_NAME:latest + # Image upload usually takes only few seconds. + # However, if we generated a new image and the build failed, it can cancel the upload. + # Let's wait for the upload to finish before starting the actual build. + waitFor: [] + timeout: 10800s # 3 hours + entrypoint: bash + env: + - BAZEL_CACHE_ARG=--remote_cache=https://storage.googleapis.com/$_BAZEL_CACHE_BUCKET --google_default_credentials + args: + - -xc + - | + cd flutter || exit $? + make ci/android/test_apk || exit $? + + mkdir -p /workspace/output/artifacts || exit $? + cp /workspace/flutter/build/app/outputs/flutter-apk/app-release.apk /workspace/output/artifacts/mlperf_app_release.apk || exit $? + cp /workspace/flutter/build/app/outputs/apk/debug/app-debug.apk /workspace/output/artifacts/mlperf_app_test_main.apk || exit $? + cp /workspace/flutter/build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk /workspace/output/artifacts/mlperf_app_test_helper.apk || exit $? +- id: instrument-test-on-firebase + name: gcr.io/cloud-builders/gcloud + waitFor: + - build-apks + timeout: 3600s # 1 hour + entrypoint: bash + args: + - -xc + - | + # redfin is Pixel 5e + gcloud firebase test android run \ + --type instrumentation \ + --app output/artifacts/mlperf_app_test_main.apk \ + --test output/artifacts/mlperf_app_test_helper.apk \ + --timeout 45m \ + --device model=redfin,version=30,locale=en,orientation=portrait +- id: crawler-test-on-firebase + name: gcr.io/cloud-builders/gcloud + waitFor: + - build-apks + timeout: 3600s # 1 hour + entrypoint: bash + args: + - -xc + - | + # x1q is SM-G981U1 (Samsung Galaxy S20 5G) + gcloud firebase test android run \ + --type robo \ + --app output/artifacts/mlperf_app_release.apk \ + --device model=x1q,version=29,locale=en,orientation=portrait \ + --timeout 90s + +# We uploaded both tags earlier, but this option also adds them to the artifacts page of the build +images: +- gcr.io/$PROJECT_ID/$_IMAGE_NAME + +artifacts: + objects: + location: gs://$_ARTIFACT_BUCKET/$_ARTIFACT_FOLDER/$COMMIT_SHA-flutter + paths: + - output/artifacts/mlperf_app_release.apk + - output/artifacts/mlperf_app_test_main.apk + - output/artifacts/mlperf_app_test_helper.apk + +timeout: 18000s # 5 hours diff --git a/flutter/Makefile b/flutter/Makefile index 7c36e5ca9..c180ce8ac 100644 --- a/flutter/Makefile +++ b/flutter/Makefile @@ -29,9 +29,7 @@ _bazel_links_arg=--symlink_prefix ${BAZEL_LINKS_DIR} --experimental_no_product_n .PHONY: cpp-ios cpp-ios: @# NOTE: add `--copt -g` for debug info (but the resulting library would be 0.5 GiB) - bazel \ - ${BAZEL_CACHE_FLAG} \ - build --config=ios_fat64 -c opt //flutter/cpp/flutter:ios_backend_fw_static + bazel ${BAZEL_OUTPUT_ROOT_ARG} build --config=ios_fat64 -c opt //flutter/cpp/flutter:ios_backend_fw_static rm -rf ${_xcode_fw} cp -a ${_bazel_ios_fw} ${_xcode_fw} @@ -48,6 +46,33 @@ debug_flags_windows=-c dbg --copt /Od --copt /Z7 --linkopt -debug .PHONY: android android: backend-bridge-android backends/tflite-android prepare-flutter +.PHONY: android/apk +android/apk: android + flutter clean + @# take results from build/app/outputs/flutter-apk/app-release.apk + flutter build apk + +.PHONY: ci/android/test_apk +ci/android/test_apk: android/apk + @# take results from build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk + cd android && ./gradlew app:assembleAndroidTest + @# take results from build/app/outputs/apk/debug/app-debug.apk + cd android && ./gradlew app:assembleDebug -Ptarget=integration_test/first_test.dart + +.PHONY: docker/android/apk +docker/android/apk: android/docker/image + @# if the build fails with java.io.IOException: Input/output error + @# remove file android/gradle/wrapper/gradle-wrapper.jar + MSYS2_ARG_CONV_EXCL="*" docker run --rm -it --user `id -u`:`id -g` -v $(CURDIR)/..:/mnt/project mlcommons/mlperf_mobile_flutter bash -c "cd /mnt/project/flutter && make prepare-flutter && make android/apk" + +.PHONY: android/docker/image +android/docker/image: build/docker/mlperf_mobile_flutter.stamp + +build/docker/mlperf_mobile_flutter.stamp: android/docker/Dockerfile + docker image build -t mlcommons/mlperf_mobile_flutter android/docker + mkdir -p build/docker + touch $@ + .PHONY: windows/docker/image windows/docker/image: docker build -t mlperf_mobile_flutter:windows-1.0 windows/docker @@ -135,7 +160,7 @@ windows/flutter-release: .PHONY: backend-bridge-windows backend-bridge-windows: - bazel build ${_bazel_links_arg} --config=windows -c opt //flutter/cpp/flutter:backend_bridge.dll + bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=windows -c opt //flutter/cpp/flutter:backend_bridge.dll cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/flutter/cpp/flutter/backend_bridge.dll mkdir -p build/win-dlls/ rm -f build/win-dlls/backend_bridge.dll @@ -143,7 +168,7 @@ backend-bridge-windows: .PHONY: backend-bridge-android backend-bridge-android: - bazel build ${_bazel_links_arg} --config=android_arm64 -c opt //flutter/cpp/flutter:libbackendbridge.so + bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=android_arm64 -c opt //flutter/cpp/flutter:libbackendbridge.so cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/flutter/cpp/flutter/libbackendbridge.so mkdir -p android/app/src/main/jniLibs/arm64-v8a rm -f android/flutter/app/src/main/jniLibs/arm64-v8a/libbackendbridge.so @@ -160,7 +185,7 @@ backend-bridge-android: .PHONY: backends/tflite-windows backends/tflite-windows: - bazel build ${_bazel_links_arg} --config=windows -c opt //mobile_back_tflite:tflitebackenddll + bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=windows -c opt //mobile_back_tflite:tflitebackenddll cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.dll mkdir -p build/win-dlls/backends rm -f build/win-dlls/backends/libtflitebackend.dll @@ -177,7 +202,7 @@ backends/tflite-windows: .PHONY: backends/tflite-android backends/tflite-android: - bazel build ${_bazel_links_arg} --config=android_arm64 -c opt //mobile_back_tflite:tflitebackend + bazel build ${BAZEL_CACHE_ARG} ${_bazel_links_arg} --config=android_arm64 -c opt //mobile_back_tflite:tflitebackend cd .. && chmod +w ${BAZEL_LINKS_DIR}bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so mkdir -p android/app/src/main/jniLibs/arm64-v8a rm -f android/app/src/main/jniLibs/arm64-v8a/libtflitebackend.so diff --git a/flutter/android/.gitignore b/flutter/android/.gitignore index ad15ec92c..9a08e12f8 100644 --- a/flutter/android/.gitignore +++ b/flutter/android/.gitignore @@ -13,3 +13,4 @@ key.properties **/*.jks /app/src/main/jniLibs/ +.kotlin diff --git a/flutter/android/app/build.gradle b/flutter/android/app/build.gradle index 32edcfd9e..a3fd074e1 100644 --- a/flutter/android/app/build.gradle +++ b/flutter/android/app/build.gradle @@ -47,6 +47,7 @@ android { targetSdkVersion 30 versionCode flutterVersionCode.toInteger() versionName flutterVersionName + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } buildTypes { @@ -64,4 +65,7 @@ flutter { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test:runner:1.2.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' } diff --git a/flutter/android/app/src/androidTest/java/org/mlcommons/android/mlperfbench/MainActivityTest.java b/flutter/android/app/src/androidTest/java/org/mlcommons/android/mlperfbench/MainActivityTest.java new file mode 100644 index 000000000..04d5815a6 --- /dev/null +++ b/flutter/android/app/src/androidTest/java/org/mlcommons/android/mlperfbench/MainActivityTest.java @@ -0,0 +1,13 @@ +package org.mlcommons.android.mlperfbench; + +import androidx.test.rule.ActivityTestRule; +import dev.flutter.plugins.integration_test.FlutterTestRunner; +import org.junit.Rule; +import org.junit.runner.RunWith; + +@RunWith(FlutterTestRunner.class) +public class MainActivityTest { + @Rule + public ActivityTestRule rule = + new ActivityTestRule<>(MainActivity.class, true, false); +} diff --git a/flutter/android/docker/Dockerfile b/flutter/android/docker/Dockerfile new file mode 100644 index 000000000..eb03d8dd6 --- /dev/null +++ b/flutter/android/docker/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:20.04 + +# Without DEBIAN_FRONTEND=noninteractive arg apt-get waits for user input. +# Docker desktop shows all previously defined args for each of the commands, +# which makes reading layer info harder, so we set this env for for individual commands only. +# JDK package downloads ~500 MB from slow mirrors, which can take a lot of time, +# so a separate layer for it makes image rebuild faster in case we change any other dependencies. +RUN apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-8-jdk-headless +RUN apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + apt-transport-https \ + curl \ + git \ + gnupg \ + make \ + protobuf-compiler \ + python3 \ + python3-pip +RUN ln -s /usr/bin/python3 /usr/bin/python +RUN python3 -m pip install numpy absl-py + +# Bazel is not present in standard repositories +RUN curl -L https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ + echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ + apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y bazel=4.2.1 + +ENV ANDROID_SDK_ROOT=/opt/android +WORKDIR $ANDROID_SDK_ROOT/cmdline-tools +# sdkmanager expects to be placed into `$ANDROID_SDK_ROOT/cmdline-tools/tools` +RUN curl -L https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip | jar x && \ + mv cmdline-tools tools && \ + chmod --recursive +x tools/bin +ENV PATH=$PATH:$ANDROID_SDK_ROOT/cmdline-tools/tools/bin + +RUN yes | sdkmanager --licenses >/dev/null +# Build tools 29.0.2 are required by Flutter 2.5.3 +# Build tools 30 are required by bazel 4.2.1 +RUN yes | sdkmanager \ + "tools" \ + "platform-tools" \ + "build-tools;29.0.2" \ + "build-tools;30.0.3" \ + "platforms;android-30" +# Install NDK in a separate layer to decrease max layer size. +RUN yes | sdkmanager "ndk;21.4.7075529" +ENV ANDROID_NDK_ROOT=$ANDROID_SDK_ROOT/ndk/21.4.7075529 +ENV ANDROID_NDK_HOME=$ANDROID_NDK_ROOT + +ENV HOME=/image-workdir +WORKDIR $HOME + +ENV PUB_CACHE=$HOME/flutter/.pub-cache +ENV PATH=$PATH:$HOME/flutter/bin:$PUB_CACHE/bin +RUN curl -L https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.5.3-stable.tar.xz | tar Jxf - +RUN dart pub global activate protoc_plugin + +ENV GRADLE_USER_HOME=$HOME/.gradle +ENV ANDROID_SDK_HOME=$HOME/.android + +RUN mkdir $ANDROID_SDK_HOME && \ + keytool -genkey -v -keystore $ANDROID_SDK_HOME/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US" + +RUN chmod --recursive a=u $HOME diff --git a/flutter/integration_test/first_test.dart b/flutter/integration_test/first_test.dart index 779bef00d..a4e214caa 100644 --- a/flutter/integration_test/first_test.dart +++ b/flutter/integration_test/first_test.dart @@ -1,7 +1,6 @@ import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:integration_test/integration_test.dart'; -import 'package:path_provider/path_provider.dart'; import 'package:shared_preferences/shared_preferences.dart'; import 'dart:io'; import 'dart:convert'; @@ -9,6 +8,8 @@ import 'dart:convert'; import 'package:mlcommons_ios_app/main.dart' as app; import 'package:mlcommons_ios_app/ui/main_screen.dart'; import 'package:mlcommons_ios_app/ui/result_screen.dart'; +import 'package:mlcommons_ios_app/benchmark/resource_manager.dart' + as resource_manager; void main() { final splashPauseSeconds = 4; @@ -63,11 +64,13 @@ void main() { reason: 'Test results were not found'); final applicationDirectory = - (await getApplicationDocumentsDirectory()).path; + await resource_manager.ResourceManager.getApplicationDirectory(); final jsonResultPath = '$applicationDirectory/result.json'; final file = File(jsonResultPath); - expect(await file.exists(), true, reason: 'Result.json does not exist'); + expect(await file.exists(), true, + reason: + 'Result.json does not exist: file $applicationDirectory/result.json is not found'); final jsonResultContent = await file.readAsString(); final results = jsonDecode(jsonResultContent); diff --git a/flutter/lib/benchmark/resource_manager.dart b/flutter/lib/benchmark/resource_manager.dart index 4148d27ff..b522f2df9 100644 --- a/flutter/lib/benchmark/resource_manager.dart +++ b/flutter/lib/benchmark/resource_manager.dart @@ -107,7 +107,7 @@ class ResourceManager { return defaultBenchmarksConfiguration; } - Future initSystemPaths() async { + static Future getApplicationDirectory() async { // applicationDirectory should be visible to user Directory? dir; if (Platform.isIOS) { @@ -116,12 +116,18 @@ class ResourceManager { dir = await getExternalStorageDirectory(); } else if (Platform.isWindows) { dir = await getDownloadsDirectory(); + } else { + throw 'unsupported platform'; } if (dir != null) { - applicationDirectory = dir.path; + return dir.path; } else { - applicationDirectory = (await getApplicationDocumentsDirectory()).path; + return (await getApplicationDocumentsDirectory()).path; } + } + + Future initSystemPaths() async { + applicationDirectory = await getApplicationDirectory(); loadedResourcesDir = '$applicationDirectory/loaded_resources'; externalResourcesDir = '$applicationDirectory/external_resources';