Patch latest od nms 1.1 (#240)

* Add Pixel backend (#77) * Add Pixel backend * add license header to resize_argmax_op* * Add mobile_back_pixel into README (#85) * Fix pixel backend hardware matching Co-authored-by: Po-Hsien Chu <[email protected]> Co-authored-by: stegben <[email protected]>
mlcommons · Feb 1, 2022 · b957f96 · b957f96
1 parent 7fd5b8d
commit b957f96
Show file tree

Hide file tree

Showing 19 changed files with 1,539 additions and 11 deletions.
diff --git a/Makefile b/Makefile
@@ -22,6 +22,7 @@ SAMSUNG_BACKEND=
 all: app
 
 include mobile_back_tflite/tflite_backend.mk
+include mobile_back_pixel/pixel_backend.mk
 include mobile_back_qti/make/qti_backend.mk
 
 
@@ -95,7 +96,7 @@ app: output/mlperf_mobile_docker_1_0.stamp ${QTI_DEPS}
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
 		${COMMON_DOCKER_FLAGS} \
-                ${QTI_BACKEND} ${SAMSUNG_BACKEND} ${MEDIATEK_BACKEND} \
+                ${QTI_BACKEND} ${SAMSUNG_BACKEND} ${MEDIATEK_BACKEND} ${PIXEL_BACKEND} \
 		--fat_apk_cpu=arm64-v8a \
 		//android/java/org/mlperf/inference:mlperf_app
 	@cp output/`readlink bazel-bin`/android/java/org/mlperf/inference/mlperf_app.apk output/mlperf_app.apk
@@ -106,7 +107,7 @@ app_x86_64: output/mlperf_mobile_docker_1_0.stamp
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
 		${COMMON_DOCKER_FLAGS} \
-                ${QTI_BACKEND} ${SAMSUNG_BACKEND} ${MEDIATEK_BACKEND} \
+                ${QTI_BACKEND} ${SAMSUNG_BACKEND} ${MEDIATEK_BACKEND} ${PIXEL_BACKEND} \
 		--fat_apk_cpu=x86_64 \
 		//android/java/org/mlperf/inference:mlperf_app
 	@cp output/`readlink bazel-bin`/android/java/org/mlperf/inference/mlperf_app.apk output/mlperf_app_x86_64.apk
@@ -117,7 +118,7 @@ test_app: output/mlperf_mobile_docker_1_0.stamp
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
 		${COMMON_DOCKER_FLAGS} \
-                ${QTI_BACKEND} ${SAMSUNG_BACKEND} ${MEDIATEK_BACKEND} \
+                ${QTI_BACKEND} ${SAMSUNG_BACKEND} ${MEDIATEK_BACKEND} ${PIXEL_BACKEND} \
 		--fat_apk_cpu=x86_64,arm64-v8a \
 		//androidTest:mlperf_test_app
 	@cp output/`readlink bazel-bin`/android/androidTest/mlperf_test_app.apk output/mlperf_test_app.apk

diff --git a/README.md b/README.md
@@ -0,0 +1,47 @@
+# MLPerf™ Mobile App
+
+This project contains the MLPerf mobile app, an app-based implementationn of
+[MLPerf Inference](https://github.com/mlperf/inference) tasks.
+
+## Overview
+
+The MLPerf app offers a simple mobile UI for executing MLPerf inference tasks
+and comparing results. The user can select a task, a supported reference model
+(float or quantized), and initiate both latency and accuracy validation for that
+task. As single-stream represents the most common inference execution on mobile
+devices, that is the default mode of inference measurement, with the results
+showing the 90%-ile latency and the task-specific accuracy metric result (e.g.,
+top-1 accuracy for image classification).
+
+Several important mobile-specific considerations are addressed in the app:
+
+*   Limited disk space - Certain datasets are quite large (multiple gigabytes),
+    which makes an exhaustive evaluation difficult. By default, the app does not
+    include the full dataset for validation. The client can optionally push part
+    or all of the task validation datasets, depending on their use-case.
+*   Device variability - The number of CPU, GPU and DSP/NPU hardware
+    permutations in the mobile ecosystem is quite large. To this end, the app
+    affords the option to customize hardware execution, e.g., adjusting the
+    number of threads for CPU inference, enabling GPU acceleration, or NN API
+    acceleration (Android’s ML abstraction layer for accelerating inference).
+
+The initial version of the app builds off of a lightweight, C++ task evaluation
+pipeline originally built for
+[TensorFlow Lite](https://www.tensorflow.org/lite/). Most of the default MLPerf
+inference reference implementations are built in Python, which is generally
+incompatible with mobile deployment. This C++ evaluation pipeline has a minimal
+set of dependencies for pre-processing datasets and post-processing, is
+compatible with iOS and Android (as well as desktop platforms), and integrates
+with the standard
+[MLPerf LoadGen library](https://github.com/mlperf/inference/tree/master/loadgen).
+While the initial version of the app uses TensorFlow Lite as the default
+inference engine, the plan is to support addition of alternative inference
+frameworks contributed by the broader MLPerf community.
+
+## Folder Structure
+
+* android - Contains the Android version of the app
+* mobile_back_qti - QTI backend for Android
+* mobile_back_samsung - Samsung backend for Android
+* mobile_back_pixel - Google Pixel backend for Android
+* mobile_back_tflite - Combined TFLite / MediaTek backends for Android
diff --git a/android/java/org/mlperf/inference/BUILD b/android/java/org/mlperf/inference/BUILD
@@ -58,6 +58,18 @@ config_setting(
     },
 )
 
+string_flag(
+    name = "with_pixel",
+    build_setting_default = "0",
+)
+
+config_setting(
+    name = "use_pixel",
+    flag_values = {
+        ":with_pixel": "1",
+    },
+)
+
 java_lite_proto_library(
     name = "mlperf_task_java_proto_lite",
     deps = ["//android/cpp/proto:mlperf_task_proto"],
@@ -73,9 +85,12 @@ genrule(
     ],
     cmd = "cat $(location Backends.java.in) " +
           select({
-              ":use_qti": "| sed -e 's/QTI/\"qti\",/' ",
-              "//conditions:default": "| sed -e 's/QTI//' ",
+              ":use_pixel": "| sed -e 's/PIXEL/\"tflitepixel\",/' ",
+              "//conditions:default": "| sed -e 's/PIXEL//' ",
           }) + select({
+        ":use_qti": "| sed -e 's/QTI/\"qti\",/' ",
+        "//conditions:default": "| sed -e 's/QTI//' ",
+    }) + select({
         ":use_samsung": "| sed -e 's/SAMSUNG/\"samsung\",/' ",
         "//conditions:default": "| sed -e 's/SAMSUNG//'",
     }) + select({
@@ -182,13 +197,14 @@ cc_library(
     deps = [
         "//mobile_back_tflite:tflitebackend",
     ] + select({
-        ":use_samsung": [
-            "//mobile_back_samsung:samsungbackend",
-        ],
+        ":use_samsung": ["//mobile_back_samsung:samsungbackend"],
         "//conditions:default": [],
     }) + select({
         ":use_mediatek": ["//mobile_back_tflite:tfliteneuronbackend"],
         "//conditions:default": [],
+    }) + select({
+        ":use_pixel": ["//mobile_back_pixel:tflitepixelbackend"],
+        "//conditions:default": [],
     }),
 )
 

diff --git a/android/java/org/mlperf/inference/Backends.java.in b/android/java/org/mlperf/inference/Backends.java.in
@@ -5,5 +5,5 @@ public final class Backends {
     private Backends() {
     }
 
-    public static final String[] BACKEND_LIST = {QTI SAMSUNG MEDIATEK "tflite"};
+    public static final String[] BACKEND_LIST = {QTI SAMSUNG MEDIATEK PIXEL "tflite"};
 }
diff --git a/mobile_back_pixel/BUILD b/mobile_back_pixel/BUILD
@@ -0,0 +1,9 @@
+package(
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],  # Apache 2.0
+)
+
+cc_library(
+    name = "tflitepixelbackend",
+    srcs = ["//mobile_back_pixel/cpp/backend_tflite:libtflitepixelbackend.so"],
+)
diff --git a/mobile_back_pixel/Makefile b/mobile_back_pixel/Makefile
@@ -0,0 +1,111 @@
+# Copyright 2020-2021 The MLPerf Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+
+this_mkfile := $(abspath $(lastword $(MAKEFILE_LIST)))
+TOPDIR := $(shell dirname ${this_mkfile})
+BUILDDIR=${TOPDIR}/build
+
+USERID=$(shell id -u)
+GROUPID=$(shell id -g)
+
+SRCFILES=\
+Makefile \
+cpp/BUILD \
+cpp/c/BUILD \
+cpp/c/version_script.lds \
+cpp/c/backend_c.h \
+cpp/c/type.h \
+cpp/backend_tflite/tflite_c.cc \
+cpp/backend_tflite/tflite_settings.h \
+cpp/backend_tflite/utils.h \
+cpp/backend_tflite/BUILD
+
+ifeq (${USE_PROXY_WORKAROUND},1)
+  export PROXY_WORKAROUND1=\
+	-v /etc/ssl/certs:/etc/ssl/certs \
+	-v /usr/share/ca-certificates:/usr/share/ca-certificates \
+	-v /usr/share/ca-certificates-java:/usr/share/ca-certificates-java
+
+  export PROXY_WORKAROUND2=--host_jvm_args -Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts
+else
+  export PROXY_WORKAROUND1=
+  export PROXY_WORKAROUND2=
+endif
+
+all: ${BUILDDIR}/lib_x86_64/libtflitebackend.so ${BUILDDIR}/lib_arm64/libtflitebackend.so
+
+.PHONY: clean docker_image
+
+USER_ID=$(shell id -u)
+
+${BUILDDIR}/mlperf_mobile_image.stamp: ${TOPDIR}/docker/mlperf_mobile/Dockerfile
+	@mkdir -p ${BUILDDIR}
+	@docker image build -t mlcommons/mlperf_mobile:1.0 docker/mlperf_mobile
+	@touch $@
+
+docker_image: ${BUILDDIR}/mlperf_mobile_image.stamp
+
+${BUILDDIR}/lib_arm64/libtflitebackend.so: ${SRCFILES} docker_image
+	@echo "Building arm64 libtflitebackend.so"
+	@mkdir -p ${BUILDDIR}/lib_arm64
+	@mkdir -p ${BUILDDIR}/cache
+	@docker run \
+	        -e USER=mlperf \
+                ${PROXY_WORKAROUND1} \
+		-v $(CURDIR):/tflite_backend \
+		-v ${BUILDDIR}/cache:/cache \
+		-w /tflite_backend \
+	        -u ${USERID}:${GROUPID} \
+		mlcommons/mlperf_mobile:1.0 bazel-3.7.2 ${PROXY_WORKAROUND2} \
+                --output_user_root=/cache/bazel build \
+		-c opt --cxxopt='--std=c++14' \
+                --cxxopt='-Wno-deprecated-declarations' \
+                --cxxopt='-Wno-unknown-attributes' \
+                --host_cxxopt='--std=c++14'  \
+                --host_cxxopt='-Wno-deprecated-declarations' \
+                --host_cxxopt='-Wno-class-memaccess' \
+		--config=android_arm64 \
+		:tflitebackend
+	@cp build/`readlink bazel-out`/arm64-v8a-opt/bin/cpp/backend_tflite/libtflitebackend.so $@
+	@chmod 777 $@
+
+${BUILDDIR}/lib_x86_64/libtflitebackend.so: ${SRCFILES} docker_image
+	@echo "Building x86_64 libtflitebackend.so"
+	@mkdir -p ${BUILDDIR}/lib_x86_64
+	@mkdir -p ${BUILDDIR}/cache
+	docker run \
+	        -e USER=mlperf \
+                ${PROXY_WORKAROUND1} \
+		-v $(CURDIR):/tflite_backend \
+		-v ${BUILDDIR}/cache:/cache \
+		-w /tflite_backend \
+	        -u ${USERID}:${GROUPID} \
+		 mlcommons/mlperf_mobile:1.0 bazel-3.7.2 ${PROXY_WORKAROUND2} \
+                --output_user_root=/cache/bazel build \
+		-c opt --cxxopt='--std=c++14' \
+                --cxxopt='-Wno-deprecated-declarations' \
+                --cxxopt='-Wno-unknown-attributes' \
+                --host_cxxopt='--std=c++14'  \
+                --host_cxxopt='-Wno-deprecated-declarations' \
+                --host_cxxopt='-Wno-class-memaccess' \
+		--config=android_x86_64 \
+		:tflitebackend
+	@cp build/`readlink bazel-out`/x86_64-opt/bin/cpp/backend_tflite/libtflitebackend.so $@
+	@chmod 777 $@
+
+clean:
+	@rm -rf bazel-bin bazel-testlogs bazel-out bazel-tflite_backend
+	@rm -rf ${BUILDDIR}
+
diff --git a/mobile_back_pixel/README.md b/mobile_back_pixel/README.md
@@ -0,0 +1,14 @@
+# Mobile backend tflite
+
+Build so-library for selected architecture:
+```bash
+bazel build -c opt \
+    --cxxopt='--std=c++14' \
+    --host_cxxopt='--std=c++14' \
+    --host_cxxopt='-Wno-deprecated-declarations' \
+    --host_cxxopt='-Wno-class-memaccess' \
+    --cxxopt='-Wno-deprecated-declarations' \
+    --cxxopt='-Wno-unknown-attributes' \
+    --fat_apk_cpu={x86_64|arm64-v8a|armeabi-v7a} \
+    //cpp/backend_tflite:libtflitebackend.so
+```
diff --git a/mobile_back_pixel/cpp/BUILD b/mobile_back_pixel/cpp/BUILD
@@ -0,0 +1,18 @@
+# Copyright 2019 The MLPerf Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+package(
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],  # Apache 2.0
+)
diff --git a/mobile_back_pixel/cpp/backend_tflite/BUILD b/mobile_back_pixel/cpp/backend_tflite/BUILD
@@ -0,0 +1,84 @@
+# Copyright 2019-2021 The MLPerf Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+load(
+    "@org_tensorflow//tensorflow/lite:build_def.bzl",
+    "tflite_copts",
+    "tflite_jni_binary",
+)
+
+package(
+    default_visibility = ["//visibility:public"],
+    licenses = ["notice"],  # Apache 2.0
+)
+
+cc_library(
+    name = "resize_bilinear_op",
+    srcs = [
+        "resize_argmax_op.cc",
+    ],
+    hdrs = [
+        "resize_argmax_op.h",
+    ],
+    deps = [
+        "@org_tensorflow//tensorflow/lite/c:common",
+        "@org_tensorflow//tensorflow/lite/kernels:kernel_util",
+        "@org_tensorflow//tensorflow/lite/core/api",
+        "@org_tensorflow//tensorflow/lite/kernels:cpu_backend_context",
+        "@org_tensorflow//tensorflow/lite/kernels:cpu_backend_threadpool",
+    ],
+)
+
+cc_library(
+    name = "tflite_pixel",
+    srcs = [
+        "tflite_pixel.cc",
+    ],
+    hdrs = [
+        "tflite_settings_pixel.h",
+        "thread_pool.h",
+    ],
+    copts = tflite_copts() + select({
+        "//android/commonlibs:use_asan": [
+            "-fsanitize=address",
+            "-g",
+            "-O1",
+            "-fno-omit-frame-pointer",
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":resize_bilinear_op",
+        "//android/cpp/c:headers",
+        "@org_tensorflow//tensorflow/lite/c:c_api",
+        "@org_tensorflow//tensorflow/lite/c:c_api_experimental",
+        "@org_tensorflow//tensorflow/lite/c:common",
+        "@org_tensorflow//tensorflow/core:tflite_portable_logging",
+    ] + select({
+        "@org_tensorflow//tensorflow:android": [
+            "@org_tensorflow//tensorflow/lite/delegates/gpu:delegate",
+        ],
+        "//conditions:default": [],
+    }),
+    alwayslink = 1,
+)
+
+tflite_jni_binary(
+    name = "libtflitepixelbackend.so",
+    exported_symbols = "//android/cpp/c:exported_symbols.lds",
+    linkscript = "//android/cpp/c:version_script.lds",
+    deps = [
+        ":tflite_pixel",
+    ],
+)