Add benchmark inference_tract

Dockerfile

@@ -4,9 +4,9 @@ ENV LD_LIBRARY_PATH=/usr/local/lib
 ENV PATH=/usr/local/bin:$PATH
 CMD ["/bin/bash"]
 ENV DEBIAN_FRONTEND="noninteractive" TZ="America"
-ARG RUST_VERSION="nightly-2023-04-01"
+ARG RUST_VERSION="nightly-2024-06-09"
 ARG WASMTIME_REPO="https://github.com/bytecodealliance/wasmtime/"
-ARG WASMTIME_COMMIT="1bfe4b5" # v9.0.1
+ARG WASMTIME_COMMIT="cedf9aa" # v21.0.1
 ARG SIGHTGLASS_REPO="https://github.com/bytecodealliance/sightglass.git"
 ARG SIGHTGLASS_BRANCH="main"
 ARG SIGHTGLASS_COMMIT="e89fce0"

benchmarks/Dockerfile.rust

@@ -1,4 +1,4 @@
-FROM rust:1.70
+FROM rust:1.75
 RUN rustup target add wasm32-wasi
 WORKDIR /usr/src
 ADD rust-benchmark rust-benchmark

benchmarks/all.suite

@@ -12,6 +12,7 @@ blind-sig/benchmark.wasm
 bz2/benchmark.wasm
 hex-simd/benchmark.wasm
 # image-classification/image-classification-benchmark.wasm
+inference_tract/benchmark.wasm
 intgemm-simd/benchmark.wasm
 libsodium/libsodium-aead_aes256gcm2.wasm
 libsodium/libsodium-aead_aes256gcm.wasm

benchmarks/build.sh

@@ -38,16 +38,18 @@ print_header "Build benchmarks"
 CONTAINER_ID=$(set -x; docker create $IMAGE_NAME)
 (set -x; docker cp $CONTAINER_ID:/benchmark/. $TMP_BENCHMARK)
 
-# Verify benchmark is a valid Sightglass benchmark.
-print_header "Verify benchmark"
+# Copy benchmark.
+print_header "Copy benchmark"
 # From https://stackoverflow.com/a/246128:
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
-SIGHTGLASS_CARGO_TOML=$(dirname $SCRIPT_DIR)/Cargo.toml
 for WASM in $TMP_BENCHMARK/*.wasm; do
-    (set -x; cargo run --manifest-path $SIGHTGLASS_CARGO_TOML --quiet -- validate $WASM)
     (set -x; mv $WASM $BENCHMARK_DIR/)
 done;
 
+for MODEL in $TMP_BENCHMARK/*.pb; do
+    (set -x; mv $MODEL $BENCHMARK_DIR/)
+done;
+
 # Clean up.
 print_header "Clean up"
 (set -x; rm $TMP_TAR)

benchmarks/inference_tract/Dockerfile

@@ -0,0 +1,24 @@
+FROM rust:1.78 AS builder
+RUN rustup target add wasm32-wasi
+RUN mkdir /benchmark
+WORKDIR /usr/src
+
+# Compile mobile_net_v2_onnx
+ADD mobile_net_v2_onnx rust-benchmark
+WORKDIR /usr/src/rust-benchmark
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+RUN (cd mobile_net_v2_onnx; cargo build --release --target wasm32-wasi)
+RUN cp target/wasm32-wasi/release/*benchmark.wasm /benchmark/mobile_net_v2_onnx_benchmark.wasm
+WORKDIR /usr/src
+RUN rm -rf rust-benchmark
+
+
+# Compile mobile_net_v2_tensorflow
+ADD mobile_net_v2_tensorflow rust-benchmark
+WORKDIR /usr/src/rust-benchmark
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+RUN (cd mobile_net_v2_tensorflow; cargo build --release --target wasm32-wasi)
+RUN cp target/wasm32-wasi/release/*benchmark.wasm /benchmark/mobile_net_v2_tensorflow_benchmark.wasm
+RUN cp assets/mobilenet_v2_1.4_224_frozen.pb /benchmark/mobilenet_v2_1.4_224_frozen.pb
+WORKDIR /usr/src
+RUN rm -rf rust-benchmark

benchmarks/inference_tract/Dockerfile.native

@@ -0,0 +1,19 @@
+FROM rust:1.78 AS builder
+RUN mkdir /benchmark
+
+# Copy onnx to the `src` directory.
+ENV SRC=/usr/src/mobile_net_v2_onnx/
+WORKDIR $SRC
+ADD mobile_net_v2_onnx rust-benchmark
+COPY mobile_net_v2_onnx_native.patch ./
+COPY build_mobile_net_v2_onnx_native.sh ./
+COPY libengine.so /usr/lib/
+
+# Compile each of the benchmarks into the `/benchmark` directory.
+RUN ./build-native.sh
+
+# We copy the shared libraries to the `/benchmark` directory, where the client
+# expects it.
+WORKDIR /benchmark
+RUN cp $SRC/*so ./mobile_net_v2_onnx_benchmark.so
+RUN rm -rf rust-benchmark

benchmarks/inference_tract/README.md

@@ -0,0 +1,7 @@
+# Image Classification Wasmtime Benchmark
+
+A benchmark that runs an image classifier in pure Wasm. This can be used to
+benchmark the performance of float heavy computations.
+
+Note that the classifier model is not included in the repo because it is large
+and is instead downloaded if needed when running the `setup.sh` script.

benchmarks/inference_tract/build_mobile_net_v2_onnx_native.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Build inference_tract benchmark as a native shared library (Linux-only).
+#
+# Usage: ./build_mobile_net_v2_onnx_native.sh
+
+(set -x;)
+(rm -rf mobile_net_v2_onnx_native);
+(cp -r mobile_net_v2_onnx mobile_net_v2_onnx_native);
+(cp mobile_net_v2_onnx_native.patch mobile_net_v2_onnx_native);
+(cd mobile_net_v2_onnx_native; patch -Np1 -i ./mobile_net_v2_onnx_native.patch; mv src/main.rs src/lib.rs; cd -);
+(cd mobile_net_v2_onnx_native; cargo build --release; cp target/release/libbenchmark.so ../mobile_net_v2_onnx_benchmark.so; cd -);
+(set +x;)

benchmarks/inference_tract/build_mobile_net_v2_tensorflow_native.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Build inference_tract benchmark as native shared library (Linux-only).
+#
+# Usage: ./build_mobile_net_v2_tensorflow_native.sh
+
+(set -x;)
+(rm -rf mobile_net_v2_tensorflow_native);
+(cp -r mobile_net_v2_tensorflow mobile_net_v2_tensorflow_native);
+(cp mobile_net_v2_tensorflow_native.patch mobile_net_v2_tensorflow_native);
+(cd mobile_net_v2_tensorflow_native; patch -Np1 -i ./mobile_net_v2_tensorflow_native.patch; mv src/main.rs src/lib.rs; cd -);
+(cd mobile_net_v2_tensorflow_native; cargo build --release; cp target/release/libbenchmark.so ../mobile_net_v2_tensorflow_benchmark.so; cd -);
+(set +x;)