Benchmarking additions to oasis-test-runner and oasis-node

.buildkite/benchmarks.pipeline.yml

@@ -0,0 +1,153 @@
+# Copied from pipeline.yml.
+docker_plugin_default_config: &docker_plugin_default_config
+  image: "oasislabs/testing:0.3.0"
+  always_pull: true
+  workdir: /workdir
+  volumes:
+    - /var/lib/buildkite-agent/.coveralls:/root/.coveralls
+    - /var/lib/buildkite-agent/.codecov:/root/.codecov
+    # Shared Rust incremental compile caches.
+    - /var/tmp/cargo_ic/debug:/var/tmp/artifacts/debug/incremental
+    - /var/tmp/cargo_ic/debug_sgx:/var/tmp/artifacts/x86_64-unknown-linux-sgx/debug/incremental
+    # Shared Rust package checkouts directory.
+    - /var/tmp/cargo_pkg/git:/root/.cargo/git
+    - /var/tmp/cargo_pkg/registry:/root/.cargo/registry
+    # Shared Rust SGX standard library artifacts cache.
+    - /var/tmp/xargo_cache:/root/.xargo
+    # Shared Go package checkouts directory.
+    - /var/tmp/go_pkg:/root/go/pkg
+    # Intel SGX Application Enclave Services Manager (AESM) daemon running on
+    # the Buildkite host.
+    - /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket
+    - /var/tmp/benchmarks:/var/tmp/benchmarks
+  environment:
+    - "LC_ALL=C.UTF-8"
+    - "LANG=C.UTF-8"
+    - "CARGO_TARGET_DIR=/var/tmp/artifacts"
+    - "CARGO_INSTALL_ROOT=/root/.cargo"
+    - "GOPROXY=https://proxy.golang.org/"
+    - "SLACK_WEBHOOK_URL"
+    - "METRICS_PUSH_ADDR"
+    - "METRICS_QUERY_ADDR"
+    - "METRICS_SOURCE_GIT_BRANCH"
+    - "METRICS_TARGET_GIT_BRANCH"
+    - "TESTS"
+  propagate-environment: true
+  unconfined: true
+
+docker_plugin: &docker_plugin
+  oasislabs/docker#v3.0.1-oasis1:
+    <<: *docker_plugin_default_config
+
+steps:
+  ###############################################################
+  # The following three steps are copied from code.pipeline.yml #
+  ###############################################################
+  ############
+  # Build jobs
+  ############
+  # This label needs to be synced with runtime-ethereum's
+  # .buildkite/scripts/download_utils.sh.
+  - label: Build Go node
+    command:
+      - .buildkite/go/build.sh
+
+      # Upload the built artifacts.
+      - cd /workdir/go/oasis-node
+      - buildkite-agent artifact upload oasis-node
+      - cd /workdir/go/oasis-node/integrationrunner
+      - buildkite-agent artifact upload integrationrunner.test
+      - cd /workdir/go/oasis-test-runner
+      - buildkite-agent artifact upload oasis-test-runner
+      - cd /workdir/go/oasis-net-runner
+      - buildkite-agent artifact upload oasis-net-runner
+      - cd /workdir/go/oasis-remote-signer
+      - buildkite-agent artifact upload oasis-remote-signer
+    plugins:
+      <<: *docker_plugin
+
+  # This label needs to be synced with runtime-ethereum's
+  # .buildkite/scripts/download_utils.sh.
+  - label: Build Rust runtime loader
+    command:
+      - .buildkite/rust/build_generic.sh /workdir -p oasis-core-runtime-loader
+      - .buildkite/rust/build_generic.sh /workdir -p test-long-term-client
+      - .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-client
+      - .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-enc-client
+      - .buildkite/rust/build_generic.sh /workdir -p simple-keyvalue-ops-client
+
+      # Upload the built artifacts.
+      - cd /var/tmp/artifacts/default/debug
+      - buildkite-agent artifact upload oasis-core-runtime-loader
+      # Clients for E2E tests.
+      - buildkite-agent artifact upload test-long-term-client
+      - buildkite-agent artifact upload simple-keyvalue-client
+      - buildkite-agent artifact upload simple-keyvalue-enc-client
+      - buildkite-agent artifact upload simple-keyvalue-ops-client
+    agents:
+      buildkite_agent_size: large
+    plugins:
+      <<: *docker_plugin
+
+  ####################
+  # Runtime build jobs
+  ####################
+  # This label needs to be synced with runtime-ethereum's
+  # .buildkite/rust/test_runtime_and_gateway.sh and .buildkite/scripts/download_utils.sh.
+  - label: Build key manager runtime
+    command:
+      - .buildkite/rust/build_runtime.sh keymanager-runtime
+      - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keyvalue
+
+      # Upload the built artifacts.
+      - cd /var/tmp/artifacts/sgx/x86_64-fortanix-unknown-sgx/debug
+      - buildkite-agent artifact upload oasis-core-keymanager-runtime.sgxs
+      - buildkite-agent artifact upload simple-keyvalue.sgxs
+      - cd /var/tmp/artifacts/default/debug
+      - buildkite-agent artifact upload oasis-core-keymanager-runtime
+      - buildkite-agent artifact upload simple-keyvalue
+    agents:
+      buildkite_agent_size: large
+    plugins:
+      <<: *docker_plugin
+
+  ##########################################################
+  # Build benchmark analysis tool, perform tests, evaluate #
+  ##########################################################
+  - label: Build benchmark analysis tool
+    command:
+      - cd /workdir/go/extra/ba
+      - go build
+      - buildkite-agent artifact upload ba
+    plugins:
+      <<: *docker_plugin
+
+  # Wait for all jobs defined before this point
+  # to finish running in parallel before continuing.
+  - wait
+
+  - label: E2E tests
+    parallelism: 1
+    timeout_in_minutes: 90
+    command:
+      - .buildkite/scripts/download_e2e_test_artifacts.sh
+      - rm -rf /var/tmp/benchmarks/*
+      - .buildkite/scripts/test_e2e.sh --metrics.address $METRICS_PUSH_ADDR --metrics.push.interval 5s --num_runs 3 -t $TESTS
+    env:
+      TEST_BASE_DIR: /var/tmp/benchmarks
+    agents:
+      buildkite_agent_size: large
+      buildkite_agent_class: stable
+    plugins:
+      <<: *docker_plugin
+
+  # Wait for all jobs defined before this point
+  # to finish running in parallel before continuing.
+  - wait
+
+  - label: Benchmark analysis
+    command:
+      - .buildkite/scripts/download_benchmark_artifacts.sh
+      - .buildkite/scripts/daily_benchmark_analysis.sh
+    plugins:
+      <<: *docker_plugin

.buildkite/scripts/daily_benchmark_analysis.sh

@@ -0,0 +1,24 @@
+#! /bin/bash
+
+set -ux
+
+# Script invoked from .buildkite/benchmarks.pipeline.yml
+
+./go/extra/ba/ba cmp --metrics.address $METRICS_QUERY_ADDR --metrics.target.git_branch $METRICS_TARGET_GIT_BRANCH --metrics.source.git_branch $METRICS_SOURCE_GIT_BRANCH -t $TESTS 2>error.txt 1>out.txt
+BA_RETURN_CODE=$?
+
+# Escape double quotes for JSON.
+BA_STDOUT=`cat out.txt | sed "s/\"/\\\\\\\\\"/g"`
+BA_STDERR=`cat error.txt | sed "s/\"/\\\\\\\\\"/g"`
+
+if [ $BA_RETURN_CODE != 0 ]; then
+  # Post error to slack channel.
+  curl -H "Content-Type: application/json" \
+       -X POST \
+        --data "{\"text\": \"Daily oasis-core benchmarks failed.\", \"attachments\":[{\"title\":\"stdout\",\"text\":\"$BA_STDOUT\"}, {\"title\":\"stderr\",\"text\":\"$BA_STDERR\"}]}" \
+        "$SLACK_WEBHOOK_URL"
+
+    # Exit with non-zero exit code, so that the buildkite build will be
+    # marked as failed.
+    exit 1
+fi

.buildkite/scripts/download_benchmark_artifacts.sh

@@ -0,0 +1,14 @@
+#! /bin/bash
+
+####################################################
+# Download artifacts needed to perform benchmarks. #
+####################################################
+
+# Helpful tips on writing build scripts:
+# https://buildkite.com/docs/pipelines/writing-build-scripts
+set -euxo pipefail
+
+source .buildkite/scripts/common.sh
+
+# Extra tools
+download_artifact ba go/extra/ba 755

.buildkite/scripts/test_e2e.sh

@@ -44,7 +44,7 @@ ${WORKDIR}/go/oasis-test-runner/oasis-test-runner \
     --e2e.runtime.binary_dir ${WORKDIR}/target/${runtime_target}/debug \
     --e2e.runtime.loader ${WORKDIR}/target/default/debug/oasis-core-runtime-loader \
     --e2e.tee_hardware ${OASIS_TEE_HARDWARE:-""} \
-    --remote_signer.binary ${WORKDIR}/go/oasis-remote-signer/oasis-remote-signer \
+    --params.remote-signer/basic.binary ${WORKDIR}/go/oasis-remote-signer/oasis-remote-signer \
     --log.level info \
     ${BUILDKITE_PARALLEL_JOB_COUNT:+--parallel.job_count ${BUILDKITE_PARALLEL_JOB_COUNT}} \
     ${BUILDKITE_PARALLEL_JOB:+--parallel.job_index ${BUILDKITE_PARALLEL_JOB}} \

bench.sh

@@ -0,0 +1,85 @@
+#/bin/bash
+
+# bench.sh and parse_bench.py are used to benchmark specific e2e test using
+# various execution parameters and parse the results.
+# Accepted environmental variables are:
+# - GS: executor group size,
+# - RT: number of runtimes used,
+# - W: number of compute workers used.
+#
+# Because of syncing and flushing caches, you need to run this script as root.
+# For example:
+# sudo GS=2 RT=4 W=2 ./bench.sh multiple-runtimes
+#
+# If no environmental variables are set, this script benchmarks all GS and RT
+# combinations from 1..128 in power of 2. W will equal GS for each run.
+
+set -x
+
+if [ "$#" -ne 1 ]
+then
+	echo "Usage: $0 <TEST NAME>"
+	exit 1
+fi
+
+if [ "$USER" != "root" ]
+then
+       echo "This script should be run as root."
+       exit 2
+fi
+
+BENCH_RESULTS_DIR=bench-results
+TEST="$1"
+
+cleanup() {
+    killall -q mpstat
+    killall -q iostat
+    killall -q sar
+}
+
+trap cleanup EXIT
+trap "exit" INT
+
+mkdir -p bench-results
+
+# Environmental variables GS and RT.
+GROUP_SIZES=$GS
+RUNTIMES=$RT
+if [ -z "$GROUP_SIZES" ]
+then
+        GROUP_SIZES="1 2 3 4 6 8"
+fi
+if [ -z "$RUNTIMES" ]
+then
+        RUNTIMES="1 2 3 4 5 6"
+fi
+
+for G in $GROUP_SIZES; do
+  for R in $RUNTIMES; do
+    if [ -z "$W" ]
+    then
+      WORKERS=$G
+    else
+      WORKERS=$W
+    fi
+
+    echo "BENCHMARK: Executing test $TEST with executor group size $G, runtime count $R, and compute workers $WORKERS..."
+    BENCH_INFO=GS_$G.RT_$R.W_$WORKERS
+
+    # Flush all to disk, clear any caches.
+    sync; echo 3 > /proc/sys/vm/drop_caches
+
+    mpstat -P ALL 1 >$BENCH_RESULTS_DIR/$TEST.$BENCH_INFO.mpstat &
+    iostat -d /dev/sda -x 1 >$BENCH_RESULTS_DIR/$TEST.$BENCH_INFO.iostat &
+    sar -r 1 >$BENCH_RESULTS_DIR/$TEST.$BENCH_INFO.sar &
+
+
+    EXECUTOR_GROUP_SIZE=$G \
+     COMPUTE_RUNTIMES=$R \
+     COMPUTE_WORKERS=$WORKERS \
+      /usr/bin/time -o $BENCH_RESULTS_DIR/$TEST.$BENCH_INFO.time \
+      .buildkite/scripts/test_e2e.sh -t $TEST
+
+    cleanup
+  done
+done

common.mk

@@ -49,6 +49,8 @@ ifeq ($(and $(LATEST_TAG),$(IS_TAG)),NO)
 endif
 export VERSION
 
+GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
+
 # Try to compute the next version based on the latest tag of the origin remote
 # using the Punch tool.
 # First, all tags from the origin remote are fetched. Next, the latest tag on
@@ -90,7 +92,7 @@ GOFLAGS ?= -trimpath -v
 
 # Add Oasis Core's version as a linker string value definition.
 ifneq ($(VERSION),)
-	export GOLDFLAGS ?= "-X github.com/oasislabs/oasis-core/go/common/version.SoftwareVersion=$(VERSION)"
+	export GOLDFLAGS ?= "-X github.com/oasislabs/oasis-core/go/common/version.SoftwareVersion=$(VERSION) -X github.com/oasislabs/oasis-core/go/common/version.GitBranch=$(GIT_BRANCH)"
 endif
 
 # Go build command to use by default.

go/Makefile

@@ -10,7 +10,7 @@ all: build
 
 # Build.
 # List of Go binaries to build.
-go-binaries := oasis-node oasis-test-runner oasis-net-runner oasis-remote-signer extra/stats
+go-binaries := oasis-node oasis-test-runner oasis-net-runner oasis-remote-signer extra/stats extra/ba
 # List of test helpers to build.
 test-helpers := urkel
 # List of test vectors to generate.

go/common/version/version.go

@@ -55,6 +55,11 @@ var (
 	// by the linker.
 	SoftwareVersion = "0.0-unset"
 
+	// GitBranch is the name of the git branch of Oasis Core.
+	//
+	// This is mostly used for reporting and metrics.
+	GitBranch = ""
+
 	// RuntimeProtocol versions the protocol between the Oasis node(s) and
 	// the runtime.
 	//

go/extra/ba/README.md

@@ -0,0 +1,10 @@
+# Benchmark analysis
+
+Connects to prometheus and compares results of benchmark results.
+
+## Basic usage
+
+```bash
+$ ba/ba cmp \
+  --metrics.address http://localhost:9090
+```