diff --git a/Makefile b/Makefile index b0d019802..ce844ce65 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ BRANCH=`git rev-parse --abbrev-ref HEAD` COMMIT=`git rev-parse --short HEAD` +LATEST_RELEASE_BRANCH := release-$(shell grep -ohE "[0-9]+.[0-9]+" version/version.go) GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)" GOFILES = $(shell find . -name \*.go) @@ -94,3 +95,11 @@ test-failpoint: test-robustness: gofail-enable build sudo env PATH=$$PATH go test -v ${TESTFLAGS} ./tests/dmflakey -test.root sudo env PATH=$(PWD)/bin:$$PATH go test -v ${TESTFLAGS} ${ROBUSTNESS_TESTFLAGS} ./tests/robustness -test.root + +.PHONY: test-benchmark-compare +# Runs benchmark tests on the current git ref and the last release and compares +# the two. +test-benchmark-compare: + @git fetch + ./tests/compare_benchmarks.sh main + ./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH} diff --git a/tests/compare_benchmarks.sh b/tests/compare_benchmarks.sh new file mode 100755 index 000000000..d05a52dbd --- /dev/null +++ b/tests/compare_benchmarks.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# exit immediately when a command fails +set -e +# only exit with zero if all commands of the pipeline exit successfully +set -o pipefail +# error on unset variables +set -u + +[[ "$#" -eq 1 ]] || echo "One argument required, $# provided." + +REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)" +REF_TO_COMPARE=$1 + +RESULT_CURRENT="$(mktemp)-${REF_CURRENT}" +RESULT_TO_COMPARE="$(mktemp)-${REF_TO_COMPARE}" + +echo "" +echo "### Testing ${REF_CURRENT}" + +go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_CURRENT}" + +echo "" +echo "### Done testing ${REF_CURRENT}" + +echo "" +echo "### Testing ${REF_TO_COMPARE}" + +git checkout "${REF_TO_COMPARE}" + +go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_TO_COMPARE}" + +echo "" +echo "### Done testing ${REF_TO_COMPARE}" + +git checkout - + +echo "" +echo "### Result" +echo "old=${REF_TO_COMPARE} new=${REF_CURRENT}" + +benchstat "${RESULT_TO_COMPARE}" "${RESULT_CURRENT}"