test(bench): run bench-diff on CI (#991)

compares bench results against `master` and `v0.7.0`

Fixes #987

Signed-off-by: Xavier Coulon <[email protected]>

.github/workflows/ci-bench.yml

@@ -0,0 +1,48 @@
+name: ci-build
+on:
+  push:
+    branches:
+      - master
+    tags-ignore:
+      - '*.*'
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.16.x,1.17.x]
+        os: [ubuntu-latest]
+    name: Benchstat with Go ${{ matrix.go-version }}
+
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Cache dependencies
+      uses: actions/cache@v2
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles ('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+
+    - name: Verify parser
+      if: runner.os == 'Linux'
+      run: |
+        make verify-parser
+
+    - name: Bench and Diff
+      run: |
+        go install golang.org/x/perf/cmd/benchstat@latest
+        make bench-diff

make/bench.mk

@@ -1,8 +1,51 @@
+REPORTS_DIR=./tmp/bench/reports
+BENCH_COUNT ?= 10
+
+# Detecting GOPATH and removing trailing "/" if any
+GOPATH = $(realpath $(shell go env GOPATH))
+$(info GOPATH: ${GOPATH})
+
 .PHONY: bench
 ## run the top-level benchmarks
 bench: clean generate-optimized
-	@mkdir -p ./tmp/bench/reports
-	@go test -tags bench -bench=. -benchmem -count=10 -run=XXX \
+	@mkdir -p $(REPORTS_DIR)
+	@go test -tags bench -bench=. -benchmem -count=$(BENCH_COUNT) -run=XXX \
 		github.com/bytesparadise/libasciidoc \
-		| tee tmp/bench/reports/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench
-	@echo "generated tmp/bench/reports/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench"
+		| tee $(REPORTS_DIR)/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench
+	@echo "generated $(REPORTS_DIR)/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench"
+
+.PHONY: bench-diff
+## run the top-level benchmarks and compares with results of 'master' and 'v0.7.0'
+bench-diff: clean generate-optimized check-git-status
+	@mkdir -p $(REPORTS_DIR)
+	@go test -tags bench -bench=. -benchmem -count=$(BENCH_COUNT) -run=XXX \
+		github.com/bytesparadise/libasciidoc \
+		| tee $(REPORTS_DIR)/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench
+	@echo "generated $(REPORTS_DIR)/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench"
+	@git checkout master
+	@go test -tags bench -bench=. -benchmem -count=$(BENCH_COUNT) -run=XXX \
+		github.com/bytesparadise/libasciidoc \
+		| tee $(REPORTS_DIR)/master.bench
+	@echo "generated $(REPORTS_DIR)/master.bench"
+	@git checkout v0.7.0
+	@go test -tags bench -bench=. -benchmem -count=$(BENCH_COUNT) -run=XXX \
+		github.com/bytesparadise/libasciidoc \
+		| tee $(REPORTS_DIR)/v0.7.0.bench
+	@echo "generated $(REPORTS_DIR)/v0.7.0.bench"
+	@git checkout $(GIT_BRANCH_NAME)
+	@echo ""
+	@echo "Comparing with 'master' branch"
+	@$(GOPATH)/bin/benchstat $(REPORTS_DIR)/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench $(REPORTS_DIR)/master.bench
+	@echo ""
+	@echo "Comparing with 'v0.7.0' tag"
+	@$(GOPATH)/bin/benchstat $(REPORTS_DIR)/$(GIT_BRANCH_NAME)-$(GIT_COMMIT_ID_SHORT).bench $(REPORTS_DIR)/v0.7.0.bench
+
+check-git-status:
+ifneq ("$(shell git status --porcelain)","")
+	@echo "Repository contains uncommitted changes:"
+	@git status --porcelain 
+	@exit 1
+else
+	@echo "Repository has no uncommitted changes"
+endif
+

make/git.mk

@@ -1,10 +1,5 @@
-GIT_COMMIT_ID := $(shell git rev-parse HEAD)
-ifneq ($(shell git status --porcelain --untracked-files=no),)
-       GIT_COMMIT_ID := $(GIT_COMMIT_ID)-dirty
-endif
-
 GIT_COMMIT_ID_SHORT := $(shell git rev-parse --short HEAD)
-ifneq ($(shell git status --porcelain --untracked-files=no),)
+ifneq ($(shell git status --porcelain),)
        GIT_COMMIT_ID_SHORT := $(GIT_COMMIT_ID_SHORT)-dirty
 endif