Merge cockroachdb#127228

127228: workflow: add cockroach-microbench-ci step in github actions r=rickystewart,herkolategan a=sambhav-jain-16

Fixes: cockroachdb#106661
Release note: None
Epic: https://cockroachlabs.atlassian.net/browse/CRDB-29662

This change intends to add a new CI step which will run `BenchmarkKV` microbenchmark for the current commit and compares the result with that of the base commit's results which is stored in a cloud bucket.

This is needed to avoid minor perf regressions so that any regression is caught in the PR itself. The current threshold is set to 5%.

There are highly 3 steps
1. Run benchmarks 
2. If this is PR branch, download the latest commit of the target branch's bench results from the cloud storage and compare the results. If there's a regression fail here itself
3. If it is a push step (PR merge to master/release), upload the result to the cloud storage with the current commit sha.

Currently, this change won't block the PR. However it will still run the microbenchmarks and upload the results during the push step.

Co-authored-by: Sambhav Jain <[email protected]>

.github/workflows/github-actions-essential-ci.yml

@@ -344,3 +344,53 @@ jobs:
       - name: clean up
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
+  cockroach-microbench-ci:
+    runs-on: [ self-hosted, basic_runner_group ]
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1  # Fetch only the latest commit
+
+      - name: Get the latest commit of the target branch
+        id: get_latest_commit_target_branch
+        run: |
+          # Get the target branch name
+          TARGET_BRANCH=${{ github.event.pull_request.base.ref }}
+          
+          # Fetch the latest commit of the target branch
+          git fetch origin $TARGET_BRANCH --depth=1
+          
+          # Get the latest commit hash of the target branch
+          LATEST_COMMIT=$(git rev-parse FETCH_HEAD)
+          
+          # Output the latest commit hash
+          echo "Latest commit on target branch ($TARGET_BRANCH): $LATEST_COMMIT"
+          
+          # Set the latest commit hash as an output variable
+          echo "::set-output name=latest_commit::$LATEST_COMMIT"
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
+      - run: ./build/github/get-engflow-keys.sh
+      - run: ./build/github/prepare-summarize-build.sh
+      - name: run scripts
+        run: ./build/github/cockroach-microbench-ci.sh
+        env:
+          COMPARE_THRESHOLD: "5.00"
+          PUSH_STEP: ${{ github.event_name == 'push' }}
+          BASE_SHA: ${{ steps.get_latest_commit_target_branch.outputs.latest_commit }}
+
+          # TODO(sambhav-jain-16): add Performance note check and use this flag. Currently set to false (https://github.com/cockroachdb/cockroach/issues/106661)
+          SKIP_COMPARISON: false
+      - name: upload build results
+        run: ./build/github/summarize-build.sh bes.bin
+        if: always()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: clean up
+        run: ./build/github/cleanup-engflow-keys.sh
+        if: always()
+

build/github/cockroach-microbench-ci.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# Directories and file names
+output_dir="./artifacts/microbench"
+cleaned_current_dir="$output_dir/current"
+cleaned_base_dir="$output_dir/base"
+
+# hardcoded the file name as of now since `compare` requires a particular format for the file name.
+benchmark_file_name="pkg_sql-report.log"
+log_output_file_path="$output_dir/microbench.log"
+storage_bucket_url="gs://cockroach-microbench-ci"
+
+# Threshold for comparison
+threshold=${COMPARE_THRESHOLD:-0}
+success_exit_status=0
+error_exit_status=1
+
+# Exit early if SKIP_COMPARISON is true and not a push step
+if $SKIP_COMPARISON && ! $PUSH_STEP; then
+  echo "Exiting since skip comparison is enabled and this is not a push step."
+  exit $success_exit_status
+fi
+
+# Build binary with Bazel
+bazel build --config crosslinux $(./build/github/engflow-args.sh) --jobs 100 //pkg/cmd/roachprod-microbench
+
+roachprod_microbench_dir="_bazel/bin/pkg/cmd/roachprod-microbench/roachprod-microbench_"
+
+mkdir -p "$output_dir"
+
+# Run benchmarks and clean output
+# running count=4 here because that's the minimum required for a comparison
+bazel test //pkg/sql/tests:tests_test \
+  --config=use_ci_timeouts \
+  --strategy=TestRunner=sandboxed \
+  --jobs 100 \
+  --config=crosslinux \
+  --remote_download_minimal \
+  $(./build/github/engflow-args.sh) \
+  --test_arg=-test.run=- \
+  --test_arg='-test.bench=^BenchmarkKV$' \
+  --test_sharding_strategy=disabled \
+  --test_arg=-test.cpu --test_arg=1 \
+  --test_arg=-test.v \
+  --test_arg=-test.count=4 \
+  --test_arg=-test.benchmem \
+  --crdb_test_off \
+  --test_output=all > "$log_output_file_path"
+
+$roachprod_microbench_dir/roachprod-microbench clean "$log_output_file_path" "$cleaned_current_dir/$benchmark_file_name"
+
+# Push artifact if this is a base merge and skip comparison
+if $PUSH_STEP; then
+  gcloud storage cp "$cleaned_current_dir/$benchmark_file_name" "$storage_bucket_url/$GITHUB_HEAD_REF/$GITHUB_SHA.log"
+  echo "Skipping comparison since this is a base merge."
+  exit $success_exit_status
+fi
+
+# Compare benchmarks
+if ! gcloud storage cp "$storage_bucket_url/$GITHUB_BASE_REF/$BASE_SHA.log" "$cleaned_base_dir/$benchmark_file_name"; then
+  echo "Couldn't download base bench file, exiting."
+  exit $success_exit_status
+fi
+
+if ! $roachprod_microbench_dir/roachprod-microbench compare "$cleaned_current_dir" "$cleaned_base_dir" --threshold "$threshold"; then
+  echo "There is an error during comparison. If it's a perf regression, please try to fix it. This won't block your change for merging currently."
+  exit $error_exit_status
+fi
+
+rm -rf "$output_dir"
+exit $success_exit_status