more automated benchmarking improvements (#105)

* move most of the benchmark logic into a script
* remove `if` on benchmarks action

.github/workflows/benchmark.yaml

@@ -35,120 +35,52 @@ on:
           - warehouse-10G
       tag:
         type: string
-
 env:
   BENCHMARK: ${{ inputs.benchmark || 'clickbench-1M' }}
   REPO: ${{ inputs.repo || 'ghcr.io/hydradatabase/hydra' }}
-  BASE_IMAGE: ${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }}
   TAG: ${{ inputs.tag || format('15-{0}', github.sha) }}
-  NAME: hydra-benchmark
-  BENCHER_PROJECT: hydra-${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }}
   BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
-  BENCHER_ADAPTER: json
   BENCHER_TESTBED: gh-4core
 
 jobs:
   benchmarks:
 
     name: Run Benchmark
-    if: github.repository == 'hydradatabase/hydra-internal'
     # 16gb ram, 4vcpu, 150gb disk
     runs-on: benchmarks-ubuntu-latest-4core
 
     steps:
-
-      - name: Configure AWS credentials
-        if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
-          aws-region: us-east-1
-          mask-aws-account-id: no
-
-      - name: Login to Amazon ECR
-        if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
-        id: spilo-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-        with:
-          registries: "011789831835"
-
       - uses: actions/setup-node@v3
         with:
           node-version: 18
 
       - uses: docker/setup-buildx-action@v2
 
+      - uses: unfor19/install-aws-cli-action@v1
+        with:
+          arch: amd64
+
+      - uses: bencherdev/bencher@main
+
       - name: Checkout benchmarks
         uses: actions/checkout@v3
         with:
           repository: hydradatabase/benchmarks
 
-      - name: Set up docker image
-        run: |
-          docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -v $PWD:/benchmarks -m 12288m --cpus=4 --shm-size=1024m --name=$NAME $REPO:$TAG
-
-      - name: Prepare to download data
-        run: |
-          set -eux
-          BENCHMARK_SRC="$(echo $BENCHMARK | cut -f 1 -d -)"
-          if [ "$BENCHMARK" != "$BENCHMARK_SRC" ]; then
-            ln -s $BENCHMARK_SRC $BENCHMARK
-          fi
-          mkdir -p $BENCHMARK/data
-          tree $BENCHMARK
-
-      - name: Download data
-        uses: keithweaver/[email protected]
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
         with:
-          command: cp
-          source: s3://hydra-benchmarks/data/${{ env.BENCHMARK }}
-          destination: ./${{ env.BENCHMARK }}/data
-          aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
-          aws_region: us-east-1
-          flags: --no-progress --recursive
-
-      - name: Prepare data
-        run: |
-          set -eux
-          pushd $BENCHMARK/data
-          for f in *.gz; do
-            TARGET="$(basename $f .gz)"
-            mkfifo $TARGET
-            nohup gzip -d -c $f >$TARGET &
-          done
-          popd
-          tree $BENCHMARK
-
-      - name: Run benchmark
-        run: |
-          docker exec $NAME /bin/sh -c "RUNTIME=now /benchmarks/run-benchmark.sh -z -b $BENCHMARK -v zstd -u postgres"
-          ./analyze.js ./results/$BENCHMARK/zstd/now > ./results.json
-
-      - name: Cleanup docker image
-        run: |
-          docker stop $NAME
-          docker rm $NAME
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY || secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY || secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+          mask-aws-account-id: no
 
-      - name: Upload result to S3
-        uses: keithweaver/[email protected]
+      - name: Login to Amazon ECR
+        if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
+        id: spilo-ecr
+        uses: aws-actions/amazon-ecr-login@v1
         with:
-          command: cp
-          source: ./results.json
-          destination: s3://hydra-benchmarks/results/${{ env.BASE_IMAGE }}-${{ env.BENCHMARK }}-zstd-${{ env.TAG }}.json
-          aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
-          aws_region: us-east-1
-          flags: --no-progress
-
-      - uses: bencherdev/bencher@main
+          registries: "011789831835"
 
-      - name: Upload result to bencher
-        run: |
-          bencher run \
-            --if-branch "$GITHUB_REF_NAME" \
-            --else-if-branch "$GITHUB_BASE_REF" \
-            --else-if-branch main \
-            --err \
-            "./analyze-bencher.js ./results/$BENCHMARK/zstd/now $BENCHMARK"
+      - name: Run benchmark
+        run: ./run-gha.sh

.github/workflows/ci.yaml

@@ -36,11 +36,11 @@ jobs:
         run: make lint_acceptance
 
   validate_columnar:
+    name: Validate Columnar ${{ matrix.postgres }}
     strategy:
       fail-fast: false
       matrix:
         postgres: ["13", "14", "15"]
-    name: Validate Columnar ${{ matrix.postgres }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -62,12 +62,12 @@ jobs:
             columnar.target=checker
 
   build_validate_postgres:
+    name: Build and Validate Postgres ${{ matrix.postgres }}
     needs: [validate_columnar]
     strategy:
       fail-fast: false
       matrix:
         postgres: ["13", "14", "15"]
-    name: Build and Validate Postgres ${{ matrix.postgres }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -121,9 +121,9 @@ jobs:
           targets: postgres
 
   build_validate_spilo:
+    name: Build and Validate Spilo
     needs: [validate_columnar]
     if: github.repository == 'hydradatabase/hydra-internal'
-    name: Build and Validate Spilo
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -196,13 +196,13 @@ jobs:
             *.platform=linux/amd64
 
   push_postgres:
+    name: Push Postgres ${{ matrix.postgres }}
     needs: [build_validate_postgres]
     if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
     strategy:
       fail-fast: false
       matrix:
         postgres: ["13", "14", "15"]
-    name: Push Postgres ${{ matrix.postgres }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -231,6 +231,7 @@ jobs:
             ${{ matrix.postgres == '14' && format('postgres.tags={0}:latest', env.POSTGRES_REPO) || '' }}
 
   push_spilo:
+    name: Push Spilo
     needs: [build_validate_spilo]
     if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
     uses: ./.github/workflows/build_spilo.yaml
@@ -240,13 +241,31 @@ jobs:
     secrets: inherit
 
   benchmark_postgres:
+    name: Benchmark Hydra Postgres image (${{ matrix.benchmark }})
     needs: [push_postgres]
     if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
     strategy:
+      fail-fast: false
       matrix:
         benchmark: ["clickbench-100M", "warehouse-10G"]
     uses: ./.github/workflows/benchmark.yaml
     with:
       benchmark: ${{ matrix.benchmark }}
       repo: ghcr.io/hydradatabase/hydra
-      tag: 15-${{ github.sha }}
+      tag: 14-${{ github.sha }}
+    secrets: inherit
+
+  benchmark_spilo:
+    name: Benchmark Hydra Spilo image (${{ matrix.benchmark }})
+    needs: [push_spilo]
+    if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
+    strategy:
+      fail-fast: false
+      matrix:
+        benchmark: ["clickbench-100M", "warehouse-10G"]
+    uses: ./.github/workflows/benchmark.yaml
+    with:
+      benchmark: ${{ matrix.benchmark }}
+      repo: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo
+      tag: ${{ github.sha }}
+    secrets: inherit