Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

automated benchmarking v3 #105

Merged
merged 2 commits into from
Jun 30, 2023
Merged

automated benchmarking v3 #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd05762
more benchmarking fixes/improvements
wuputah Jun 28, 2023
36b89b1
move most of the benchmark logic into a script
wuputah Jun 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 19 additions & 87 deletions .github/workflows/benchmark.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,120 +35,52 @@ on:
- warehouse-10G
tag:
type: string

env:
BENCHMARK: ${{ inputs.benchmark || 'clickbench-1M' }}
REPO: ${{ inputs.repo || 'ghcr.io/hydradatabase/hydra' }}
BASE_IMAGE: ${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }}
TAG: ${{ inputs.tag || format('15-{0}', github.sha) }}
NAME: hydra-benchmark
BENCHER_PROJECT: hydra-${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }}
BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
BENCHER_ADAPTER: json
BENCHER_TESTBED: gh-4core

jobs:
benchmarks:

name: Run Benchmark
if: github.repository == 'hydradatabase/hydra-internal'
# 16gb ram, 4vcpu, 150gb disk
runs-on: benchmarks-ubuntu-latest-4core

steps:

- name: Configure AWS credentials
if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
aws-region: us-east-1
mask-aws-account-id: no

- name: Login to Amazon ECR
if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
id: spilo-ecr
uses: aws-actions/amazon-ecr-login@v1
with:
registries: "011789831835"

- uses: actions/setup-node@v3
with:
node-version: 18

- uses: docker/setup-buildx-action@v2

- uses: unfor19/install-aws-cli-action@v1
with:
arch: amd64

- uses: bencherdev/bencher@main

- name: Checkout benchmarks
uses: actions/checkout@v3
with:
repository: hydradatabase/benchmarks

- name: Set up docker image
run: |
docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -v $PWD:/benchmarks -m 12288m --cpus=4 --shm-size=1024m --name=$NAME $REPO:$TAG

- name: Prepare to download data
run: |
set -eux
BENCHMARK_SRC="$(echo $BENCHMARK | cut -f 1 -d -)"
if [ "$BENCHMARK" != "$BENCHMARK_SRC" ]; then
ln -s $BENCHMARK_SRC $BENCHMARK
fi
mkdir -p $BENCHMARK/data
tree $BENCHMARK

- name: Download data
uses: keithweaver/[email protected]
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
command: cp
source: s3://hydra-benchmarks/data/${{ env.BENCHMARK }}
destination: ./${{ env.BENCHMARK }}/data
aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
aws_region: us-east-1
flags: --no-progress --recursive

- name: Prepare data
run: |
set -eux
pushd $BENCHMARK/data
for f in *.gz; do
TARGET="$(basename $f .gz)"
mkfifo $TARGET
nohup gzip -d -c $f >$TARGET &
done
popd
tree $BENCHMARK

- name: Run benchmark
run: |
docker exec $NAME /bin/sh -c "RUNTIME=now /benchmarks/run-benchmark.sh -z -b $BENCHMARK -v zstd -u postgres"
./analyze.js ./results/$BENCHMARK/zstd/now > ./results.json

- name: Cleanup docker image
run: |
docker stop $NAME
docker rm $NAME
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY || secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY || secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
mask-aws-account-id: no

- name: Upload result to S3
uses: keithweaver/[email protected]
- name: Login to Amazon ECR
if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
id: spilo-ecr
uses: aws-actions/amazon-ecr-login@v1
with:
command: cp
source: ./results.json
destination: s3://hydra-benchmarks/results/${{ env.BASE_IMAGE }}-${{ env.BENCHMARK }}-zstd-${{ env.TAG }}.json
aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
aws_region: us-east-1
flags: --no-progress

- uses: bencherdev/bencher@main
registries: "011789831835"

- name: Upload result to bencher
run: |
bencher run \
--if-branch "$GITHUB_REF_NAME" \
--else-if-branch "$GITHUB_BASE_REF" \
--else-if-branch main \
--err \
"./analyze-bencher.js ./results/$BENCHMARK/zstd/now $BENCHMARK"
- name: Run benchmark
run: ./run-gha.sh
29 changes: 24 additions & 5 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ jobs:
run: make lint_acceptance

validate_columnar:
name: Validate Columnar ${{ matrix.postgres }}
strategy:
fail-fast: false
matrix:
postgres: ["13", "14", "15"]
name: Validate Columnar ${{ matrix.postgres }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
Expand All @@ -62,12 +62,12 @@ jobs:
columnar.target=checker

build_validate_postgres:
name: Build and Validate Postgres ${{ matrix.postgres }}
needs: [validate_columnar]
strategy:
fail-fast: false
matrix:
postgres: ["13", "14", "15"]
name: Build and Validate Postgres ${{ matrix.postgres }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
Expand Down Expand Up @@ -121,9 +121,9 @@ jobs:
targets: postgres

build_validate_spilo:
name: Build and Validate Spilo
needs: [validate_columnar]
if: github.repository == 'hydradatabase/hydra-internal'
name: Build and Validate Spilo
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
Expand Down Expand Up @@ -196,13 +196,13 @@ jobs:
*.platform=linux/amd64

push_postgres:
name: Push Postgres ${{ matrix.postgres }}
needs: [build_validate_postgres]
if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
strategy:
fail-fast: false
matrix:
postgres: ["13", "14", "15"]
name: Push Postgres ${{ matrix.postgres }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
Expand Down Expand Up @@ -231,6 +231,7 @@ jobs:
${{ matrix.postgres == '14' && format('postgres.tags={0}:latest', env.POSTGRES_REPO) || '' }}

push_spilo:
name: Push Spilo
needs: [build_validate_spilo]
if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
uses: ./.github/workflows/build_spilo.yaml
Expand All @@ -240,13 +241,31 @@ jobs:
secrets: inherit

benchmark_postgres:
name: Benchmark Hydra Postgres image (${{ matrix.benchmark }})
needs: [push_postgres]
if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
strategy:
fail-fast: false
matrix:
benchmark: ["clickbench-100M", "warehouse-10G"]
uses: ./.github/workflows/benchmark.yaml
with:
benchmark: ${{ matrix.benchmark }}
repo: ghcr.io/hydradatabase/hydra
tag: 15-${{ github.sha }}
tag: 14-${{ github.sha }}
secrets: inherit

benchmark_spilo:
name: Benchmark Hydra Spilo image (${{ matrix.benchmark }})
needs: [push_spilo]
if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
strategy:
fail-fast: false
matrix:
benchmark: ["clickbench-100M", "warehouse-10G"]
uses: ./.github/workflows/benchmark.yaml
with:
benchmark: ${{ matrix.benchmark }}
repo: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo
tag: ${{ github.sha }}
secrets: inherit