diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 06759323..2b7292c9 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -35,120 +35,52 @@ on: - warehouse-10G tag: type: string - env: BENCHMARK: ${{ inputs.benchmark || 'clickbench-1M' }} REPO: ${{ inputs.repo || 'ghcr.io/hydradatabase/hydra' }} - BASE_IMAGE: ${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }} TAG: ${{ inputs.tag || format('15-{0}', github.sha) }} - NAME: hydra-benchmark - BENCHER_PROJECT: hydra-${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }} BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - BENCHER_ADAPTER: json BENCHER_TESTBED: gh-4core jobs: benchmarks: name: Run Benchmark - if: github.repository == 'hydradatabase/hydra-internal' # 16gb ram, 4vcpu, 150gb disk runs-on: benchmarks-ubuntu-latest-4core steps: - - - name: Configure AWS credentials - if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }} - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} - aws-region: us-east-1 - mask-aws-account-id: no - - - name: Login to Amazon ECR - if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }} - id: spilo-ecr - uses: aws-actions/amazon-ecr-login@v1 - with: - registries: "011789831835" - - uses: actions/setup-node@v3 with: node-version: 18 - uses: docker/setup-buildx-action@v2 + - uses: unfor19/install-aws-cli-action@v1 + with: + arch: amd64 + + - uses: bencherdev/bencher@main + - name: Checkout benchmarks uses: actions/checkout@v3 with: repository: hydradatabase/benchmarks - - name: Set up docker image - run: | - docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -v $PWD:/benchmarks -m 12288m --cpus=4 --shm-size=1024m --name=$NAME $REPO:$TAG - - - name: Prepare to download data - run: | - set -eux - BENCHMARK_SRC="$(echo $BENCHMARK | cut -f 1 -d -)" - if [ "$BENCHMARK" != "$BENCHMARK_SRC" ]; then - ln -s $BENCHMARK_SRC $BENCHMARK - fi - mkdir -p $BENCHMARK/data - tree $BENCHMARK - - - name: Download data - uses: keithweaver/aws-s3-github-action@v1.0.0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 with: - command: cp - source: s3://hydra-benchmarks/data/${{ env.BENCHMARK }} - destination: ./${{ env.BENCHMARK }}/data - aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }} - aws_region: us-east-1 - flags: --no-progress --recursive - - - name: Prepare data - run: | - set -eux - pushd $BENCHMARK/data - for f in *.gz; do - TARGET="$(basename $f .gz)" - mkfifo $TARGET - nohup gzip -d -c $f >$TARGET & - done - popd - tree $BENCHMARK - - - name: Run benchmark - run: | - docker exec $NAME /bin/sh -c "RUNTIME=now /benchmarks/run-benchmark.sh -z -b $BENCHMARK -v zstd -u postgres" - ./analyze.js ./results/$BENCHMARK/zstd/now > ./results.json - - - name: Cleanup docker image - run: | - docker stop $NAME - docker rm $NAME + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY || secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY || secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + mask-aws-account-id: no - - name: Upload result to S3 - uses: keithweaver/aws-s3-github-action@v1.0.0 + - name: Login to Amazon ECR + if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }} + id: spilo-ecr + uses: aws-actions/amazon-ecr-login@v1 with: - command: cp - source: ./results.json - destination: s3://hydra-benchmarks/results/${{ env.BASE_IMAGE }}-${{ env.BENCHMARK }}-zstd-${{ env.TAG }}.json - aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }} - aws_region: us-east-1 - flags: --no-progress - - - uses: bencherdev/bencher@main + registries: "011789831835" - - name: Upload result to bencher - run: | - bencher run \ - --if-branch "$GITHUB_REF_NAME" \ - --else-if-branch "$GITHUB_BASE_REF" \ - --else-if-branch main \ - --err \ - "./analyze-bencher.js ./results/$BENCHMARK/zstd/now $BENCHMARK" + - name: Run benchmark + run: ./run-gha.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 684bdd4e..097a54db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,11 +36,11 @@ jobs: run: make lint_acceptance validate_columnar: + name: Validate Columnar ${{ matrix.postgres }} strategy: fail-fast: false matrix: postgres: ["13", "14", "15"] - name: Validate Columnar ${{ matrix.postgres }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -62,12 +62,12 @@ jobs: columnar.target=checker build_validate_postgres: + name: Build and Validate Postgres ${{ matrix.postgres }} needs: [validate_columnar] strategy: fail-fast: false matrix: postgres: ["13", "14", "15"] - name: Build and Validate Postgres ${{ matrix.postgres }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -121,9 +121,9 @@ jobs: targets: postgres build_validate_spilo: + name: Build and Validate Spilo needs: [validate_columnar] if: github.repository == 'hydradatabase/hydra-internal' - name: Build and Validate Spilo runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -196,13 +196,13 @@ jobs: *.platform=linux/amd64 push_postgres: + name: Push Postgres ${{ matrix.postgres }} needs: [build_validate_postgres] if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main' strategy: fail-fast: false matrix: postgres: ["13", "14", "15"] - name: Push Postgres ${{ matrix.postgres }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -231,6 +231,7 @@ jobs: ${{ matrix.postgres == '14' && format('postgres.tags={0}:latest', env.POSTGRES_REPO) || '' }} push_spilo: + name: Push Spilo needs: [build_validate_spilo] if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main' uses: ./.github/workflows/build_spilo.yaml @@ -240,13 +241,31 @@ jobs: secrets: inherit benchmark_postgres: + name: Benchmark Hydra Postgres image (${{ matrix.benchmark }}) needs: [push_postgres] if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main' strategy: + fail-fast: false matrix: benchmark: ["clickbench-100M", "warehouse-10G"] uses: ./.github/workflows/benchmark.yaml with: benchmark: ${{ matrix.benchmark }} repo: ghcr.io/hydradatabase/hydra - tag: 15-${{ github.sha }} + tag: 14-${{ github.sha }} + secrets: inherit + + benchmark_spilo: + name: Benchmark Hydra Spilo image (${{ matrix.benchmark }}) + needs: [push_spilo] + if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main' + strategy: + fail-fast: false + matrix: + benchmark: ["clickbench-100M", "warehouse-10G"] + uses: ./.github/workflows/benchmark.yaml + with: + benchmark: ${{ matrix.benchmark }} + repo: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo + tag: ${{ github.sha }} + secrets: inherit