Skip to content
Back to pull request #5696

H-3624: Fix AI worker github build action #4452

Sign in to view logs

H-3624: Fix AI worker github build action

H-3624: Fix AI worker github build action #4452

Workflow file for this run

.github/workflows/bench.yml at 801897c
name: Bench
on:
pull_request:
push:
branches:
- main
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: hashintel
TURBO_REMOTE_ONLY: true
VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
setup:
runs-on: ubuntu-24.04
outputs:
unit: ${{ steps.packages.outputs.unit }}
integration: ${{ steps.packages.outputs.integration }}
steps:
- name: Checkout source code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 2
- name: Install turbo
uses: ./.github/actions/install-turbo
- name: Determine changed packages
id: packages
run: |
UNIT_BENCH_FILTER=$(turbo run bench:unit --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]')
UNIT_BENCH_TASKS=$(sh -c "turbo run bench:unit --dry-run=json $UNIT_BENCH_FILTER" | jq -c '.tasks[]')
UNIT_BENCH_PACKAGES=$(echo "$UNIT_BENCH_TASKS" \
| jq 'select(.task == "bench:unit" and .command != "<NONEXISTENT>")' \
| jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }')
INTEGRATION_BENCH_FILTER=$(turbo run bench:integration --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]')
INTEGRATION_BENCH_TASKS=$(sh -c "turbo run bench:integration --dry-run=json $INTEGRATION_BENCH_FILTER" | jq -c '.tasks[]')
INTEGRATION_BENCH_PACKAGES=$(echo "$INTEGRATION_BENCH_TASKS" \
| jq 'select(.task == "bench:integration" and .command != "<NONEXISTENT>")' \
| jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }')
set -x
echo "unit=$UNIT_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT
echo "integration=$INTEGRATION_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT
unit-benches:
name: Unit
needs: [setup]
permissions:
id-token: write
contents: read
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.unit) }}
fail-fast: false
if: needs.setup.outputs.unit != '{"package":[],"include":[]}'
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 2
- name: Install turbo
uses: ./.github/actions/install-turbo
- name: Find bench steps to run
id: benches
run: |
if [[ "${{ github.event_name == 'pull_request' }}" == 'true' ]]; then
echo "create-baseline=true" | tee -a $GITHUB_OUTPUT
else
echo "create-baseline=false" | tee -a $GITHUB_OUTPUT
fi
HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
if [[ $HAS_RUST = 'true' ]]; then
if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then
RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml"
else
RUST_TOOLCHAIN_FILE="rust-toolchain.toml"
fi
echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
fi
- name: Install Rust tools
if: steps.benches.outputs.has-rust == 'true'
uses: taiki-e/install-action@5d427d86f088a6cedcddb92b3ad038d30721b52f # v2.44.71
with:
tool: [email protected],[email protected]
- name: Install Protobuf
if: steps.benches.outputs.has-rust == 'true'
run: sudo apt install protobuf-compiler
- name: Checkout base branch
if: steps.benches.outputs.create-baseline == 'true'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
clean: false
ref: ${{ github.event.pull_request.base.sha }}
- name: Install Rust toolchain
if: steps.benches.outputs.has-rust == 'true' && steps.benches.outputs.create-baseline == 'true'
uses: ./.github/actions/install-rust-toolchain
with:
toolchain: ${{ steps.benches.outputs.rust-toolchain }}
working-directory: ${{ matrix.directory }}
- name: Prune repository
if: steps.benches.outputs.create-baseline == 'true'
uses: ./.github/actions/prune-repository
with:
scope: |
${{ matrix.package }}
@rust/hash-repo-chores
@apps/hash-external-services
- name: Warm up repository
if: steps.benches.outputs.create-baseline == 'true'
uses: ./.github/actions/warm-up-repo
- name: Cache Rust dependencies
if: steps.benches.outputs.has-rust == 'true'
uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
with:
workspaces: ${{ matrix.directory }}
save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
- name: Run base benches
if: steps.benches.outputs.create-baseline == 'true'
run: turbo run bench:unit --filter "${{ matrix.package }}"
- name: Checkout head branch
if: steps.benches.outputs.create-baseline == 'true'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Install Rust toolchain
if: steps.benches.outputs.has-rust == 'true'
uses: ./.github/actions/install-rust-toolchain
with:
toolchain: ${{ steps.benches.outputs.rust-toolchain }}
working-directory: ${{ matrix.directory }}
- name: Prune repository
uses: ./.github/actions/prune-repository
with:
scope: |
${{ matrix.package }}
@rust/hash-repo-chores
@apps/hash-external-services
- name: Warm up repository
uses: ./.github/actions/warm-up-repo
- name: Run head benches
run: turbo run bench:unit --filter "${{ matrix.package }}"
- name: Analyze benchmarks
if: steps.benches.outputs.create-baseline == 'true'
run: |
TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')
echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV
echo '<details><summary><h2>${{ matrix.package }} – Units</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md"
echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph
cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
- name: Authenticate Vault
id: secrets
uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
with:
exportToken: true
url: ${{ env.VAULT_ADDR }}
method: jwt
role: dev
secrets: |
aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID;
aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY;
aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN;
- name: Upload benchmark results
run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph
- name: Upload benchmark summary
if: steps.benches.outputs.create-baseline == 'true'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }}
path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md
integration-benches:
name: Integration
needs: [setup]
permissions:
id-token: write
contents: read
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.integration) }}
fail-fast: false
if: needs.setup.outputs.integration != '{"package":[],"include":[]}'
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 2
- name: Install turbo
uses: ./.github/actions/install-turbo
- name: Find bench steps to run
id: benches
run: |
if [[ "${{ github.event_name == 'pull_request' }}" == 'true' ]]; then
echo "create-baseline=true" | tee -a $GITHUB_OUTPUT
else
echo "create-baseline=false" | tee -a $GITHUB_OUTPUT
fi
HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
if [[ $HAS_RUST = 'true' ]]; then
if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then
RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml"
else
RUST_TOOLCHAIN_FILE="rust-toolchain.toml"
fi
echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
fi
- name: Install Rust tools
if: steps.benches.outputs.has-rust == 'true'
uses: taiki-e/install-action@5d427d86f088a6cedcddb92b3ad038d30721b52f # v2.44.71
with:
tool: [email protected],[email protected]
- name: Install Protobuf
if: steps.benches.outputs.has-rust == 'true'
run: sudo apt install protobuf-compiler
- name: Checkout base branch
if: steps.benches.outputs.create-baseline == 'true'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
clean: false
ref: ${{ github.event.pull_request.base.sha }}
- name: Install Rust toolchain
if: steps.benches.outputs.has-rust == 'true' && steps.benches.outputs.create-baseline == 'true'
uses: ./.github/actions/install-rust-toolchain
with:
toolchain: ${{ steps.benches.outputs.rust-toolchain }}
working-directory: ${{ matrix.directory }}
- name: Prune repository
if: steps.benches.outputs.create-baseline == 'true'
uses: ./.github/actions/prune-repository
with:
scope: |
${{ matrix.package }}
@rust/hash-repo-chores
@apps/hash-external-services
- name: Warm up repository
if: steps.benches.outputs.create-baseline == 'true'
uses: ./.github/actions/warm-up-repo
- name: Cache Rust dependencies
if: steps.benches.outputs.has-rust == 'true'
uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
with:
workspaces: ${{ matrix.directory }}
save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
- name: Launch external services
if: steps.benches.outputs.create-baseline == 'true'
run: |
turbo codegen --filter '@apps/hash-external-services'
SERVICES="hydra kratos redis spicedb"
echo "Running services: $SERVICES"
yarn workspace @apps/hash-external-services deploy:test up $SERVICES --wait
- name: Run base benches
if: steps.benches.outputs.create-baseline == 'true'
run: turbo run bench:integration --filter "${{ matrix.package }}"
- name: Tear down external services
if: steps.benches.outputs.create-baseline == 'true'
run: yarn workspace @apps/hash-external-services deploy:test down
- name: Checkout head branch
if: steps.benches.outputs.create-baseline == 'true'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
clean: false
ref: ${{ github.event.pull_request.head.sha }}
- name: Install Rust toolchain
if: steps.benches.outputs.has-rust == 'true'
uses: ./.github/actions/install-rust-toolchain
with:
toolchain: ${{ steps.benches.outputs.rust-toolchain }}
working-directory: ${{ matrix.directory }}
- name: Prune repository
uses: ./.github/actions/prune-repository
with:
scope: |
${{ matrix.package }}
@rust/hash-repo-chores
@apps/hash-external-services
- name: Warm up repository
uses: ./.github/actions/warm-up-repo
- name: Launch external services
run: |
turbo codegen --filter '@apps/hash-external-services'
SERVICES="hydra kratos redis spicedb"
echo "Running services: $SERVICES"
yarn workspace @apps/hash-external-services deploy:test up $SERVICES --wait
- name: Run head benches
run: turbo run bench:integration --filter "${{ matrix.package }}"
- name: Analyze benchmarks
if: steps.benches.outputs.create-baseline == 'true'
run: |
TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')
echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV
echo '<details><summary><h2>${{ matrix.package }} – Integrations</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md"
echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph
cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md"
- name: Authenticate Vault
id: secrets
if: github.event.pull_request.head.repo.full_name == github.repository
uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
with:
exportToken: true
url: ${{ env.VAULT_ADDR }}
method: jwt
role: dev
secrets: |
aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID;
aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY;
aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN;
- name: Upload benchmark results
if: github.event.pull_request.head.repo.full_name == github.repository
run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.directory }}/out" --enforce-flame-graph
- name: Upload benchmark summary
if: steps.benches.outputs.create-baseline == 'true'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }}
path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md
- name: Show container logs
if: ${{ success() || failure() }}
run: yarn workspace @apps/hash-external-services deploy logs --timestamps
passed:
name: Benches passed
needs: [setup, unit-benches, integration-benches]
if: always()
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Check setup script
run: |
[[ ${{ needs.setup.result }} = success ]]
- name: Check unit benches
run: |
[[ ${{ needs.unit-benches.result }} =~ success|skipped ]]
- name: Check integration benches
run: |
[[ ${{ needs.integration-benches.result }} =~ success|skipped ]]
- name: Notify Slack on failure
uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
if: ${{ failure() && github.event_name == 'merge_group' }}
env:
SLACK_LINK_NAMES: true
SLACK_MESSAGE: "At least one bench job failed for a Pull Request in the Merge Queue failed <@U02NLJY0FGX>" # Notifies T
SLACK_TITLE: Tests failed
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_USERNAME: GitHub
VAULT_ADDR: ""
VAULT_TOKEN: ""
- name: Checkout
if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Download reports' artifacts
if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: downloads
- name: Create comment
if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success'
run: |
echo "## Benchmark results" > /tmp/summary.md
cat downloads/*/*.md >> /tmp/summary.md
cat /tmp/summary.md
- name: Post comment
uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' && github.event.pull_request.head.repo.full_name == github.repository
with:
message-path: /tmp/summary.md
message-id: benchmark-results
refresh-message-position: true