Skip to content

Update GitHub Action returntocorp/semgrep to v1.97.0 #12379

Update GitHub Action returntocorp/semgrep to v1.97.0

Update GitHub Action returntocorp/semgrep to v1.97.0 #12379

Workflow file for this run

name: Test
on:
pull_request:
push:
branches:
- main
merge_group:
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: hashintel
TURBO_REMOTE_ONLY: true
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
setup:
runs-on: ubuntu-24.04
outputs:
unit-tests: ${{ steps.packages.outputs.unit-tests }}
integration-tests: ${{ steps.packages.outputs.integration-tests }}
system-tests: ${{ steps.packages.outputs.system-tests }}
dockers: ${{ steps.packages.outputs.dockers }}
publish: ${{ steps.packages.outputs.publish }}
steps:
- name: Checkout source code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 2
- name: Install turbo
uses: ./.github/actions/install-turbo
- name: Determine changed packages
id: packages
run: |
UNIT_TEST_FILTER=$(turbo run test:unit --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]')
UNIT_TEST_TASKS=$(sh -c "turbo run test:unit --dry-run=json $UNIT_TEST_FILTER" | jq -c '.tasks[]')
UNIT_TEST_PACKAGES=$(echo "$UNIT_TEST_TASKS" \
| jq 'select(.task == "test:unit" and .command != "<NONEXISTENT>")' \
| jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }')
INTEGRATION_TEST_FILTER=$(turbo run test:integration --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]')
INTEGRATION_TEST_TASKS=$(sh -c "turbo run test:integration --dry-run=json $INTEGRATION_TEST_FILTER" | jq -c '.tasks[]')
INTEGRATION_TEST_PACKAGES=$(echo "$INTEGRATION_TEST_TASKS" \
| jq 'select(.task == "test:integration" and .command != "<NONEXISTENT>")' \
| jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }')
DOCKER_PACKAGES_FILTER=$(turbo run build:docker --dry-run=json --filter '...[HEAD^]' | jq -e '.packages | contains(["//"])' > /dev/null && echo '' || echo '--filter ...[HEAD^]')
DOCKER_PACKAGES=$(sh -c "turbo run build:docker --dry-run=json $DOCKER_PACKAGES_FILTER" \
| jq '.tasks[]' \
| jq 'select(.task == "build:docker" and .command != "<NONEXISTENT>")' \
| jq --compact-output --slurp '{ package: [.[].package] | unique, include: [( .[] | {package: .package, directory: .directory })] | unique }')
PUBLISH_PACKAGES=[]
PUBLISH_INCLUDES=[]
while IFS= read -r file; do
if [[ -n "$file" ]]; then
PACKAGE_NAME=$(yq '.package.name' -p toml -oy "$file")
if [[ "$PACKAGE_NAME" == "null" ]]; then
continue
fi
if [[ "$(yq '.package.publish' -p toml -oy "$file")" == "false" ]]; then
continue
fi
if [[ "$(yq '.package.publish.workspace' -p toml -oy "$file")" == "true" ]]; then
if [[ "$(yq '.workspace.package.publish' -p toml -oy "Cargo.toml")" == "false" ]]; then
continue
fi
fi
OLD_VERSION=$(git show HEAD^:"$file" | yq '.package.version' -p toml)
if [[ "$OLD_VERSION" == "null" ]]; then
continue
fi
NEW_VERSION=$(yq '.package.version' -p toml -oy "$file")
if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then
continue
fi
DIR=$(dirname "$file")
PUBLISH_PACKAGES=$(echo "$PUBLISH_PACKAGES" | jq -c --arg package "$PACKAGE_NAME" '. += [$package]')
PUBLISH_INCLUDES=$(echo "$PUBLISH_INCLUDES" | jq -c --arg package "$PACKAGE_NAME" --arg directory "$DIR" '. += [{package: $package, directory: $directory}]')
fi
done <<< "$(find . -name 'Cargo.toml' -type f | sed 's|\./||')"
PUBLISH_PACKAGES=$(jq -c -n --argjson packages "$PUBLISH_PACKAGES" --argjson includes "$PUBLISH_INCLUDES" '{package: $packages, include: $includes}')
echo "unit-tests=$UNIT_TEST_PACKAGES" | tee -a $GITHUB_OUTPUT
echo "integration-tests=$INTEGRATION_TEST_PACKAGES" | tee -a $GITHUB_OUTPUT
echo "system-tests=$SYSTEM_TEST_PACKAGES" | tee -a $GITHUB_OUTPUT
echo "dockers=$DOCKER_PACKAGES" | tee -a $GITHUB_OUTPUT
echo "publish=$PUBLISH_PACKAGES" | tee -a $GITHUB_OUTPUT
unit-tests:
name: Unit
needs: [setup]
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.unit-tests) }}
fail-fast: false
if: needs.setup.outputs.unit-tests != '{"package":[],"include":[]}'
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Install turbo
uses: ./.github/actions/install-turbo
- name: Find test steps to run
id: tests
run: |
HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
if [[ $HAS_RUST = 'true' ]]; then
if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then
RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml"
else
RUST_TOOLCHAIN_FILE="rust-toolchain.toml"
fi
echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
echo "has-miri=$(yq '.toolchain.components | contains(["miri"])' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
fi
- name: Prune repository
uses: ./.github/actions/prune-repository
with:
scope: ${{ matrix.package }}
- name: Install Protobuf
if: always() && steps.tests.outputs.has-rust == 'true'
run: sudo apt install protobuf-compiler
- name: Install PDFium
if: matrix.package == '@rust/chonky'
env:
GH_TOKEN: ${{ github.token }}
run: |
temp_dir=$(mktemp -d)
gh release download chromium/6721 --repo bblanchon/pdfium-binaries --pattern 'pdfium-linux-x64.tgz' --dir $temp_dir
tar -xzf $temp_dir/pdfium-linux-x64.tgz -C $temp_dir
mv $temp_dir/lib/* "${{ matrix.directory }}/libs/"
rm -rf $temp_dir
echo "PDFIUM_DYNAMIC_LIB_PATH=$(pwd)/${{ matrix.directory }}/libs/" >> $GITHUB_ENV
- name: Install Rust toolchain
if: always() && steps.tests.outputs.has-rust == 'true'
uses: ./.github/actions/install-rust-toolchain
with:
toolchain: ${{ steps.tests.outputs.rust-toolchain }}
working-directory: ${{ matrix.directory }}
- name: Install Rust tools
if: always() && steps.tests.outputs.has-rust == 'true'
uses: taiki-e/install-action@5d427d86f088a6cedcddb92b3ad038d30721b52f # v2.44.71
with:
tool: [email protected],[email protected],[email protected],[email protected]
- name: Warm up repository
uses: ./.github/actions/warm-up-repo
- name: Cache Rust dependencies
if: always() && steps.tests.outputs.has-rust == 'true'
uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
with:
workspaces: ${{ matrix.directory }}
save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
- name: Show disk usage
run: df -h
- name: Run tests
continue-on-error: ${{ steps.tests.outputs.allow-failure == 'true' }}
env:
TEST_COVERAGE: ${{ github.event_name != 'merge_group' }}
run: |
turbo run test:unit --env-mode=loose --filter "${{ matrix.package }}"
echo "TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')" >> $GITHUB_ENV
- name: Show disk usage
run: df -h
- uses: codecov/codecov-action@5c47607acb93fed5485fdbf7232e8a31425f672a # v5.0.2
name: Upload coverage to https://app.codecov.io/gh/hashintel/hash
with:
flags: ${{ env.TRIMMED_PACKAGE_NAME }}
token: ${{ secrets.CODECOV_TOKEN }} ## not required for public repos, can be removed when https://github.com/codecov/codecov-action/issues/837 is resolved
- name: Run miri
if: always() && steps.tests.outputs.has-miri == 'true'
run: |
turbo run test:miri --filter "${{ matrix.package }}"
build:
name: Build
runs-on: ubuntu-24.04
needs: [setup]
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.dockers) }}
fail-fast: false
if: needs.setup.outputs.dockers != '{"package":[],"include":[]}'
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Build image
uses: ./.github/actions/build-docker-images
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
hash-graph: ${{ matrix.package == '@apps/hash-graph' }}
hash-ai-worker-ts: ${{ matrix.package == '@apps/hash-ai-worker-ts' }}
hash-integration-worker: ${{ matrix.package == '@apps/hash-integration-worker' }}
GOOGLE_CLOUD_WORKLOAD_IDENTITY_FEDERATION_CONFIG_JSON: ${{ secrets.GOOGLE_CLOUD_WORKLOAD_IDENTITY_FEDERATION_CONFIG_JSON }}
hash-api: ${{ matrix.package == '@apps/hash-api' }}
integration-tests:
name: Integration
needs: [setup]
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.integration-tests) }}
fail-fast: false
if: needs.setup.outputs.integration-tests != '{"package":[],"include":[]}'
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Install turbo
uses: ./.github/actions/install-turbo
- name: Prune repository
uses: ./.github/actions/prune-repository
with:
scope: |
${{ matrix.package }}
@apps/hash-external-services
- name: Find test steps to run
id: tests
run: |
TEST_TASKS=$(turbo run test:integration --env-mode=loose --dry-run=json --filter "${{ matrix.package }}" | jq -c '.tasks[]')
HAS_BACKGROUND_TASKS=$(turbo run start:test --dry-run json | jq -c '[.tasks[] | select(.command != "<NONEXISTENT>" and .package != "@apps/hash-external-services") | .package] != []' || echo 'false')
echo "has-background-tasks=$HAS_BACKGROUND_TASKS" | tee -a $GITHUB_OUTPUT
HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then
RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml"
else
RUST_TOOLCHAIN_FILE="rust-toolchain.toml"
fi
echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
if [[ $HAS_RUST = 'true' ]]; then
echo "has-miri=$(yq '.toolchain.components | contains(["miri"])' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
fi
- name: Create temp files and folders
run: mkdir -p var/logs
- name: Install Rust toolchain
uses: ./.github/actions/install-rust-toolchain
with:
toolchain: ${{ steps.tests.outputs.rust-toolchain }}
working-directory: ${{ matrix.directory }}
- name: Install Rust tools
if: steps.tests.outputs.has-rust == 'true'
uses: taiki-e/install-action@5d427d86f088a6cedcddb92b3ad038d30721b52f # v2.44.71
with:
tool: [email protected],[email protected],[email protected],[email protected]
- name: Warm up repository
uses: ./.github/actions/warm-up-repo
- name: Install Protobuf
run: sudo apt install protobuf-compiler
- name: Install PDFium
if: matrix.package == '@rust/chonky'
env:
GH_TOKEN: ${{ github.token }}
run: |
temp_dir=$(mktemp -d)
gh release download chromium/6721 --repo bblanchon/pdfium-binaries --pattern 'pdfium-linux-x64.tgz' --dir $temp_dir
tar -xzf $temp_dir/pdfium-linux-x64.tgz -C $temp_dir
mv $temp_dir/lib/* "${{ matrix.directory }}/libs/"
rm -rf $temp_dir
echo "PDFIUM_DYNAMIC_LIB_PATH=$(pwd)/${{ matrix.directory }}/libs/" >> $GITHUB_ENV
- name: Install playwright
if: matrix.package == '@tests/hash-playwright'
uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
with:
max_attempts: 3
timeout_minutes: 10
shell: bash
command: npx playwright install --with-deps chromium
- name: Cache Rust dependencies
if: steps.tests.outputs.has-rust == 'true'
uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
with:
workspaces: ${{ matrix.directory }}
save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
- name: Show disk usage
run: df -h
- name: Launch external services
run: |
touch .env.local
echo 'OPENAI_API_KEY=dummy' >> .env.local
echo 'ANTHROPIC_API_KEY=dummy' >> .env.local
echo 'HASH_TEMPORAL_WORKER_AI_AWS_ACCESS_KEY_ID=dummy' >> .env.local
echo 'HASH_TEMPORAL_WORKER_AI_AWS_SECRET_ACCESS_KEY=dummy' >> .env.local
echo 'HASH_GRAPH_PG_DATABASE=graph' >> .env.local
cp .env.local .env.test.local
yarn external-services:test up hydra kratos redis spicedb temporal-ui --wait
- name: Show disk usage
run: df -h
- name: Start background tasks
id: background-tasks
if: steps.tests.outputs.has-background-tasks == 'true'
run: |
# Optimistically compile background tasks. Ideally, we also `build` them but this includes
# `@apps/plugin-browser` which needs `build:test` instead. We cannot filter it out because
# except when running PlayWright tests, the plugin-browser is already pruned out so the
# command would fail.
turbo run compile --env-mode=loose
mkdir -p logs
turbo run start:test --env-mode=loose --log-order stream >var/logs/background-task.log 2>&1 &
PID=$!
echo "pid=$PID" | tee -a $GITHUB_OUTPUT
# Not strictly needed to run the healthchecks here as they are also run in the integration
# tests but it this way we can fail the job early if the background tasks are not healthy.
# Also, the recorded time for the different steps is more accurate.
turbo run start:test:healthcheck --env-mode=loose
- name: Run tests
continue-on-error: ${{ steps.tests.outputs.allow-failure == 'true' }}
run: |
turbo run test:integration --env-mode=loose --filter "${{ matrix.package }}"
echo "TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.package }}" | sed 's|@||g' | sed 's|/|.|g')" >> $GITHUB_ENV
- name: Show disk usage
run: df -h
- uses: codecov/codecov-action@5c47607acb93fed5485fdbf7232e8a31425f672a # v5.0.2
name: Upload coverage to https://app.codecov.io/gh/hashintel/hash
with:
flags: ${{ env.TRIMMED_PACKAGE_NAME }}
token: ${{ secrets.CODECOV_TOKEN }} ## not required for public repos, can be removed when https://github.com/codecov/codecov-action/issues/837 is resolved
- name: Show container logs
if: ${{ success() || failure() }}
run: yarn workspace @apps/hash-external-services deploy logs --timestamps
- name: Show background tasks logs
if: always() && steps.tests.outputs.has-background-tasks == 'true'
run: |
kill -15 ${{ steps.background-tasks.outputs.pid }} || true
cat var/logs/background-task.log
- name: Upload artifact playwright-report
if: matrix.package == '@tests/hash-playwright' && success() || failure()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: ${{ env.TRIMMED_PACKAGE_NAME }}-report
path: tests/hash-playwright/playwright-report
- name: Upload logs
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: success() || failure()
with:
name: ${{ env.TRIMMED_PACKAGE_NAME }}-logs
path: |
var/api
var/logs
publish:
name: Publish
needs: [setup]
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.setup.outputs.publish) }}
if: needs.setup.outputs.publish != '{"package":[],"include":[]}'
runs-on: ubuntu-24.04
steps:
- name: Checkout source code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Find publish jobs to run
id: publish
run: |
HAS_RUST=$([[ -f "${{ matrix.directory }}/Cargo.toml" || ${{ matrix.directory }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
if [[ $HAS_RUST = 'true' ]]; then
if [[ -f "${{ matrix.directory }}/rust-toolchain.toml" ]]; then
RUST_TOOLCHAIN_FILE="${{ matrix.directory }}/rust-toolchain.toml"
else
RUST_TOOLCHAIN_FILE="rust-toolchain.toml"
fi
echo "rust-toolchain=$(yq '.toolchain.channel' $RUST_TOOLCHAIN_FILE)" | tee -a $GITHUB_OUTPUT
fi
- name: Install Rust toolchain
if: always() && steps.publish.outputs.has-rust == 'true'
uses: ./.github/actions/install-rust-toolchain
with:
toolchain: ${{ steps.publish.outputs.rust-toolchain }}
working-directory: ${{ matrix.directory }}
- name: Login
run: |
[[ -n "${{ secrets.CARGO_REGISTRY_TOKEN }}" ]]
cargo login "${{ secrets.CARGO_REGISTRY_TOKEN }}"
- name: Publish (dry run)
if: always() && steps.publish.outputs.has-rust == 'true' && github.event_name == 'pull_request' || github.event_name == 'merge_group'
working-directory: ${{ matrix.directory }}
run: cargo publish --all-features --dry-run --no-verify
- name: Publish
if: always() && steps.publish.outputs.has-rust == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
working-directory: ${{ matrix.directory }}
run: cargo publish --all-features --no-verify
passed:
name: Tests passed
needs: [setup, unit-tests, build, integration-tests, publish]
if: always()
runs-on: ubuntu-latest
steps:
- name: Check setup script
run: |
[[ ${{ needs.setup.result }} = success ]]
- name: Check unit tests
run: |
[[ ${{ needs.unit-tests.result }} =~ success|skipped ]]
- name: Check builds
run: |
[[ ${{ needs.build.result }} =~ success|skipped ]]
- name: Check integration tests
run: |
[[ ${{ needs.integration-tests.result }} =~ success|skipped ]]
- name: Check publish results
run: |
[[ ${{ needs.publish.result }} =~ success|skipped ]]
- name: Notify Slack on failure
uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
if: ${{ failure() && github.event_name == 'merge_group' }}
env:
SLACK_LINK_NAMES: true
SLACK_MESSAGE: "At least one test job failed for a Pull Request in the Merge Queue failed <@U0143NL4GMP> <@U02NLJY0FGX>" # Notifies C & T
SLACK_TITLE: Tests failed
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_USERNAME: GitHub
VAULT_ADDR: ""
VAULT_TOKEN: ""