[Storage] Implement shard id extraction in a different way. #57931
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## IMPORTANT NOTE TO EDITORS OF THIS FILE ## | |
| ## Note that when you create a PR the jobs in this file are triggered off the | |
| ## `pull_request_target` event instead of `pull_request` event. This is because | |
| ## the `pull_request` event makes secrets only available to PRs from branches, | |
| ## not from forks, and some of these jobs require secrets. So with `pull_request_target` | |
| ## we're making secrets available to fork-based PRs too. Using `pull_request_target" | |
| ## has a side effect, which is that the workflow execution will be driven by the | |
| ## state of the <workflow>.yaml on the `main` (=target) branch, even if you edited | |
| ## the <workflow>.yaml in your PR. So when you for example add a new job here, you | |
| ## won't see that job appear in the PR itself. It will only become effective once | |
| ## you merge the PR to main. Therefore, if you want to add a new job here and want | |
| ## to test it's functionality prior to a merge to main, you have to to _temporarily_ | |
| ## change the trigger event from `pull_request_target` to `pull_request`. | |
| ## Additionally, because `pull_request_target` gets secrets injected for forked PRs | |
| ## we use `https://github.com/sushichop/action-repository-permission` to ensure these | |
| ## jobs are only executed when a repo member with "write" permission has triggered | |
| ## the workflow (directly through a push or indirectly by applying a label or enabling | |
| ## auto_merge). | |
| name: "Build+Test Docker Images" | |
| on: # build on main branch OR when a PR is labeled with `CICD:build-images` | |
| # Allow us to run this specific workflow without a PR | |
| workflow_dispatch: | |
| pull_request_target: | |
| types: [labeled, opened, synchronize, reopened, auto_merge_enabled] | |
| push: | |
| branches: | |
| - main | |
| # release branches | |
| - devnet | |
| - testnet | |
| - mainnet | |
| - aptos-node-v* | |
| - aptos-release-v* | |
| # experimental branches | |
| - performance_benchmark | |
| - preview | |
| - preview-networking | |
| # grpc release branches | |
| - aptos-indexer-grpc-devnet | |
| - aptos-indexer-grpc-testnet | |
| - aptos-indexer-grpc-mainnet | |
| - aptos-indexer-grpc-v* | |
| # cancel redundant builds | |
| concurrency: | |
| # for push and workflow_dispatch events we use `github.sha` in the concurrency group and don't really cancel each other out/limit concurrency | |
| # for pull_request events newer jobs cancel earlier jobs to save on CI etc. | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha || github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| AWS_ECR_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }} | |
| # In case of pull_request events by default github actions merges main into the PR branch and then runs the tests etc | |
| # on the prospective merge result instead of only on the tip of the PR. | |
| # For more info also see https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit | |
| GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| # TARGET_CACHE_ID is used as part of the docker tag / cache key inside our bake.hcl docker bake files. | |
| # The goal here is to have a branch or PR-local cache such that consecutive pushes to a shared branch or a specific PR can | |
| # reuse layers from a previous docker build/commit. | |
| # We use `pr-<pr_number>` as cache-id for PRs and simply <branch_name> otherwise. | |
| TARGET_CACHE_ID: ${{ github.event.number && format('pr-{0}', github.event.number) || github.ref_name }} | |
| # On PRs, only build and push to GCP | |
| # On push, build and push to all remote registries | |
| TARGET_REGISTRY: ${{ github.event_name == 'pull_request_target' && 'remote' || 'remote-all' }} | |
| permissions: | |
| contents: read | |
| id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry | |
| issues: write | |
| pull-requests: write | |
| # Note on the job-level `if` conditions: | |
| # This workflow is designed such that: | |
| # 1. Run ALL jobs when a 'push', 'workflow_dispatch' triggered the workflow or on 'pull_request's which have set auto_merge=true or have the label "CICD:run-e2e-tests". | |
| # 2. Run ONLY the docker image building jobs on PRs with the "CICD:build[-<PROFILE/FEATURE>]-images" label. | |
| # 3. Run ONLY the forge-e2e-test job on PRs with the "CICD:run-forge-e2e-perf" label. | |
| # 4. Run NOTHING when neither 1. or 2. or 3. conditions are satisfied. | |
| jobs: | |
| permission-check: | |
| if: | | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(join(github.event.pull_request.labels.*.name, ','), 'CICD:build-') || | |
| contains(join(github.event.pull_request.labels.*.name, ','), 'CICD:run-') || | |
| github.event.pull_request.auto_merge != null || | |
| contains(github.event.pull_request.body, '#e2e') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check repository permission for user which triggered workflow | |
| uses: sushichop/action-repository-permission@13d208f5ae7a6a3fc0e5a7c2502c214983f0241c | |
| with: | |
| required-permission: write | |
| comment-not-permitted: Sorry, you don't have permission to trigger this workflow. | |
| # Because the docker build happens in a reusable workflow, have a separate job that collects the right metadata | |
| # for the subsequent docker builds. Reusable workflows do not currently have the "env" context: https://github.com/orgs/community/discussions/26671 | |
| determine-docker-build-metadata: | |
| needs: [permission-check] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: collect metadata | |
| run: | | |
| echo "GIT_SHA: ${GIT_SHA}" | |
| echo "TARGET_CACHE_ID: ${TARGET_CACHE_ID}" | |
| echo "TARGET_REGISTRY: ${TARGET_REGISTRY}" | |
| outputs: | |
| gitSha: ${{ env.GIT_SHA }} | |
| targetCacheId: ${{ env.TARGET_CACHE_ID }} | |
| targetRegistry: ${{ env.TARGET_REGISTRY }} | |
| # This job determines which files were changed | |
| file_change_determinator: | |
| needs: [permission-check] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| only_docs_changed: ${{ steps.determine_file_changes.outputs.only_docs_changed }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Run the file change determinator | |
| id: determine_file_changes | |
| uses: ./.github/actions/file-change-determinator | |
| # This job determines which tests to run | |
| test-target-determinator: | |
| needs: [permission-check] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run_framework_upgrade_test: ${{ steps.determine_test_targets.outputs.run_framework_upgrade_test }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Run the test target determinator | |
| id: determine_test_targets | |
| uses: ./.github/actions/test-target-determinator | |
| # This is a PR required job. | |
| rust-images: | |
| needs: [permission-check, determine-docker-build-metadata] | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-docker-rust-build.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| PROFILE: release | |
| BUILD_ADDL_TESTING_IMAGES: true | |
| TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} | |
| rust-images-failpoints: | |
| needs: [permission-check, determine-docker-build-metadata] | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-docker-rust-build.yaml@main | |
| if: | | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:build-failpoints-images') | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| PROFILE: release | |
| FEATURES: failpoints | |
| BUILD_ADDL_TESTING_IMAGES: true | |
| TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} | |
| rust-images-performance: | |
| needs: [permission-check, determine-docker-build-metadata] | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-docker-rust-build.yaml@main | |
| if: | | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:build-performance-images') | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| PROFILE: performance | |
| BUILD_ADDL_TESTING_IMAGES: true | |
| TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} | |
| rust-images-consensus-only-perf-test: | |
| needs: [permission-check, determine-docker-build-metadata] | |
| if: | | |
| contains(github.event.pull_request.labels.*.name, 'CICD:build-consensus-only-image') || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-consensus-only-perf-test') | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-docker-rust-build.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| PROFILE: release | |
| FEATURES: consensus-only-perf-test | |
| BUILD_ADDL_TESTING_IMAGES: true | |
| TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} | |
| # This is a PR required job. | |
| node-api-compatibility-tests: | |
| needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images" | |
| if: | | |
| ( | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || | |
| github.event.pull_request.auto_merge != null) || | |
| contains(github.event.pull_request.body, '#e2e' | |
| ) | |
| uses: ./.github/workflows/node-api-compatibility-tests.yaml | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} | |
| # This is a PR required job. | |
| cli-e2e-tests: | |
| needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images" | |
| if: | | |
| ( | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || | |
| github.event.pull_request.auto_merge != null) || | |
| contains(github.event.pull_request.body, '#e2e' | |
| ) | |
| uses: aptos-labs/aptos-core/.github/workflows/cli-e2e-tests.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} | |
| faucet-tests-main: | |
| needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images" | |
| if: | | |
| ( | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || | |
| github.event.pull_request.auto_merge != null) || | |
| contains(github.event.pull_request.body, '#e2e' | |
| ) | |
| uses: ./.github/workflows/faucet-tests-main.yaml | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} | |
| indexer-grpc-e2e-tests: | |
| needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images" | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || | |
| contains(github.event.pull_request.body, '#e2e') | |
| uses: aptos-labs/aptos-core/.github/workflows/docker-indexer-grpc-test.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| # This is a PR required job. | |
| forge-e2e-test: | |
| needs: | |
| - permission-check | |
| - determine-docker-build-metadata | |
| - rust-images | |
| - rust-images-failpoints | |
| - rust-images-performance | |
| - rust-images-consensus-only-perf-test | |
| - file_change_determinator | |
| if: | | |
| !failure() && !cancelled() && needs.permission-check.result == 'success' && ( | |
| (github.event_name == 'push' && github.ref_name != 'main') || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-forge-e2e-perf') || | |
| github.event.pull_request.auto_merge != null || | |
| contains(github.event.pull_request.body, '#e2e') | |
| ) | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_TEST_SUITE: realistic_env_max_load | |
| IMAGE_TAG: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_RUNNER_DURATION_SECS: 480 | |
| COMMENT_HEADER: forge-e2e | |
| # Use the cache ID as the Forge namespace so we can limit Forge test concurrency on k8s, since Forge | |
| # test lifecycle is separate from that of GHA. This protects us from the case where many Forge tests are triggered | |
| # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace. | |
| FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} | |
| # Run e2e compat test against testnet branch. This is a PR required job. | |
| forge-compat-test: | |
| needs: | |
| - permission-check | |
| - determine-docker-build-metadata | |
| - rust-images | |
| - rust-images-failpoints | |
| - rust-images-performance | |
| - rust-images-consensus-only-perf-test | |
| - file_change_determinator | |
| if: | | |
| !failure() && !cancelled() && needs.permission-check.result == 'success' && ( | |
| (github.event_name == 'push' && github.ref_name != 'main') || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || | |
| github.event.pull_request.auto_merge != null || | |
| contains(github.event.pull_request.body, '#e2e') | |
| ) | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_TEST_SUITE: compat | |
| IMAGE_TAG: 1c2ee7082d6eff8c811ee25d6f5a7d00860a75d5 #aptos-node-v1.16.0 | |
| FORGE_RUNNER_DURATION_SECS: 300 | |
| COMMENT_HEADER: forge-compat | |
| FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} | |
| # Run forge framework upgradability test. This is a PR required job. | |
| forge-framework-upgrade-test: | |
| needs: | |
| - permission-check | |
| - determine-docker-build-metadata | |
| - rust-images | |
| - rust-images-failpoints | |
| - rust-images-performance | |
| - rust-images-consensus-only-perf-test | |
| - test-target-determinator | |
| if: | | |
| !failure() && !cancelled() && needs.permission-check.result == 'success' && ( | |
| (github.event_name == 'push' && github.ref_name != 'main') || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-framework-upgrade-test') || | |
| github.event.pull_request.auto_merge != null | |
| ) | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_TEST_SUITE: framework_upgrade | |
| IMAGE_TAG: 1c2ee7082d6eff8c811ee25d6f5a7d00860a75d5 #aptos-node-v1.16.0 | |
| FORGE_RUNNER_DURATION_SECS: 3600 | |
| COMMENT_HEADER: forge-framework-upgrade | |
| FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| SKIP_JOB: ${{ !contains(github.event.pull_request.labels.*.name, 'CICD:run-framework-upgrade-test') && (needs.test-target-determinator.outputs.run_framework_upgrade_test == 'false') }} | |
| forge-consensus-only-perf-test: | |
| needs: | |
| - permission-check | |
| - determine-docker-build-metadata | |
| - rust-images | |
| - rust-images-failpoints | |
| - rust-images-performance | |
| - rust-images-consensus-only-perf-test | |
| if: | | |
| !failure() && !cancelled() && needs.permission-check.result == 'success' && | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-consensus-only-perf-test') | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_TEST_SUITE: consensus_only_realistic_env_max_tps | |
| IMAGE_TAG: consensus_only_perf_test_${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_RUNNER_DURATION_SECS: 300 | |
| COMMENT_HEADER: consensus-only-realistic-env-max-tps | |
| FORGE_NAMESPACE: forge-consensus-only-realistic-env-max-tps-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| # Run forge multiregion test. This test uses the multiregion forge cluster that deploys pods in three GCP regions. | |
| forge-multiregion-test: | |
| needs: | |
| - permission-check | |
| - determine-docker-build-metadata | |
| - rust-images | |
| - rust-images-failpoints | |
| - rust-images-performance | |
| - rust-images-consensus-only-perf-test | |
| if: | | |
| !failure() && !cancelled() && needs.permission-check.result == 'success' && | |
| contains(github.event.pull_request.labels.*.name, 'CICD:run-multiregion-test') | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main | |
| secrets: inherit | |
| with: | |
| GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_TEST_SUITE: multiregion_benchmark_test | |
| IMAGE_TAG: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} | |
| FORGE_RUNNER_DURATION_SECS: 300 | |
| COMMENT_HEADER: forge-multiregion-test | |
| FORGE_NAMESPACE: forge-multiregion-test-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} | |
| FORGE_CLUSTER_NAME: forge-multiregion |