From af67b21a40d1b5cc6f60b27cc60199e169b27c85 Mon Sep 17 00:00:00 2001 From: Josh Lind Date: Wed, 9 Aug 2023 19:13:19 -0400 Subject: [PATCH] [GHA] Run smoke and performance tests when appropriate. --- .github/workflows/cli-e2e-tests.yaml | 19 ++++++++- .github/workflows/docker-build-test.yaml | 25 +++++++++-- .github/workflows/execution-performance.yaml | 9 ++++ .../indexer-grpc-integration-tests.yaml | 1 - .github/workflows/lint-test.yaml | 8 ++++ .../node-api-compatibility-tests.yaml | 41 +++++++++++++++---- .github/workflows/workflow-run-forge.yaml | 27 ++++++++++-- 7 files changed, 112 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cli-e2e-tests.yaml b/.github/workflows/cli-e2e-tests.yaml index 20c50ef37302e..aaa8f215fafdf 100644 --- a/.github/workflows/cli-e2e-tests.yaml +++ b/.github/workflows/cli-e2e-tests.yaml @@ -11,7 +11,14 @@ on: required: true type: string description: Use this to override the git SHA1, branch name (e.g. devnet) or tag + SKIP_JOB: + required: false + default: false + type: boolean + description: Set to true to skip this job. Useful for PRs that don't require this workflow. +# TODO: should we migrate this to a composite action, so that we can skip it +# at the call site, and don't need to wrap each step in an if statement? jobs: # Run the Aptos CLI examples. We run the CLI on this commit / PR against a # local testnet using the devnet, testnet, and mainnet branches. This way @@ -24,10 +31,12 @@ jobs: id-token: write steps: - uses: actions/checkout@v3 + if: ${{ !inputs.SKIP_JOB }} with: ref: ${{ inputs.GIT_SHA }} - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main + if: ${{ !inputs.SKIP_JOB }} with: GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} @@ -37,11 +46,13 @@ jobs: GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} - uses: ./.github/actions/python-setup + if: ${{ !inputs.SKIP_JOB }} with: pyproject_directory: crates/aptos/e2e # Run CLI tests against local testnet built from devnet branch. - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2 + if: ${{ !inputs.SKIP_JOB }} name: devnet-tests with: max_attempts: 5 @@ -50,6 +61,7 @@ jobs: # Run CLI tests against local testnet built from testnet branch. - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2 + if: ${{ !inputs.SKIP_JOB }} name: testnet-tests with: max_attempts: 5 @@ -58,6 +70,7 @@ jobs: # Run CLI tests against local testnet built from mainnet branch. - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2 + if: ${{ !inputs.SKIP_JOB }} name: mainnet-tests with: max_attempts: 5 @@ -65,6 +78,10 @@ jobs: command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network mainnet --image-repo-with-project ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-mainnet - name: Print local testnet logs on failure - if: ${{ failure() }} + if: ${{ !inputs.SKIP_JOB && failure() }} working-directory: docker/compose/validator-testnet run: docker logs aptos-tools-devnet && docker logs aptos-tools-testnet && docker logs aptos-tools-mainnet + + # Print out whether the job was skipped. + - run: echo "Skipping CLI E2E tests!" + if: ${{ inputs.SKIP_JOB }} diff --git a/.github/workflows/docker-build-test.yaml b/.github/workflows/docker-build-test.yaml index 28a752a883f63..7240ca7a97a49 100644 --- a/.github/workflows/docker-build-test.yaml +++ b/.github/workflows/docker-build-test.yaml @@ -109,6 +109,17 @@ jobs: targetCacheId: ${{ env.TARGET_CACHE_ID }} targetRegistry: ${{ env.TARGET_REGISTRY }} + # This job determines which files were changed + file_change_determinator: + runs-on: ubuntu-latest + outputs: + only_docs_changed: ${{ steps.determine_file_changes.outputs.only_docs_changed }} + steps: + - uses: actions/checkout@v3 + - name: Run the file change determinator + id: determine_file_changes + uses: ./.github/actions/file-change-determinator + # This is a PR required job. rust-images: needs: [permission-check, determine-docker-build-metadata] @@ -185,7 +196,7 @@ jobs: # This is a PR required job. node-api-compatibility-tests: - needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images" + needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images" if: | ( github.event_name == 'push' || @@ -198,10 +209,11 @@ jobs: secrets: inherit with: GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} + SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} # This is a PR required job. cli-e2e-tests: - needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images" + needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images" if: | ( github.event_name == 'push' || @@ -214,14 +226,13 @@ jobs: secrets: inherit with: GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }} + SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} indexer-grpc-e2e-tests: needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images" if: | - (github.event_name == 'push' && github.ref_name != 'main') || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || - github.event.pull_request.auto_merge != null || contains(github.event.pull_request.body, '#e2e') uses: aptos-labs/aptos-core/.github/workflows/docker-indexer-grpc-test.yaml@main secrets: inherit @@ -238,6 +249,7 @@ jobs: - rust-images-failpoints - rust-images-performance - rust-images-consensus-only-perf-test + - file_change_determinator if: | !failure() && !cancelled() && needs.permission-check.result == 'success' && ( (github.event_name == 'push' && github.ref_name != 'main') || @@ -258,6 +270,7 @@ jobs: # test lifecycle is separate from that of GHA. This protects us from the case where many Forge tests are triggered # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace. FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} + SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} # Run e2e compat test against testnet branch. This is a PR required job. forge-compat-test: @@ -269,6 +282,7 @@ jobs: - rust-images-failpoints - rust-images-performance - rust-images-consensus-only-perf-test + - file_change_determinator if: | !failure() && !cancelled() && needs.permission-check.result == 'success' && ( (github.event_name == 'push' && github.ref_name != 'main') || @@ -286,6 +300,7 @@ jobs: FORGE_RUNNER_DURATION_SECS: 300 COMMENT_HEADER: forge-compat FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} + SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} # Run forge framework upgradability test. This is a PR required job. forge-framework-upgrade-test: @@ -297,6 +312,7 @@ jobs: - rust-images-failpoints - rust-images-performance - rust-images-consensus-only-perf-test + - file_change_determinator if: | !failure() && !cancelled() && needs.permission-check.result == 'success' && ( (github.event_name == 'push' && github.ref_name != 'main') || @@ -314,6 +330,7 @@ jobs: FORGE_RUNNER_DURATION_SECS: 300 COMMENT_HEADER: forge-framework-upgrade FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} + SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }} forge-consensus-only-perf-test: needs: diff --git a/.github/workflows/execution-performance.yaml b/.github/workflows/execution-performance.yaml index 02c7fc4c94c0f..32bc330857014 100644 --- a/.github/workflows/execution-performance.yaml +++ b/.github/workflows/execution-performance.yaml @@ -2,11 +2,20 @@ name: "execution-performance" on: workflow_dispatch: pull_request: + types: [labeled, opened, synchronize, reopened, auto_merge_enabled] schedule: - cron: "0 12 * * *" # This runs every day at 12pm UTC. jobs: execution-performance: + if: | # Only run on each PR once an appropriate event occurs + ( + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || + github.event.pull_request.auto_merge != null) || + contains(github.event.pull_request.body, '#e2e' + ) uses: aptos-labs/aptos-core/.github/workflows/workflow-run-execution-performance.yaml@main secrets: inherit with: diff --git a/.github/workflows/indexer-grpc-integration-tests.yaml b/.github/workflows/indexer-grpc-integration-tests.yaml index 4dbad1259f90d..5cd1929856fc5 100644 --- a/.github/workflows/indexer-grpc-integration-tests.yaml +++ b/.github/workflows/indexer-grpc-integration-tests.yaml @@ -19,7 +19,6 @@ concurrency: jobs: permission-check: - if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')) runs-on: ubuntu-latest steps: - name: Check repository permission for user which triggered workflow diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 5040a0e1eaa9b..26f98ba3b862e 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -97,6 +97,14 @@ jobs: # Run all rust smoke tests. This is a PR required job. rust-smoke-tests: needs: file_change_determinator + if: | # Only run on each PR once an appropriate event occurs + ( + github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || + contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') || + github.event.pull_request.auto_merge != null) || + contains(github.event.pull_request.body, '#e2e' + ) runs-on: high-perf-docker steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/node-api-compatibility-tests.yaml b/.github/workflows/node-api-compatibility-tests.yaml index 5eccb18865862..cfd01e7fd5863 100644 --- a/.github/workflows/node-api-compatibility-tests.yaml +++ b/.github/workflows/node-api-compatibility-tests.yaml @@ -24,6 +24,11 @@ on: required: true type: string description: Use this to override the git SHA1, branch name (e.g. devnet) or tag to release the SDK from + SKIP_JOB: + required: false + default: false + type: boolean + description: Set to true to skip this job. Useful for PRs that don't require this workflow. env: # This is the docker image tag that will be used for the SDK release. @@ -31,6 +36,8 @@ env: IMAGE_TAG: ${{ inputs.GIT_SHA || 'devnet' }} # default to "devnet" tag when not running on workflow_call GIT_SHA: ${{ inputs.GIT_SHA || github.event.pull_request.head.sha || github.sha }} # default to PR branch sha when not running on workflow_call +# TODO: should we migrate this to a composite action, so that we can skip it +# at the call site, and don't need to wrap each step in an if statement? jobs: # Confirm that the generated client within the TS SDK has been re-generated # if there are any changes that would affect it within the PR / commit. If @@ -42,10 +49,12 @@ jobs: id-token: write steps: - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3 + if: ${{ !inputs.SKIP_JOB }} with: ref: ${{ env.GIT_SHA }} - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main + if: ${{ !inputs.SKIP_JOB }} with: GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} @@ -55,6 +64,7 @@ jobs: GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} - uses: actions/setup-node@969bd2663942d722d85b6a8626225850c2f7be4b # pin@v3 + if: ${{ !inputs.SKIP_JOB }} with: node-version-file: .node-version registry-url: "https://registry.npmjs.org" @@ -62,6 +72,7 @@ jobs: # Self hosted runners don't have pnpm preinstalled. # https://github.com/actions/setup-node/issues/182 - uses: pnpm/action-setup@537643d491d20c2712d11533497cb47b2d0eb9d5 # pin https://github.com/pnpm/action-setup/releases/tag/v2.2.3 + if: ${{ !inputs.SKIP_JOB }} # When using high-perf-docker, the CI is actually run with two containers # in a k8s pod, one for docker commands run in the CI steps (docker), and @@ -69,9 +80,11 @@ jobs: # mounts, ${{ runner.temp }} is one of them. Writing the specs here ensures # the docker run step writes to a same place that the runner can read from. - run: mkdir -p ${{ runner.temp }}/specs + if: ${{ !inputs.SKIP_JOB }} # Build the API specs. - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2 + if: ${{ !inputs.SKIP_JOB }} name: generate-yaml-spec with: max_attempts: 3 @@ -79,6 +92,7 @@ jobs: command: docker run --rm --mount=type=bind,source=${{ runner.temp }}/specs,target=/specs ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/tools:${IMAGE_TAG} aptos-openapi-spec-generator -f yaml -o /specs/spec.yaml - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2 + if: ${{ !inputs.SKIP_JOB }} name: generate-json-spec with: max_attempts: 3 @@ -86,18 +100,27 @@ jobs: command: docker run --rm --mount=type=bind,source=${{ runner.temp }}/specs,target=/specs ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/tools:${IMAGE_TAG} aptos-openapi-spec-generator -f json -o /specs/spec.json # Confirm that the specs we built here are the same as those checked in. - - run: echo "If this step fails, run the following commands locally to fix it:" - - run: echo "cargo run -p aptos-openapi-spec-generator -- -f yaml -o api/doc/spec.yaml" - - run: echo "cargo run -p aptos-openapi-spec-generator -- -f json -o api/doc/spec.json" - - run: git diff --no-index --ignore-space-at-eol --ignore-blank-lines ${{ runner.temp }}/specs/spec.yaml api/doc/spec.yaml - - run: git diff --no-index --ignore-space-at-eol --ignore-blank-lines ${{ runner.temp }}/specs/spec.json api/doc/spec.json + - run: | + echo "If this step fails, run the following commands locally to fix it:" + echo "cargo run -p aptos-openapi-spec-generator -- -f yaml -o api/doc/spec.yaml" + echo "cargo run -p aptos-openapi-spec-generator -- -f json -o api/doc/spec.json" + git diff --no-index --ignore-space-at-eol --ignore-blank-lines ${{ runner.temp }}/specs/spec.yaml api/doc/spec.yaml + git diff --no-index --ignore-space-at-eol --ignore-blank-lines ${{ runner.temp }}/specs/spec.json api/doc/spec.json + if: ${{ !inputs.SKIP_JOB }} # Run package install. If install fails, it probably means the lockfile # was not included in the commit. - run: cd ./ecosystem/typescript/sdk && pnpm install --frozen-lockfile + if: ${{ !inputs.SKIP_JOB }} # Ensure any changes to the generated client were checked in. - - run: cd ./ecosystem/typescript/sdk && pnpm generate-client -o /tmp/generated_client - - run: echo "If this step fails, run the following command locally to fix it:" - - run: echo "cd ecosystem/typescript/sdk && pnpm generate-client" - - run: git diff --no-index --ignore-space-at-eol --ignore-blank-lines ./ecosystem/typescript/sdk/src/generated/ /tmp/generated_client/ + - run: | + cd ./ecosystem/typescript/sdk && pnpm generate-client -o /tmp/generated_client + echo "If this step fails, run the following command locally to fix it:" + echo "cd ecosystem/typescript/sdk && pnpm generate-client" + git diff --no-index --ignore-space-at-eol --ignore-blank-lines ./ecosystem/typescript/sdk/src/generated/ /tmp/generated_client/ + if: ${{ !inputs.SKIP_JOB }} + + # Print out whether the job was skipped. + - run: echo "Skipping node API compatibility tests!" + if: ${{ inputs.SKIP_JOB }} diff --git a/.github/workflows/workflow-run-forge.yaml b/.github/workflows/workflow-run-forge.yaml index 5712cca393090..338d7c6a2a5a2 100644 --- a/.github/workflows/workflow-run-forge.yaml +++ b/.github/workflows/workflow-run-forge.yaml @@ -67,6 +67,11 @@ on: default: forge description: A unique ID for Forge sticky comment on your PR. See https://github.com/marocchino/sticky-pull-request-comment#keep-more-than-one-comment + SKIP_JOB: + required: false + default: false + type: boolean + description: Set to true to skip this job. Useful for PRs that don't require this workflow. env: AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }} @@ -96,32 +101,39 @@ env: VERBOSE: true COMMENT_ON_PR: ${{ inputs.COMMENT_ON_PR }} +# TODO: should we migrate this to a composite action, so that we can skip it +# at the call site, and don't need to wrap each step in an if statement? jobs: forge: runs-on: ubuntu-latest timeout-minutes: ${{ inputs.TIMEOUT_MINUTES }} steps: - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3 + if: ${{ !inputs.SKIP_JOB }} with: ref: ${{ inputs.GIT_SHA }} # get the last 10 commits if GIT_SHA is not specified fetch-depth: inputs.GIT_SHA != null && 0 || 10 - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # pin@v4 + if: ${{ !inputs.SKIP_JOB }} - name: Install python deps run: pip3 install click==8.1.3 psutil==5.9.1 + if: ${{ !inputs.SKIP_JOB }} # Calculate the auth duration based on the test duration # If the test duration is less than the default 90 minutes, use the default # otherwise add 30 minutes to the length of the Forge test run - name: Calculate Forge Auth Duration + if: ${{ !inputs.SKIP_JOB }} id: calculate-auth-duration run: | auth_duration=$(( $FORGE_RUNNER_DURATION_SECS > 5400 ? $FORGE_RUNNER_DURATION_SECS + 30 * 60 : 5400 )) echo "auth_duration=${auth_duration}" >> $GITHUB_OUTPUT - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main + if: ${{ !inputs.SKIP_JOB }} id: docker-setup with: GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} @@ -140,28 +152,32 @@ jobs: GCP_AUTH_DURATION: ${{ steps.calculate-auth-duration.outputs.auth_duration }} - name: "Install GCloud SDK" + if: ${{ !inputs.SKIP_JOB }} uses: "google-github-actions/setup-gcloud@62d4898025f6041e16b1068643bfc5a696863587" # pin@v1 with: version: ">= 418.0.0" install_components: "kubectl,gke-gcloud-auth-plugin" - name: "Export GCloud auth token" + if: ${{ !inputs.SKIP_JOB }} id: gcloud-auth run: echo "CLOUDSDK_AUTH_ACCESS_TOKEN=${{ steps.docker-setup.outputs.CLOUDSDK_AUTH_ACCESS_TOKEN }}" >> $GITHUB_ENV shell: bash - name: "Setup GCloud project" + if: ${{ !inputs.SKIP_JOB }} shell: bash run: gcloud config set project aptos-forge-gcp-0 - name: Run pre-Forge checks + if: ${{ !inputs.SKIP_JOB }} shell: bash env: FORGE_RUNNER_MODE: pre-forge run: testsuite/run_forge.sh - name: Post pre-Forge comment - if: env.COMMENT_ON_PR == 'true' && github.event.number != null + if: ${{ !inputs.SKIP_JOB && env.COMMENT_ON_PR == 'true' && github.event.number != null }} uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443 # pin@39c5b5dc7717447d0cba270cd115037d32d2844 with: header: ${{ env.COMMENT_HEADER }} @@ -170,12 +186,13 @@ jobs: path: ${{ env.FORGE_PRE_COMMENT }} - name: Run Forge + if: ${{ !inputs.SKIP_JOB }} shell: bash run: testsuite/run_forge.sh - name: Post forge result comment # Post a Github comment if the run has not been cancelled and if we're running on a PR - if: env.COMMENT_ON_PR == 'true' && github.event.number != null && !cancelled() + if: ${{ !inputs.SKIP_JOB && env.COMMENT_ON_PR == 'true' && github.event.number != null && !cancelled() }} uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443 # pin@39c5b5dc7717447d0cba270cd115037d32d2844 with: header: ${{ env.COMMENT_HEADER }} @@ -185,7 +202,7 @@ jobs: - name: Post to a Slack channel on failure # Post a Slack comment if the run has not been cancelled and the envs are set - if: env.POST_TO_SLACK == 'true' && failure() + if: ${{ !inputs.SKIP_JOB && env.POST_TO_SLACK == 'true' && failure() }} id: slack uses: slackapi/slack-github-action@936158bbe252e9a6062e793ea4609642c966e302 # pin@v1.21.0 with: @@ -196,3 +213,7 @@ jobs: } env: SLACK_WEBHOOK_URL: ${{ secrets.FORGE_SLACK_WEBHOOK_URL }} + + # Print out whether the job was skipped. + - run: echo "Skipping forge test!" + if: ${{ inputs.SKIP_JOB }}