Merge branch 'main' into michael/P256

aptos-labs · Aug 11, 2023 · 82ec02c · 82ec02c
2 parents 9ab4170 + 21b7429
commit 82ec02c
Show file tree

Hide file tree

Showing 175 changed files with 6,530 additions and 1,790 deletions.
diff --git a/.github/actions/docker-setup/action.yaml b/.github/actions/docker-setup/action.yaml
@@ -83,6 +83,13 @@ runs:
         service_account: ${{ inputs.GCP_SERVICE_ACCOUNT_EMAIL }}
         export_environment_variables: ${{ inputs.EXPORT_GCP_PROJECT_VARIABLES }}
 
+    - name: Login to us-west1 Google Artifact Registry
+      uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # pin@v2
+      with:
+        registry: us-west1-docker.pkg.dev
+        username: oauth2accesstoken
+        password: ${{ steps.auth.outputs.access_token }}
+
     - name: Login to US multi-region Google Artifact Registry
       uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # pin@v2
       with:

diff --git a/.github/actions/file-change-determinator/action.yaml b/.github/actions/file-change-determinator/action.yaml
@@ -13,4 +13,5 @@ runs:
       continue-on-error: true # Avoid skipping any checks if this job fails (see: https://github.com/fkirc/skip-duplicate-actions/issues/301)
       uses: fkirc/skip-duplicate-actions@v5
       with:
+        skip_after_successful_duplicate: false # Don't skip if the action is a duplicate (this may cause false positives)
         paths_ignore: '["**/*.md", "developer-docs-site/**"]'
diff --git a/.github/actions/general-lints/action.yaml b/.github/actions/general-lints/action.yaml
@@ -10,6 +10,8 @@ runs:
   steps:
     # Checkout the repository
     - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # get all the history because cargo xtest --change-since origin/main requires it.
 
     # Install shellcheck and run it on the dev_setup.sh script
     - name: Run shell lints

diff --git a/.github/linters/semgrep/pull-request-target-code-checkout.yaml b/.github/linters/semgrep/pull-request-target-code-checkout.yaml
@@ -0,0 +1,59 @@
+rules:
+  - id: pull-request-target-code-checkout
+    languages:
+      - yaml
+    message: This GitHub Actions workflow file uses `pull_request_target` and checks
+      out code from the incoming pull request. When using `pull_request_target`,
+      the Action runs in the context of the target repository, which includes
+      access to all repository secrets. Please ensure you have `permission-check`
+      enabled for the jobs that check out code. Please see
+      https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+      for additional mitigations.
+    metadata:
+      category: security
+      owasp:
+        - A01:2021 - Broken Access Control
+      cwe:
+        - "CWE-913: Improper Control of Dynamically-Managed Code Resources"
+      references:
+        - https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+        - https://github.com/justinsteven/advisories/blob/master/2021_github_actions_checkspelling_token_leak_via_advice_symlink.md
+      technology:
+        - github-actions
+      subcategory:
+        - audit
+      likelihood: MEDIUM
+      impact: LOW
+      confidence: MEDIUM
+      license: Commons Clause License Condition v1.0[LGPL-2.1-only]
+      vulnerability_class:
+        - Code Injection
+    patterns:
+      - pattern-either:
+          - pattern-inside: |
+              on:
+                ...
+                pull_request_target: ...
+                ...
+              ...
+          - pattern-inside: |
+              on: [..., pull_request_target, ...]
+              ...
+          - pattern-inside: |
+              on: pull_request_target
+              ...
+      - pattern-inside: |
+          jobs:
+            ...
+            $JOBNAME:
+              ...
+      - pattern-not-inside: |
+          needs: [permission-check]
+          ...
+      - pattern: |
+          ...
+          uses: "$ACTION"
+      - metavariable-regex:
+          metavariable: $ACTION
+          regex: actions/checkout@.*
+    severity: WARNING
diff --git a/.github/workflows/cli-e2e-tests.yaml b/.github/workflows/cli-e2e-tests.yaml
@@ -11,7 +11,14 @@ on:
         required: true
         type: string
         description: Use this to override the git SHA1, branch name (e.g. devnet) or tag
+      SKIP_JOB:
+        required: false
+        default: false
+        type: boolean
+        description: Set to true to skip this job. Useful for PRs that don't require this workflow.
 
+# TODO: should we migrate this to a composite action, so that we can skip it
+# at the call site, and don't need to wrap each step in an if statement?
 jobs:
   # Run the Aptos CLI examples. We run the CLI on this commit / PR against a
   # local testnet using the devnet, testnet, and mainnet branches. This way
@@ -24,10 +31,12 @@ jobs:
       id-token: write
     steps:
       - uses: actions/checkout@v3
+        if: ${{ !inputs.SKIP_JOB }}
         with:
           ref: ${{ inputs.GIT_SHA }}
 
       - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
+        if: ${{ !inputs.SKIP_JOB }}
         with:
           GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
           GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
@@ -37,11 +46,13 @@ jobs:
           GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
 
       - uses: ./.github/actions/python-setup
+        if: ${{ !inputs.SKIP_JOB }}
         with:
           pyproject_directory: crates/aptos/e2e
 
       # Run CLI tests against local testnet built from devnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
+        if: ${{ !inputs.SKIP_JOB }}
         name: devnet-tests
         with:
           max_attempts: 5
@@ -50,6 +61,7 @@ jobs:
 
       # Run CLI tests against local testnet built from testnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
+        if: ${{ !inputs.SKIP_JOB }}
         name: testnet-tests
         with:
           max_attempts: 5
@@ -58,13 +70,18 @@ jobs:
 
       # Run CLI tests against local testnet built from mainnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
+        if: ${{ !inputs.SKIP_JOB }}
         name: mainnet-tests
         with:
           max_attempts: 5
           timeout_minutes: 20
           command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network mainnet --image-repo-with-project ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-mainnet
 
       - name: Print local testnet logs on failure
-        if: ${{ failure() }}
+        if: ${{ !inputs.SKIP_JOB && failure() }}
         working-directory: docker/compose/validator-testnet
         run: docker logs aptos-tools-devnet && docker logs aptos-tools-testnet && docker logs aptos-tools-mainnet
+
+      # Print out whether the job was skipped.
+      - run: echo "Skipping CLI E2E tests!"
+        if: ${{ inputs.SKIP_JOB }}
diff --git a/.github/workflows/copy-images-to-dockerhub.yaml b/.github/workflows/copy-images-to-dockerhub.yaml
@@ -59,6 +59,7 @@ jobs:
           FORCE_COLOR: 3 # Force color output as per https://github.com/google/zx#using-github-actions
           GIT_SHA: ${{ inputs.GIT_SHA || github.sha }} # If GIT_SHA is not provided, use the sha of the triggering branch
           GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+          GCP_DOCKER_ARTIFACT_REPO_US: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO_US }}
           AWS_ACCOUNT_ID: ${{ secrets.AWS_ECR_ACCOUNT_NUM }}
           IMAGE_TAG_PREFIX: ${{ inputs.image_tag_prefix }}
         run: ./docker/release-images.mjs --wait-for-image-seconds=3600
diff --git a/.github/workflows/docker-build-test.yaml b/.github/workflows/docker-build-test.yaml
@@ -48,6 +48,7 @@ concurrency:
 
 env:
   GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+  GCP_DOCKER_ARTIFACT_REPO_US: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO_US }}
   AWS_ECR_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
   # In case of pull_request events by default github actions merges main into the PR branch and then runs the tests etc
   # on the prospective merge result instead of only on the tip of the PR.
@@ -108,6 +109,17 @@ jobs:
       targetCacheId: ${{ env.TARGET_CACHE_ID }}
       targetRegistry: ${{ env.TARGET_REGISTRY }}
 
+  # This job determines which files were changed
+  file_change_determinator:
+    runs-on: ubuntu-latest
+    outputs:
+      only_docs_changed: ${{ steps.determine_file_changes.outputs.only_docs_changed }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run the file change determinator
+        id: determine_file_changes
+        uses: ./.github/actions/file-change-determinator
+
   # This is a PR required job.
   rust-images:
     needs: [permission-check, determine-docker-build-metadata]
@@ -184,7 +196,7 @@ jobs:
 
   # This is a PR required job.
   node-api-compatibility-tests:
-    needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
+    needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images"
     if: |
       (
         github.event_name == 'push' ||
@@ -197,10 +209,11 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
   # This is a PR required job.
   cli-e2e-tests:
-    needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
+    needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images"
     if: |
       (
         github.event_name == 'push' ||
@@ -213,14 +226,13 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
   indexer-grpc-e2e-tests:
     needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
     if: |
-      (github.event_name == 'push' && github.ref_name != 'main') ||
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
-      github.event.pull_request.auto_merge != null ||
       contains(github.event.pull_request.body, '#e2e')
     uses: aptos-labs/aptos-core/.github/workflows/docker-indexer-grpc-test.yaml@main
     secrets: inherit
@@ -237,6 +249,7 @@ jobs:
       - rust-images-failpoints
       - rust-images-performance
       - rust-images-consensus-only-perf-test
+      - file_change_determinator
     if: |
       !failure() && !cancelled() && needs.permission-check.result == 'success' && (
         (github.event_name == 'push' && github.ref_name != 'main') ||
@@ -257,6 +270,7 @@ jobs:
       # test lifecycle is separate from that of GHA. This protects us from the case where many Forge tests are triggered
       # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace.
       FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
   # Run e2e compat test against testnet branch. This is a PR required job.
   forge-compat-test:
@@ -268,6 +282,7 @@ jobs:
       - rust-images-failpoints
       - rust-images-performance
       - rust-images-consensus-only-perf-test
+      - file_change_determinator
     if: |
       !failure() && !cancelled() && needs.permission-check.result == 'success' && (
         (github.event_name == 'push' && github.ref_name != 'main') ||
@@ -285,6 +300,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 300
       COMMENT_HEADER: forge-compat
       FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
   # Run forge framework upgradability test. This is a PR required job.
   forge-framework-upgrade-test:
@@ -296,6 +312,7 @@ jobs:
       - rust-images-failpoints
       - rust-images-performance
       - rust-images-consensus-only-perf-test
+      - file_change_determinator
     if: |
       !failure() && !cancelled() && needs.permission-check.result == 'success' && (
         (github.event_name == 'push' && github.ref_name != 'main') ||
@@ -313,6 +330,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 300
       COMMENT_HEADER: forge-framework-upgrade
       FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
   forge-consensus-only-perf-test:
     needs:

diff --git a/.github/workflows/execution-performance.yaml b/.github/workflows/execution-performance.yaml
@@ -2,11 +2,20 @@ name: "execution-performance"
 on:
   workflow_dispatch:
   pull_request:
+    types: [labeled, opened, synchronize, reopened, auto_merge_enabled]
   schedule:
     - cron: "0 12 * * *" # This runs every day at 12pm UTC.
 
 jobs:
   execution-performance:
+    if: | # Only run on each PR once an appropriate event occurs
+      (
+        github.event_name == 'workflow_dispatch' ||
+        github.event_name == 'schedule' ||
+        contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
+        github.event.pull_request.auto_merge != null) ||
+        contains(github.event.pull_request.body, '#e2e'
+      )
     uses: aptos-labs/aptos-core/.github/workflows/workflow-run-execution-performance.yaml@main
     secrets: inherit
     with:

diff --git a/.github/workflows/forge-stable.yaml b/.github/workflows/forge-stable.yaml
@@ -127,11 +127,23 @@ jobs:
       FORGE_TEST_SUITE: realistic_env_load_sweep
       POST_TO_SLACK: true
 
-  run-forge-realistic-env-graceful-overload:
+  run-forge-realistic-env-workload-sweep:
     if: ${{ github.event_name != 'pull_request' && always() }}
     needs: [determine-test-metadata, run-forge-realistic-env-load-sweep] # Only run after the previous job completes
     uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
     secrets: inherit
+    with:
+      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
+      FORGE_NAMESPACE: forge-realistic-env-workload-sweep-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
+      FORGE_RUNNER_DURATION_SECS: 1600 # Run for 26 minutes (4 tests, each for 400 seconds)
+      FORGE_TEST_SUITE: realistic_env_workload_sweep
+      POST_TO_SLACK: true
+
+  run-forge-realistic-env-graceful-overload:
+    if: ${{ github.event_name != 'pull_request' && always() }}
+    needs: [determine-test-metadata, run-forge-realistic-env-workload-sweep] # Only run after the previous job completes
+    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
+    secrets: inherit
     with:
       IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
       FORGE_NAMESPACE: forge-realistic-env-graceful-overload-${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}

diff --git a/.github/workflows/indexer-grpc-integration-tests.yaml b/.github/workflows/indexer-grpc-integration-tests.yaml
@@ -19,7 +19,6 @@ concurrency:
 
 jobs:
   permission-check:
-    if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests'))
     runs-on: ubuntu-latest
     steps:
       - name: Check repository permission for user which triggered workflow

diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml
@@ -97,6 +97,14 @@ jobs:
   # Run all rust smoke tests. This is a PR required job.
   rust-smoke-tests:
     needs: file_change_determinator
+    if: | # Only run on each PR once an appropriate event occurs
+      (
+        github.event_name == 'workflow_dispatch' ||
+        github.event_name == 'push' ||
+        contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
+        github.event.pull_request.auto_merge != null) ||
+        contains(github.event.pull_request.body, '#e2e'
+      )
     runs-on: high-perf-docker
     steps:
       - uses: actions/checkout@v3

diff --git a/.github/workflows/module-verify.yaml b/.github/workflows/module-verify.yaml
@@ -35,9 +35,9 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ inputs.GIT_SHA }}
-      BUCKET: aptos-testnet-backup-2223d95b
+      BUCKET: aptos-testnet-backup-b7b1ad7a
       SUB_DIR: e1
-      BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml
+      BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml
       # workflow config
       RUNS_ON: high-perf-docker-with-local-ssd
       TIMEOUT_MINUTES: 20
@@ -48,9 +48,9 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ inputs.GIT_SHA }}
-      BUCKET: aptos-mainnet-backup-backup-831a69a8
+      BUCKET: aptos-mainnet-backup-backup-e098483d
       SUB_DIR: e1
-      BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml
+      BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml
       # workflow config
       RUNS_ON: high-perf-docker-with-local-ssd
       TIMEOUT_MINUTES: 20
@@ -61,9 +61,9 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ github.event.pull_request.head.sha }}
-      BUCKET: aptos-testnet-backup-2223d95b
+      BUCKET: aptos-testnet-backup-b7b1ad7a
       SUB_DIR: e1
-      BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml
+      BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/gcs.yaml
       # workflow config
       RUNS_ON: "high-perf-docker-with-local-ssd"
       TIMEOUT_MINUTES: 20