Skip to content

Commit

Permalink
Merge branch 'main' into rosetta-fa-support
Browse files Browse the repository at this point in the history
  • Loading branch information
gregnazario authored Oct 8, 2024
2 parents eb88e1d + ebc1b64 commit bc91c4e
Show file tree
Hide file tree
Showing 488 changed files with 18,351 additions and 6,534 deletions.
3 changes: 2 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@
!crates/aptos/src/move_tool/*.bpl
!crates/aptos/src/node/local_testnet/hasura_metadata.json
!crates/aptos-faucet/doc/
!crates/transaction-emitter-lib/src/emitter/test_proofs_for_localnet_txn_emitter.txt
!api/doc/
!crates/indexer/migrations/**/*.sql
!ecosystem/indexer-grpc/indexer-grpc-parser/migrations/**/*.sql
!ecosystem/nft-metadata-crawler-parser/migrations/**/*.sql
!ecosystem/nft-metadata-crawler/migrations/**/*.sql
!rust-toolchain.toml
!scripts/
!terraform/helm/aptos-node/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
name: "Determine or Use Target Branch and Get Latest Docker Image"
description: |
Determine the branch to fetch the latest docker image tag from, or use a target branch directly.
- If base-branch is set, determine the target branch to fetch the latest docker image tag from
- Determine the IMAGE_TAG based on the latest release R and check for images built on branch R - 1:
- For commits on the aptos-release-v1.19 branch, the IMAGE_TAG should be the latest commit built on aptos-release-v1.18
- For commits on the main branch, the IMAGE_TAG should be the latest commit on the max release aptos-release-v<X.X> branch
- For commits on other branches, the IMAGE_TAG should be the latest commit on the branch's last release aptos-release-v<X.X> branch
- If branch is set, use it directly
inputs:
base-branch:
description: "The base branch to determine the target from, or use a target branch directly"
required: false
branch:
description: "Use this branch directly if set"
required: false
variants:
description: "The variants to check, as a space-separated string, e.g. 'performance failpoints'"
required: false

outputs:
TARGET_BRANCH:
description: "The determined or target target branch"
value: ${{ steps.set-target-branch.outputs.TARGET_BRANCH }}
IMAGE_TAG:
description: "The latest docker image tag for the given branch and variants"
value: ${{ steps.determine-test-image-tag.outputs.IMAGE_TAG }}

runs:
using: composite
steps:
# Checkout repository based on base branch or target branch
- name: Checkout branch
uses: actions/checkout@v4
with:
ref: ${{ inputs.base-branch || inputs.branch }}
path: checkout_branch
fetch-depth: 0

- name: Setup Python environment
uses: ./checkout_branch/.github/actions/python-setup
with:
pyproject_directory: checkout_branch/testsuite

# Determine the target branch if base-branch is used
- name: Set target branch
id: set-target-branch
run: |
if [[ -n "${{ inputs.base-branch }}" ]]; then
base_branch="${{ inputs.base-branch }}"
echo "Determining target branch from base branch: $base_branch"
./testrun determine_target_branch_to_fetch_last_released_image.py "$base_branch"
else
echo "Using target branch: ${{ inputs.branch }}"
echo "TARGET_BRANCH=${{ inputs.branch }}" >> $GITHUB_OUTPUT
fi
shell: bash
working-directory: checkout_branch/testsuite

# Checkout the determined or target branch
- name: Checkout target branch
if: ${{ steps.set-target-branch.outputs.TARGET_BRANCH != inputs.branch }}
uses: actions/checkout@v4
with:
ref: ${{ steps.set-target-branch.outputs.TARGET_BRANCH }}
path: checkout_branch
fetch-depth: 0

# Setup Python environment again after second checkout, as branches are different
- name: Setup Python environment again
if: ${{ steps.set-target-branch.outputs.TARGET_BRANCH != inputs.branch }}
uses: ./checkout_branch/.github/actions/python-setup
with:
pyproject_directory: checkout_branch/testsuite

# Determine image tag using the target branch
- name: Determine image tag
id: determine-test-image-tag
run: |
variants=(${{ inputs.variants }}) # split the variants string into an array
variants_args=()
for variant in "${variants[@]}"; do
variants_args+=("--variant" "$variant")
done
./testrun find_latest_image.py "${variants_args[@]}"
shell: bash
working-directory: checkout_branch/testsuite
3 changes: 2 additions & 1 deletion .github/runs-on.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ images:
aptos-ubuntu-x64:
platform: "linux"
arch: "x64"
ami: "ami-05f29bbd9fcfcca20"
ami: "ami-09f7d4bfb08b771be" # image for testing, with gcloud sdk and kubectl
owner: "058264343338" # aptos-ci AWS account id
aptos-ubuntu-x64-latest:
platform: "linux"
arch: "x64"
Expand Down
15 changes: 7 additions & 8 deletions .github/workflows/adhoc-forge.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,10 @@ on:
required: false
type: string
description: The Forge k8s cluster to be used for test
FORGE_ENABLE_HAPROXY:
FORGE_ENABLE_INDEXER:
required: false
default: false
type: boolean
description: enable haproxy for the forge test
description: Whether to use indexer
FORGE_NUM_VALIDATORS:
required: false
type: string
Expand Down Expand Up @@ -67,7 +66,7 @@ jobs:
echo "FORGE_RUNNER_DURATION_SECS: ${{ inputs.FORGE_RUNNER_DURATION_SECS }}"
echo "FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}"
echo "FORGE_CLUSTER_NAME: ${{ inputs.FORGE_CLUSTER_NAME }}"
echo "FORGE_ENABLE_HAPROXY: ${{ inputs.FORGE_ENABLE_HAPROXY }}"
echo "FORGE_ENABLE_INDEXER: ${{ inputs.FORGE_ENABLE_INDEXER }}"
echo "FORGE_NUM_VALIDATORS: ${{ inputs.FORGE_NUM_VALIDATORS }}"
echo "FORGE_NUM_VALIDATOR_FULLNODES: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}"
echo "FORGE_RETAIN_DEBUG_LOGS: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}"
Expand All @@ -78,10 +77,10 @@ jobs:
forgeRunnerDurationSecs: ${{ inputs.FORGE_RUNNER_DURATION_SECS || 600 }}
forgeTestSuite: ${{ inputs.FORGE_TEST_SUITE }}
forgeClusterName: ${{ inputs.FORGE_CLUSTER_NAME }}
forgeEnableHaproxy: ${{ inputs.FORGE_ENABLE_HAPROXY }}
forgeEnableIndexer: ${{ inputs.FORGE_ENABLE_INDEXER }}
forgeNumValidators: ${{ inputs.FORGE_NUM_VALIDATORS }}
forgeNumValidatorFullnodes: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}

forgeRetainDebugLogs: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}
adhoc-forge-test:
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
needs: [determine-forge-run-metadata]
Expand All @@ -93,7 +92,7 @@ jobs:
FORGE_TEST_SUITE: ${{ needs.determine-forge-run-metadata.outputs.forgeTestSuite }}
FORGE_RUNNER_DURATION_SECS: ${{ fromJSON(needs.determine-forge-run-metadata.outputs.forgeRunnerDurationSecs) }} # fromJSON converts to integer
FORGE_CLUSTER_NAME: ${{ needs.determine-forge-run-metadata.outputs.forgeClusterName }}
FORGE_ENABLE_HAPROXY: ${{ needs.determine-forge-run-metadata.outputs.forgeEnableHaproxy }}
FORGE_ENABLE_INDEXER: ${{ needs.determine-forge-run-metadata.outputs.forgeEnableIndexer == 'true' }}
FORGE_NUM_VALIDATORS: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidators }}
FORGE_NUM_VALIDATOR_FULLNODES: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidatorFullnodes }}
FORGE_RETAIN_DEBUG_LOGS: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}
FORGE_RETAIN_DEBUG_LOGS: ${{ needs.determine-forge-run-metadata.outputs.forgeRetainDebugLogs == 'true' }}
6 changes: 3 additions & 3 deletions .github/workflows/cargo-metadata-upload.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ permissions:
id-token: write
jobs:
cargo-metadata:
runs-on: ubuntu-latest
runs-on: runs-on,cpu=4,ram=16,family=m7a+m7i-flex,image=aptos-ubuntu-x64,run-id=${{ github.run_id }},spot=co
steps:
- uses: actions/checkout@v4
- uses: dsherret/rust-toolchain-file@v1
Expand All @@ -20,7 +20,7 @@ jobs:
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
- uses: "google-github-actions/setup-gcloud@v2"
- shell: bash
run: |
cargo metadata --all-features | gsutil cp - gs://aptos-core-cargo-metadata-public/metadata-${{ github.sha }}.json
set -ex
cargo metadata --all-features | gcloud storage cp - gs://aptos-core-cargo-metadata-public/metadata-${{ github.sha }}.json
70 changes: 55 additions & 15 deletions .github/workflows/docker-build-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -245,17 +245,6 @@ jobs:
GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}

indexer-grpc-e2e-tests:
needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
if: |
github.event_name == 'workflow_dispatch' ||
contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
contains(github.event.pull_request.body, '#e2e')
uses: aptos-labs/aptos-core/.github/workflows/docker-indexer-grpc-test.yaml@main
secrets: inherit
with:
GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}

# This is a PR required job.
forge-e2e-test:
needs:
Expand Down Expand Up @@ -288,11 +277,61 @@ jobs:
# by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace.
FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}

# This job determines the last released docker image tag, which is used by forge compat test.
fetch-last-released-docker-image-tag:
needs:
- permission-check
# runs only when need to run forge-compat-test or forge-framework-upgrade-test
if: |
!failure() && !cancelled() && needs.permission-check.result == 'success' && (
(github.event_name == 'push' && github.ref_name != 'main') ||
github.event_name == 'workflow_dispatch' ||
contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
contains(github.event.pull_request.labels.*.name, 'CICD:run-framework-upgrade-test') ||
github.event.pull_request.auto_merge != null ||
contains(github.event.pull_request.body, '#e2e')
)
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 1
ref: ${{ github.ref }}

# actions/get-latest-docker-image-tag requires docker utilities and having authenticated to internal docker image registries
- uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
id: docker-setup
with:
GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
EXPORT_GCP_PROJECT_VARIABLES: "false"
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DOCKER_ARTIFACT_REPO: ${{ secrets.AWS_DOCKER_ARTIFACT_REPO }}
GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}

- name: Get Docker Image Tag
uses: ./.github/actions/determine-or-use-target-branch-and-get-last-released-image
id: get-docker-image-tag
with:
base-branch: ${{ github.base_ref }}
variants: "failpoints performance"

- name: Add Image Tag to Step Summary
run: |
echo "## Image Tag for compat tests" >> $GITHUB_STEP_SUMMARY
echo "IMAGE_TAG: ${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}" >> $GITHUB_STEP_SUMMARY
echo "TARGET_BRANCH: ${{ steps.get-docker-image-tag.outputs.TARGET_BRANCH }}" >> $GITHUB_STEP_SUMMARY
outputs:
IMAGE_TAG: ${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}

# Run e2e compat test against testnet branch. This is a PR required job.
forge-compat-test:
needs:
needs:
- permission-check
- fetch-last-released-docker-image-tag
- determine-docker-build-metadata
- rust-images
- rust-images-failpoints
Expand All @@ -312,16 +351,17 @@ jobs:
with:
GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
FORGE_TEST_SUITE: compat
IMAGE_TAG: 25a081116546670e62ca927ba90478de78557056 #aptos-node-v1.20
IMAGE_TAG: ${{ needs.fetch-last-released-docker-image-tag.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 300
COMMENT_HEADER: forge-compat
FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}

# Run forge framework upgradability test. This is a PR required job.
forge-framework-upgrade-test:
needs:
- permission-check
- fetch-last-released-docker-image-tag
- determine-docker-build-metadata
- rust-images
- rust-images-failpoints
Expand All @@ -340,7 +380,7 @@ jobs:
with:
GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
FORGE_TEST_SUITE: framework_upgrade
IMAGE_TAG: 25a081116546670e62ca927ba90478de78557056 #aptos-node-v1.20
IMAGE_TAG: ${{ needs.fetch-last-released-docker-image-tag.outputs.IMAGE_TAG }}
FORGE_RUNNER_DURATION_SECS: 3600
COMMENT_HEADER: forge-framework-upgrade
FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/docker-indexer-grpc-test.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# THIS WORKFLOW IS DEPRECATED. Keep it around for branches that still reference it on the main branch.
# This file should eventually be deleted.
name: "Docker Indexer gRPC test"
on:
pull_request:
Expand Down
13 changes: 11 additions & 2 deletions .github/workflows/forge-stable.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
IMAGE_TAG: ${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}
IMAGE_TAG_FOR_COMPAT_TEST: ${{ steps.get-last-released-image-tag-for-compat-test.outputs.IMAGE_TAG }}
BRANCH: ${{ steps.determine-test-branch.outputs.BRANCH }}
BRANCH_HASH: ${{ steps.hash-branch.outputs.BRANCH_HASH }}
steps:
Expand Down Expand Up @@ -110,9 +111,17 @@ jobs:
branch: ${{ steps.determine-test-branch.outputs.BRANCH }}
variants: "failpoints performance"

- uses: ./.github/actions/determine-or-use-target-branch-and-get-last-released-image
id: get-last-released-image-tag-for-compat-test
with:
base-branch: ${{ steps.determine-test-branch.outputs.BRANCH }}
variants: "failpoints performance"

- name: Write summary
run: |
IMAGE_TAG=${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}
IMAGE_TAG_FOR_COMPAT_TEST=${{ steps.get-last-released-image-tag-for-compat-test.outputs.IMAGE_TAG }}
TARGET_BRANCH_TO_FETCH_IMAGE_FOR_COMPAT_TEST=${{ steps.get-last-released-image-tag-for-compat-test.outputs.TARGET_BRANCH }}
BRANCH=${{ steps.determine-test-branch.outputs.BRANCH }}
if [ -n "${BRANCH}" ]; then
echo "BRANCH: [${BRANCH}](https://github.com/${{ github.repository }}/tree/${BRANCH})" >> $GITHUB_STEP_SUMMARY
Expand All @@ -128,7 +137,7 @@ jobs:
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: 25a081116546670e62ca927ba90478de78557056 #aptos-node-v1.20
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG_FOR_COMPAT_TEST }}
FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
FORGE_TEST_SUITE: framework_upgrade
Expand Down Expand Up @@ -269,7 +278,7 @@ jobs:
FORGE_RUNNER_DURATION_SECS: 300 # Run for 5 minutes
# This will upgrade from testnet branch to the latest main
FORGE_TEST_SUITE: compat
IMAGE_TAG: 25a081116546670e62ca927ba90478de78557056 #aptos-node-v1.20
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG_FOR_COMPAT_TEST }}
GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout
POST_TO_SLACK: true

Expand Down
Loading

0 comments on commit bc91c4e

Please sign in to comment.