diff --git a/.github/workflows/docker-build-test.yaml b/.github/workflows/docker-build-test.yaml index 4f079c829e087e..4ad334e77d7f1b 100644 --- a/.github/workflows/docker-build-test.yaml +++ b/.github/workflows/docker-build-test.yaml @@ -61,6 +61,10 @@ env: # We use `pr-` as cache-id for PRs and simply otherwise. TARGET_CACHE_ID: ${{ github.event.number && format('pr-{0}', github.event.number) || github.ref_name }} + # On PRs, only build and push to GCP + # On push, build and push to all remote registries + TARGET_REGISTRY: ${{ github.event_name == 'pull_request_target' && 'remote' || 'remote-all' }} + permissions: contents: read id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry @@ -99,9 +103,11 @@ jobs: run: | echo "GIT_SHA: ${GIT_SHA}" echo "TARGET_CACHE_ID: ${TARGET_CACHE_ID}" + echo "TARGET_REGISTRY: ${TARGET_REGISTRY}" outputs: gitSha: ${{ env.GIT_SHA }} targetCacheId: ${{ env.TARGET_CACHE_ID }} + targetRegistry: ${{ env.TARGET_REGISTRY }} rust-images: needs: [permission-check, determine-docker-build-metadata] @@ -112,6 +118,7 @@ jobs: TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} PROFILE: release BUILD_ADDL_TESTING_IMAGES: true + TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} rust-images-indexer: needs: [permission-check, determine-docker-build-metadata] @@ -127,6 +134,7 @@ jobs: PROFILE: release FEATURES: indexer BUILD_ADDL_TESTING_IMAGES: true + TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} rust-images-failpoints: needs: [permission-check, determine-docker-build-metadata] @@ -142,6 +150,7 @@ jobs: PROFILE: release FEATURES: failpoints BUILD_ADDL_TESTING_IMAGES: true + TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} rust-images-performance: needs: [permission-check, determine-docker-build-metadata] @@ -156,6 +165,7 @@ jobs: TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }} PROFILE: performance BUILD_ADDL_TESTING_IMAGES: true + TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} rust-images-consensus-only-perf-test: needs: [permission-check, determine-docker-build-metadata] @@ -170,6 +180,7 @@ jobs: PROFILE: release FEATURES: consensus-only-perf-test BUILD_ADDL_TESTING_IMAGES: true + TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }} sdk-release: needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images" diff --git a/.github/workflows/forge-stable.yaml b/.github/workflows/forge-stable.yaml index 475655abe3a414..6eaaef78abb7ca 100644 --- a/.github/workflows/forge-stable.yaml +++ b/.github/workflows/forge-stable.yaml @@ -25,6 +25,7 @@ on: pull_request: paths: - ".github/workflows/forge-stable.yaml" + - "testsuite/find_latest_image.py" push: branches: - aptos-release-v* # the aptos release branches @@ -75,6 +76,18 @@ jobs: with: cancel-workflow: ${{ github.event_name == 'schedule' }} # Cancel the workflow if it is scheduled on a fork + # find_latest_images.py requires docker utilities and having authenticated to internal docker image registries + - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main + id: docker-setup + with: + GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} + EXPORT_GCP_PROJECT_VARIABLES: "false" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DOCKER_ARTIFACT_REPO: ${{ secrets.AWS_DOCKER_ARTIFACT_REPO }} + GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} + - uses: ./.github/actions/python-setup with: pyproject_directory: testsuite diff --git a/.github/workflows/forge-unstable.yaml b/.github/workflows/forge-unstable.yaml index f7dd7e47f75f1c..299318a0f23b8e 100644 --- a/.github/workflows/forge-unstable.yaml +++ b/.github/workflows/forge-unstable.yaml @@ -69,6 +69,18 @@ jobs: with: cancel-workflow: ${{ github.event_name == 'schedule' }} # Cancel the workflow if it is scheduled on a fork + # find_latest_images.py requires docker utilities and having authenticated to internal docker image registries + - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main + id: docker-setup + with: + GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} + EXPORT_GCP_PROJECT_VARIABLES: "false" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DOCKER_ARTIFACT_REPO: ${{ secrets.AWS_DOCKER_ARTIFACT_REPO }} + GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} + - uses: ./.github/actions/python-setup with: pyproject_directory: testsuite diff --git a/.github/workflows/workflow-run-docker-rust-build.yaml b/.github/workflows/workflow-run-docker-rust-build.yaml index 8208263bfe427a..f11a01352f60c9 100644 --- a/.github/workflows/workflow-run-docker-rust-build.yaml +++ b/.github/workflows/workflow-run-docker-rust-build.yaml @@ -25,6 +25,12 @@ on: required: false type: boolean description: Whether to build additional testing images. If not specified, only the base release images will be built + TARGET_REGISTRY: + default: gcp + required: false + type: string + description: The target docker registry to push to + workflow_dispatch: inputs: GIT_SHA: @@ -45,6 +51,11 @@ on: required: false type: boolean description: Whether to build additional testing images. If not specified, only the base release images will be built + TARGET_REGISTRY: + default: gcp + required: false + type: string + description: The target docker registry to push to env: GIT_SHA: ${{ inputs.GIT_SHA }} @@ -55,6 +66,11 @@ env: GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }} GCP_DOCKER_ARTIFACT_REPO_US: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO_US }} AWS_ECR_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }} + TARGET_REGISTRY: ${{ inputs.TARGET_REGISTRY }} + +permissions: + contents: read + id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry jobs: rust-all: @@ -80,3 +96,4 @@ jobs: FEATURES: ${{ env.FEATURES }} BUILD_ADDL_TESTING_IMAGES: ${{ env.BUILD_ADDL_TESTING_IMAGES }} GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }} + TARGET_REGISTRY: ${{ env.TARGET_REGISTRY }} diff --git a/docker/builder/docker-bake-rust-all.hcl b/docker/builder/docker-bake-rust-all.hcl index 115dd2e5eaa12a..651d8053aad4b5 100644 --- a/docker/builder/docker-bake-rust-all.hcl +++ b/docker/builder/docker-bake-rust-all.hcl @@ -27,7 +27,7 @@ variable "GCP_DOCKER_ARTIFACT_REPO_US" {} variable "AWS_ECR_ACCOUNT_NUM" {} variable "TARGET_REGISTRY" { - // must be "aws" | "remote" | "local", informs which docker tags are being generated + // must be "gcp" | "local" | "remote-all" | "remote" (deprecated, but kept for backwards compatibility. Same as "gcp"), informs which docker tags are being generated default = CI == "true" ? "remote" : "local" } @@ -74,8 +74,8 @@ target "debian-base" { target "builder-base" { dockerfile = "docker/builder/builder.Dockerfile" - target = "builder-base" - context = "." + target = "builder-base" + context = "." contexts = { rust = "docker-image://rust:1.66.1-bullseye@sha256:f72949bcf1daf8954c0e0ed8b7e10ac4c641608f6aa5f0ef7c172c49f35bd9b5" } @@ -92,7 +92,7 @@ target "builder-base" { target "aptos-node-builder" { dockerfile = "docker/builder/builder.Dockerfile" - target = "aptos-node-builder" + target = "aptos-node-builder" contexts = { builder-base = "target:builder-base" } @@ -103,9 +103,9 @@ target "aptos-node-builder" { target "tools-builder" { dockerfile = "docker/builder/builder.Dockerfile" - target = "tools-builder" + target = "tools-builder" contexts = { - builder-base = "target:builder-base" + builder-base = "target:builder-base" } secret = [ "id=GIT_CREDENTIALS" @@ -114,8 +114,8 @@ target "tools-builder" { target "_common" { contexts = { - debian-base = "target:debian-base" - node-builder = "target:aptos-node-builder" + debian-base = "target:debian-base" + node-builder = "target:aptos-node-builder" tools-builder = "target:tools-builder" } labels = { @@ -124,12 +124,12 @@ target "_common" { "org.label-schema.git-sha" = "${GIT_SHA}" } args = { - PROFILE = "${PROFILE}" - FEATURES = "${FEATURES}" - GIT_SHA = "${GIT_SHA}" - GIT_BRANCH = "${GIT_BRANCH}" - GIT_TAG = "${GIT_TAG}" - BUILD_DATE = "${BUILD_DATE}" + PROFILE = "${PROFILE}" + FEATURES = "${FEATURES}" + GIT_SHA = "${GIT_SHA}" + GIT_BRANCH = "${GIT_BRANCH}" + GIT_TAG = "${GIT_TAG}" + BUILD_DATE = "${BUILD_DATE}" } } @@ -137,7 +137,7 @@ target "validator-testing" { inherits = ["_common"] dockerfile = "docker/builder/validator-testing.Dockerfile" target = "validator-testing" - cache-from = generate_cache_from("validator-testing") + cache-from = generate_cache_from("validator-testing") cache-to = generate_cache_to("validator-testing") tags = generate_tags("validator-testing") } @@ -146,7 +146,7 @@ target "tools" { inherits = ["_common"] dockerfile = "docker/builder/tools.Dockerfile" target = "tools" - cache-from = generate_cache_from("tools") + cache-from = generate_cache_from("tools") cache-to = generate_cache_to("tools") tags = generate_tags("tools") } @@ -155,7 +155,7 @@ target "forge" { inherits = ["_common"] dockerfile = "docker/builder/forge.Dockerfile" target = "forge" - cache-from = generate_cache_from("forge") + cache-from = generate_cache_from("forge") cache-to = generate_cache_to("forge") tags = generate_tags("forge") } @@ -164,7 +164,7 @@ target "validator" { inherits = ["_common"] dockerfile = "docker/builder/validator.Dockerfile" target = "validator" - cache-from = generate_cache_from("validator") + cache-from = generate_cache_from("validator") cache-to = generate_cache_to("validator") tags = generate_tags("validator") } @@ -173,7 +173,7 @@ target "tools" { inherits = ["_common"] dockerfile = "docker/builder/tools.Dockerfile" target = "tools" - cache-from = generate_cache_from("tools") + cache-from = generate_cache_from("tools") cache-to = generate_cache_to("tools") tags = generate_tags("tools") } @@ -182,7 +182,7 @@ target "node-checker" { inherits = ["_common"] dockerfile = "docker/builder/node-checker.Dockerfile" target = "node-checker" - cache-from = generate_cache_from("node-checker") + cache-from = generate_cache_from("node-checker") cache-to = generate_cache_to("node-checker") tags = generate_tags("node-checker") } @@ -191,8 +191,8 @@ target "faucet" { inherits = ["_common"] dockerfile = "docker/builder/faucet.Dockerfile" target = "faucet" - cache-from = generate_cache_from("faucet") - cache-to = generate_cache_to("faucet") + cache-from = generate_cache_from("faucet") + cache-to = generate_cache_to("faucet") tags = generate_tags("faucet") } @@ -200,17 +200,17 @@ target "telemetry-service" { inherits = ["_common"] dockerfile = "docker/builder/telemetry-service.Dockerfile" target = "telemetry-service" - cache-from = generate_cache_from("telemetry-service") - cache-to = generate_cache_to("telemetry-service") - tags = generate_tags("telemetry-service") + cache-from = generate_cache_from("telemetry-service") + cache-to = generate_cache_to("telemetry-service") + tags = generate_tags("telemetry-service") } target "indexer-grpc" { - inherits = ["_common"] + inherits = ["_common"] dockerfile = "docker/builder/indexer-grpc.Dockerfile" - target = "indexer-grpc" - cache-to = generate_cache_to("indexer-grpc") - tags = generate_tags("indexer-grpc") + target = "indexer-grpc" + cache-to = generate_cache_to("indexer-grpc") + tags = generate_tags("indexer-grpc") } function "generate_cache_from" { @@ -224,7 +224,7 @@ function "generate_cache_from" { function "generate_cache_to" { params = [target] - result = TARGET_REGISTRY == "remote" ? [ + result = TARGET_REGISTRY != "local" ? [ "type=registry,ref=${GCP_DOCKER_ARTIFACT_REPO}/${target}:cache-${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}", "type=registry,ref=${GCP_DOCKER_ARTIFACT_REPO}/${target}:cache-${IMAGE_TAG_PREFIX}${GIT_SHA}" ] : [] @@ -232,15 +232,22 @@ function "generate_cache_to" { function "generate_tags" { params = [target] - result = TARGET_REGISTRY == "remote" ? [ + result = TARGET_REGISTRY == "remote-all" ? [ + "${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}", + "${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}", + "${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}", + "${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}", + "${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}", + "${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}", + ] : ( + TARGET_REGISTRY == "gcp" || TARGET_REGISTRY == "remote" ? [ "${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}", "${GCP_DOCKER_ARTIFACT_REPO}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}", "${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}", "${GCP_DOCKER_ARTIFACT_REPO_US}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}", - "${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}", - "${ecr_base}/${target}:${IMAGE_TAG_PREFIX}${NORMALIZED_GIT_BRANCH_OR_PR}", - ] : [ - "aptos-core/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}-from-local", - "aptos-core/${target}:${IMAGE_TAG_PREFIX}from-local", - ] + ] : [ // "local" or any other value + "aptos-core/${target}:${IMAGE_TAG_PREFIX}${GIT_SHA}-from-local", + "aptos-core/${target}:${IMAGE_TAG_PREFIX}from-local", + ] + ) } diff --git a/docker/builder/docker-bake-rust-all.sh b/docker/builder/docker-bake-rust-all.sh index 4589fd1fa20ad6..dc62f1f02ab286 100755 --- a/docker/builder/docker-bake-rust-all.sh +++ b/docker/builder/docker-bake-rust-all.sh @@ -51,9 +51,9 @@ echo "To build only a specific target, run: docker/builder/docker-bake-rust-all. echo "E.g. docker/builder/docker-bake-rust-all.sh forge-images" if [ "$CI" == "true" ]; then - TARGET_REGISTRY=remote docker buildx bake --progress=plain --file docker/builder/docker-bake-rust-all.hcl --push $BUILD_TARGET + docker buildx bake --progress=plain --file docker/builder/docker-bake-rust-all.hcl --push $BUILD_TARGET else - TARGET_REGISTRY=local docker buildx bake --file docker/builder/docker-bake-rust-all.hcl $BUILD_TARGET + docker buildx bake --file docker/builder/docker-bake-rust-all.hcl $BUILD_TARGET fi echo "Build complete. Docker buildx cache usage:" diff --git a/testsuite/find_latest_image.py b/testsuite/find_latest_image.py index 4cbfac2d88cce0..4ea90743a59d5a 100644 --- a/testsuite/find_latest_image.py +++ b/testsuite/find_latest_image.py @@ -13,6 +13,7 @@ from forge import find_recent_images, image_exists from test_framework.shell import LocalShell from test_framework.git import Git +from test_framework.cluster import Cloud # gh output logic from determinator from determinator import GithubOutput, write_github_output @@ -34,7 +35,7 @@ def main() -> None: "--image-name", "-i", help="The name of the image to search for", - default="aptos/validator-testing", + default="validator-testing", ) parser.add_argument( "--variant", @@ -44,13 +45,22 @@ def main() -> None: dest="variants", default=[], ) + parser.add_argument( + "--cloud", + "-c", + help="The cloud to use", + choices=[c.value for c in Cloud], + default=Cloud.GCP.value, + ) args = parser.parse_args() image_name = args.image_name + cloud = Cloud(args.cloud) + log.info(f"Using cloud: {cloud}") # If the IMAGE_TAG environment variable is set, check that if IMAGE_TAG_ENV in os.environ and os.environ[IMAGE_TAG_ENV]: image_tag = os.environ[IMAGE_TAG_ENV] - if not image_exists(shell, image_name, image_tag): + if not image_exists(shell, image_name, image_tag, cloud=cloud): sys.exit(1) variants = args.variants @@ -63,7 +73,9 @@ def main() -> None: # Find the latest image from git history num_images_to_find = 1 # for the purposes of this script, this is always 1 images = list( - find_recent_images(shell, git, num_images_to_find, image_name, variant_prefixes) + find_recent_images( + shell, git, num_images_to_find, image_name, variant_prefixes, cloud=cloud + ) ) log.info(f"Found latest images: {images}") diff --git a/testsuite/fixtures/testMain.fixture b/testsuite/fixtures/testMain.fixture index 233d7d5c072938..0fd5f0bcbc28e7 100644 --- a/testsuite/fixtures/testMain.fixture +++ b/testsuite/fixtures/testMain.fixture @@ -1,9 +1,13 @@ Looking for cluster aptos-forge-big-1 in cloud AWS Found cluster: Cloud.AWS/us-west-2/aptos-forge-big-1 +Checking if image exists in GCP: aptos/validator-testing:banana Using the following image tags: forge: banana swarm: banana swarm upgrade (if applicable): banana +Checking if image exists in GCP: aptos/validator-testing:banana +Checking if image exists in GCP: aptos/validator-testing:banana +Checking if image exists in GCP: aptos/forge:banana === Start temp-pre-comment === ### Forge is running suite `banana-test` on `banana` * [Grafana dashboard (auto-refresh)](https://aptoslabs.grafana.net/d/overview/overview?orgId=1&refresh=10s&var-Datasource=VictoriaMetrics%20Global%20%28Non-mainnet%29&var-BigQuery=Google%20BigQuery&var-namespace=forge-perry-1659078000&var-metrics_source=All&var-chain_name=forge-big-1&refresh=10s&from=now-15m&to=now) diff --git a/testsuite/forge.py b/testsuite/forge.py index 0fd7f96fa66ffd..d9ef684b12cdb4 100644 --- a/testsuite/forge.py +++ b/testsuite/forge.py @@ -47,15 +47,18 @@ "release": "", # the default release profile has no tag prefix } -VALIDATOR_IMAGE_NAME = "aptos/validator" -VALIDATOR_TESTING_IMAGE_NAME = "aptos/validator-testing" -FORGE_IMAGE_NAME = "aptos/forge" +VALIDATOR_IMAGE_NAME = "validator" +VALIDATOR_TESTING_IMAGE_NAME = "validator-testing" +FORGE_IMAGE_NAME = "forge" +ECR_REPO_PREFIX = "aptos" DEFAULT_CONFIG = "forge-wrapper-config" DEFAULT_CONFIG_KEY = "forge-wrapper-config.json" FORGE_TEST_RUNNER_TEMPLATE_PATH = "forge-test-runner-template.yaml" +GAR_REPO_NAME = "us-west1-docker.pkg.dev/aptos-global/aptos-internal" + @dataclass class RunResult: @@ -676,12 +679,12 @@ def run(self, context: ForgeContext) -> ForgeResult: # determine the interal image repos based on the context of where the cluster is located if context.cloud == Cloud.AWS: - forge_image_full = f"{context.aws_account_num}.dkr.ecr.{context.aws_region}.amazonaws.com/aptos/forge:{context.forge_image_tag}" + forge_image_full = f"{context.aws_account_num}.dkr.ecr.{context.aws_region}.amazonaws.com/{ECR_REPO_PREFIX}/forge:{context.forge_image_tag}" validator_node_selector = "eks.amazonaws.com/nodegroup: validators" elif ( context.cloud == Cloud.GCP ): # the GCP project for images is separate than the cluster - forge_image_full = f"us-west1-docker.pkg.dev/aptos-global/aptos-internal/forge:{context.forge_image_tag}" + forge_image_full = f"{GAR_REPO_NAME}/forge:{context.forge_image_tag}" validator_node_selector = "" # no selector # TODO: also no NAP node selector yet # TODO: also registries need to be set up such that the default compute service account can access it: $PROJECT_ID-compute@developer.gserviceaccount.com @@ -852,6 +855,7 @@ def find_recent_images_by_profile_or_features( num_images: int, enable_failpoints: Optional[bool], enable_performance_profile: Optional[bool], + cloud: Cloud = Cloud.GCP, ) -> Sequence[str]: image_tag_prefix = "" if enable_failpoints and enable_performance_profile: @@ -870,6 +874,7 @@ def find_recent_images_by_profile_or_features( num_images, image_name=VALIDATOR_TESTING_IMAGE_NAME, image_tag_prefixes=[image_tag_prefix], + cloud=cloud, ) @@ -880,6 +885,7 @@ def find_recent_images( image_name: str, image_tag_prefixes: List[str] = [""], commit_threshold: int = 100, + cloud: Cloud = Cloud.GCP, ) -> Sequence[str]: """ Find the last `num_images` images built from the current git repo by searching the git commit history @@ -901,7 +907,7 @@ def find_recent_images( temp_ret = [] # count variants for this revision for prefix in image_tag_prefixes: image_tag = f"{prefix}{revision}" - exists = image_exists(shell, image_name, image_tag) + exists = image_exists(shell, image_name, image_tag, cloud=cloud) if exists: temp_ret.append(image_tag) if len(temp_ret) >= num_variants: @@ -916,19 +922,40 @@ def find_recent_images( return ret -def image_exists(shell: Shell, image_name: str, image_tag: str) -> bool: - result = shell.run( - [ - "aws", - "ecr", - "describe-images", - "--repository-name", - f"{image_name}", - "--image-ids", - f"imageTag={image_tag}", - ] - ) - return result.exit_code == 0 +def image_exists( + shell: Shell, + image_name: str, + image_tag: str, + cloud: Cloud = Cloud.GCP, +) -> bool: + """Check if an image exists in a given repository""" + if cloud == Cloud.GCP: + full_image = f"{GAR_REPO_NAME}/{image_name}:{image_tag}" + return shell.run( + [ + "crane", + "manifest", + full_image, + ], + stream_output=True, + ).succeeded() + elif cloud == Cloud.AWS: + full_image = f"{ECR_REPO_PREFIX}/{image_name}:{image_tag}" + log.info(f"Checking if image exists in GCP: {full_image}") + return shell.run( + [ + "aws", + "ecr", + "describe-images", + "--repository-name", + f"{ECR_REPO_PREFIX}/{image_name}", + "--image-ids", + f"imageTag={image_tag}", + ], + stream_output=True, + ).succeeded() + else: + raise Exception(f"Unknown cloud repo type: {cloud}") def sanitize_forge_resource_name(forge_resource: str) -> str: @@ -1338,6 +1365,7 @@ def test( 2, enable_failpoints=enable_failpoints, enable_performance_profile=enable_performance_profile, + cloud=cloud_enum, ) ) # This might not work as intended because we dont know if that revision @@ -1354,6 +1382,7 @@ def test( 1, enable_failpoints=enable_failpoints, enable_performance_profile=enable_performance_profile, + cloud=cloud_enum, )[0] image_tag = image_tag or default_latest_image @@ -1378,13 +1407,13 @@ def test( # finally, whether we've derived the image tags or used the user-inputted ones, check if they exist assert image_exists( - shell, VALIDATOR_TESTING_IMAGE_NAME, image_tag + shell, VALIDATOR_TESTING_IMAGE_NAME, image_tag, cloud=cloud_enum ), f"swarm (validator) image does not exist: {image_tag}" assert image_exists( - shell, VALIDATOR_TESTING_IMAGE_NAME, upgrade_image_tag + shell, VALIDATOR_TESTING_IMAGE_NAME, upgrade_image_tag, cloud=cloud_enum ), f"swarm upgrade (validator) image does not exist: {upgrade_image_tag}" assert image_exists( - shell, FORGE_IMAGE_NAME, forge_image_tag + shell, FORGE_IMAGE_NAME, forge_image_tag, cloud=cloud_enum ), f"forge (test runner) image does not exist: {forge_image_tag}" forge_args = create_forge_command( diff --git a/testsuite/forge_test.py b/testsuite/forge_test.py index b5b3510b15de2e..ff3a2fcfc5e368 100644 --- a/testsuite/forge_test.py +++ b/testsuite/forge_test.py @@ -39,6 +39,7 @@ main, sanitize_forge_resource_name, validate_forge_config, + GAR_REPO_NAME, ) from click.testing import CliRunner, Result @@ -56,6 +57,7 @@ from test_framework.shell import SpyShell, FakeShell, FakeCommand, RunResult from test_framework.time import FakeTime +from test_framework.cluster import Cloud # Show the entire diff when unittest fails assertion unittest.util._MAX_LENGTH = 2000 # type: ignore @@ -278,7 +280,31 @@ def testFindRecentImage(self) -> None: ] ) git = Git(shell) - image_tags = find_recent_images(shell, git, 1, "aptos/validator-testing") + image_tags = find_recent_images( + shell, git, 1, "validator-testing", cloud=Cloud.AWS + ) + self.assertEqual(list(image_tags), ["lychee"]) + shell.assert_commands(self) + + def testFindRecentImageGcp(self) -> None: + shell = SpyShell( + [ + FakeCommand("git rev-parse HEAD~0", RunResult(0, b"potato\n")), + FakeCommand( + f"crane manifest {GAR_REPO_NAME}/validator-testing:potato", + RunResult(1, b""), + ), + FakeCommand("git rev-parse HEAD~1", RunResult(0, b"lychee\n")), + FakeCommand( + f"crane manifest {GAR_REPO_NAME}/validator-testing:lychee", + RunResult(0, b""), + ), + ] + ) + git = Git(shell) + image_tags = find_recent_images( + shell, git, 1, "validator-testing", cloud=Cloud.GCP + ) self.assertEqual(list(image_tags), ["lychee"]) shell.assert_commands(self) @@ -294,7 +320,12 @@ def testFindRecentFailpointsImage(self) -> None: ) git = Git(shell) image_tags = find_recent_images_by_profile_or_features( - shell, git, 1, enable_performance_profile=False, enable_failpoints=True + shell, + git, + 1, + enable_performance_profile=False, + enable_failpoints=True, + cloud=Cloud.AWS, ) self.assertEqual(list(image_tags), ["failpoints_tomato"]) shell.assert_commands(self) @@ -316,6 +347,7 @@ def testFindRecentPerformanceImage(self) -> None: 1, enable_performance_profile=True, enable_failpoints=False, + cloud=Cloud.AWS, ) self.assertEqual(list(image_tags), ["performance_potato"]) shell.assert_commands(self) @@ -368,7 +400,7 @@ def testFindRecentFewImages( ] ) git = Git(shell) - images = find_recent_images(shell, git, 2, "aptos/validator") + images = find_recent_images(shell, git, 2, "validator", cloud=Cloud.AWS) self.assertEqual(list(images), ["crab", "shrimp"]) def testFailpointsProvidedImageTag(self) -> None: