fix: add retry around docker login and revive spot_run_test_script (#…

…2346)

- couldn't use retry directly as I didn't want the password to be part
of a bash -c command, and otherwise couldnt pass the full string to
retry
- deleted script was still used

build-system/scripts/build

@@ -57,7 +57,16 @@ function try_fetch_image() {
   return 0
 }
 
-echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin
+function docker_login() {
+  # Retries up to 3 times with 10 second intervals
+  for i in $(seq 1 3); do
+    echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin && return || sleep 10
+  done
+  echo "$@ failed docker_login after 3 attempts"
+  exit 1
+}
+
+docker_login
 
 # Ensure ECR repository exists.
 retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle

build-system/scripts/create_dockerhub_manifest

@@ -49,8 +49,17 @@ IMAGE_TAG=$COMMIT_TAG_VERSION
 MANIFEST_DEPLOY_URI=$ACCOUNT/$REPOSITORY:$IMAGE_TAG
 MANIFEST_LATEST_URI=$ACCOUNT/$REPOSITORY:latest
 
+function docker_login() {
+  # Retries up to 3 times with 10 second intervals
+  for i in $(seq 1 3); do
+    echo "$DOCKERHUB_PASSWORD" | docker login -u $USERNAME --password-stdin && return || sleep 10
+  done
+  echo "$@ failed docker_login after 3 attempts"
+  exit 1
+}
+
 # Login to dockerhub.
-echo "$DOCKERHUB_PASSWORD" | docker login -u $USERNAME --password-stdin
+docker_login
 
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
@@ -79,4 +88,4 @@ docker manifest push --purge $MANIFEST_DEPLOY_URI
 
 echo "Pushing manifest list $MANIFEST_LATEST_URI..."
 # Push the latest tagged list
-docker manifest push --purge $MANIFEST_LATEST_URI
+docker manifest push --purge $MANIFEST_LATEST_URI

build-system/scripts/deploy_dockerhub

@@ -55,8 +55,17 @@ echo "Deploying to dockerhub: $IMAGE_DEPLOY_URI"
 # Login.
 retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION
 
+function docker_login() {
+  # Retries up to 3 times with 10 second intervals
+  for i in $(seq 1 3); do
+    echo "$DOCKERHUB_PASSWORD" | docker login -u $USERNAME --password-stdin && return || sleep 10
+  done
+  echo "$@ failed docker_login after 3 attempts"
+  exit 1
+}
+
 # Login to dockerhub.
-echo "$DOCKERHUB_PASSWORD" | docker login -u $USERNAME --password-stdin
+docker_login
 
 echo "Pulling $IMAGE_COMMIT_URI"
 # Pull image.

build-system/scripts/ensure_repo

@@ -26,7 +26,19 @@ REPOSITORY=$1
 REGION=$2
 REFRESH_LIFECYCLE=${3:-}
 
-aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null
+function docker_login() {
+  # Retries up to 3 times with 10 second intervals
+  for i in $(seq 1 3); do
+    aws ecr get-login-password --region $REGION \
+      | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \
+      && return || sleep 10
+  done
+  echo "$@ failed docker_login after 3 attempts"
+  exit 1
+}
+
+# Login to dockerhub.
+docker_login
 
 # Create the repository if it doesn't exist.
 if ! aws ecr describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then

build-system/scripts/spot_run_test_script

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -eu
+SCRIPT_PATH=$1
+REPOSITORY=$2
+shift
+shift
+
+cd $(query_manifest projectDir $REPOSITORY)
+
+mkdir -p /tmp/test-logs
+
+set -o pipefail
+
+CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
+echo "Content hash: $CONTENT_HASH"
+spot_run_script $CONTENT_HASH 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log"

yarn-project/canary/scripts/run_tests

@@ -12,7 +12,19 @@ if [ "$TEST" = "uniswap_trade_on_l1_from_l2.test.ts" ]; then
   export FORK_BLOCK_NUMBER=17514288
 fi
 
-aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com
+function docker_login() {
+  # Retries up to 3 times with 10 second intervals
+  for i in $(seq 1 3); do
+    aws ecr get-login-password --region us-east-2 \
+      | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com \
+      && return || sleep 10
+  done
+  echo "$@ failed docker_login after 3 attempts"
+  exit 1
+}
+
+# Login to dockerhub.
+docker_login
 
 export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"
 IMAGE_URI=$(calculate_image_uri $IMAGE)

yarn-project/end-to-end/scripts/run_tests_local

@@ -6,7 +6,19 @@ set -exu
 export TEST=$1
 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml}
 
-aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com
+function docker_login() {
+  # Retries up to 3 times with 10 second intervals
+  for i in $(seq 1 3); do
+    aws ecr get-login-password --region us-east-2 \
+      | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com \
+      && return || sleep 10
+  done
+  echo "$@ failed docker_login after 3 attempts"
+  exit 1
+}
+
+# Login to dockerhub.
+docker_login
 
 export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts"