Revert "Merge master (#2)"

This reverts commit 9d7ea0b.

.circleci/build

@@ -0,0 +1,132 @@
+#!/bin/bash
+#
+# Builds a docker image and pushes it to it's repository. Leverages caches where possible.
+# Cached images include previous successfully built images built on this branch.
+# The images output are cache images, meaning they will eventually get purged.
+# The deploy phase will tag the images such that they become permanent.
+#
+# usage: ./build <repository>
+# example: ./build barretenberg-x86_64-linux-clang
+# output image:
+#   278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang:cache-deadbeefcafebabe1337c0d3
+
+set -e
+
+REPOSITORY=$1
+DOCKERFILE=$(query_manifest dockerfile $REPOSITORY)
+PROJECT_DIR=$(query_manifest projectDir $REPOSITORY)
+
+echo "Repository: $REPOSITORY"
+echo "Working directory: $PWD"
+echo "Dockerfile: $DOCKERFILE"
+
+cd $(query_manifest buildDir $REPOSITORY)
+
+function fetch_image() {
+  echo "Pulling: $1"
+  if ! docker pull $1 > /dev/null 2>&1; then
+    echo "Image not found: $1"
+    return 1
+  fi
+  return 0
+}
+
+# Ensure ECR repository exists.
+ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle
+
+LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY)
+echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT"
+
+# If we have previously successful commit, we can early out if nothing relevant has changed since.
+if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then
+  echo "No rebuild necessary. Retagging..."
+  STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p')
+  for STAGE in $STAGES; do
+    tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true
+  done
+  tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH
+  untag_remote_image $REPOSITORY tainted
+  exit 0
+fi
+
+# Validate any terraform if it exists.
+if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then
+  ensure_terraform
+  export TF_IN_AUTOMATION=1
+  pushd $ROOT_PATH/$PROJECT_DIR/terraform
+  for DIR in . $(find . -maxdepth 1 -type d); do
+    pushd $DIR
+    if [ -f ./main.tf ]; then
+      terraform init -input=false -backend-config="key=dummy"
+      terraform validate
+    fi
+    popd
+  done
+  popd
+fi
+
+# Pull latest parents that are not ours.
+echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin
+PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | grep -v $ECR_DEPLOY_URL | sort | uniq)
+for PARENT in $PARENTS; do
+  fetch_image $PARENT
+done
+
+# For each parent that's ours, pull in the latest image.
+PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+\).*/\1/p")
+for PARENT in $PARENTS; do
+  # Extract repository name (i.e. discard tag).
+  PARENT_REPO=${PARENT%:*}
+  PARENT_COMMIT_HASH=$(last_successful_commit $PARENT_REPO)
+  # There must be a parent image to continue.
+  if [ -z "$PARENT_COMMIT_HASH" ]; then
+    echo "No parent image found for $PARENT_REPO"
+    exit 1
+  fi
+  PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_COMMIT_HASH
+  echo "Pulling dependency $PARENT_REPO..."
+  fetch_image $PARENT_IMAGE_URI
+  # Tag it to look like an official release as that's what we use in Dockerfiles.
+  docker tag $PARENT_IMAGE_URI $ECR_DEPLOY_URL/$PARENT
+done
+
+# Pull, build and push each named stage to cache.
+STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p')
+for STAGE in $STAGES; do
+  # Get the last build of this stage to leverage layer caching.
+  if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then
+    echo "Pulling stage: $STAGE"
+    STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT-$STAGE
+    if fetch_image $STAGE_IMAGE_LAST_URI; then
+      STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI"
+    fi
+  fi
+
+  echo "Building stage: $STAGE"
+  STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH-$STAGE
+  docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE .
+
+  # We don't want to have redo this stages work when building the final image. Use it as a layer cache.
+  CACHE_FROM="--cache-from $STAGE_IMAGE_COMMIT_URI $CACHE_FROM"
+
+  echo "Pushing stage: $STAGE"
+  docker push $STAGE_IMAGE_COMMIT_URI > /dev/null 2>&1
+  echo
+done
+
+# Pull previous image to use it as a layer cache if it exists.
+if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then
+  LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT
+  echo "Pulling previous build of $REPOSITORY..."
+  fetch_image $LAST_SUCCESSFUL_URI || true
+  CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM"
+  echo
+fi
+
+# Build the actual image and give it a commit tag.
+IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH
+echo "Building image: $IMAGE_COMMIT_URI"
+docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG .
+echo "Pushing image: $IMAGE_COMMIT_URI"
+docker push $IMAGE_COMMIT_URI > /dev/null 2>&1
+untag_remote_image $REPOSITORY tainted

.circleci/build_local

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Builds the PROJECTS in the given order.
+# Will terminate build at TARGET_PROJECT (if given).
+# Will only build TARGET_PROJECT if ONLY_TARGET given.
+# Will push TARGET_PROJECT to ECR with PUSH_LABEL (if given).
+# PROJECT elements have structure WORKING_DIR:PROJECT:DOCKERFILE:REPO.
+# If DOCKERFILE is excluded it tries to default to ./Dockerfile then ./REPO/Dockerfile
+
+set -e
+
+TARGET_PROJECT=$1
+ONLY_TARGET=$2
+PUSH_LABEL=$3
+
+PROJECTS=(
+  cpp:barretenberg-cpp:./dockerfiles/Dockerfile.x86_64-linux-clang:barretenberg-x86_64-linux-clang
+  cpp:barretenberg-cpp:./dockerfiles/Dockerfile.wasm-linux-clang:barretenberg-wasm-linux-clang
+  # js:barretenberg-js
+)
+
+for E in ${PROJECTS[@]}; do
+  ARR=(${E//:/ })
+  WORKING_DIR=${ARR[0]}
+  PROJECT=${ARR[1]}
+  DOCKERFILE=${ARR[2]}
+  REPO=${ARR[3]:-$PROJECT}
+
+  if [ -n "$ONLY_TARGET" -a ! "$PROJECT" = "$TARGET_PROJECT" ]; then
+    continue
+  fi
+
+  pushd $WORKING_DIR
+
+  if [ ! -f "$DOCKERFILE" ]; then
+    DOCKERFILE=./Dockerfile
+    if [ ! -f "$DOCKERFILE" ]; then
+      DOCKERFILE=./$REPO/Dockerfile
+      if [ ! -f "$DOCKERFILE" ]; then
+        echo "Dockerfile not found."
+        exit 1
+      fi
+    fi
+  fi
+
+  echo
+  echo
+  echo
+  echo "*** Building $PROJECT:$DOCKERFILE -> $REPO ***"
+  echo
+
+  time docker build -f $DOCKERFILE -t $ECR_URL/$REPO:latest .
+
+  if [ -n "$TERM_AT_TARGET" -a "$PROJECT" = "$TARGET_PROJECT" ]; then
+    break
+  fi
+
+  popd
+done

.circleci/changed

@@ -0,0 +1,10 @@
+#!/bin/bash
+LAST_SUCCESSFUL_COMMIT=$1
+
+[ -n "$COMMIT_HASH" ] || exit 1
+
+if git diff --name-only $LAST_SUCCESSFUL_COMMIT $COMMIT_HASH | grep -q $2; then
+  exit 0
+else
+  exit 1
+fi

.circleci/check_rebuild

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Succeeds if any files matching the rebuild patterns, have changed since the base commit.
+set -e
+
+BASE_COMMIT=$1
+REPOSITORY=$2
+
+# If given nothing, then return true to rebuild.
+[ -n "$BASE_COMMIT" ] || exit 0
+
+# If a tainted tag exists, remove it and return true to rebuild.
+if image_exists $REPOSITORY tainted; then
+  echo "$REPOSITORY has been tainted. Will rebuild."
+  exit 0
+fi
+
+# Compute .rebuild_patterns from the build manifest.
+query_manifest rebuildPatterns $REPOSITORY > .rebuild_patterns
+
+echo "Rebuild patterns:"
+cat .rebuild_patterns
+
+git config diff.renameLimit 999999
+
+if git diff --name-only ${BASE_COMMIT} ${COMMIT_HASH} | grep -q -f .rebuild_patterns; then
+  exit 0
+else
+  exit 1
+fi

.circleci/cond_spot_run_build

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+REPOSITORY=$1
+shift
+SPEC=$1
+shift
+DOCKERFILE=$(query_manifest dockerfile $REPOSITORY)
+
+cd $(query_manifest buildDir $REPOSITORY)
+
+LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY)
+echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT"
+
+if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then
+  spot_run_script $SPEC ../.circleci/remote_build/remote_build $REPOSITORY $@
+else
+  echo "No rebuild necessary. Retagging..."
+  STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p')
+  for STAGE in $STAGES; do
+    tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true
+  done
+  tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH
+fi

.circleci/cond_spot_run_script

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Conditionally runs a script on a remote spot instance if any dependent code has changed between
+# the last successful run and the present commit.
+#
+# It's expected to be run from the project directory, and that there be a directory called `scripts`
+# containing the given named script to execute on the remote machine.
+#
+# This script is only useful if there is nothing to do in the event there is no rebuild. This is fine
+# for running a suite of tests for example, but is not useful for performing a build, as even if a
+# build has nothing to do, the previous images are retagged with the new commit hash for upstream jobs.
+#
+# Arguments are:
+# 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success.
+# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a
+#    successful run. The script will only run if there were relevant code changes since the last successful commit.
+# 3... ARGS: Arguments to pass to spot_run_script.
+set -e
+
+REPOSITORY=$1
+shift
+SUCCESS_TAG=$1
+shift
+
+LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $SUCCESS_TAG)
+
+echo "Last successful commit for $SUCCESS_TAG: $LAST_SUCCESSFUL_COMMIT"
+
+if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then
+  spot_run_script $@
+  tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG
+fi

.circleci/cond_spot_run_test_script → .circleci/cond_spot_run_tests

@@ -1,13 +1,12 @@
 #!/bin/bash
 set -e
+set -o pipefail
+
 REPOSITORY=$1
-SCRIPT_PATH=$2
-shift
 shift
 
 cd $(query_manifest projectDir $REPOSITORY)
 
 mkdir -p /tmp/test-logs
 
-set -o pipefail
-cond_spot_run_script $REPOSITORY $JOB_NAME 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log"
+cond_spot_run_script $REPOSITORY $JOB_NAME 32 ./scripts/run_tests $@ | tee "/tmp/test-logs/$JOB_NAME.log"