feat(ci): use content hash in build system, restrict docs build to *.…

…ts or *.cpp (#1953)

Benefits:
- merges and squashes in your branch don't cause rebuilds
- O(1) tag lookup, no need for tag renaming

Should at least reduce the number of builds by half (stuff that passed
will revalidate quickly on master etc)

.circleci/config.yml

@@ -57,10 +57,8 @@ checkout: &checkout
       git remote add origin $CIRCLE_REPOSITORY_URL
 
       # Only download metadata when fetching.
-      retry_10 git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1
+      retry_10 git fetch --depth 1 --filter=blob:none origin $CIRCLE_SHA1
       git checkout FETCH_HEAD
-      # Initialize submodules recursively (retry 10 times on failure)
-      retry_10 git submodule update --init --recursive
 
 # Called setup_env to setup a bunch of global variables used throughout the rest of the build process.
 # It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of
@@ -1460,4 +1458,3 @@ workflows:
           requires:
             - build-deployment-canary
           <<: *deploy_defaults
-

bootstrap_docker.sh

@@ -32,11 +32,11 @@ if [ -z "$TARGET_PROJECT" ]; then
   fi
 fi
 
-source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel)
+source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER 
 build_local $TARGET_PROJECT $ONLY_TARGET
 
 if [ -z "$TARGET_PROJECT" ]; then
   echo
   echo "Success! You could now run e.g.:"
   echo "  docker run -ti --rm aztecprotocol/end-to-end:latest e2e_private_token_contract.test"
-fi
+fi

build-system/remote_build/remote_build

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eu
 
 ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
 
@@ -14,7 +14,7 @@ git remote add origin $GIT_REPOSITORY_URL
 # Only download metadata when fetching.
 git config remote.origin.promisor true
 git config remote.origin.partialclonefilter blob:none
-git fetch --depth 50 origin $COMMIT_HASH
+git fetch --depth 1 origin $COMMIT_HASH
 git checkout FETCH_HEAD
 # Checkout barretenberg submodule only.
 git submodule update --init build-system
@@ -25,4 +25,4 @@ BASH_ENV=/tmp/bash_env
 echo "Calling setup env..."
 source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL"
 echo "Calling build..."
-build $@
+build $@

build-system/scripts/build

@@ -35,8 +35,6 @@ echo "Working directory: $PWD"
 echo "Dockerfile: $DOCKERFILE"
 echo "Build directory: $BUILD_DIR"
 
-init_submodules $REPOSITORY
-
 function fetch_image() {
   echo "Pulling: $1"
   if ! docker pull $1 > /dev/null 2>&1; then
@@ -49,23 +47,20 @@ function fetch_image() {
 # Ensure ECR repository exists.
 ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle
 
-LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY)
-echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT"
+CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
+echo "Content hash: $CONTENT_HASH"
 
 cd $BUILD_DIR
 
 # If we have previously successful commit, we can early out if nothing relevant has changed since.
-if [[ $FORCE_BUILD == 'false' ]] && check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then
-  echo "No rebuild necessary. Retagging..."
-  STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p')
-  for STAGE in $STAGES; do
-    tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true
-  done
-  tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH
+if [[ $FORCE_BUILD == 'false' ]] && check_rebuild cache-"$CONTENT_HASH" $REPOSITORY; then
+  echo "No rebuild necessary."
   untag_remote_image $REPOSITORY tainted
   exit 0
 fi
 
+init_submodules $REPOSITORY
+
 # Validate any terraform if it exists.
 if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then
   ensure_terraform
@@ -94,13 +89,13 @@ PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+
 for PARENT in $PARENTS; do
   # Extract repository name (i.e. discard tag).
   PARENT_REPO=${PARENT%:*}
-  PARENT_COMMIT_HASH=$(last_successful_commit $PARENT_REPO)
+  PARENT_CONTENT_HASH=$(calculate_content_hash $PARENT_REPO)
   # There must be a parent image to continue.
-  if [ -z "$PARENT_COMMIT_HASH" ]; then
+  if [ -z "$PARENT_CONTENT_HASH" ]; then
     echo "No parent image found for $PARENT_REPO"
     exit 1
   fi
-  PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_COMMIT_HASH
+  PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_CONTENT_HASH
   echo "Pulling dependency $PARENT_REPO..."
   fetch_image $PARENT_IMAGE_URI
   # Tag it to look like an official release as that's what we use in Dockerfiles.
@@ -113,18 +108,18 @@ CACHE_FROM=""
 STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p')
 for STAGE in $STAGES; do
   # Get the last build of this stage to leverage layer caching.
-  if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then
+  if [ -n "$CONTENT_HASH" ]; then
     echo "Pulling stage: $STAGE"
-    STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT-$STAGE
+    STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE
     if fetch_image $STAGE_IMAGE_LAST_URI; then
       STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI"
     fi
   fi
 
   echo "Building stage: $STAGE"
-  STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH-$STAGE
+  STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE
   # Build our dockerfile, add timing information
-  docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg ARG_COMMIT_HASH=$COMMIT_HASH . \
+  docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \
     | while read line ; do echo "$(date "+%H:%M:%S")| $line"; done
 
   # We don't want to have redo this stages work when building the final image. Use it as a layer cache.
@@ -136,19 +131,19 @@ for STAGE in $STAGES; do
 done
 
 # Pull previous image to use it as a layer cache if it exists.
-if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then
-  LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT
+if [ -n "$CONTENT_HASH" ]; then
+  LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH
   echo "Pulling previous build of $REPOSITORY..."
   fetch_image $LAST_SUCCESSFUL_URI || true
   CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM"
   echo
 fi
 
 # Build the actual image and give it a commit tag.
-IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH
+IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH
 echo "Building image: $IMAGE_COMMIT_URI"
 # Build our dockerfile, add timing information
-docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG --build-arg ARG_COMMIT_HASH=$COMMIT_HASH . \
+docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \
   | while read line ; do echo "$(date "+%H:%M:%S")| $line"; done
 echo "Pushing image: $IMAGE_COMMIT_URI"
 docker push $IMAGE_COMMIT_URI > /dev/null 2>&1

build-system/scripts/build_local

@@ -6,7 +6,7 @@
 # If DOCKERFILE is excluded it tries to default to ./Dockerfile then ./<PROJECT_DIR_NAME>/Dockerfile
 # If REPO is excluded it defaults to PROJECT_DIR_NAME.
 
-set -e
+set -eu
 
 TARGET_PROJECT=$1
 ONLY_TARGET=$2

build-system/scripts/calculate_content_hash

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -eu
+
+REPOSITORY=$1
+COMMIT_HASH=${2:-$COMMIT_HASH}
+
+# Compute REBUILD_PATTERNS from the build manifest
+REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY)
+
+AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g')
+# an example line is
+#   100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0       yarn-project/world-state/tsconfig.json
+# this format is beneficial as it grabs the hashes from git efficiently
+# we will next filter by our rebuild patterns 
+# then we pipe the hash portion of each file to git hash-object to produce our content hash
+git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin

build-system/scripts/check_npm_version

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eu
 
 readonly LOCAL_VERSION=$(node -pe "require('./package.json').version")
 readonly PACKAGE_NAME=${1:-./}
@@ -14,4 +14,4 @@ elif [ "$LOCAL_VERSION" != "$PUBLISHED_VERSION" ] && [ "$LOCAL_VERSION" == "$HIG
 else
   echo "Expect npm version number to be higher than '$PUBLISHED_VERSION'. Current local version is '$LOCAL_VERSION'."
   exit 1
-fi
+fi

build-system/scripts/check_rebuild

@@ -4,36 +4,20 @@
 # The rebuild patterns are taken from the build manifest (computed from set of dependencies).
 set -euo pipefail
 
-BASE_COMMIT=$1
+TAG=$1
 REPOSITORY=$2
 
 # If given nothing, then exit with failure to rebuild
-[ -n "$BASE_COMMIT" ] || exit 1
+[ -n "$TAG" ] || exit 1
 
-# If a tainted tag exists, remove it exit with failure to rebuild.
-if image_exists $REPOSITORY tainted; then
-  echo "$REPOSITORY has been tainted. Will rebuild."
-  exit 1
-fi
-
-# Compute .rebuild_patterns from the build manifest.
-query_manifest rebuildPatterns $REPOSITORY > .rebuild_patterns
-
-echo "Rebuild patterns:"
-cat .rebuild_patterns
-
-git config diff.renameLimit 999999
-
-# Get list of files that differ
-different_files=$(list_file_diff ${BASE_COMMIT} ${COMMIT_HASH}) || {
-  echo "list_file_diff failed. Rebuild required.";
-  exit 1;
-}
-
-if grep -f .rebuild_patterns <<< "$different_files" &> /dev/null; then
+if ! image_exists $REPOSITORY $TAG; then
   echo "Rebuild required."
   exit 1
+elif image_exists $REPOSITORY tainted; then
+  # If a tainted tag exists, remove it exit with failure to rebuild.
+  echo "$REPOSITORY has been tainted. Will rebuild."
+  exit 1
 else
   echo "No rebuild required."
   exit 0
-fi
+fi

build-system/scripts/cond_spot_run_build

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eu
 set -o pipefail
 
 REPOSITORY=$1
@@ -8,20 +8,13 @@ SPEC=$1
 shift
 DOCKERFILE=$(query_manifest dockerfile $REPOSITORY)
 
-init_submodules $REPOSITORY
-
-LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY)
-echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT"
+CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
+echo "Content hash: $CONTENT_HASH"
 
 cd $(query_manifest buildDir $REPOSITORY)
 
-if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then
-  spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@
-else
-  echo "No rebuild necessary. Retagging..."
-  STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p')
-  for STAGE in $STAGES; do
-    tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true
-  done
-  tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH
-fi
+if ! check_rebuild cache-$CONTENT_HASH $REPOSITORY; then
+  init_submodules $REPOSITORY
+  spot_run_script $CONTENT_HASH $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@
+  tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH
+fi

build-system/scripts/cond_spot_run_script

@@ -11,21 +11,21 @@
 #
 # Arguments are:
 # 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success.
-# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a
+# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a
 #    successful run. The script will only run if there were relevant code changes since the last successful commit.
 # 3... ARGS: Arguments to pass to spot_run_script.
-set -e
+set -eu
 
 REPOSITORY=$1
 shift
 SUCCESS_TAG=$1
 shift
 
-LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $SUCCESS_TAG)
+CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
+echo "Content hash: $CONTENT_HASH"
 
-echo "Last successful commit for $SUCCESS_TAG: $LAST_SUCCESSFUL_COMMIT"
-
-if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then
-  spot_run_script $@
-  tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG
-fi
+if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then
+  init_submodules $REPOSITORY
+  spot_run_script $CONTENT_HASH $@
+  tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG
+fi

build-system/scripts/cond_spot_run_test_script

@@ -1,12 +1,10 @@
 #!/bin/bash
-set -e
+set -eu
 SCRIPT_PATH=$1
 REPOSITORY=$2
 shift
 shift
 
-init_submodules $REPOSITORY
-
 cd $(query_manifest projectDir $REPOSITORY)
 
 mkdir -p /tmp/test-logs

build-system/scripts/cond_spot_run_tests

@@ -1,4 +1,4 @@
 #!/bin/bash
-set -e
+set -eu
 
 cond_spot_run_test_script ./scripts/run_tests $@

build-system/scripts/deploy

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eu
 
 REPOSITORY=$1
 SERVICES=${2:-$REPOSITORY}
@@ -11,9 +11,9 @@ cd $(query_manifest projectDir $REPOSITORY)
 deploy_ecr $REPOSITORY
 
 # Bail out if nothing changed.
-LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $DEPLOY_TAG-deployed)
-echo "Last successfully deployed commit: $LAST_SUCCESSFUL_COMMIT"
-if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then
+CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
+echo "Last successfully deployed commit: $CONTENT_HASH"
+if check_rebuild cache-$CONTENT_HASH-$DEPLOY_TAG-deployed $REPOSITORY; then
   echo "No changes detected, skipping deployment."
   exit 0
 fi
@@ -26,4 +26,4 @@ for SERVICE in $SERVICES; do
 done
 
 # Tag the image as deployed.
-tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$DEPLOY_TAG-deployed
+tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed

build-system/scripts/deploy_dockerhub

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eu
 
 if [ -z "$COMMIT_TAG" ]; then
   echo "Will only push tagged builds to dockerhub. Skipping."
@@ -56,4 +56,4 @@ docker tag $IMAGE_COMMIT_URI $IMAGE_LATEST_URI
 # Push tagged image to dockerhub.
 docker push $IMAGE_DEPLOY_URI
 # Push :latest image to dockerhub
-docker push $IMAGE_LATEST_URI
+docker push $IMAGE_LATEST_URI