diff --git a/.circleci/bin/jq b/.circleci/bin/jq deleted file mode 100755 index 93a5da26c4..0000000000 Binary files a/.circleci/bin/jq and /dev/null differ diff --git a/.circleci/build b/.circleci/build deleted file mode 100755 index 3db9f403af..0000000000 --- a/.circleci/build +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -# -# Builds a docker image and pushes it to it's repository. Leverages caches where possible. -# Cached images include previous successfully built images built on this branch. -# The images output are cache images, meaning they will eventually get purged. -# The deploy phase will tag the images such that they become permanent. -# -# usage: ./build -# example: ./build barretenberg-x86_64-linux-clang -# output image: -# 278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang:cache-deadbeefcafebabe1337c0d3 - -set -e - -REPOSITORY=$1 -DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) -PROJECT_DIR=$(query_manifest projectDir $REPOSITORY) - -echo "Repository: $REPOSITORY" -echo "Working directory: $PWD" -echo "Dockerfile: $DOCKERFILE" - -cd $(query_manifest buildDir $REPOSITORY) - -function fetch_image() { - echo "Pulling: $1" - if ! docker pull $1 > /dev/null 2>&1; then - echo "Image not found: $1" - return 1 - fi - return 0 -} - -# Ensure ECR repository exists. -ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle - -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" - -# If we have previously successful commit, we can early out if nothing relevant has changed since. -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH - untag_remote_image $REPOSITORY tainted - exit 0 -fi - -# Validate any terraform if it exists. -if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then - ensure_terraform - export TF_IN_AUTOMATION=1 - pushd $ROOT_PATH/$PROJECT_DIR/terraform - for DIR in . $(find . -maxdepth 1 -type d); do - pushd $DIR - if [ -f ./main.tf ]; then - terraform init -input=false -backend-config="key=dummy" - terraform validate - fi - popd - done - popd -fi - -# Pull latest parents that are not ours. -echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin -PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | grep -v $ECR_DEPLOY_URL | sort | uniq) -for PARENT in $PARENTS; do - fetch_image $PARENT -done - -# For each parent that's ours, pull in the latest image. -PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+\).*/\1/p") -for PARENT in $PARENTS; do - # Extract repository name (i.e. discard tag). - PARENT_REPO=${PARENT%:*} - PARENT_COMMIT_HASH=$(last_successful_commit $PARENT_REPO) - # There must be a parent image to continue. - if [ -z "$PARENT_COMMIT_HASH" ]; then - echo "No parent image found for $PARENT_REPO" - exit 1 - fi - PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_COMMIT_HASH - echo "Pulling dependency $PARENT_REPO..." - fetch_image $PARENT_IMAGE_URI - # Tag it to look like an official release as that's what we use in Dockerfiles. - docker tag $PARENT_IMAGE_URI $ECR_DEPLOY_URL/$PARENT -done - -# Pull, build and push each named stage to cache. -STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') -for STAGE in $STAGES; do - # Get the last build of this stage to leverage layer caching. - if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then - echo "Pulling stage: $STAGE" - STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT-$STAGE - if fetch_image $STAGE_IMAGE_LAST_URI; then - STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI" - fi - fi - - echo "Building stage: $STAGE" - STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH-$STAGE - docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE . - - # We don't want to have redo this stages work when building the final image. Use it as a layer cache. - CACHE_FROM="--cache-from $STAGE_IMAGE_COMMIT_URI $CACHE_FROM" - - echo "Pushing stage: $STAGE" - docker push $STAGE_IMAGE_COMMIT_URI > /dev/null 2>&1 - echo -done - -# Pull previous image to use it as a layer cache if it exists. -if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then - LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT - echo "Pulling previous build of $REPOSITORY..." - fetch_image $LAST_SUCCESSFUL_URI || true - CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM" - echo -fi - -# Build the actual image and give it a commit tag. -IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH -echo "Building image: $IMAGE_COMMIT_URI" -docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG . -echo "Pushing image: $IMAGE_COMMIT_URI" -docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 -untag_remote_image $REPOSITORY tainted \ No newline at end of file diff --git a/.circleci/build_local b/.circleci/build_local deleted file mode 100755 index 84e7224123..0000000000 --- a/.circleci/build_local +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Builds the PROJECTS in the given order. -# Will terminate build at TARGET_PROJECT (if given). -# Will only build TARGET_PROJECT if ONLY_TARGET given. -# Will push TARGET_PROJECT to ECR with PUSH_LABEL (if given). -# PROJECT elements have structure WORKING_DIR:PROJECT:DOCKERFILE:REPO. -# If DOCKERFILE is excluded it tries to default to ./Dockerfile then ./REPO/Dockerfile - -set -e - -TARGET_PROJECT=$1 -ONLY_TARGET=$2 -PUSH_LABEL=$3 - -PROJECTS=( - cpp:barretenberg-cpp:./dockerfiles/Dockerfile.x86_64-linux-clang:barretenberg-x86_64-linux-clang - cpp:barretenberg-cpp:./dockerfiles/Dockerfile.wasm-linux-clang:barretenberg-wasm-linux-clang - # js:barretenberg-js -) - -for E in ${PROJECTS[@]}; do - ARR=(${E//:/ }) - WORKING_DIR=${ARR[0]} - PROJECT=${ARR[1]} - DOCKERFILE=${ARR[2]} - REPO=${ARR[3]:-$PROJECT} - - if [ -n "$ONLY_TARGET" -a ! "$PROJECT" = "$TARGET_PROJECT" ]; then - continue - fi - - pushd $WORKING_DIR - - if [ ! -f "$DOCKERFILE" ]; then - DOCKERFILE=./Dockerfile - if [ ! -f "$DOCKERFILE" ]; then - DOCKERFILE=./$REPO/Dockerfile - if [ ! -f "$DOCKERFILE" ]; then - echo "Dockerfile not found." - exit 1 - fi - fi - fi - - echo - echo - echo - echo "*** Building $PROJECT:$DOCKERFILE -> $REPO ***" - echo - - time docker build -f $DOCKERFILE -t $ECR_URL/$REPO:latest . - - if [ -n "$TERM_AT_TARGET" -a "$PROJECT" = "$TARGET_PROJECT" ]; then - break - fi - - popd -done diff --git a/.circleci/changed b/.circleci/changed deleted file mode 100755 index 341557dbf8..0000000000 --- a/.circleci/changed +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -LAST_SUCCESSFUL_COMMIT=$1 - -[ -n "$COMMIT_HASH" ] || exit 1 - -if git diff --name-only $LAST_SUCCESSFUL_COMMIT $COMMIT_HASH | grep -q $2; then - exit 0 -else - exit 1 -fi \ No newline at end of file diff --git a/.circleci/check_rebuild b/.circleci/check_rebuild deleted file mode 100755 index d7d7f2c2dd..0000000000 --- a/.circleci/check_rebuild +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Succeeds if any files matching the rebuild patterns, have changed since the base commit. -set -e - -BASE_COMMIT=$1 -REPOSITORY=$2 - -# If given nothing, then return true to rebuild. -[ -n "$BASE_COMMIT" ] || exit 0 - -# If a tainted tag exists, remove it and return true to rebuild. -if image_exists $REPOSITORY tainted; then - echo "$REPOSITORY has been tainted. Will rebuild." - exit 0 -fi - -# Compute .rebuild_patterns from the build manifest. -query_manifest rebuildPatterns $REPOSITORY > .rebuild_patterns - -echo "Rebuild patterns:" -cat .rebuild_patterns - -git config diff.renameLimit 999999 - -if git diff --name-only ${BASE_COMMIT} ${COMMIT_HASH} | grep -q -f .rebuild_patterns; then - exit 0 -else - exit 1 -fi \ No newline at end of file diff --git a/.circleci/cond_spot_run_build b/.circleci/cond_spot_run_build deleted file mode 100755 index 6aa18b7ffe..0000000000 --- a/.circleci/cond_spot_run_build +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -e - -REPOSITORY=$1 -shift -SPEC=$1 -shift -DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) - -cd $(query_manifest buildDir $REPOSITORY) - -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" - -if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ -else - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH -fi \ No newline at end of file diff --git a/.circleci/cond_spot_run_script b/.circleci/cond_spot_run_script deleted file mode 100755 index 0882bae7e0..0000000000 --- a/.circleci/cond_spot_run_script +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Conditionally runs a script on a remote spot instance if any dependent code has changed between -# the last successful run and the present commit. -# -# It's expected to be run from the project directory, and that there be a directory called `scripts` -# containing the given named script to execute on the remote machine. -# -# This script is only useful if there is nothing to do in the event there is no rebuild. This is fine -# for running a suite of tests for example, but is not useful for performing a build, as even if a -# build has nothing to do, the previous images are retagged with the new commit hash for upstream jobs. -# -# Arguments are: -# 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success. -# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a -# successful run. The script will only run if there were relevant code changes since the last successful commit. -# 3... ARGS: Arguments to pass to spot_run_script. -set -e - -REPOSITORY=$1 -shift -SUCCESS_TAG=$1 -shift - -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $SUCCESS_TAG) - -echo "Last successful commit for $SUCCESS_TAG: $LAST_SUCCESSFUL_COMMIT" - -if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - spot_run_script $@ - tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG -fi \ No newline at end of file diff --git a/.circleci/cond_spot_run_tests b/.circleci/cond_spot_run_tests deleted file mode 100755 index f214624885..0000000000 --- a/.circleci/cond_spot_run_tests +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -REPOSITORY=$1 -shift - -cd $(query_manifest projectDir $REPOSITORY) - -mkdir -p /tmp/test-logs - -cond_spot_run_script $REPOSITORY $JOB_NAME 32 ./scripts/run_tests $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/.circleci/config.yml b/.circleci/config.yml index d626441352..b760b64c67 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,3 @@ -# This config consists of currently 3 workflows. -# - system: The main Aztec infrastructure, services, frontends etc. -# - metrics: The metrics grafana and prometheus instances, and some prometheus data sources. -# - website: The company website. -# -# The default workflow is system. To trigger the other workflows, trigger a workflow from CCI -# setting a string variable called `workflow` to another name. -# # This file uses YAML anchors and aliases to prevent repetition of blocks of config: # https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ # @@ -24,6 +16,14 @@ version: 2.1 +orbs: + slack: circleci/slack@4.12.1 + +parameters: + workflow: + type: string + default: "system" + # This build step checks out the code from the repository. It has a hardcoded readonly key to allow the checkout. # Initially it just fetches the repo metadata for the current commit hash to a depth of 50 commits. # We need historical commit hashes to calculate diffs between previous and current commits. @@ -42,13 +42,17 @@ checkout: &checkout chmod 0600 .ssh/id_rsa + # IF YOU'RE CHANGING THIS, YOU ALSO WANT TO CHANGE: build-system/remote_build/remote_build # Shallow checkout this commit. mkdir -p project cd project git init git remote add origin $CIRCLE_REPOSITORY_URL + # Only download metadata when fetching. git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1 git checkout FETCH_HEAD + # Initialize submodules recursively + git submodule update --init --recursive # This build step checks out the code from the benchmark-archive repository. The key is saved in CircleCi environment in base64 format. # Initially it just fetches the latest version. @@ -82,8 +86,7 @@ benchmark_add_keys: &benchmark_add_keys setup_env: &setup_env run: name: "Setup environment" - command: | - cd .circleci && ./setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" + command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" setup_aztec_commit: &setup_aztec_commit run: @@ -278,7 +281,7 @@ jobs: - *setup_aztec_commit - run: name: "Build" - command: cond_spot_run_test_script barretenberg-circuits-wasm-linux-clang-builder-runner ./scripts/run_aztec_circuits_tests "$AZTEC_COMMIT" 1 wasm scripts/a3-tests -*.skip*:*.circuit* + command: cond_spot_run_test_script ./scripts/run_aztec_circuits_tests barretenberg-circuits-wasm-linux-clang-builder-runner "$AZTEC_COMMIT" 1 wasm scripts/a3-tests -*.skip*:*.circuit* circuits-x86_64-tests: docker: @@ -290,22 +293,35 @@ jobs: - *setup_aztec_commit - run: name: "Build" - command: cond_spot_run_test_script barretenberg-circuits-x86_64-linux-clang-builder-runner ./scripts/run_aztec_circuits_tests "$AZTEC_COMMIT" 1 x86_64 scripts/a3-tests -*.skip* + command: cond_spot_run_test_script ./scripts/run_aztec_circuits_tests barretenberg-circuits-x86_64-linux-clang-builder-runner "$AZTEC_COMMIT" 1 x86_64 scripts/a3-tests -*.skip* # End Aztec integration tests ################################### # Repeatable config for defining the workflow below. +tag_regex: &tag_regex /v[0-9]+(\.[0-9]+)*(-[a-zA-Z-]+\.[0-9]+)?/ +defaults: &defaults + filters: + tags: + only: *tag_regex + context: + - slack + post-steps: + - slack/notify: + event: fail + branch_pattern: "master" + bb_test: &bb_test requires: - x86_64-linux-clang-assert + <<: *defaults workflows: system: jobs: - - x86_64-linux-gcc - - x86_64-linux-clang - - x86_64-linux-clang-assert - - wasm-linux-clang + - x86_64-linux-gcc: *defaults + - x86_64-linux-clang: *defaults + - x86_64-linux-clang-assert: *defaults + - wasm-linux-clang: *defaults - honk-tests: *bb_test - barretenberg-tests: *bb_test - stdlib-primitives-tests: *bb_test @@ -325,11 +341,14 @@ workflows: branches: only: - master - - circuits-wasm-linux-clang-builder-runner - - circuits-x86_64-linux-clang-builder-runner + <<: *defaults + - circuits-wasm-linux-clang-builder-runner: *defaults + - circuits-x86_64-linux-clang-builder-runner: *defaults - circuits-wasm-tests: requires: - circuits-wasm-linux-clang-builder-runner + <<: *defaults - circuits-x86_64-tests: requires: - - circuits-x86_64-linux-clang-builder-runner \ No newline at end of file + - circuits-x86_64-linux-clang-builder-runner + <<: *defaults diff --git a/.circleci/ensure_repo b/.circleci/ensure_repo deleted file mode 100755 index c1d579d1f0..0000000000 --- a/.circleci/ensure_repo +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e - -LIFECYCLE_POLICY='{ - "rules": [ - { - "rulePriority": 1, - "description": "No more than 200 cache images.", - "selection": { - "tagStatus": "tagged", - "tagPrefixList": ["cache-"], - "countType": "imageCountMoreThan", - "countNumber": 200 - }, - "action": { - "type": "expire" - } - } - ] -} -' - -REPOSITORY=$1 -REGION=$2 -REFRESH_LIFECYCLE=$3 - -aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null - -# Create the repository if it doesn't exist. -if ! aws ecr describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then - echo "Creating repo: $REPOSITORY" - aws ecr create-repository --region $REGION --repository-name $REPOSITORY > /dev/null 2>&1 - aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 -elif [ -n "$REFRESH_LIFECYCLE" ]; then - echo "Refreshing lifecycle rules for repo: $REPOSITORY" - aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 -fi -echo \ No newline at end of file diff --git a/.circleci/ensure_terraform b/.circleci/ensure_terraform deleted file mode 100755 index 982e28b565..0000000000 --- a/.circleci/ensure_terraform +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e - -[ ! -f /usr/local/bin/terraform ] || exit 0 - -cd $HOME -TERRAFORM_VERSION=0.13.3 -curl -sSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -o terraform.zip -sudo apt install -y unzip -unzip terraform.zip -sudo mv terraform /usr/local/bin/ -rm terraform.zip \ No newline at end of file diff --git a/.circleci/erase_image_tags b/.circleci/erase_image_tags deleted file mode 100755 index 7183e3099b..0000000000 --- a/.circleci/erase_image_tags +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Erase the image tag associated with the last commit for the given repository. -# If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. - -REPOSITORY=$1 -TILL_COMMIT_HASH=$2 - -if [ -n "$3" ]; then - TAG_POSTFIX=-$3 -fi - -for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do - TAG=cache-${COMMIT_HASH}$TAG_POSTFIX - if image_exists $REPOSITORY $TAG; then - echo "Erasing image tag: $REPOSITORY:$TAG" - aws ecr batch-delete-image --region=$ECR_REGION --repository-name $1 --image-ids imageTag=$TAG > /dev/null - if [ -z "$TILL_COMMIT_HASH" -o "$COMMIT_HASH" = "$TILL_COMMIT_HASH" ]; then - break - fi - fi -done \ No newline at end of file diff --git a/.circleci/image_exists b/.circleci/image_exists deleted file mode 100755 index e926b22537..0000000000 --- a/.circleci/image_exists +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 \ No newline at end of file diff --git a/.circleci/last_successful_commit b/.circleci/last_successful_commit deleted file mode 100755 index 03b0fe625a..0000000000 --- a/.circleci/last_successful_commit +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -REPOSITORY=$1 - -if [ -n "$2" ]; then - TAG_POSTFIX=-$2 -fi - -# We are assuming that there has been a successful build of the given repo -# within the last 50 commits. If not return nothing. -for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do - >&2 echo "Checking if image exists for commit $COMMIT_HASH..." - if image_exists $REPOSITORY cache-${COMMIT_HASH}$TAG_POSTFIX; then - >&2 echo "Found." - echo $COMMIT_HASH - exit 0 - fi -done \ No newline at end of file diff --git a/.circleci/lib/libjq.so.1 b/.circleci/lib/libjq.so.1 deleted file mode 100644 index d3b71eabca..0000000000 Binary files a/.circleci/lib/libjq.so.1 and /dev/null differ diff --git a/.circleci/lib/libonig.so.5 b/.circleci/lib/libonig.so.5 deleted file mode 100644 index 2408aea05d..0000000000 Binary files a/.circleci/lib/libonig.so.5 and /dev/null differ diff --git a/.circleci/query_manifest b/.circleci/query_manifest deleted file mode 100755 index 9e9629b156..0000000000 --- a/.circleci/query_manifest +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -CMD=$1 -REPO=$2 - -ROOT=$(git rev-parse --show-toplevel) - -case "$CMD" in - dockerfile) - jq -r ".\"$REPO\".dockerfile // \"Dockerfile\"" $ROOT/build_manifest.json - ;; - buildDir) - jq -r ".\"$REPO\".buildDir" $ROOT/build_manifest.json - ;; - projectDir) - jq -r ".\"$REPO\".projectDir // .\"$REPO\".buildDir" $ROOT/build_manifest.json - ;; - dependencies) - ALL_DEPS=() - add_deps() { - DEPS=($(jq -r ".\"$1\".dependencies // [] | .[]" $ROOT/build_manifest.json)) - ALL_DEPS=(${ALL_DEPS[@]} ${DEPS[@]}) - for DEP in "${DEPS[@]}"; do - add_deps $DEP - done - } - add_deps $REPO - printf "%s\n" "${ALL_DEPS[@]}" | sort | uniq - ;; - rebuildPatterns) - ALL_PATTERNS=($(jq -r ".\"$REPO\".rebuildPatterns | .[]" $ROOT/build_manifest.json)) - DEPS=($($0 dependencies $REPO)) - for DEP in "${DEPS[@]}"; do - PATTERNS=($(jq -r ".\"$DEP\".rebuildPatterns | .[]" $ROOT/build_manifest.json)) - ALL_PATTERNS=(${ALL_PATTERNS[@]} ${PATTERNS[@]}) - done - printf "%s\n" "${ALL_PATTERNS[@]}" | sort | uniq - ;; -esac \ No newline at end of file diff --git a/.circleci/remote/32core.json b/.circleci/remote/32core.json deleted file mode 100644 index 79f0f396d8..0000000000 --- a/.circleci/remote/32core.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ImageId": "ami-0e5df77ac318c7a18", - "KeyName": "build-instance", - "SecurityGroupIds": ["sg-0ccd4e5df0dcca0c9"], - "InstanceType": "r5.8xlarge", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 16 - } - } - ] -} diff --git a/.circleci/remote/64core.json b/.circleci/remote/64core.json deleted file mode 100644 index 13df8fd353..0000000000 --- a/.circleci/remote/64core.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "ImageId": "ami-0e5df77ac318c7a18", - "KeyName": "build-instance", - "SecurityGroupIds": ["sg-0ccd4e5df0dcca0c9"], - "InstanceType": "r5.16xlarge", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 16 - } - } - ] -} diff --git a/.circleci/remote/ssh_config b/.circleci/remote/ssh_config deleted file mode 100644 index 96cc4d910e..0000000000 --- a/.circleci/remote/ssh_config +++ /dev/null @@ -1,3 +0,0 @@ -IdentityFile ~/.ssh/build_instance_key -StrictHostKeyChecking no -User ubuntu \ No newline at end of file diff --git a/.circleci/remote_build/remote_build b/.circleci/remote_build/remote_build deleted file mode 100755 index 92fe12a6af..0000000000 --- a/.circleci/remote_build/remote_build +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e - -ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - -echo "Initialising remote build..." - -# Shallow checkout this commit. -mkdir project -cd project -git init -git remote add origin $GIT_REPOSITORY_URL -# Configure to only download metadata when fetching. -git config remote.origin.promisor true -git config remote.origin.partialclonefilter blob:none -git fetch --depth 50 origin $COMMIT_HASH -git checkout FETCH_HEAD - -echo "Git operations completed..." - -cd .circleci -BASH_ENV=/tmp/bash_env -echo "Calling setup env..." -source ./setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" -cd .. -echo "Calling build..." -build $@ \ No newline at end of file diff --git a/.circleci/remote_run_script b/.circleci/remote_run_script deleted file mode 100755 index b62b6b9cf1..0000000000 --- a/.circleci/remote_run_script +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e - -IP=$1 -FULL_PATH=$2 -shift -shift - -SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} -DIR_NAME=$(dirname $FULL_PATH) -SCRIPT_NAME=$(basename $FULL_PATH) - -# Copy all files in script directory to spot instance. -scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. - -# Run script on remote instance. -ssh -A -F $SSH_CONFIG_PATH $IP "COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ./$SCRIPT_NAME $@" \ No newline at end of file diff --git a/.circleci/request_spot b/.circleci/request_spot deleted file mode 100755 index 027037ce65..0000000000 --- a/.circleci/request_spot +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -set -e - -NAME=$1 -CPUS=${2:-32} - -export AWS_DEFAULT_REGION=us-east-2 - -BID_PER_CPU_HOUR=0.03125 -PRICE=$(jq -n "$BID_PER_CPU_HOUR*$CPUS*100000 | round / 100000") - ->&2 echo "Requesting $CPUS cpu instance (bid: $PRICE)..." -SIR=$(aws ec2 request-spot-instances \ - --spot-price "$PRICE" \ - --instance-count 1 \ - --type "one-time" \ - --launch-specification file://$BUILD_SYSTEM_PATH/remote/${CPUS}core.json \ - --query "SpotInstanceRequests[*].[SpotInstanceRequestId]" \ - --output text) - ->&2 echo "Waiting for instance id for spot request: $SIR..." -sleep 5 -for I in {1..6}; do - IID=$(aws ec2 describe-spot-instance-requests \ - --spot-instance-request-ids $SIR \ - --query "SpotInstanceRequests[*].[InstanceId]" \ - --output text) - - [ -z "$IID" -o "$IID" == "None" ] || break - - if [ $I -eq 6 ]; then - # Cancel spot request. We may still get allocated an instance if it's *just* happened. - aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SIR > /dev/null - fi - - sleep 5 -done - -if [ -z "$IID" -o "$IID" == "None" ]; then - # Request on-demand instance. - >&2 echo "Falling back to on-demand instance..." - IID=$(aws ec2 run-instances \ - --cli-input-json file://$BUILD_SYSTEM_PATH/remote/${CPUS}core.json \ - --query "Instances[*].[InstanceId]" \ - --output text) -fi - -aws ec2 create-tags --resources $IID --tags "Key=Name,Value=$NAME" -aws ec2 create-tags --resources $IID --tags "Key=Group,Value=build-instance" - -while [ -z "$IP" ]; do - sleep 1 - IP=$(aws ec2 describe-instances \ - --filter "Name=instance-id,Values=$IID" \ - --query "Reservations[*].Instances[*].PublicIpAddress" \ - --output=text) -done - -# Wait till ssh port is open. ->&2 echo "Waiting for SSH at $IP..." -while ! nc -z $IP 22; do sleep 1; done; - -echo $IP \ No newline at end of file diff --git a/.circleci/setup_env b/.circleci/setup_env deleted file mode 100755 index 471ed9597c..0000000000 --- a/.circleci/setup_env +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash -# This script sets up the global build environment. This should be called before any other build scripts, -# as the other build scripts assume these global variables are set. The global variables are written to -# the file in $BASH_ENV, which means that any new bash shells launched within the lifetime of the machine -# context will have these variables set. This happens for example when CircleCI runs a new "step". -# The script should be sourced, e.g: -# source setup_env -# This ensures the resultant variables are set in the calling shell. -set -e - -COMMIT_HASH=$1 -COMMIT_TAG=$2 -JOB_NAME=$3 -GIT_REPOSITORY_URL=$4 -BRANCH=$5 - -BUILD_SYSTEM_PATH=$(pwd) -# PROJECT=$(cat ../PROJECT) -# VERSION=$(cat ../VERSION) - -echo "Setting up environment COMMIT_HASH=$COMMIT_HASH, COMMIT_TAG=$COMMIT_TAG, JOB_NAME=$JOB_NAME, GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL" - -# if [ -n "$COMMIT_TAG" ]; then -# # We're tagged e.g. v2.1.123 or v2.1.123-testnet.0. -# # First we sanity check that the tag matches the VERSION file. -# echo "Extracting commit tag version..." -# # Strips the e.g. '-testnet.XX' from the COMMIT_TAG -# WITHOUT_VERSION_TAG=${COMMIT_TAG%%-*} -# # Strips the subversion, leaving just the 'vMAJ.MIN' e.g. v2.1 -# COMMIT_TAG_VERSION=${WITHOUT_VERSION_TAG%.*} -# echo "WITHOUT_VERSION_TAG=$WITHOUT_VERSION_TAG" -# echo "COMMIT_TAG_VERSION=$COMMIT_TAG_VERSION" -# # COMMIT_TAG_VERSION=$(echo "$COMMIT_TAG" | grep -oE "v\d+.\d+") -# if [ "$COMMIT_TAG_VERSION" != "$VERSION" ]; then -# echo "Commit tag $COMMIT_TAG does not match repo version $VERSION." -# exit 1 -# fi - -# # Extract the version tag from the commit tag, if it has one, e.g. testnet. -# # If we have one, we look something like v2.1.123-testnet.0. This is a "non production" release. -# if [[ "$COMMIT_TAG" == *"-"* ]]; then -# # Strips the trailing '.XX' from the end of the commit tag -# TEMP=${COMMIT_TAG%.*} -# # Strips the 'vX.Y.ZZZ-' from the front of the commit tag, leaving the e.g. 'testnet' -# VERSION_TAG=${TEMP##*-} -# else -# VERSION_TAG=prod -# fi -# else -# # We have no commit tag. This is one of our development project branches. -# VERSION_TAG=dev -# fi - -# # - The deploy tag (used in api paths, subdomains), is a concatenation of the project name and version tag, -# # e.g. aztec-connect-dev, aztec-connect-testnet, or aztec-connect-prod -# DEPLOY_TAG=$PROJECT-$VERSION_TAG - -if [ -z "$BASH_ENV" ]; then - BASH_ENV=$(mktemp) -fi - -echo export ROOT_PATH=$(cd .. && echo ${PWD}) >> $BASH_ENV -echo export BUILD_SYSTEM_PATH=$BUILD_SYSTEM_PATH >> $BASH_ENV -echo export SSH_CONFIG_PATH=$BUILD_SYSTEM_PATH/remote/ssh_config >> $BASH_ENV -echo export PATH=$PATH:$BUILD_SYSTEM_PATH >> $BASH_ENV -echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV -echo export ECR_REGION=us-east-2 >> $BASH_ENV -echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV -echo export ECR_URL=278380418400.dkr.ecr.us-east-2.amazonaws.com >> $BASH_ENV -# echo export ECR_DEPLOY_REGION=eu-west-2 >> $BASH_ENV -# echo export ECR_DEPLOY_URL=278380418400.dkr.ecr.eu-west-2.amazonaws.com >> $BASH_ENV -# echo export PROJECT=$PROJECT >> $BASH_ENV -echo export COMMIT_HASH=$COMMIT_HASH >> $BASH_ENV -echo export COMMIT_TAG=$COMMIT_TAG >> $BASH_ENV -echo export JOB_NAME=$JOB_NAME >> $BASH_ENV -echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV -# echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV -# echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV -echo export BRANCH=$BRANCH >> $BASH_ENV -# Our alpine build image has jq installed, ubuntu build instances use our committed version. -if [ ! -f /etc/alpine-release ]; then - echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/bin >> $BASH_ENV - echo export LD_LIBRARY_PATH=$BUILD_SYSTEM_PATH/lib >> $BASH_ENV -fi -cat $BASH_ENV - -# Having written the variables to $BASH_ENV, we now want to set them in this shell context. -source $BASH_ENV - -# Only run the following if we're the result of a commit (i.e. not being run manually). -if [ -n "$COMMIT_HASH" ]; then - # Install and ensure correct permissions on build instance key. - mkdir -p ~/.ssh - echo $BUILD_INSTANCE_KEY | base64 -d > ~/.ssh/build_instance_key - chmod 600 ~/.ssh/build_instance_key -fi - -set +e diff --git a/.circleci/spot_run_script b/.circleci/spot_run_script deleted file mode 100755 index 550019df32..0000000000 --- a/.circleci/spot_run_script +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Runs a test script on a remote spot instance. Arguments are: -# 1. SPEC: Instance specification filename. -# 2... ARGS: Arguments to pass to remote_run_script. -set -e - -SPEC=$1 -shift - -# Get spot instance. -IP=$(request_spot $COMMIT_HASH:$JOB_NAME $SPEC) - -# Run script remotely on spot instance, capturing success or failure. -set +e -remote_run_script $IP $@ -CODE=$? - -# Shutdown spot. -echo "Terminating spot instance..." -ssh -F $SSH_CONFIG_PATH $IP sudo halt -p > /dev/null 2>&1 - -exit $CODE \ No newline at end of file diff --git a/.circleci/store_test_benchmark_logs b/.circleci/store_test_benchmark_logs deleted file mode 100755 index 424a9ff318..0000000000 --- a/.circleci/store_test_benchmark_logs +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e - -REPOSITORY=$1 -shift - -cd $(query_manifest projectDir $REPOSITORY) - -mkdir -p /tmp/csv -export SAVED_TIMESTAMP=$(date +%s) -export HUMAN_READABLE_TIME=$(date -u -d @${SAVED_TIMESTAMP}) - -# Pick logs from test-logs and convert all information into CSV format including the current timestamp, branch, commit and tag information -for file in $(ls /tmp/test-logs); do - echo $file - cat /tmp/test-logs/$file | grep "##BENCHMARK_INFO_PREFIX##" | sed "s/.*##BENCHMARK_INFO_PREFIX##\(.*\)##BENCHMARK_INFO_SUFFIX##.*/\1/" | sed "s/#/,/g" | sed "s_\(.*\)_$SAVED_TIMESTAMP,$HUMAN_READABLE_TIME,$BRANCH,$COMMIT_HASH,$COMMIT_TAG,\1_" | tee -a /tmp/csv/new.csv 1>/dev/null -done -echo "Parsed from logs:" -cat /tmp/csv/new.csv - -# We have lots of repeated entries, no need to put them into repository. Unfortunately build times differ a bit and uniq only works with space as separator -cat /tmp/csv/new.csv | sort | sed "s_ _%_g" | sed "s_^\(.*\),\(.*\)\$_\2 \1_" | uniq -f 1 | sed "s_^\(.*\) \(.*\)\$_\2,\1_" | sed "s_%_ _g" >/tmp/csv/trimmed.csv - -# If there actually were any logs, update the information in the benchmark repository -if [ -s /tmp/csv/trimmed.csv ]; then - cd /tmp - git clone --depth 1 git@github.com-logs:AztecProtocol/benchmark-archive.git - - cd benchmark-archive - git config user.email "circleci@bot" - git config user.name "CircleCi Bot" - cat /tmp/csv/trimmed.csv >>benchmarks.csv - git add benchmarks.csv - git commit -m "Added information from branch $BRANCH commit $COMMIT_HASH" - git push -fi diff --git a/.circleci/tag_remote_image b/.circleci/tag_remote_image deleted file mode 100755 index ee42f32cb8..0000000000 --- a/.circleci/tag_remote_image +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -e - -REPOSITORY=$1 -EXISTING_TAG=$2 -NEW_TAG=$3 -REGION=${4:-$ECR_REGION} - -EXISTING_TAG_MANIFEST=$(aws ecr batch-get-image \ - --region $REGION \ - --repository-name $REPOSITORY \ - --image-ids imageTag=$EXISTING_TAG \ - --query images[].imageManifest \ - --output text) - -if [ -z "$EXISTING_TAG_MANIFEST" ]; then - echo "Existing tag for image not found: $1:$EXISTING_TAG" - exit 1 -fi - -NEW_TAG_MANIFEST=$(aws ecr batch-get-image \ - --region $REGION \ - --repository-name $REPOSITORY \ - --image-ids imageTag=$NEW_TAG \ - --query images[].imageManifest \ - --output text) - -if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then - echo "Tagging $1:$EXISTING_TAG as $1:$NEW_TAG..." - aws ecr put-image \ - --region $REGION \ - --repository-name $REPOSITORY \ - --image-tag $NEW_TAG \ - --image-manifest "$EXISTING_TAG_MANIFEST" > /dev/null 2>&1 -fi \ No newline at end of file diff --git a/.circleci/untag_remote_image b/.circleci/untag_remote_image deleted file mode 100755 index f79fee729b..0000000000 --- a/.circleci/untag_remote_image +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -REPOSITORY=$1 -TAG=$2 -aws ecr batch-delete-image --region=$ECR_REGION --repository-name $REPOSITORY --image-ids imageTag=$2 > /dev/null \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index d07f11dee8..ae63c99f6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "sol/lib/openzeppelin-contracts"] path = sol/lib/openzeppelin-contracts url = https://github.com/OpenZeppelin/openzeppelin-contracts +[submodule "build-system"] + path = build-system + url = git@github.com:AztecProtocol/build-system.git diff --git a/PROJECT b/PROJECT index abebeb417d..adcaa3c7f5 100644 --- a/PROJECT +++ b/PROJECT @@ -1 +1 @@ -barretenberg.js \ No newline at end of file +barretenberg diff --git a/VERSION b/VERSION new file mode 100644 index 0000000000..d9fb9c7365 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v3.0 diff --git a/build-system b/build-system new file mode 160000 index 0000000000..a109f3aef2 --- /dev/null +++ b/build-system @@ -0,0 +1 @@ +Subproject commit a109f3aef28cea4a50481cdf2d74fc3909212c0b