add build-system and all relevant CI (#9)

circuits/.circleci/config.yml

@@ -0,0 +1,188 @@
+# This config consists of currently 1 workflow.
+# - system: The main Aztec 3 circuits and C++
+#
+# The default workflow is system. To trigger the other workflows, trigger a workflow from CCI
+# setting a string variable called `workflow` to another name.
+#
+# This file uses YAML anchors and aliases to prevent repetition of blocks of config:
+# https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/
+#
+# Two primary anchors are checkout and setup_env, called as the first step of almost all jobs:
+# - checkout: A custom checkout step to reduce the amount of data downloaded to improve speed.
+# - setup_env: Sets up the common environment used by all build steps.
+#
+# Two CCI executors are used:
+# - docker (small): Used only to launch external EC2 instances for big workloads. It's the cheapest option.
+# - machine (large): Used for building in CCI itself. 4cpus, 15GB has the optimal power/cost ratio.
+#
+# The docker executor uses a custom image build in `build_image`. It's specifically streamlined for fast download
+# with just enough tools to execute the build system, and launch EC2 instances etc.
+#
+# There are some `join` steps that are just noops. They are just used to produce cleaner graph rendering in CCI.
+
+version: 2.1
+
+# This build step checks out the code from the repository. It has a hardcoded readonly key to allow the checkout.
+# Initially it just fetches the repo metadata for the current commit hash to a depth of 50 commits.
+# We need historical commit hashes to calculate diffs between previous and current commits.
+# It then checks out the fetched head to actually download the data.
+checkout: &checkout
+  run:
+    name: 'Checkout code'
+    command: |
+      cd $HOME
+      mkdir -p .ssh
+      chmod 0700 .ssh
+      ssh-keyscan -t rsa github.com >> .ssh/known_hosts
+
+      # A read only key for cloning the repository.
+      echo $GIT_CHECKOUT_KEY | base64 -d > .ssh/id_rsa
+
+      chmod 0600 .ssh/id_rsa
+
+      # IF YOU'RE CHANGING THIS, YOU ALSO WANT TO CHANGE: build-system/remote_build/remote_build
+      # Shallow checkout this commit.
+      mkdir -p project
+      cd project
+      git init
+      git remote add origin $CIRCLE_REPOSITORY_URL
+      # Only download metadata when fetching.
+      git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1
+      git checkout FETCH_HEAD
+      # Pull in build-system submodule.
+      git submodule update --init build-system
+
+# This build step checks out the code from the benchmark-archive repository.
+# The key is saved in CircleCi environment in base64 format.
+# Initially it just fetches the latest version.
+benchmark_add_keys: &benchmark_add_keys
+  run:
+    name: 'Add keys for getting the benchmark archive'
+    command: |
+      cd $HOME
+      mkdir -p .ssh
+      chmod 0700 .ssh
+      ssh-keyscan -t rsa github.com >> .ssh/known_hosts
+
+      # A read-write key for updating the repository.
+      echo "$GITHUB_BENCMARK_REPOSITORY_SSH_KEY" | base64 -d > .ssh/id_ed25519
+
+      # This allows github to discern wich key to use.
+      echo "Host github.com
+        Hostname github.com
+        IdentityFile=/root/.ssh/id_rsa
+
+      Host github.com-logs
+        Hostname github.com
+        IdentityFile=/root/.ssh/id_ed25519" > .ssh/config
+
+      chmod 0600 .ssh/id_ed25519
+      ssh-add .ssh/id_ed25519
+
+# Called setup_env to setup a bunch of global variables used throughout the rest of the build process.
+# It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of
+# the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds.
+setup_env: &setup_env
+  run:
+    name: 'Setup environment'
+    command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH"
+
+# This step is used to save logs from various barretenberg test to the workspace so that they can be used later to parse benchmark values out of them
+save_logs: &save_logs
+  persist_to_workspace:
+    root: /tmp/test-logs
+    paths:
+      - ./*
+
+jobs:
+  wasm-linux-clang:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: 'Build'
+          command: cond_spot_run_build aztec3-circuits-wasm-linux-clang 64
+
+  x86_64-linux-clang:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: 'Build'
+          command: cond_spot_run_build aztec3-circuits-x86_64-linux-clang 64
+
+  x86_64-linux-clang-assert:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: 'Build'
+          command: cond_spot_run_build aztec3-circuits-x86_64-linux-clang-assert 64
+
+  x86_64-linux-gcc:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: 'Build'
+          command: cond_spot_run_build aztec3-circuits-x86_64-linux-gcc 64
+
+  aztec3-circuits-tests:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: 'Test'
+          command: cond_spot_run_tests aztec3-circuits-x86_64-linux-clang-assert a3-tests
+      - *save_logs
+
+  #aztec3-circuits-<individual>-tests:
+  #  docker:
+  #    - image: aztecprotocol/alpine-build-image
+  #  resource_class: small
+  #  steps:
+  #    - *checkout
+  #    - *setup_env
+  #    - run:
+  #        name: 'Test'
+  #        command: cond_spot_run_tests aztec3-circuits-x86_64-linux-clang-assert
+  #    - *save_logs
+
+# Repeatable config for defining the workflow below.
+tag_regex: &tag_regex /v[0-9]+(\.[0-9]+)*(-[a-zA-Z-]+\.[0-9]+)?/
+tag_filter: &tag_filter
+  tags:
+    only: *tag_regex
+a3_test: &a3_test
+  requires:
+    - x86_64-linux-clang-assert
+  filters: *tag_filter
+
+workflows:
+  system:
+    jobs:
+      - wasm-linux-clang:
+          filters: *tag_filter
+      - x86_64-linux-clang:
+          filters: *tag_filter
+      - x86_64-linux-clang-assert:
+          filters: *tag_filter
+      - x86_64-linux-gcc:
+          filters: *tag_filter
+      - aztec3-circuits-tests: *a3_test
+      #- aztec3-circuit-<individual>-tests: *a3_test

circuits/.dockerignore

@@ -0,0 +1,18 @@
+build
+build-wasm
+docker*
+scripts
+.*
+src/wasi-sdk*
+barretenberg/cpp/build*
+barretenberg/cpp/docker*
+barretenberg/cpp/scripts
+barretenberg/cpp/.*
+barretenberg/cpp/src/wasi-sdk*
+barretenberg/cpp/src/aztec/rollup/proofs/root_*/fixtures/account
+barretenberg/cpp/src/aztec/rollup/proofs/root_*/fixtures/join_split
+barretenberg/cpp/src/aztec/rollup/proofs/root_*/fixtures/**/proving_key
+barretenberg/cpp/srs_db/ignition/transcript*
+barretenberg/cpp/srs_db/lagrange
+barretenberg/cpp/srs_db/coset_lagrange
+barretenberg/cpp/srs_db/modified_lagrange

circuits/.gitignore

@@ -1,5 +1,6 @@
 .cache/
-build*/
+build/
+build-wasm/
 src/wasi-sdk-*
 src/aztec/proof_system/proving_key/fixtures
 src/aztec/rollup/proofs/*/fixtures

circuits/.gitmodules

@@ -2,3 +2,6 @@
 	path = barretenberg
 	url = [email protected]:AztecProtocol/barretenberg.git
 	branch = aztec3
+[submodule "aztec-build-system"]
+	path = build-system
+	url = [email protected]:AztecProtocol/build-system.git

circuits/bootstrap_docker.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# This script builds the monorepo projects listed in the build_local script, terminating when it reaches TARGET_PROJECT.
+# It performs the build inside a docker image that can launch further docker builds by mapping the users docker daemon
+# socket into the container. A kind of simulated "docker-in-docker".
+# We copy the monorepo entire working tree into this container, excluding any build output.
+# The mechanics of this are, we mount the users repo into the container, then clone it into a directory within the
+# container, and also apply modified/untracked/deleted changes to the internal repo.
+# The result is we have a fresh copy of the repo, with only the working changes applied, that we can modify as we wish.
+
+set -e
+
+TARGET_PROJECT=$1
+REPO=aztec3-circuits
+COMMIT_HASH=$(git rev-parse HEAD)
+
+# If we're calling this script from within a project directory, that's the target project.
+if [ -z "$TARGET_PROJECT" ]; then
+  TARGET_PROJECT=$(git rev-parse --show-prefix)
+  if [ -n "$TARGET_PROJECT" ]; then
+    # We are in a project folder.
+    ONLY_TARGET=true
+    TARGET_PROJECT=$(basename $TARGET_PROJECT)
+    cd $(git rev-parse --show-cdup)
+  fi
+fi
+
+docker build -t $REPO-build - <<EOF
+FROM ubuntu:latest
+RUN apt update && apt install -y git rsync docker.io
+EOF
+
+docker run -ti --rm -v/run/user/$UID/docker.sock:/var/run/docker.sock -v$(git rev-parse --show-toplevel):/repo:ro $REPO-build /bin/bash -c "
+# Checkout head.
+mkdir /$REPO
+cd /$REPO
+git init
+git remote add origin /repo
+git fetch --depth 1 origin $COMMIT_HASH
+git checkout FETCH_HEAD
+
+# Copy untracked and modified files, and remove deleted files, from our current repo.
+cd /repo
+{ git ls-files --others --exclude-standard ; git diff --name-only --diff-filter=TMAR HEAD ; } | rsync -a --files-from=- . /$REPO
+for F in \$(git ls-files --deleted); do rm /$REPO/\$F > /dev/null 2>&1; done
+
+# Setup build environment.
+source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER /repo
+
+build_local $TARGET_PROJECT $ONLY_TARGET
+"

circuits/build_manifest.json

@@ -0,0 +1,26 @@
+{
+    "aztec3-circuits-wasm-linux-clang": {
+      "buildDir": ".",
+      "dockerfile": "dockerfiles/Dockerfile.wasm-linux-clang",
+      "rebuildPatterns": ["^./"],
+      "dependencies": []
+    },
+    "aztec3-circuits-x86_64-linux-clang": {
+      "buildDir": ".",
+      "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang",
+      "rebuildPatterns": ["^./"],
+      "dependencies": []
+    },
+    "aztec3-circuits-x86_64-linux-clang-assert": {
+      "buildDir": ".",
+      "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang-assert",
+      "rebuildPatterns": ["^./"],
+      "dependencies": []
+    },
+    "aztec3-circuits-x86_64-linux-gcc": {
+      "buildDir": "./",
+      "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-gcc",
+      "rebuildPatterns": ["^./"],
+      "dependencies": []
+    }
+  }

circuits/build_manifest.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Source this file to define the PROJECTS variable.
+# PROJECT elements have structure PROJECT:WORKING_DIR:DOCKERFILE:REPO.
+#
+# TODO: Generate this from build_manifest.json
+
+# Commenting out a few projects, as the main use case is now to build the images needed to run end-to-end tests.
+# If wanting to just see if docker images actually build, you can temporarily uncomment required projects.
+PROJECTS=(
+  aztec3-circuits-wasm:./:./dockerfiles/Dockerfile.wasm-linux-clang:aztec3-circuits-wasm-linux-clang
+  aztec3-circuits-x86_64-clang:./:./dockerfiles/Dockerfile.x86_64-linux-clang:aztec3-circuits-x86_64-linux-clang
+  aztec3-circuits-x86_64-clang-assert:./:./dockerfiles/Dockerfile.x86_64-linux-clang-assert:aztec3-circuits-x86_64-linux-clang-assert
+  aztec3-circuits-x86_64-gcc:./:./dockerfiles/Dockerfile.x86_64-linux-gcc:aztec3-circuits-x86_64-linux-gcc
+)

circuits/dockerfiles/Dockerfile.arm64-linux-gcc

@@ -0,0 +1,6 @@
+FROM aztecprotocol/crosstool-ng-arm64:latest
+WORKDIR /usr/src/aztec3-circuits
+COPY . .
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=arm64-linux-gcc .. && cmake --build . --parallel
+RUN cd build && for test in ./bin/*_tests; do qemu-aarch64 $test; done
+ENTRYPOINT /bin/bash

circuits/dockerfiles/Dockerfile.crosstool-ng

@@ -0,0 +1,14 @@
+FROM ubuntu:latest
+
+RUN apt-get update && apt-get install -y autoconf flex texinfo unzip help2man file gawk libtool-bin ncurses-dev bison python-dev
+
+# Download and install the "crosstool-ng" source.
+ENV REV 1.24.0
+RUN wget https://github.com/crosstool-ng/crosstool-ng/archive/crosstool-ng-${REV}.tar.gz \
+  && tar -xf "crosstool-ng-${REV}.tar.gz" \
+  && cd crosstool-ng-crosstool-ng-${REV} \
+  && ./bootstrap \
+  && ./configure --prefix=/usr/local \
+  && make -j$(nproc) \
+  && make install \
+  && rm -rf /crosstool-*

circuits/dockerfiles/Dockerfile.crosstool-ng-arm64

@@ -0,0 +1,26 @@
+FROM aztecprotocol/crosstool-ng:1.24.0
+
+# Build toolchain.
+ENV CT_PREFIX /usr/xcc
+WORKDIR /usr/src/toolchain
+COPY ./crosstool/arm64-linux.config .config
+RUN CT_ALLOW_BUILD_AS_ROOT_SURE=1 ct-ng build && cd / && rm -rf /usr/src/toolchain
+
+# The cross-compiling emulator.
+RUN apt-get update \
+&& apt-get install -y \
+  qemu-user \
+  qemu-user-static \
+&& apt-get clean --yes
+
+ENV CROSS_TRIPLE aarch64-unknown-linux-gnu
+ENV CROSS_ROOT ${CT_PREFIX}/${CROSS_TRIPLE}
+ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
+    AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
+    CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
+    CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
+    CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
+    LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld \
+    FC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gfortran
+ENV QEMU_LD_PREFIX "${CROSS_ROOT}/${CROSS_TRIPLE}/sysroot"
+ENV QEMU_SET_ENV "LD_LIBRARY_PATH=${CROSS_ROOT}/lib:${QEMU_LD_PREFIX}"

circuits/dockerfiles/Dockerfile.wasm-linux-clang

@@ -0,0 +1,14 @@
+FROM ubuntu:kinetic AS builder
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential wget git libssl-dev cmake curl binaryen
+RUN curl https://wasmtime.dev/install.sh -sSf | bash /dev/stdin --version v3.0.1
+WORKDIR /usr/src/aztec3-circuits/src
+RUN curl -s -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz | tar zxfv -
+WORKDIR /usr/src/aztec3-circuits
+COPY . .
+# Build both honk_tests barretenberg.wasm
+# This ensures that we aren't using features that would be incompatible with WASM for Honk
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && cmake --build . --parallel --target honk_tests --target aztec3.wasm
+
+FROM alpine:3.17
+COPY --from=builder /usr/src/aztec3-circuits/build/bin/aztec3.wasm /usr/src/aztec3-circuits/build/bin/aztec3.wasm
+COPY --from=builder /usr/src/aztec3-circuits/build/bin/*_tests /usr/src/aztec3-circuits/build/bin/