feat(ci): build noir artifacts in ci (#1251)

# Description

closes: #1248

- Adds a build step before yarn project base that compiles the noir
contracts using the nightly compiler version.
- Adds an ephemeral layer to yarn project base to process the noir
artifacts.

## Dependencies
Uses the following build system pr in order to use multi stage builds of
yarn project base.
AztecProtocol/build-system#16

# Checklist:

- [x] I have reviewed my diff in github, line by line.
- [x] Every change is related to the PR description.
- [x] I have
[linked](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue)
this pull request to the issue(s) that it resolves.
- [x] There are no unexpected formatting changes, superfluous debug
logs, or commented-out code.
- [ ] The branch has been merged or rebased against the head of its
merge target.
- [ ] I'm happy for the PR to be merged at the reviewer's next
convenience.

---------

Co-authored-by: ludamad <[email protected]>

.circleci/config.yml

@@ -489,15 +489,15 @@ jobs:
           command: build key-store
 
   noir-contracts:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
           name: "Build and test"
-          command: build noir-contracts
+          command: cond_spot_run_build noir-contracts 32
 
   noir-compiler:
     machine:
@@ -701,7 +701,6 @@ jobs:
           command: ./scripts/cond_run_script end-to-end $JOB_NAME ./scripts/run_tests_local e2e_lending_contract.test.ts
           working_directory: yarn-project/end-to-end
 
-
   e2e-zk-token-contract:
     machine:
       image: ubuntu-2004:202010-01
@@ -917,12 +916,6 @@ jobs:
           command: |
             deploy_ecr circuits.js
             deploy_npm circuits.js
-      - run:
-          name: "noir-contracts"
-          working_directory: noir-contracts
-          command: |
-            deploy_ecr noir-contracts
-            deploy_npm noir-contracts
       - run:
           name: "l1-artifacts"
           working_directory: l1-artifacts
@@ -1024,7 +1017,7 @@ circuits-x86_64-test: &circuits-x86_64-test
 workflows:
   system:
     when:
-      equal: [ system, << pipeline.parameters.workflow >> ]
+      equal: [system, << pipeline.parameters.workflow >>]
     jobs:
       - barretenberg-x86_64-linux-gcc: *defaults
       - barretenberg-x86_64-linux-clang: *defaults
@@ -1088,11 +1081,13 @@ workflows:
           <<: *defaults
 
       - l1-contracts: *defaults
+      - noir-contracts: *defaults
 
       - yarn-project-base:
           requires:
             - circuits-wasm-linux-clang
             - l1-contracts
+            - noir-contracts
           <<: *defaults
 
       - aztec-js: *yarn_project
@@ -1110,7 +1105,6 @@ workflows:
       - p2p: *yarn_project
       - p2p-bootstrap: *yarn_project
       - l1-artifacts: *yarn_project
-      - noir-contracts: *yarn_project
       - noir-compiler: *yarn_project
       - sequencer-client: *yarn_project
       - types: *yarn_project
@@ -1135,7 +1129,6 @@ workflows:
             - p2p
             - p2p-bootstrap
             - l1-artifacts
-            - noir-contracts
             - noir-compiler
             - sequencer-client
             - types

bootstrap.sh

@@ -36,6 +36,7 @@ fi
 
 circuits/cpp/bootstrap.sh
 l1-contracts/bootstrap.sh
+(cd yarn-project/noir-contracts && ./bootstrap.sh)
 
 if [ "$(uname)" = "Darwin" ]; then
   # works around https://github.com/AztecProtocol/aztec3-packages/issues/158

build_manifest.json

@@ -118,6 +118,7 @@
       "^circuits/",
       "^l1-contracts/",
       "^yarn-project/l1-artifacts/",
+      "^yarn-project/noir-contracts/",
       "^yarn-project/yarn-project-base/",
       "^yarn-project/yarn.lock"
     ],

yarn-project/.dockerignore

@@ -10,4 +10,7 @@
 */dest
 */.tsbuildinfo
 **/Dockerfile*
-**/node_modules
+**/node_modules
+
+noir-contracts/src/types
+noir-contracts/src/artifacts

yarn-project/noir-contracts/.dockerignore

@@ -0,0 +1,6 @@
+src/artifacts/*
+src/types/*
+Dockerfile.build
+Dockerfile.lint
+Dockerfile.types
+README.md

yarn-project/noir-contracts/.gitignore

@@ -1,4 +1,6 @@
+artifacts/
 target/
 proofs/
+types/
 Prover.toml
 Verifier.toml

yarn-project/noir-contracts/Dockerfile

@@ -1,15 +1,20 @@
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base AS builder
+# Running on ubuntu until noir supports an alpine build
 
+# Builder stage to build the noir artifacts
+FROM ubuntu:kinetic
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    sed
+
+WORKDIR /usr/src/yarn-project
 COPY . .
 
-WORKDIR /usr/src/yarn-project/noir-contracts
-RUN yarn build && yarn formatting && yarn test
+WORKDIR /usr/src/yarn-project/noir-contracts 
 
-# Prune dev dependencies. See comment in base image.
-RUN yarn cache clean
-RUN yarn workspaces focus --production > /dev/null
+# Download and extract nargo
+RUN ./scripts/install_noir.sh
+ENV PATH="/usr/src/yarn-project/noir-contracts/.nargo/bin:${PATH}"
 
-FROM node:18-alpine
-COPY --from=builder /usr/src/yarn-project/noir-contracts /usr/src/yarn-project/noir-contracts
-WORKDIR /usr/src/yarn-project/noir-contracts
-ENTRYPOINT ["yarn"]
+RUN ./scripts/compile_ci.sh

yarn-project/noir-contracts/bootstrap.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Install noir if it is not installed already
+if ! command -v nargo &> /dev/null
+then
+    echo "Installing noir"
+    ./scripts/install_noir.sh
+fi
+
+
+# Use yarn script to compile and create types
+yarn 
+yarn noir:build:all

yarn-project/noir-contracts/package.json

@@ -17,7 +17,8 @@
     "formatting:fix": "run -T prettier -w ./src",
     "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests",
     "noir:build": "./src/scripts/compile.sh",
-    "noir:build:all": "./src/scripts/compile.sh $(ls -d src/contracts/*_contract/Nargo.toml | sed -r 's/src\\/contracts\\/(.+)_contract\\/Nargo.toml/\\1/')"
+    "noir:build:all": "./src/scripts/compile.sh $(./scripts/get_all_contracts.sh)",
+    "noir:types:all": "./scripts/types.sh $(./scripts/get_all_contracts.sh)"
   },
   "inherits": [
     "../package.common.json"

yarn-project/noir-contracts/scripts/compile.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Compiles noir contracts in parallel, bubbling any compilation errors
+
+ROOT=$(pwd)
+
+# Error flag file
+error_file="/tmp/error.$$"
+# Array of child PIDs
+pids=()
+
+# Handler for SIGCHLD, cleanup if child exit with error
+handle_sigchld() {
+    for pid in "${pids[@]}"; do
+        # If process is no longer running
+        if ! kill -0 "$pid" 2>/dev/null; then
+            # Wait for the process and get exit status
+            wait "$pid"
+            status=$?
+
+            # If exit status is error
+            if [ $status -ne 0 ]; then
+                # Create error file
+                touch "$error_file"
+            fi
+        fi
+    done
+}
+
+# Set SIGCHLD handler
+trap handle_sigchld SIGCHLD # Trap any ERR signal and call the custom error handler
+
+build() {
+  CONTRACT_NAME=$1
+  CONTRACT_FOLDER="${CONTRACT_NAME}_contract"
+  echo "Compiling $CONTRACT_NAME..."
+  cd src/contracts/$CONTRACT_FOLDER
+  rm -f target/*
+
+  # If VERBOSE is not set, compile with 'nargo' and redirect standard error (stderr) to /dev/null and standard output (stdout) to /dev/null.
+  # If the compilation fails, rerun the compilation with 'nargo' and show the compiler output.
+  nargo compile main --contracts;
+}
+
+process() {
+  CONTRACT_NAME=$1
+
+  cd $ROOT
+  echo "Copying output for $CONTRACT_NAME"
+  NODE_OPTIONS=--no-warnings yarn ts-node --esm src/scripts/copy_output.ts $CONTRACT_NAME
+}
+
+echo "Using $(nargo --version)"
+
+# Build contracts
+for CONTRACT_NAME in "$@"; do
+  build $CONTRACT_NAME &
+  pids+=($!)
+done
+
+# Wait for all background processes to finish
+wait
+
+# If error file exists, exit with error
+if [ -f "$error_file" ]; then
+    rm "$error_file"
+    echo "Error occurred in one or more child processes. Exiting..."
+    exit 1
+fi

yarn-project/noir-contracts/scripts/compile_ci.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Runs the compile scripts for all contracts.
+
+./scripts/compile.sh $(./scripts/get_all_contracts.sh)

yarn-project/noir-contracts/scripts/get_all_contracts.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# Utility to get the names of all contracts
+echo $(ls -d src/contracts/*_contract/Nargo.toml | sed -r "s/src\\/contracts\\/(.+)_contract\\/Nargo.toml/\\1/")

yarn-project/noir-contracts/scripts/install_noir.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Script to install noirup and the latest nargo
+set -eu
+
+VERSION="nightly"
+
+export NARGO_HOME="$(pwd)/.nargo"
+NARGO_BIN_DIR="$NARGO_HOME/bin"
+BIN_URL="https://raw.githubusercontent.com/noir-lang/noirup/master/noirup"
+BIN_PATH="$NARGO_BIN_DIR/noirup"
+NARGO_MAN_DIR="$NARGO_HOME/share/man/man1"
+
+# Clean
+rm -rf $NARGO_HOME
+
+# Install noirup.
+mkdir -p $NARGO_BIN_DIR
+mkdir -p $NARGO_MAN_DIR
+
+curl -# -L $BIN_URL -o $BIN_PATH
+chmod +x $BIN_PATH
+export PATH=$NARGO_BIN_DIR:$PATH
+
+# Install nargo
+noirup -v $VERSION

yarn-project/noir-contracts/scripts/types.sh

@@ -0,0 +1,80 @@
+
+#!/bin/bash
+
+# Example:
+# - this script will automatically be run when running `yarn noir:build`
+# - it exists on its own to allow ci to compile and format in different contexts, as the noir compiler is not available in yarn project base ( by choice )
+# - you can run `yarn noir:types:all` to create all noir artifacts and types consumed by aztec packages.
+
+# Enable strict mode:
+# Exit on error (set -e), treat unset variables as an error (set -u),
+set -eu;
+
+artifacts_dir="src/artifacts"
+types_dir="src/types"
+
+# Create output directories
+mkdir -p $types_dir
+mkdir -p $artifacts_dir
+
+
+ROOT=$(pwd)
+
+write_import() {
+    CONTRACT_NAME=$1
+    NAME=`echo $CONTRACT_NAME | sed -r 's/(^|_)(.)/\U\2/g'`
+    echo "import ${NAME}Json from './${CONTRACT_NAME}_contract.json' assert { type: 'json' };"  >> "$artifacts_dir/index.ts";
+}
+
+write_export() {
+    CONTRACT_NAME=$1
+    NAME=`echo $CONTRACT_NAME | sed -r 's/(^|_)(.)/\U\2/g'`
+
+    # artifacts
+    echo "export const ${NAME}ContractAbi = ${NAME}Json as ContractAbi;"  >> "$artifacts_dir/index.ts";
+    echo "Written typescript for $NAME"
+
+    # types
+    echo "export * from './${CONTRACT_NAME}.js';" >> "$types_dir/index.ts";
+}
+
+
+process() {
+  CONTRACT=$1
+
+  cd $ROOT
+  echo "Creating types for $CONTRACT"
+  NODE_OPTIONS=--no-warnings yarn ts-node --esm src/scripts/copy_output.ts $CONTRACT_NAME
+}
+
+format(){
+  echo "Formatting contract folders"
+  yarn run -T prettier -w  ../aztec.js/src/abis/*.json ./$types_dir/*.ts
+  echo -e "Done\n"
+}
+
+# Make type files
+for CONTRACT_NAME in "$@"; do
+  process $CONTRACT_NAME &
+done
+
+# Wait for all background processes to finish
+wait
+
+# Write the index ts stuff
+# Remove the output file
+rm $artifacts_dir/index.ts || true
+
+# Generate artifacts package index.ts
+echo "// Auto generated module\n" > "$artifacts_dir/index.ts";
+echo "import { ContractAbi } from '@aztec/foundation/abi';"  >> "$artifacts_dir/index.ts";
+
+# Generate types package index.ts
+echo "// Auto generated module\n" > "$types_dir/index.ts";
+for CONTRACT_NAME in "$@"; do
+    write_import $CONTRACT_NAME
+    write_export $CONTRACT_NAME
+done
+
+# only run the rest when the full flag is set
+format

yarn-project/noir-contracts/scripts/types_ci.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# Run the types script for all files
+./scripts/types.sh $(./scripts/get_all_contracts.sh)