squash commits

.github/.gitignore

@@ -0,0 +1 @@
+.secrets

.github/workflows/ci.yml

@@ -208,6 +208,7 @@ jobs:
         uses: ./.github/ensure-tester-with-images
         env:
           USERNAME: ${{ needs.configure.outputs.username }}
+          PULL_REQUEST: ${{ github.event.pull_request.number }}
         with:
           runner_type: ${{ steps.runner_type.outputs.type }}
           builder_type: builder-x86
@@ -443,7 +444,7 @@ jobs:
         timeout-minutes: 40
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb-ultra-honk
 
-  bb-acir-tests-bb-mega-honk:
+  bb-acir-tests-bb-client-ivc:
     needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
@@ -452,11 +453,11 @@ jobs:
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
-          concurrency_key: barretenberg-acir-tests-bb-mega-honk-x86
-      - name: "BB Native Acir Tests (Megahonk)"
+          concurrency_key: barretenberg-acir-tests-bb-client-ivc-x86
+      - name: "BB Native Acir Tests (ClientIVC)"
         working-directory: ./barretenberg/
         timeout-minutes: 40
-        run: earthly-ci --no-output ./+barretenberg-acir-tests-bb-mega-honk
+        run: earthly-ci --no-output ./+barretenberg-acir-tests-bb-client-ivc
 
   bb-acir-tests-sol:
     needs: [noir-build-acir-tests, build, configure]
@@ -779,8 +780,7 @@ jobs:
         with:
           concurrency_key: docs-preview-x86
       - name: "Docs Preview"
-        if: github.event.number
-        timeout-minutes: 40
+        timeout-minutes: 60
         run: |
           earthly-ci --no-output \
           ./docs/+deploy-preview --ENV=staging --PR=${{ github.event.number }} \
@@ -875,6 +875,19 @@ jobs:
         timeout-minutes: 40
         run: earthly-ci -P --no-output +test --box=${{ matrix.box }} --browser=${{ matrix.browser }} --mode=cache
 
+  rough-rhino-installer:
+    needs: [configure]
+    runs-on: ${{ needs.configure.outputs.username }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: rough-rhino-installer
+      - name: Rough Rhino Installer Helper Script
+        working-directory: ./spartan/releases/rough-rhino
+        run: earthly-ci +test-all
+
   protocol-circuits-gates-report:
     needs: [build, configure]
     if: needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true'
@@ -969,7 +982,7 @@ jobs:
       - bb-acir-tests-bb
       - bb-acir-tests-bb-ultra-plonk
       - bb-acir-tests-bb-ultra-honk
-      - bb-acir-tests-bb-mega-honk
+      - bb-acir-tests-bb-client-ivc
       - bb-acir-tests-sol
       - bb-acir-tests-sol-honk
       - bb-acir-tests-bb-js

.github/workflows/network-deploy.yml

@@ -1,18 +1,17 @@
-name: Aztec Network EKS Deployment
-
-# Manual trigerring of this workflow is intentionally disabled
-# Helm deployments do not support lock files
-# Without a lockfile, manual trigerring can lead to corrupted or partial deployments
+name: Aztec Network Deployment
 
 on:
-  push:
-    branches:
-      - staging
-      - production
-  pull_request:
-    branches:
-      - staging
-      - production
+  workflow_dispatch:
+    inputs:
+      namespace:
+        description: The namespace to deploy to, e.g. smoke
+        required: true
+      values_file:
+        description: The values file to use, e.g. 1-validators.yaml
+        required: true
+      aztec_docker_image:
+        description: The Aztec Docker image to use, e.g. aztecprotocol/aztec:da809c58290f9590836f45ec59376cbf04d3c4ce-x86_64
+        required: true
 
 jobs:
   network_deployment:
@@ -24,34 +23,67 @@ jobs:
 
     # Set up a variable based on the branch name
     env:
-      NAMESPACE: ${{ github.ref == 'refs/heads/production' && 'production' || 'staging' }}
+      AZTEC_DOCKER_IMAGE: ${{ inputs.aztec_docker_image }}
+      NAMESPACE: ${{ inputs.namespace }}
+      VALUES_FILE: ${{ inputs.values_file }}
       CHART_PATH: ./spartan/aztec-network
+      CLUSTER_NAME: aztec-gke
+      REGION: us-west1-a
+      TF_STATE_BUCKET: aztec-terraform
+      GKE_CLUSTER_CONTEXT: gke_testnet-440309_us-west1-a_aztec-gke
 
     steps:
-      # Step 1: Check out the repository's code
       - name: Checkout code
         uses: actions/checkout@v3
 
-      # Step 2: Configure AWS credentials using GitHub Secrets
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-1
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
 
-      # Step 3: Set up Kubernetes context for AWS EKS
-      - name: Configure kubectl with EKS cluster
+      - name: Install GKE Auth Plugin
         run: |
-          aws eks update-kubeconfig --region us-east-1 --name spartan
+          gcloud components install gke-gcloud-auth-plugin --quiet
 
-      # Step 4: Install Helm
-      - name: Install Helm
+      - name: Configure kubectl with GKE cluster
         run: |
-          curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+          gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --region ${{ env.REGION }}
 
-      # Step 5: Apply Helm Chart
-      - name: Deploy Helm chart
+      - name: Ensure Terraform state bucket exists
         run: |
-          helm dependency update ${{ env.CHART_PATH }}
-          helm upgrade --install ${{ env.NAMESPACE }} ${{ env.CHART_PATH }} --namespace ${{ env.NAMESPACE }} --set network.public=true --atomic --create-namespace --timeout 20m
+          if ! gsutil ls gs://${{ env.TF_STATE_BUCKET }} >/dev/null 2>&1; then
+            echo "Creating GCS bucket for Terraform state..."
+            gsutil mb -l us-east4 gs://${{ env.TF_STATE_BUCKET }}
+            gsutil versioning set on gs://${{ env.TF_STATE_BUCKET }}
+          else
+            echo "Terraform state bucket already exists"
+          fi
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: "1.5.0" # Specify your desired version
+
+      - name: Terraform Init
+        working-directory: ./spartan/terraform/deploy-release
+        run: |
+          terraform init \
+            -backend-config="bucket=${{ env.TF_STATE_BUCKET }}" \
+            -backend-config="prefix=network-deploy/${{ env.REGION }}/${{ env.CLUSTER_NAME }}/${{ env.NAMESPACE }}/terraform.tfstate" \
+
+      - name: Terraform Plan
+        working-directory: ./spartan/terraform/deploy-release
+        run: |
+          terraform plan \
+            -var="release_name=${{ env.NAMESPACE }}" \
+            -var="values_file=${{ env.VALUES_FILE }}" \
+            -var="gke_cluster_context=${{ env.GKE_CLUSTER_CONTEXT }}" \
+            -var="aztec_docker_image=${{ env.AZTEC_DOCKER_IMAGE }}" \
+            -out=tfplan
+
+      - name: Terraform Apply
+        working-directory: ./spartan/terraform/deploy-release
+        run: terraform apply -auto-approve tfplan

.github/workflows/network-test.yml

@@ -0,0 +1,86 @@
+name: Aztec Network Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      namespace:
+        description: The namespace to deploy to, e.g. smoke
+        required: true
+      test:
+        description: The test to run, e.g. spartan/smoke.test.ts
+        required: true
+      aztec_e2e_docker_image:
+        description: The Aztec E2E Docker image to use, e.g. aztecprotocol/end-to-end:da809c58290f9590836f45ec59376cbf04d3c4ce-x86_64
+        required: true
+
+jobs:
+  network_test:
+    runs-on: ubuntu-latest
+
+    env:
+      TEST_DOCKER_IMAGE: ${{ inputs.aztec_e2e_docker_image }}
+      NAMESPACE: ${{ inputs.namespace }}
+      TEST: ${{ inputs.test }}
+      CHART_PATH: ./spartan/aztec-network
+      CLUSTER_NAME: aztec-gke
+      REGION: us-west1-a
+      PROJECT_ID: testnet-440309
+      GKE_CLUSTER_CONTEXT: gke_testnet-440309_us-west1-a_aztec-gke
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          install_components: gke-gcloud-auth-plugin
+
+      - name: Configure kubectl with GKE cluster
+        run: |
+          gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --region ${{ env.REGION }}
+
+      - name: Run test
+        run: |
+
+          # Find 3 free ports between 9000 and 10000
+          FREE_PORTS=$(comm -23 <(seq 9000 10000 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 3)
+
+          # Extract the free ports from the list
+          PXE_PORT=$(echo $FREE_PORTS | awk '{print $1}')
+          ANVIL_PORT=$(echo $FREE_PORTS | awk '{print $2}')
+          METRICS_PORT=$(echo $FREE_PORTS | awk '{print $3}')
+
+          export GRAFANA_PASSWORD=$(kubectl get secrets -n metrics metrics-grafana -o jsonpath='{.data.admin-password}' | base64 --decode)
+
+          gcloud config set project ${{ env.PROJECT_ID }}
+
+          GCLOUD_CONFIG_DIR=$(gcloud info --format='value(config. paths. global_config_dir)')
+
+          echo "gcloud config dir: [$GCLOUD_CONFIG_DIR]"
+
+          docker run --rm --network=host \
+            -v ~/.kube:/root/.kube \
+            -v $GCLOUD_CONFIG_DIR:/root/.config/gcloud \
+            -e K8S=gcloud \
+            -e CLUSTER_NAME=${{ env.CLUSTER_NAME }} \
+            -e REGION=${{ env.REGION }} \
+            -e INSTANCE_NAME=${{ env.NAMESPACE }} \
+            -e SPARTAN_DIR="/usr/src/spartan" \
+            -e NAMESPACE=${{ env.NAMESPACE }} \
+            -e HOST_PXE_PORT=$PXE_PORT \
+            -e CONTAINER_PXE_PORT=8081 \
+            -e HOST_ETHEREUM_PORT=$ANVIL_PORT \
+            -e CONTAINER_ETHEREUM_PORT=8545 \
+            -e HOST_METRICS_PORT=$METRICS_PORT \
+            -e CONTAINER_METRICS_PORT=80 \
+            -e GRAFANA_PASSWORD=$GRAFANA_PASSWORD \
+            -e DEBUG="aztec:*" \
+            -e LOG_JSON=1 \
+            -e LOG_LEVEL=debug \
+            ${{ env.TEST_DOCKER_IMAGE }} ${{ env.TEST }}

.github/workflows/publish-aztec-packages.yml

@@ -100,13 +100,13 @@ jobs:
         with:
           concurrency_key: build-aztec
           dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
-      - name: Build & Push Aztec x86_64
+      - name: Build & Push Aztec and End-to-End x86_64
         timeout-minutes: 40
         run: |
           earthly-ci \
             --no-output \
             --push \
-            ./yarn-project+export-aztec-arch \
+            ./yarn-project+export-images-arch \
             --DIST_TAG=${{ env.GIT_COMMIT }} \
             --ARCH=x86_64
 
@@ -275,7 +275,7 @@ jobs:
         with:
           concurrency_key: publish-npm
           dockerhub_password: "${{ env.DOCKERHUB_PASSWORD }}"
-      
+
       - name: Publish bb.js NPM package
         run: |
           DEPLOY_TAG=${{ env.DEPLOY_TAG }}
@@ -312,6 +312,18 @@ jobs:
             --VERSION=$VERSION \
             --DRY_RUN=${{ (github.event.inputs.publish == 'false') && '1' || '0' }}
 
+      - name: Publish spartan NPM package
+        run: |
+          DEPLOY_TAG=${{ env.DEPLOY_TAG }}
+          VERSION=${DEPLOY_TAG#aztec-packages-v}
+          earthly-ci \
+            --no-output \
+            --secret NPM_TOKEN=${{ env.NPM_TOKEN }} \
+            ./spartan/releases/rough-rhino+publish-npm \
+            --DIST_TAG=latest \
+            --VERSION=$VERSION \
+            --DRY_RUN=${{ (github.event.inputs.publish == 'false') && '1' || '0' }}
+
   publish-aztec-up:
     needs: [configure, publish-manifests]
     runs-on: ubuntu-latest
@@ -335,10 +347,10 @@ jobs:
         working-directory: ./aztec-up/terraform
         run: |
           terraform init
-          if [ "${{ github.ref_name }}" == "master" ]; then
-            TAG=master
-          else
+          if [ -n "${{ env.DEPLOY_TAG }}" ]; then
             TAG=${{ env.DEPLOY_TAG }}
+          else
+            TAG=${{ github.ref_name }}
           fi
           export TF_VAR_VERSION=${TAG#aztec-packages-v}
           terraform apply -auto-approve

.github/workflows/publish-docs.yml

@@ -29,7 +29,7 @@ jobs:
         with:
           concurrency_key: docs-preview-${{ inputs.username || github.actor }}-x86
 
-      - timeout-minutes: 25
+      - timeout-minutes: 60 # in case of full build
         run: |
           earthly-ci --no-output ./docs/+deploy-prod \
           --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} \

.gitignore

@@ -25,8 +25,6 @@ terraform.tfstate*
 .secret
 .bb_tmp
 
-# Terraform
-*.tfvars
 
 # tmux
 tmux-client-*.log

.release-please-manifest.json

@@ -1,7 +1,7 @@
 {
-  ".": "0.63.1",
+  ".": "0.65.2",
   "yarn-project/cli": "0.35.1",
-  "yarn-project/aztec": "0.63.1",
-  "barretenberg": "0.63.1",
-  "barretenberg/ts": "0.63.1"
+  "yarn-project/aztec": "0.65.2",
+  "barretenberg": "0.65.2",
+  "barretenberg/ts": "0.65.2"
 }