fix: include ignition data in package or save after 1st download (#2591)

Fixes #2445

barretenberg/cpp/srs_db/download_ignition.sh

@@ -20,6 +20,9 @@ cd ignition
 mkdir -p monomial
 cd monomial
 NUM=${1:-19}
+RANGE_START=${2:-}
+RANGE_END=${3:-}
+APPEND=${4:-"false"}
 
 if command -v sha256sum > /dev/null; then
   SHASUM=sha256sum
@@ -33,12 +36,26 @@ checksum() {
 }
 
 download() {
-  curl https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat
+  # Initialize an empty variable for the Range header
+  RANGE_HEADER=""
+
+  # If both RANGE_START and RANGE_END are set, add them to the Range header
+  if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then
+    RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END"
+  fi
+
+  # Download the file
+  if [ "$APPEND" = "true" ]; then
+    curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat >> transcript${1}.dat
+  else
+    curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat
+  fi
+
 }
 
 for TRANSCRIPT in $(seq 0 $NUM); do
   NUM=$(printf %02d $TRANSCRIPT)
-  if [ -f checksums ]; then
+  if [ -f checksums  ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then
     checksum $NUM && continue
     download $NUM
     checksum $NUM || exit 1

yarn-project/Dockerfile

@@ -19,4 +19,13 @@ RUN yarn tsc -b
 
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base
 COPY --from=builder /usr/src/yarn-project /usr/src/yarn-project
+
+# Download minimal CRS requirements
+COPY --from=builder /usr/src/barretenberg/cpp/srs_db/download_ignition.sh /usr/src/yarn-project/circuits.js/resources/download_ignition.sh
+WORKDIR /usr/src/yarn-project/circuits.js/resources
+RUN ./download_ignition.sh 0 28 3200027
+RUN ./download_ignition.sh 0 322560092 322560219 true
+
+WORKDIR /usr/src/yarn-project
+
 ENTRYPOINT ["yarn"]

yarn-project/circuits.js/.gitignore

@@ -0,0 +1 @@
+resources/ignition/

yarn-project/circuits.js/resources/download_ignition.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+# Downloads the ignition trusted setup transcripts.
+#
+# See here for details of the contents of the transcript.dat files:
+#  https://github.com/AztecProtocol/ignition-verification/blob/master/Transcript_spec.md
+#
+# To download all transcripts.
+#  ./download_ignition.sh
+#
+# To download a range of transcripts, e.g. 0, 1 and 2.
+#  ./download_ignition.sh 2
+#
+# If a checksums file is available, it will be used to validate if a download is required
+# and also check the validity of the downloaded transcripts. If not the script downloads
+# whatever is requested but does not check the validity of the downloads.
+set -eu
+
+mkdir -p ignition
+cd ignition
+mkdir -p monomial
+cd monomial
+NUM=${1:-19}
+RANGE_START=${2:-}
+RANGE_END=${3:-}
+APPEND=${4:-"false"}
+
+if command -v sha256sum > /dev/null; then
+  SHASUM=sha256sum
+else
+  SHASUM="shasum -a 256"
+fi
+
+checksum() {
+  grep transcript${1}.dat checksums | $SHASUM -c
+  return $?
+}
+
+download() {
+  # Initialize an empty variable for the Range header
+  RANGE_HEADER=""
+
+  # If both RANGE_START and RANGE_END are set, add them to the Range header
+  if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then
+    RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END"
+  fi
+
+  # Download the file
+  if [ "$APPEND" = "true" ]; then
+    curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat >> transcript${1}.dat
+  else
+    curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat
+  fi
+
+}
+
+for TRANSCRIPT in $(seq 0 $NUM); do
+  NUM=$(printf %02d $TRANSCRIPT)
+  if [ -f checksums  ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then
+    checksum $NUM && continue
+    download $NUM
+    checksum $NUM || exit 1
+  else
+    download $NUM
+  fi
+done

yarn-project/circuits.js/src/crs/index.ts

@@ -1,9 +1,10 @@
+import { createDebugLogger } from '@aztec/foundation/log';
 import { fileURLToPath } from '@aztec/foundation/url';
 
 import isNode from 'detect-node';
 import { existsSync } from 'fs';
 import { open } from 'fs/promises';
-import { dirname } from 'path';
+import { dirname, join } from 'path';
 
 /**
  * Downloader for CRS from the web or local.
@@ -26,7 +27,6 @@ export class NetCrs {
     // We need numPoints number of g1 points.
     const g1Start = 28;
     const g1End = g1Start + this.numPoints * 64 - 1;
-
     // Download required range of data.
     const response = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat', {
       headers: {
@@ -43,7 +43,8 @@ export class NetCrs {
    * Download the G2 points data.
    */
   async downloadG2Data() {
-    const g2Start = 28 + 5040001 * 64;
+    const g2Start = 28 + 5_040_001 * 64; // = 322_560_092
+
     const g2End = g2Start + 128 - 1;
 
     const response2 = await fetch('https://aztec-ignition.s3.amazonaws.com/MAIN%20IGNITION/monomial/transcript00.dat', {
@@ -85,17 +86,18 @@ export class FileCrs {
      */
     public readonly numPoints: number,
     private path: string,
+    private readonly offsetStart = true,
   ) {}
 
   /**
    * Read the data file.
    */
   async init() {
     // We need numPoints number of g1 points.
-    const g1Start = 28;
+    const g1Start = this.offsetStart ? 28 : 0;
     const g1Length = this.numPoints * 64;
 
-    const g2Start = 28 + 5040001 * 64;
+    const g2Start = 28 + 5_040_001 * 64; // = 322_560_092
     const g2Length = 128;
     // Lazily seek our data
     const fileHandle = await open(this.path, 'r');
@@ -104,7 +106,7 @@ export class FileCrs {
       await fileHandle.read(this.data, 0, g1Length, g1Start);
 
       this.g2Data = Buffer.alloc(g2Length);
-      await fileHandle.read(this.g2Data, 0, g2Length, g2Start);
+      await fileHandle.read(this.g2Data, 0, g2Length, this.offsetStart ? g2Start : g1Start + g1Length);
     } finally {
       await fileHandle.close();
     }
@@ -132,21 +134,40 @@ export class FileCrs {
  */
 export class Crs {
   private crs: FileCrs | NetCrs;
+  private logger = createDebugLogger('circuits:crs');
+  /**
+   * The path to our SRS object, assuming that we are in the aztec3-packages folder structure.
+   */
+  private devPath = '/../../../../barretenberg/cpp/srs_db/ignition/monomial/transcript00.dat';
+  /**
+   * The path of our SRS object, if we downloaded on init.
+   */
+  private localPath = `/../../resources/ignition/monomial/transcript00.dat`;
 
   constructor(
     /**
      * The number of circuit gates.
      */
     public readonly numPoints: number,
+
+    /**
+     * Option to save downloaded SRS on file.
+     */
+    private readonly saveOnFile = true,
   ) {
     if (isNode) {
-      /**
-       * The path to our SRS object, assuming that we are in the aztec3-packages folder structure.
-       */
-      const SRS_DEV_PATH =
-        dirname(fileURLToPath(import.meta.url)) +
-        '/../../../../barretenberg/cpp/srs_db/ignition/monomial/transcript00.dat';
-      this.crs = existsSync(SRS_DEV_PATH) ? new FileCrs(numPoints, SRS_DEV_PATH) : new NetCrs(numPoints);
+      const devPath = join(fileURLToPath(import.meta.url), this.devPath);
+      const localPath = join(dirname(fileURLToPath(import.meta.url)), this.localPath);
+      const existsDev = existsSync(devPath);
+      const existsLocal = existsSync(localPath);
+
+      if (existsDev) {
+        this.crs = new FileCrs(numPoints, devPath);
+      } else if (existsLocal) {
+        this.crs = new FileCrs(numPoints, localPath, false);
+      } else {
+        this.crs = new NetCrs(numPoints);
+      }
     } else {
       this.crs = new NetCrs(numPoints);
     }
@@ -157,6 +178,29 @@ export class Crs {
    */
   async init() {
     await this.crs.init();
+    if (isNode) {
+      const localPath = dirname(fileURLToPath(import.meta.url)) + this.localPath;
+      // save downloaded CRS on file
+      if (this.saveOnFile && !existsSync(localPath)) {
+        const fileHandle = await open(localPath, 'w');
+        const g1Data = Buffer.from(this.crs.getG1Data());
+        try {
+          await fileHandle.write(g1Data);
+        } catch (err: any) {
+          this.logger.warn('Failed to save CRS data: ', err.message);
+        }
+
+        const g2Data = Buffer.from(this.crs.getG2Data());
+        try {
+          await fileHandle.write(g2Data, 0, g2Data.length, g1Data.length);
+          // appendFileSync(localPath, Buffer.from(g2Data));
+        } catch (err: any) {
+          this.logger.warn('Failed to append G2 data: ', err.message);
+        } finally {
+          await fileHandle.close();
+        }
+      }
+    }
   }
 
   /**

yarn-project/deploy_npm.sh

@@ -74,4 +74,3 @@ deploy_package world-state
 deploy_package key-store
 deploy_package aztec-node
 deploy_package aztec-sandbox
-

yarn-project/yarn-project-base/Dockerfile

@@ -44,7 +44,8 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/circuits-wasm-linux-clang as c
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts
 
 FROM node:18-alpine
-RUN apk update && apk add --no-cache bash jq
+RUN apk update && apk add --no-cache bash jq curl
+
 WORKDIR /usr/src/yarn-project
 
 # The dockerignore file ensures the context only contains package.json and tsconfig.json files.
@@ -69,6 +70,9 @@ RUN yarn prepare:check
 # Bring in circuits wasms.
 COPY --from=circuits /usr/src/circuits/cpp/build-wasm/bin /usr/src/circuits/cpp/build-wasm/bin
 
+# Copy ignition download script.
+COPY --from=circuits /usr/src/barretenberg/cpp/srs_db/download_ignition.sh /usr/src/barretenberg/cpp/srs_db/download_ignition.sh
+
 # Generate L1 contract TypeScript artifacts.
 COPY --from=contracts /usr/src/l1-contracts/out /usr/src/l1-contracts/out
 RUN cd l1-artifacts && ./scripts/generate-artifacts.sh