feat: add aztec-srs (#2854)

* add aztec-srs

* use upgraded ark-srs api, and only cached srs upon nix-shell entry

* address comments

* update ci to download srs

* fix linting and quiet wget

* use KZG_TEST and vid_scheme_for_test() instead

* minor update on err msg

.github/workflows/build-and-test.yml

@@ -3,11 +3,11 @@ name: Build and Test
 on:
   push:
     branches:
-      - 'develop'
-      - 'main'
+      - "develop"
+      - "main"
   pull_request:
   schedule:
-    - cron: '0 0 * * 1'
+    - cron: "0 0 * * 1"
   workflow_dispatch:
 
 concurrency:
@@ -52,11 +52,13 @@ jobs:
 
       - name: Unit and integration tests for all crates in workspace
         run: |
+          export AZTEC_SRS_PATH="$PWD/data/aztec20/kzg10-aztec20-srs-1048584.bin"
+          ./scripts/download_srs_aztec.sh
           just ${{ matrix.just_variants }} ${{ matrix.test_suites }}
         timeout-minutes: 60
         env:
           RUST_BACKTRACE: full
-      
+
   test-examples:
     strategy:
       matrix:
@@ -86,7 +88,10 @@ jobs:
           sudo cp just /usr/bin/just
 
       - name: Test examples
-        run: just ${{ matrix.just_variants }} example all-webserver -- --config_file ./crates/orchestrator/run-config.toml
+        run: |
+          export AZTEC_SRS_PATH="$PWD/data/aztec20/kzg10-aztec20-srs-1048584.bin"
+          ./scripts/download_srs_aztec.sh
+          just ${{ matrix.just_variants }} example all-webserver -- --config_file ./crates/orchestrator/run-config.toml
 
   build-release:
     strategy:

.gitignore

@@ -17,3 +17,4 @@
 .github/workflows/preserve-build-and-test.yml
 config/ValidatorConfigOutput
 scripts/preserve_ci_ecs_benchmarks.sh
+**/*.bin

Cargo.toml

@@ -59,9 +59,7 @@ futures = "0.3"
 # TODO generic-array should not be a direct dependency
 # https://github.com/EspressoSystems/HotShot/issues/1850
 generic-array = { version = "0.14.7", features = ["serde"] }
-jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3", features = [
-  "test-srs",
-] }
+jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3" }
 jf-plonk = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3" }
 jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3" }
 jf-utils = { git = "https://github.com/espressosystems/jellyfish", tag = "0.4.3" }

crates/types/Cargo.toml

@@ -5,11 +5,6 @@ edition = "2021"
 name = "hotshot-types"
 version = "0.1.11"
 
-[features]
-gpu-vid = ["jf-primitives/gpu-vid"]
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 anyhow = { workspace = true }
 ark-bls12-381 = { workspace = true }
@@ -18,6 +13,7 @@ ark-ec = { workspace = true }
 ark-ed-on-bn254 = { workspace = true }
 ark-ff = { workspace = true }
 ark-serialize = { workspace = true }
+ark-srs = { version = "0.2.0" }
 ark-std = { workspace = true }
 async-compatibility-layer = { workspace = true }
 async-lock = { workspace = true }
@@ -60,6 +56,10 @@ url = "2.5.0"
 [dev-dependencies]
 serde_json = { workspace = true }
 
+[features]
+gpu-vid = ["jf-primitives/gpu-vid"]
+test-srs = ["jf-primitives/test-srs"]
+
 [target.'cfg(all(async_executor_impl = "async-std"))'.dependencies]
 async-std = { workspace = true }
 

crates/types/src/constants.rs

@@ -53,3 +53,10 @@ pub type WebServerVersion = StaticVersion<WEB_SERVER_MAJOR_VERSION, WEB_SERVER_M
 
 /// Constant for Web Server CDN Version
 pub const WEB_SERVER_VERSION: WebServerVersion = StaticVersion {};
+
+/// For `STAKE_TABLE_CAPACITY=200`, the light client prover (a.k.a. `hotshot-state-prover`)
+/// would need to generate proof for a circuit of slightly below 2^20 gates.
+/// Thus we need to support this upperbounded degree in our Structured Reference String (SRS),
+/// the `+2` is just an artifact from the jellyfish's Plonk proof system.
+#[allow(clippy::cast_possible_truncation)]
+pub const SRS_DEGREE: usize = 2u64.pow(20) as usize + 2;

crates/types/src/vid.rs

@@ -13,7 +13,6 @@ use std::{fmt::Debug, ops::Range};
 use ark_bn254::Bn254;
 use jf_primitives::{
     pcs::{
-        checked_fft_size,
         prelude::{UnivariateKzgPCS, UnivariateUniversalParams},
         PolynomialCommitmentScheme,
     },
@@ -31,6 +30,8 @@ use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 
+use crate::constants::SRS_DEGREE;
+
 /// VID scheme constructor.
 ///
 /// Returns an opaque type that impls jellyfish traits:
@@ -58,12 +59,37 @@ pub fn vid_scheme(num_storage_nodes: usize) -> VidSchemeType {
 
     #[allow(clippy::panic)]
     let num_storage_nodes = u32::try_from(num_storage_nodes).unwrap_or_else(|err| {
-        panic!("num_storage_nodes {num_storage_nodes} should fit into u32\n\terror: : {err}")
+        panic!(
+            "num_storage_nodes {num_storage_nodes} should fit into u32; \
+                error: {err}"
+        )
     });
 
     // TODO panic, return `Result`, or make `new` infallible upstream (eg. by panicking)?
     #[allow(clippy::panic)]
-    VidSchemeType(Advz::new(num_storage_nodes, recovery_threshold, &*KZG_SRS).unwrap_or_else(|err| panic!("advz construction failure:\n\t(num_storage nodes,recovery_threshold)=({num_storage_nodes},{recovery_threshold})\n\terror: : {err}")))
+    VidSchemeType(
+        Advz::new(num_storage_nodes, recovery_threshold, &*KZG_SRS).unwrap_or_else(|err| {
+              panic!("advz construction failure: (num_storage nodes,recovery_threshold)=({num_storage_nodes},{recovery_threshold}); \
+                      error: {err}")
+        })
+    )
+}
+
+/// Similar to [`vid_scheme()`], but with `KZG_SRS_TEST` for testing purpose only.
+#[cfg(feature = "test-srs")]
+pub fn vid_scheme_for_test(num_storage_nodes: usize) -> VidSchemeType {
+    let recovery_threshold = 1 << num_storage_nodes.ilog2();
+    #[allow(clippy::panic)]
+    let num_storage_nodes = u32::try_from(num_storage_nodes).unwrap_or_else(|err| {
+        panic!("num_storage_nodes {num_storage_nodes} should fit into u32; error: {err}")
+    });
+    #[allow(clippy::panic)]
+    VidSchemeType(
+        Advz::new(num_storage_nodes, recovery_threshold, &*KZG_SRS_TEST).unwrap_or_else(|err| {
+           panic!("advz construction failure: (num_storage nodes,recovery_threshold)=({num_storage_nodes},{recovery_threshold});\
+                   error: {err}")
+        })
+    )
 }
 
 /// VID commitment type
@@ -116,22 +142,34 @@ pub struct SmallRangeProofType(
     SmallRangeProof<<UnivariateKzgPCS<E> as PolynomialCommitmentScheme>::Proof>,
 );
 
+#[cfg(feature = "test-srs")]
 lazy_static! {
-    /// SRS comment
-    ///
-    /// TODO use a proper SRS
-    /// https://github.com/EspressoSystems/HotShot/issues/1686
-    static ref KZG_SRS: UnivariateUniversalParams<E> = {
+    /// SRS for testing only
+    static ref KZG_SRS_TEST: UnivariateUniversalParams<E> = {
         let mut rng = jf_utils::test_rng();
         UnivariateKzgPCS::<E>::gen_srs_for_testing(
             &mut rng,
-            // TODO what's the maximum possible SRS size?
-            checked_fft_size(200).unwrap(),
+            SRS_DEGREE,
         )
         .unwrap()
     };
 }
 
+// By default, use SRS from Aztec's ceremony
+lazy_static! {
+    /// SRS comment
+    static ref KZG_SRS: UnivariateUniversalParams<E> = {
+        let srs = ark_srs::kzg10::aztec20::setup(SRS_DEGREE)
+            .expect("Aztec SRS failed to load");
+        UnivariateUniversalParams {
+            powers_of_g: srs.powers_of_g,
+            h: srs.h,
+            beta_h: srs.beta_h,
+            powers_of_h: vec![srs.h, srs.beta_h],
+        }
+    };
+}
+
 /// Private type alias for the EC pairing type parameter for [`Advz`].
 type E = Bn254;
 /// Private type alias for the hash type parameter for [`Advz`].

flake.nix

@@ -198,6 +198,11 @@
         devShell = pkgs.mkShell {
           inherit CARGO_TARGET_DIR;
           buildInputs = [ fenixStable ] ++ buildDeps;
+          shellHook = ''
+            export ROOT_DIR=$(dirname "$(realpath ./flake.nix)")
+            export AZTEC_SRS_PATH="$ROOT_DIR/data/aztec20/kzg10-aztec20-srs-1048584.bin"
+            ./scripts/download_srs_aztec.sh
+          '';
         };
 
         devShells = {

scripts/download_srs_aztec.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [ -f "$AZTEC_SRS_PATH" ]; then
+    echo "SRS file $AZTEC_SRS_PATH exists"
+else
+    echo "SRS file $AZTEC_SRS_PATH does not exist, downloading ..."
+    wget -q -P "$(dirname $AZTEC_SRS_PATH)" "https://github.com/EspressoSystems/ark-srs/releases/download/v0.2.0/$(basename $AZTEC_SRS_PATH)"
+fi