From 02012a296342f5efe4e3bd1e7c4c004c7c9d3f9a Mon Sep 17 00:00:00 2001 From: Alex Xiong Date: Thu, 18 Apr 2024 21:55:31 +0800 Subject: [PATCH] feat: add aztec-srs (#2854) * add aztec-srs * use upgraded ark-srs api, and only cached srs upon nix-shell entry * address comments * update ci to download srs * fix linting and quiet wget * use KZG_TEST and vid_scheme_for_test() instead * minor update on err msg --- .github/workflows/build-and-test.yml | 15 ++++--- .gitignore | 1 + Cargo.lock | 42 ++++++++++++++++++++ Cargo.toml | 4 +- crates/types/Cargo.toml | 10 ++--- crates/types/src/constants.rs | 7 ++++ crates/types/src/vid.rs | 58 +++++++++++++++++++++++----- flake.nix | 5 +++ scripts/download_srs_aztec.sh | 10 +++++ 9 files changed, 129 insertions(+), 23 deletions(-) create mode 100755 scripts/download_srs_aztec.sh diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f9c71b74c0..c1b2a312c8 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -3,11 +3,11 @@ name: Build and Test on: push: branches: - - 'develop' - - 'main' + - "develop" + - "main" pull_request: schedule: - - cron: '0 0 * * 1' + - cron: "0 0 * * 1" workflow_dispatch: concurrency: @@ -52,11 +52,13 @@ jobs: - name: Unit and integration tests for all crates in workspace run: | + export AZTEC_SRS_PATH="$PWD/data/aztec20/kzg10-aztec20-srs-1048584.bin" + ./scripts/download_srs_aztec.sh just ${{ matrix.just_variants }} ${{ matrix.test_suites }} timeout-minutes: 60 env: RUST_BACKTRACE: full - + test-examples: strategy: matrix: @@ -86,7 +88,10 @@ jobs: sudo cp just /usr/bin/just - name: Test examples - run: just ${{ matrix.just_variants }} example all-webserver -- --config_file ./crates/orchestrator/run-config.toml + run: | + export AZTEC_SRS_PATH="$PWD/data/aztec20/kzg10-aztec20-srs-1048584.bin" + ./scripts/download_srs_aztec.sh + just ${{ matrix.just_variants }} example all-webserver -- --config_file ./crates/orchestrator/run-config.toml build-release: strategy: diff --git a/.gitignore b/.gitignore index 52303589c6..a9159c5ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ .github/workflows/preserve-build-and-test.yml config/ValidatorConfigOutput scripts/preserve_ci_ecs_benchmarks.sh +**/*.bin diff --git a/Cargo.lock b/Cargo.lock index 8b7527c414..d43d8a4c6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -416,6 +416,24 @@ dependencies = [ "rayon", ] +[[package]] +name = "ark-poly-commit" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a741492629ffcd228337676dc223a28551aa6792eedb8a2a22c767f00df6c89" +dependencies = [ + "ark-crypto-primitives", + "ark-ec", + "ark-ff", + "ark-poly", + "ark-relations", + "ark-serialize", + "ark-std", + "derivative", + "digest 0.10.7", + "rayon", +] + [[package]] name = "ark-relations" version = "0.4.0" @@ -463,6 +481,23 @@ dependencies = [ "ark-std", ] +[[package]] +name = "ark-srs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9d2dbff870743404933d3ef8a3be3adc3d1926f205e5aec2e43d3ee6885431" +dependencies = [ + "anyhow", + "ark-bn254", + "ark-ec", + "ark-ff", + "ark-poly-commit", + "ark-serialize", + "ark-std", + "hex-literal", + "sha2 0.10.8", +] + [[package]] name = "ark-std" version = "0.4.0" @@ -2950,6 +2985,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" + [[package]] name = "hex_fmt" version = "0.3.0" @@ -3373,6 +3414,7 @@ dependencies = [ "ark-ed-on-bn254", "ark-ff", "ark-serialize", + "ark-srs", "ark-std", "async-compatibility-layer", "async-lock 2.8.0", diff --git a/Cargo.toml b/Cargo.toml index 930b4c5ec4..8548eaf2ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,9 +59,7 @@ futures = "0.3" # TODO generic-array should not be a direct dependency # https://github.com/EspressoSystems/HotShot/issues/1850 generic-array = { version = "0.14.7", features = ["serde"] } -jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3", features = [ - "test-srs", -] } +jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3" } jf-plonk = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3" } jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.3" } jf-utils = { git = "https://github.com/espressosystems/jellyfish", tag = "0.4.3" } diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 5ad087f24f..4fc5621445 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -5,11 +5,6 @@ edition = "2021" name = "hotshot-types" version = "0.1.11" -[features] -gpu-vid = ["jf-primitives/gpu-vid"] - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] anyhow = { workspace = true } ark-bls12-381 = { workspace = true } @@ -18,6 +13,7 @@ ark-ec = { workspace = true } ark-ed-on-bn254 = { workspace = true } ark-ff = { workspace = true } ark-serialize = { workspace = true } +ark-srs = { version = "0.2.0" } ark-std = { workspace = true } async-compatibility-layer = { workspace = true } async-lock = { workspace = true } @@ -60,6 +56,10 @@ url = "2.5.0" [dev-dependencies] serde_json = { workspace = true } +[features] +gpu-vid = ["jf-primitives/gpu-vid"] +test-srs = ["jf-primitives/test-srs"] + [target.'cfg(all(async_executor_impl = "async-std"))'.dependencies] async-std = { workspace = true } diff --git a/crates/types/src/constants.rs b/crates/types/src/constants.rs index 06b9adf24b..8f93667ca8 100644 --- a/crates/types/src/constants.rs +++ b/crates/types/src/constants.rs @@ -53,3 +53,10 @@ pub type WebServerVersion = StaticVersion VidSchemeType { #[allow(clippy::panic)] let num_storage_nodes = u32::try_from(num_storage_nodes).unwrap_or_else(|err| { - panic!("num_storage_nodes {num_storage_nodes} should fit into u32\n\terror: : {err}") + panic!( + "num_storage_nodes {num_storage_nodes} should fit into u32; \ + error: {err}" + ) }); // TODO panic, return `Result`, or make `new` infallible upstream (eg. by panicking)? #[allow(clippy::panic)] - VidSchemeType(Advz::new(num_storage_nodes, recovery_threshold, &*KZG_SRS).unwrap_or_else(|err| panic!("advz construction failure:\n\t(num_storage nodes,recovery_threshold)=({num_storage_nodes},{recovery_threshold})\n\terror: : {err}"))) + VidSchemeType( + Advz::new(num_storage_nodes, recovery_threshold, &*KZG_SRS).unwrap_or_else(|err| { + panic!("advz construction failure: (num_storage nodes,recovery_threshold)=({num_storage_nodes},{recovery_threshold}); \ + error: {err}") + }) + ) +} + +/// Similar to [`vid_scheme()`], but with `KZG_SRS_TEST` for testing purpose only. +#[cfg(feature = "test-srs")] +pub fn vid_scheme_for_test(num_storage_nodes: usize) -> VidSchemeType { + let recovery_threshold = 1 << num_storage_nodes.ilog2(); + #[allow(clippy::panic)] + let num_storage_nodes = u32::try_from(num_storage_nodes).unwrap_or_else(|err| { + panic!("num_storage_nodes {num_storage_nodes} should fit into u32; error: {err}") + }); + #[allow(clippy::panic)] + VidSchemeType( + Advz::new(num_storage_nodes, recovery_threshold, &*KZG_SRS_TEST).unwrap_or_else(|err| { + panic!("advz construction failure: (num_storage nodes,recovery_threshold)=({num_storage_nodes},{recovery_threshold});\ + error: {err}") + }) + ) } /// VID commitment type @@ -116,22 +142,34 @@ pub struct SmallRangeProofType( SmallRangeProof< as PolynomialCommitmentScheme>::Proof>, ); +#[cfg(feature = "test-srs")] lazy_static! { - /// SRS comment - /// - /// TODO use a proper SRS - /// https://github.com/EspressoSystems/HotShot/issues/1686 - static ref KZG_SRS: UnivariateUniversalParams = { + /// SRS for testing only + static ref KZG_SRS_TEST: UnivariateUniversalParams = { let mut rng = jf_utils::test_rng(); UnivariateKzgPCS::::gen_srs_for_testing( &mut rng, - // TODO what's the maximum possible SRS size? - checked_fft_size(200).unwrap(), + SRS_DEGREE, ) .unwrap() }; } +// By default, use SRS from Aztec's ceremony +lazy_static! { + /// SRS comment + static ref KZG_SRS: UnivariateUniversalParams = { + let srs = ark_srs::kzg10::aztec20::setup(SRS_DEGREE) + .expect("Aztec SRS failed to load"); + UnivariateUniversalParams { + powers_of_g: srs.powers_of_g, + h: srs.h, + beta_h: srs.beta_h, + powers_of_h: vec![srs.h, srs.beta_h], + } + }; +} + /// Private type alias for the EC pairing type parameter for [`Advz`]. type E = Bn254; /// Private type alias for the hash type parameter for [`Advz`]. diff --git a/flake.nix b/flake.nix index 364c2ff4ba..d435385312 100644 --- a/flake.nix +++ b/flake.nix @@ -198,6 +198,11 @@ devShell = pkgs.mkShell { inherit CARGO_TARGET_DIR; buildInputs = [ fenixStable ] ++ buildDeps; + shellHook = '' + export ROOT_DIR=$(dirname "$(realpath ./flake.nix)") + export AZTEC_SRS_PATH="$ROOT_DIR/data/aztec20/kzg10-aztec20-srs-1048584.bin" + ./scripts/download_srs_aztec.sh + ''; }; devShells = { diff --git a/scripts/download_srs_aztec.sh b/scripts/download_srs_aztec.sh new file mode 100755 index 0000000000..7ec16c68f7 --- /dev/null +++ b/scripts/download_srs_aztec.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ -f "$AZTEC_SRS_PATH" ]; then + echo "SRS file $AZTEC_SRS_PATH exists" +else + echo "SRS file $AZTEC_SRS_PATH does not exist, downloading ..." + wget -q -P "$(dirname $AZTEC_SRS_PATH)" "https://github.com/EspressoSystems/ark-srs/releases/download/v0.2.0/$(basename $AZTEC_SRS_PATH)" +fi