From 9ca622e0755bd723bc4a92a6cacd05ed86a1e67f Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Thu, 19 Oct 2023 11:48:02 -0700 Subject: [PATCH 01/54] restore simple bytes-to-field from https://github.com/EspressoSystems/jellyfish/pull/381/commits/03d7d87e24a05804b293c40b611df0fd076c8bf4 --- utilities/src/conversion.rs | 303 +++++++++++------------------------- 1 file changed, 94 insertions(+), 209 deletions(-) diff --git a/utilities/src/conversion.rs b/utilities/src/conversion.rs index 377901059..079e6b3fb 100644 --- a/utilities/src/conversion.rs +++ b/utilities/src/conversion.rs @@ -9,9 +9,9 @@ use ark_ff::{BigInteger, Field, PrimeField}; use ark_std::{ borrow::Borrow, cmp::min, - iter::{once, repeat, Peekable, Take}, + iter::{once, repeat, Take}, marker::PhantomData, - mem, vec, + mem, vec::{IntoIter, Vec}, }; use sha2::{Digest, Sha512}; @@ -293,21 +293,18 @@ fn compile_time_checks() -> (usize, usize, usize) { /// Deterministic, infallible, invertible iterator adaptor to convert from /// arbitrary bytes to field elements. /// -/// # TODO doc test +/// The final field element is padded with zero bytes as needed. /// -/// # How it works +/// # Example /// -/// Returns an iterator over [`PrimeField`] items defined as follows: -/// - For each call to `next()`: -/// - Consume P-1 items from `bytes` where P is the field characteristic byte -/// length. (Consume all remaining B items from `bytes` if B < P-1.) -/// - Convert the consumed bytes into a [`PrimeField`] via -/// [`from_le_bytes_mod_order`]. Reduction modulo the field characteristic -/// is guaranteed not to occur because we consumed at most P-1 bytes. -/// - Return the resulting [`PrimeField`] item. -/// - The returned iterator has an additional item that encodes the number of -/// input items consumed in order to produce the final output item. -/// - If `bytes` is empty then result is empty. +/// ``` +/// # use jf_utils::bytes_to_field; +/// # use ark_ed_on_bn254::Fr as Fr254; +/// let bytes = [1, 2, 3]; +/// let mut elems_iter = bytes_to_field::<_, Fr254>(bytes); +/// assert_eq!(elems_iter.next(), Some(Fr254::from(197121u64))); +/// assert_eq!(elems_iter.next(), None); +/// ``` /// /// # Panics /// @@ -328,7 +325,24 @@ where /// Deterministic, infallible inverse of [`bytes_to_field`]. /// -/// This function is not invertible because [`bytes_to_field`] is not onto. +/// The composition of [`field_to_bytes`] with [`bytes_to_field`] might contain +/// extra zero bytes. +/// +/// # Example +/// +/// ``` +/// # use jf_utils::{bytes_to_field, field_to_bytes}; +/// # use ark_ed_on_bn254::Fr as Fr254; +/// let bytes = [1, 2, 3]; +/// let mut bytes_iter = field_to_bytes(bytes_to_field::<_, Fr254>(bytes)); +/// assert_eq!(bytes_iter.next(), Some(1)); +/// assert_eq!(bytes_iter.next(), Some(2)); +/// assert_eq!(bytes_iter.next(), Some(3)); +/// for _ in 0..28 { +/// assert_eq!(bytes_iter.next(), Some(0)); +/// } +/// assert_eq!(bytes_iter.next(), None); +/// ``` /// /// ## Panics /// @@ -342,31 +356,22 @@ where FieldToBytes::new(elems.into_iter()) } -struct BytesToField -where - I: Iterator, -{ - bytes_iter: Peekable, - final_byte_len: Option, - done: bool, - new: bool, - _phantom: PhantomData, +struct BytesToField { + bytes_iter: I, primefield_bytes_len: usize, + _phantom: PhantomData, } -impl BytesToField +impl BytesToField where - I: Iterator, + F: Field, { - fn new(iter: I) -> Self { + fn new(bytes_iter: I) -> Self { let (primefield_bytes_len, ..) = compile_time_checks::(); Self { - bytes_iter: iter.peekable(), - final_byte_len: None, - done: false, - new: true, - _phantom: PhantomData, + bytes_iter, primefield_bytes_len, + _phantom: PhantomData, } } } @@ -380,77 +385,42 @@ where type Item = F; fn next(&mut self) -> Option { - if self.done { - // we don't support iterators that return `Some` after returning `None` - return None; - } - - if let Some(len) = self.final_byte_len { - // iterator is done. final field elem encodes length. - self.done = true; - return Some(F::from(len as u64)); - } - - if self.new && self.bytes_iter.peek().is_none() { - // zero-length iterator - self.done = true; - return None; - } - - // TODO const generics: use [u8; primefield_bytes_len] - let mut field_elem_bytes = vec![0u8; self.primefield_bytes_len]; - for (i, b) in field_elem_bytes.iter_mut().enumerate() { + let mut elem_bytes = Vec::with_capacity(self.primefield_bytes_len); + for _ in 0..elem_bytes.capacity() { if let Some(byte) = self.bytes_iter.next() { - *b = *byte.borrow(); + elem_bytes.push(*byte.borrow()); } else { - self.final_byte_len = Some(i); break; } } - Some(F::from_le_bytes_mod_order(&field_elem_bytes)) + if elem_bytes.is_empty() { + None + } else { + Some(F::from_le_bytes_mod_order(&elem_bytes)) + } } } struct FieldToBytes { elems_iter: I, - state: FieldToBytesState, + bytes_iter: Take>, primefield_bytes_len: usize, + _phantom: PhantomData, } -enum FieldToBytesState { - New, - Typical { - bytes_iter: Take>, - next_elem: F, - next_next_elem: F, - }, - Final { - bytes_iter: Take>, - }, -} - -impl FieldToBytes { +impl FieldToBytes +where + F: Field, +{ fn new(elems_iter: I) -> Self { let (primefield_bytes_len, ..) = compile_time_checks::(); Self { elems_iter, - state: FieldToBytesState::New, + bytes_iter: Vec::new().into_iter().take(0), primefield_bytes_len, + _phantom: PhantomData, } } - - fn elem_to_usize(elem: F) -> usize { - usize::try_from(u64::from_le_bytes( - elem.into_bigint().to_bytes_le()[..mem::size_of::()] - .try_into() - .expect("conversion from [u8] to u64 should succeed"), - )) - .expect("result len conversion from u64 to usize should succeed") - } - - fn elem_to_bytes_iter(elem: F) -> IntoIter { - elem.into_bigint().to_bytes_le().into_iter() - } } impl Iterator for FieldToBytes @@ -462,87 +432,19 @@ where type Item = u8; fn next(&mut self) -> Option { - use FieldToBytesState::{Final, New, Typical}; - match &mut self.state { - New => { - let cur_elem = if let Some(elem) = self.elems_iter.next() { - *elem.borrow() - } else { - // length-0 iterator - // move to `Final` state with an empty iterator - self.state = Final { - bytes_iter: Vec::new().into_iter().take(0), - }; - return None; - }; - - let bytes_iter = Self::elem_to_bytes_iter(cur_elem); - - let next_elem = if let Some(elem) = self.elems_iter.next() { - *elem.borrow() - } else { - // length-1 iterator: we never produced this - // move to `Final` state with primefield_bytes_len bytes from the sole elem - let mut bytes_iter = bytes_iter.take(self.primefield_bytes_len); - let ret = bytes_iter.next(); - self.state = Final { bytes_iter }; - return ret; - }; - - let next_next_elem = if let Some(elem) = self.elems_iter.next() { - *elem.borrow() - } else { - // length-2 iterator - let final_byte_len = Self::elem_to_usize(next_elem); - let mut bytes_iter = bytes_iter.take(final_byte_len); - let ret = bytes_iter.next(); - self.state = Final { bytes_iter }; - return ret; - }; - - // length >2 iterator - let mut bytes_iter = bytes_iter.take(self.primefield_bytes_len); - let ret = bytes_iter.next(); - self.state = Typical { - bytes_iter, - next_elem, - next_next_elem, - }; - ret - }, - Typical { - bytes_iter, - next_elem, - next_next_elem, - } => { - let ret = bytes_iter.next(); - if ret.is_some() { - return ret; - } - - let bytes_iter = Self::elem_to_bytes_iter(*next_elem); - - if let Some(elem) = self.elems_iter.next() { - // advance to the next field element - let mut bytes_iter = bytes_iter.take(self.primefield_bytes_len); - let ret = bytes_iter.next(); - self.state = Typical { - bytes_iter, - next_elem: *next_next_elem, - next_next_elem: *elem.borrow(), - }; - return ret; - } - - // done - let final_byte_len = Self::elem_to_usize(*next_next_elem); - let mut bytes_iter = bytes_iter.take(final_byte_len); - let ret = bytes_iter.next(); - self.state = Final { bytes_iter }; - ret - }, - Final { bytes_iter } => bytes_iter.next(), + if let Some(byte) = self.bytes_iter.next() { + return Some(byte); + } + if let Some(elem) = self.elems_iter.next() { + self.bytes_iter = elem + .borrow() + .into_bigint() + .to_bytes_le() + .into_iter() + .take(self.primefield_bytes_len); + return self.bytes_iter.next(); } + None } } @@ -621,49 +523,33 @@ mod tests { } } - fn bytes_field_elems_iter() { - // copied from bytes_field_elems() + fn bytes_to_field_iter() { + let byte_lens = [0, 1, 2, 16, 31, 32, 33, 48, 65, 100, 200, 5000]; - let lengths = [0, 1, 2, 16, 31, 32, 33, 48, 65, 100, 200, 5000]; - let trailing_zeros_lengths = [0, 1, 2, 5, 50]; - - let max_len = *lengths.iter().max().unwrap(); - let max_trailing_zeros_len = *trailing_zeros_lengths.iter().max().unwrap(); - let mut bytes = Vec::with_capacity(max_len + max_trailing_zeros_len); - let mut elems: Vec = Vec::with_capacity(max_len); + let max_len = *byte_lens.iter().max().unwrap(); + let mut bytes = Vec::with_capacity(max_len); + // TODO pre-allocate space for elems, owned, borrowed let mut rng = test_rng(); - for len in lengths { - for trailing_zeros_len in trailing_zeros_lengths { - // fill bytes with random bytes and trailing zeros - bytes.resize(len + trailing_zeros_len, 0); - rng.fill_bytes(&mut bytes[..len]); - bytes[len..].fill(0); - - // debug - // println!("byte_len: {}, trailing_zeros: {}", len, trailing_zeros_len); - // println!("bytes: {:?}", bytes); - // let encoded: Vec = bytes_to_field(bytes.iter()).collect(); - // println!("encoded: {:?}", encoded); - // let result: Vec<_> = bytes_from_field(encoded).collect(); - // println!("result: {:?}", result); - - // round trip: bytes as Iterator, elems as Iterator - let result_clone: Vec<_> = - field_to_bytes(bytes_to_field::<_, F>(bytes.clone())).collect(); - assert_eq!(result_clone, bytes); - - // round trip: bytes as Iterator, elems as Iterator - let encoded: Vec<_> = bytes_to_field::<_, F>(bytes.iter()).collect(); - let result_borrow: Vec<_> = field_to_bytes::<_, F>(encoded.iter()).collect(); - assert_eq!(result_borrow, bytes); - } - - // test infallibility of bytes_from_field - // with random field elements - elems.resize(len, F::zero()); - elems.iter_mut().for_each(|e| *e = F::rand(&mut rng)); - let _: Vec = field_to_bytes::<_, F>(elems.iter()).collect(); + for len in byte_lens { + // fill bytes with random bytes and trailing zeros + bytes.resize(len, 0); + rng.fill_bytes(&mut bytes); + + // round trip, owned: + // bytes as Iterator, elems as Iterator + let owned: Vec<_> = field_to_bytes(bytes_to_field::<_, F>(bytes.clone())) + .take(bytes.len()) + .collect(); + assert_eq!(owned, bytes); + + // round trip, borrowed: + // bytes as Iterator, elems as Iterator + let elems: Vec<_> = bytes_to_field::<_, F>(bytes.iter()).collect(); + let borrowed: Vec<_> = field_to_bytes::<_, F>(elems.iter()) + .take(bytes.len()) + .collect(); + assert_eq!(borrowed, bytes); } // empty input -> empty output @@ -672,11 +558,10 @@ mod tests { let mut elems_iter = bytes_to_field::<_, F>(bytes.iter()); assert!(elems_iter.next().is_none()); - // smallest non-empty input -> 2-item output + // 1-item input -> 1-item output let bytes = [42u8; 1]; let mut elems_iter = bytes_to_field::<_, F>(bytes.iter()); assert_eq!(elems_iter.next().unwrap(), F::from(42u64)); - assert_eq!(elems_iter.next().unwrap(), F::from(1u64)); assert!(elems_iter.next().is_none()); } @@ -692,8 +577,8 @@ mod tests { #[test] fn test_bytes_field_elems_iter() { - bytes_field_elems_iter::(); - bytes_field_elems_iter::(); - bytes_field_elems_iter::(); + bytes_to_field_iter::(); + bytes_to_field_iter::(); + bytes_to_field_iter::(); } } From 8f0d7f5efe3e62458ec8e34aeca303552b22524c Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 23 Oct 2023 13:43:23 -0400 Subject: [PATCH 02/54] rename vid/mod.rs -> vid.rs --- primitives/src/{vid/mod.rs => vid.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename primitives/src/{vid/mod.rs => vid.rs} (100%) diff --git a/primitives/src/vid/mod.rs b/primitives/src/vid.rs similarity index 100% rename from primitives/src/vid/mod.rs rename to primitives/src/vid.rs From f25b9e177c09336a055480b8c6137e7af09a9f8b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 24 Oct 2023 14:55:36 -0400 Subject: [PATCH 03/54] new assoc type VidScheme::Payload instead of Vec --- primitives/benches/advz.rs | 6 +- primitives/src/vid.rs | 23 +-- primitives/src/vid/advz.rs | 215 +++++++++++++---------------- primitives/src/vid/advz/payload.rs | 70 ++++++++++ primitives/tests/advz.rs | 7 +- primitives/tests/vid/mod.rs | 2 + 6 files changed, 192 insertions(+), 131 deletions(-) create mode 100644 primitives/src/vid/advz/payload.rs diff --git a/primitives/benches/advz.rs b/primitives/benches/advz.rs index 7017e7a17..d5e0dc4b6 100644 --- a/primitives/benches/advz.rs +++ b/primitives/benches/advz.rs @@ -23,7 +23,10 @@ mod feature_gated { use digest::{crypto_common::generic_array::ArrayLength, Digest, DynDigest, OutputSizeUser}; use jf_primitives::{ pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}, - vid::{advz::Advz, VidScheme}, + vid::{ + advz::{payload::Payload, Advz}, + VidScheme, + }, }; use sha2::Sha256; @@ -63,6 +66,7 @@ mod feature_gated { // random payload data let mut payload_bytes = vec![0u8; len]; rng.fill_bytes(&mut payload_bytes); + let payload_bytes = Payload::from_vec(payload_bytes); let benchmark_group_name = |op_name| format!("advz_{}_{}_{}KB", pairing_name, op_name, len / KB); diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index 00d1a7a40..1d20899e3 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -7,7 +7,7 @@ //! Trait and implementation for a Verifiable Information Retrieval (VID). /// See section 1.3--1.4 for intro to VID semantics. use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; -use ark_std::{borrow::Borrow, error::Error, fmt::Debug, hash::Hash, string::String, vec::Vec}; +use ark_std::{error::Error, fmt::Debug, hash::Hash, string::String, vec::Vec}; use displaydoc::Display; use serde::{Deserialize, Serialize}; @@ -44,6 +44,9 @@ pub type VidResult = Result; /// VID: Verifiable Information Dispersal pub trait VidScheme { + /// Payload + type Payload: Debug + PartialEq; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + /// Payload commitment. type Commit: Clone + Debug + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 @@ -54,16 +57,10 @@ pub trait VidScheme { type Common: CanonicalSerialize + CanonicalDeserialize + Clone + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 /// Compute a payload commitment - fn commit_only(&self, payload: I) -> VidResult - where - I: IntoIterator, - I::Item: Borrow; + fn commit_only(&self, payload: &Self::Payload) -> VidResult; /// Compute shares to send to the storage nodes - fn disperse(&self, payload: I) -> VidResult> - where - I: IntoIterator, - I::Item: Borrow; + fn disperse(&self, payload: &Self::Payload) -> VidResult>; /// Verify a share. Used by both storage node and retrieval client. /// Why is return type a nested `Result`? See @@ -76,7 +73,13 @@ pub trait VidScheme { /// Recover payload from shares. /// Do not verify shares or check recovered payload against anything. - fn recover_payload(&self, shares: &[Self::Share], common: &Self::Common) -> VidResult>; + /// + /// TODO: return type [`Payload`]? + fn recover_payload( + &self, + shares: &[Self::Share], + common: &Self::Common, + ) -> VidResult; } /// Convenience struct to aggregate disperse data. diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index acc14b855..fe9b9bdfc 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -8,6 +8,7 @@ //! //! `advz` named for the authors Alhaddad-Duan-Varia-Zhang. +use self::payload::Payload; use super::{vid, VidDisperse, VidError, VidResult, VidScheme}; use crate::{ merkle_tree::{hasher::HasherMerkleTree, MerkleCommitment, MerkleTreeScheme}, @@ -41,6 +42,8 @@ use itertools::Itertools; use jf_utils::{bytes_to_field, canonical, field_to_bytes}; use serde::{Deserialize, Serialize}; +pub mod payload; + /// The [ADVZ VID scheme](https://eprint.iacr.org/2021/1500), a concrete impl for [`VidScheme`]. /// /// - `H` is any [`Digest`]-compatible hash function @@ -168,7 +171,7 @@ where #[serde(with = "canonical")] poly_commits: Vec, all_evals_digest: V::NodeValue, - elems_len: usize, + bytes_len: usize, } // We take great pains to maintain abstraction by relying only on traits and not @@ -194,12 +197,9 @@ where type Commit = Output; type Share = Share; type Common = Common; + type Payload = Payload; - fn commit_only(&self, payload: I) -> VidResult - where - I: IntoIterator, - I::Item: Borrow, - { + fn commit_only(&self, payload: &Self::Payload) -> VidResult { let mut hasher = H::new(); let elems_iter = bytes_to_field::<_, P::Evaluation>(payload); for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { @@ -218,105 +218,12 @@ where Ok(hasher.finalize()) } - fn disperse(&self, payload: I) -> VidResult> - where - I: IntoIterator, - I::Item: Borrow, - { - self.disperse_from_elems(bytes_to_field::<_, P::Evaluation>(payload)) - } + fn disperse(&self, payload: &Self::Payload) -> VidResult> { + let bytes_len = payload.as_slice().len(); - fn verify_share( - &self, - share: &Self::Share, - common: &Self::Common, - ) -> VidResult> { - // check arguments - if share.evals.len() != common.poly_commits.len() { - return Err(VidError::Argument(format!( - "(share eval, common poly commit) lengths differ ({},{})", - share.evals.len(), - common.poly_commits.len() - ))); - } - if share.index >= self.num_storage_nodes { - return Ok(Err(())); // not an arg error - } - - // verify eval proof - if V::verify( - common.all_evals_digest, - &V::Index::from(share.index as u64), - &share.evals_proof, - ) - .map_err(vid)? - .is_err() - { - return Ok(Err(())); - } - - let pseudorandom_scalar = Self::pseudorandom_scalar(common)?; - - // Compute aggregate polynomial [commitment|evaluation] - // as a pseudorandom linear combo of [commitments|evaluations] - // via evaluation of the polynomial whose coefficients are - // [commitments|evaluations] and whose input point is the pseudorandom - // scalar. - let aggregate_poly_commit = P::Commitment::from( - polynomial_eval( - common - .poly_commits - .iter() - .map(|x| CurveMultiplier(x.as_ref())), - pseudorandom_scalar, - ) - .into(), - ); - let aggregate_eval = - polynomial_eval(share.evals.iter().map(FieldMultiplier), pseudorandom_scalar); - - // verify aggregate proof - Ok(P::verify( - &self.vk, - &aggregate_poly_commit, - &self.multi_open_domain.element(share.index), - &aggregate_eval, - &share.aggregate_proof, - ) - .map_err(vid)? - .then_some(()) - .ok_or(())) - } - - fn recover_payload(&self, shares: &[Self::Share], common: &Self::Common) -> VidResult> { - // TODO can we avoid collect() here? - Ok(field_to_bytes(self.recover_elems(shares, common)?).collect()) - } -} - -impl GenericAdvz -where - P: UnivariatePCS::Evaluation>, - P::Evaluation: PrimeField, - P::Polynomial: DenseUVPolynomial, - P::Commitment: From + AsRef, - T: AffineRepr, - H: Digest + DynDigest + Default + Clone + Write, - V: MerkleTreeScheme>, - V::MembershipProof: Sync + Debug, /* TODO https://github.com/EspressoSystems/jellyfish/issues/253 */ - V::Index: From, -{ - /// Same as [`VidScheme::disperse`] except `payload` iterates over - /// field elements. - pub fn disperse_from_elems(&self, payload: I) -> VidResult> - where - I: IntoIterator, - I::Item: Borrow, - { // partition payload into polynomial coefficients // and count `elems_len` for later - let elems_iter = payload.into_iter().map(|elem| *elem.borrow()); - let mut elems_len = 0; + let elems_iter = bytes_to_field::<_, P::Evaluation>(payload).map(|elem| *elem.borrow()); let mut polys = Vec::new(); for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { // TODO TEMPORARY: use FFT to encode polynomials in eval form @@ -334,7 +241,6 @@ where assert_eq!(coeffs.len(), pre_fft_len); } - elems_len += pre_fft_len; polys.push(DenseUVPolynomial::from_coefficients_vec(coeffs)); } @@ -388,7 +294,7 @@ where .collect::>() .map_err(vid)?, all_evals_digest: all_evals_commit.commitment().digest(), - elems_len, + bytes_len, }; let commit = { @@ -444,13 +350,73 @@ where }) } - /// Same as [`VidScheme::recover_payload`] except returns a [`Vec`] of field - /// elements. - pub fn recover_elems( + fn verify_share( &self, - shares: &[::Share], - common: &::Common, - ) -> VidResult> { + share: &Self::Share, + common: &Self::Common, + ) -> VidResult> { + // check arguments + if share.evals.len() != common.poly_commits.len() { + return Err(VidError::Argument(format!( + "(share eval, common poly commit) lengths differ ({},{})", + share.evals.len(), + common.poly_commits.len() + ))); + } + if share.index >= self.num_storage_nodes { + return Ok(Err(())); // not an arg error + } + + // verify eval proof + if V::verify( + common.all_evals_digest, + &V::Index::from(share.index as u64), + &share.evals_proof, + ) + .map_err(vid)? + .is_err() + { + return Ok(Err(())); + } + + let pseudorandom_scalar = Self::pseudorandom_scalar(common)?; + + // Compute aggregate polynomial [commitment|evaluation] + // as a pseudorandom linear combo of [commitments|evaluations] + // via evaluation of the polynomial whose coefficients are + // [commitments|evaluations] and whose input point is the pseudorandom + // scalar. + let aggregate_poly_commit = P::Commitment::from( + polynomial_eval( + common + .poly_commits + .iter() + .map(|x| CurveMultiplier(x.as_ref())), + pseudorandom_scalar, + ) + .into(), + ); + let aggregate_eval = + polynomial_eval(share.evals.iter().map(FieldMultiplier), pseudorandom_scalar); + + // verify aggregate proof + Ok(P::verify( + &self.vk, + &aggregate_poly_commit, + &self.multi_open_domain.element(share.index), + &aggregate_eval, + &share.aggregate_proof, + ) + .map_err(vid)? + .then_some(()) + .ok_or(())) + } + + fn recover_payload( + &self, + shares: &[Self::Share], + common: &Self::Common, + ) -> VidResult { if shares.len() < self.payload_chunk_size { return Err(VidError::Argument(format!( "not enough shares {}, expected at least {}", @@ -497,10 +463,25 @@ where result.append(&mut coeffs); } assert_eq!(result.len(), result_len); - result.truncate(common.elems_len); - Ok(result) + + let mut result_bytes: Vec<_> = field_to_bytes(result).collect(); + result_bytes.truncate(common.bytes_len); + Ok(Payload::from_vec(result_bytes)) } +} +impl GenericAdvz +where + P: UnivariatePCS::Evaluation>, + P::Evaluation: PrimeField, + P::Polynomial: DenseUVPolynomial, + P::Commitment: From + AsRef, + T: AffineRepr, + H: Digest + DynDigest + Default + Clone + Write, + V: MerkleTreeScheme>, + V::MembershipProof: Sync + Debug, /* TODO https://github.com/EspressoSystems/jellyfish/issues/253 */ + V::Index: From, +{ fn pseudorandom_scalar(common: &::Common) -> VidResult { let mut hasher = H::new(); for poly_commit in common.poly_commits.iter() { @@ -831,7 +812,7 @@ mod tests { let payload_elems_len = num_polys * payload_chunk_size; let payload_bytes_len = payload_elems_len * modulus_byte_len::(); let mut rng = jf_utils::test_rng(); - let payload_bytes = init_random_bytes(payload_bytes_len, &mut rng); + let payload_bytes = init_random_payload(payload_bytes_len, &mut rng); let srs = init_srs(payload_elems_len, &mut rng); let advz = Advz::::new(payload_chunk_size, num_storage_nodes, srs).unwrap(); @@ -881,12 +862,12 @@ mod tests { /// Returns the following tuple: /// 1. An initialized [`Advz`] instance. /// 2. A `Vec` filled with random bytes. - fn avdz_init() -> (Advz, Vec) { + fn avdz_init() -> (Advz, Payload) { let (payload_chunk_size, num_storage_nodes) = (4, 6); let mut rng = jf_utils::test_rng(); let srs = init_srs(payload_chunk_size, &mut rng); let advz = Advz::new(payload_chunk_size, num_storage_nodes, srs).unwrap(); - let bytes_random = init_random_bytes(4000, &mut rng); + let bytes_random = init_random_payload(4000, &mut rng); (advz, bytes_random) } @@ -895,13 +876,13 @@ mod tests { assert!(matches!(res, Err(Argument(_))), "{}", msg); } - fn init_random_bytes(len: usize, rng: &mut R) -> Vec + fn init_random_payload(len: usize, rng: &mut R) -> Payload where R: RngCore + CryptoRng, { let mut bytes_random = vec![0u8; len]; rng.fill_bytes(&mut bytes_random); - bytes_random + Payload::from_vec(bytes_random) } fn init_srs(num_coeffs: usize, rng: &mut R) -> UnivariateUniversalParams diff --git a/primitives/src/vid/advz/payload.rs b/primitives/src/vid/advz/payload.rs new file mode 100644 index 000000000..457d63a28 --- /dev/null +++ b/primitives/src/vid/advz/payload.rs @@ -0,0 +1,70 @@ +//! paylod module doc + +// use ark_ff::PrimeField; +use ark_std::{ + // borrow::Borrow, + slice::Iter, + vec::{IntoIter, Vec}, +}; + +/// payload +#[derive(Debug, PartialEq)] +pub struct Payload { + // TODO store as Vec instead? + payload: Vec, +} + +impl Payload { + /// from_iter + // pub fn from_iter(payload: I) -> Self + // where + // I: IntoIterator, + // I::Item: Borrow, + // { + // Self { + // payload: payload.into_iter().map(|b| *b.borrow()).collect(), + // } + // } + + /// from_vec + pub fn from_vec(payload: Vec) -> Self { + Self { payload } + } + + /// as_slice + pub fn as_slice(&self) -> &[u8] { + &self.payload + } + + /// iter + pub fn iter(&self) -> Iter<'_, u8> { + self.payload.iter() + } + + // fn field_iter(&self) -> impl Iterator + // where + // F: PrimeField, + // { + // todo!() + // } +} + +// delegation boilerplate +impl<'a> IntoIterator for &'a Payload { + type Item = &'a u8; + type IntoIter = Iter<'a, u8>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +// delegation boilerplate +impl IntoIterator for Payload { + type Item = u8; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.payload.into_iter() + } +} diff --git a/primitives/tests/advz.rs b/primitives/tests/advz.rs index 88db3a096..eff1f731c 100644 --- a/primitives/tests/advz.rs +++ b/primitives/tests/advz.rs @@ -3,7 +3,7 @@ use ark_bls12_381::Bls12_381; use ark_ff::{Field, PrimeField}; use jf_primitives::{ pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}, - vid::advz::Advz, + vid::advz::{payload::Payload, Advz}, }; use sha2::Sha256; @@ -13,7 +13,7 @@ mod vid; fn round_trip() { // play with these items let vid_sizes = [(2, 3), (8, 11)]; - let byte_lens = [0, 1, 2, 16, 32, 47, 48, 49, 64, 100, 400]; + let payload_byte_lens = [0, 1, 2, 16, 32, 47, 48, 49, 64, 100, 400]; // more items as a function of the above let supported_degree = vid_sizes.iter().max_by_key(|v| v.0).unwrap().0 - 1; @@ -34,8 +34,9 @@ fn round_trip() { |payload_chunk_size, num_storage_nodes| { Advz::::new(payload_chunk_size, num_storage_nodes, &srs).unwrap() }, + |bytes| Payload::from_vec(bytes), &vid_sizes, - &byte_lens, + &payload_byte_lens, &mut rng, ); } diff --git a/primitives/tests/vid/mod.rs b/primitives/tests/vid/mod.rs index 2152a3d84..0e0096284 100644 --- a/primitives/tests/vid/mod.rs +++ b/primitives/tests/vid/mod.rs @@ -13,6 +13,7 @@ use ark_std::{ /// pub fn round_trip( vid_factory: impl Fn(usize, usize) -> V, + payload_factory: impl Fn(Vec) -> V::Payload, vid_sizes: &[(usize, usize)], payload_byte_lens: &[usize], rng: &mut R, @@ -31,6 +32,7 @@ pub fn round_trip( let mut bytes_random = vec![0u8; len]; rng.fill_bytes(&mut bytes_random); + let bytes_random = payload_factory(bytes_random); let disperse = vid.disperse(&bytes_random).unwrap(); let (mut shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); From 20ac68f7bdae5edec5343086a31f685c13fc79f0 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 24 Oct 2023 17:03:25 -0400 Subject: [PATCH 04/54] tidy: move boilerplate to the bottom --- primitives/src/vid.rs | 64 ++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index 1d20899e3..b6c115ec2 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -11,37 +11,6 @@ use ark_std::{error::Error, fmt::Debug, hash::Hash, string::String, vec::Vec}; use displaydoc::Display; use serde::{Deserialize, Serialize}; -pub mod advz; - -/// The error type for `VidScheme` methods. -#[derive(Display, Debug)] -pub enum VidError { - /// invalid args: {0} - Argument(String), - /// internal error: {0} - Internal(anyhow::Error), -} - -impl Error for VidError {} - -/// Convenience wrapper to convert any error into a [`VidError`]. -/// -/// Private fn so as not to expose error conversion API outside this crate -/// as per [stackoverflow](https://stackoverflow.com/a/70057677). -/// -/// # No-std support -/// `no_std` mode requires `.map_err(vid)` to convert from a non-`anyhow` error -/// as per [`anyhow` docs](https://docs.rs/anyhow/latest/anyhow/index.html#no-std-support), -fn vid(e: E) -> VidError -where - E: ark_std::fmt::Display + Debug + Send + Sync + 'static, -{ - VidError::Internal(anyhow::anyhow!(e)) -} - -/// Convenience [`Result`] wrapper for [`VidError`]. -pub type VidResult = Result; - /// VID: Verifiable Information Dispersal pub trait VidScheme { /// Payload @@ -108,3 +77,36 @@ pub struct VidDisperse { /// VID payload commitment. pub commit: V::Commit, } + +pub mod advz; // instantiation of `VidScheme` + +// BOILERPLATE: error handling + +/// The error type for `VidScheme` methods. +#[derive(Display, Debug)] +pub enum VidError { + /// invalid args: {0} + Argument(String), + /// internal error: {0} + Internal(anyhow::Error), +} + +impl Error for VidError {} + +/// Convenience wrapper to convert any error into a [`VidError`]. +/// +/// Private fn so as not to expose error conversion API outside this crate +/// as per [stackoverflow](https://stackoverflow.com/a/70057677). +/// +/// # No-std support +/// `no_std` mode requires `.map_err(vid)` to convert from a non-`anyhow` error +/// as per [`anyhow` docs](https://docs.rs/anyhow/latest/anyhow/index.html#no-std-support), +fn vid(e: E) -> VidError +where + E: ark_std::fmt::Display + Debug + Send + Sync + 'static, +{ + VidError::Internal(anyhow::anyhow!(e)) +} + +/// Convenience [`Result`] wrapper for [`VidError`]. +pub type VidResult = Result; From a1070c966907ab346927023751022543a4771574 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 25 Oct 2023 14:26:25 -0400 Subject: [PATCH 05/54] new Namespacer trait skeleton --- primitives/src/vid.rs | 2 + primitives/src/vid/advz.rs | 1 + primitives/src/vid/advz/namespace.rs | 58 ++++++++++++++++++++++++++++ primitives/src/vid/namespace.rs | 47 ++++++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 primitives/src/vid/advz/namespace.rs create mode 100644 primitives/src/vid/namespace.rs diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index b6c115ec2..efaa8bef7 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -78,6 +78,8 @@ pub struct VidDisperse { pub commit: V::Commit, } +pub mod namespace; + pub mod advz; // instantiation of `VidScheme` // BOILERPLATE: error handling diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index fe9b9bdfc..d93a25d56 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -42,6 +42,7 @@ use itertools::Itertools; use jf_utils::{bytes_to_field, canonical, field_to_bytes}; use serde::{Deserialize, Serialize}; +pub mod namespace; pub mod payload; /// The [ADVZ VID scheme](https://eprint.iacr.org/2021/1500), a concrete impl for [`VidScheme`]. diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs new file mode 100644 index 000000000..f39cd763d --- /dev/null +++ b/primitives/src/vid/advz/namespace.rs @@ -0,0 +1,58 @@ +// Copyright (c) 2023 Espresso Systems (espressosys.com) +// This file is part of the Jellyfish library. + +// You should have received a copy of the MIT License +// along with the Jellyfish library. If not, see . + +//! Implementation of [`Namespacer`] for [`Advz`]. + +use super::{ + AffineRepr, Debug, DenseUVPolynomial, Digest, DynDigest, GenericAdvz, MerkleTreeScheme, + PolynomialCommitmentScheme, PrimeField, UnivariatePCS, Vec, Write, +}; +use crate::vid::namespace::Namespacer; + +impl Namespacer for GenericAdvz +where + // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 + P: UnivariatePCS::Evaluation>, + P::Evaluation: PrimeField, + P::Polynomial: DenseUVPolynomial, + P::Commitment: From + AsRef, + T: AffineRepr, + H: Digest + DynDigest + Default + Clone + Write, + V: MerkleTreeScheme>, + V::MembershipProof: Sync + Debug, + V::Index: From, +{ + type DataProof = (); + + fn data_proof( + &self, + _payload: &Self::Payload, + _start: usize, + _len: usize, + ) -> crate::vid::VidResult { + todo!() + } + + fn data_verify( + &self, + _payload: &Self::Payload, + _start: usize, + _len: usize, + _proof: Self::DataProof, + ) -> crate::vid::VidResult> { + todo!() + } + + fn namespace_verify( + &self, + _payload: &Self::Payload, + _namespace_index: usize, + _commit: &Self::Commit, + _common: &Self::Common, + ) -> crate::vid::VidResult> { + todo!() + } +} diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs new file mode 100644 index 000000000..86e44460d --- /dev/null +++ b/primitives/src/vid/namespace.rs @@ -0,0 +1,47 @@ +// Copyright (c) 2023 Espresso Systems (espressosys.com) +// This file is part of the Jellyfish library. + +// You should have received a copy of the MIT License +// along with the Jellyfish library. If not, see . + +//! Trait for namespace functionality in Verifiable Information Retrieval (VID). + +use super::{VidResult, VidScheme}; + +/// Namespace functionality for [`VidScheme`]. +pub trait Namespacer: VidScheme { + /// data proof + type DataProof; + + /// Compute a proof for `payload` for data index range `start` to `start + + /// len`. + fn data_proof( + &self, + payload: &Self::Payload, + start: usize, + len: usize, + ) -> VidResult; + + /// Verify a proof for `payload` for data index range `start` to `start + + /// len`. + fn data_verify( + &self, + payload: &Self::Payload, + start: usize, + len: usize, + proof: Self::DataProof, + ) -> VidResult>; + + /// Verify the `payload` namespace indexed by `namespace_index` against + /// `commit`, `common`. + /// + /// TODO: Seems ugly to include `common` in this API but [`advz`] impl needs + /// it. + fn namespace_verify( + &self, + payload: &Self::Payload, + namespace_index: usize, + commit: &Self::Commit, + common: &Self::Common, + ) -> VidResult>; +} From ddccddb07f4aea61eb78158f4415da25b2dc71d7 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 25 Oct 2023 17:02:15 -0400 Subject: [PATCH 06/54] wip: untested impl for namespace_verify --- primitives/src/vid/advz/namespace.rs | 80 +++++++++++++++++++++++++--- utilities/src/conversion.rs | 4 +- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index f39cd763d..59b61075e 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -6,11 +6,19 @@ //! Implementation of [`Namespacer`] for [`Advz`]. +use ark_poly::EvaluationDomain; +use ark_serialize::CanonicalSerialize; +use jf_utils::{bytes_to_field, compile_time_checks}; + use super::{ AffineRepr, Debug, DenseUVPolynomial, Digest, DynDigest, GenericAdvz, MerkleTreeScheme, PolynomialCommitmentScheme, PrimeField, UnivariatePCS, Vec, Write, }; -use crate::vid::namespace::Namespacer; +use crate::{ + alloc::string::ToString, + vid::{namespace::Namespacer, vid, VidError}, +}; +use ark_std::{borrow::Borrow, format}; impl Namespacer for GenericAdvz where @@ -48,11 +56,71 @@ where fn namespace_verify( &self, - _payload: &Self::Payload, - _namespace_index: usize, - _commit: &Self::Commit, - _common: &Self::Common, + payload: &Self::Payload, + namespace_index: usize, + commit: &Self::Commit, + common: &Self::Common, ) -> crate::vid::VidResult> { - todo!() + // check args: `namespace_index`` in bounds for `common`. + if namespace_index >= common.poly_commits.len() { + return Err(VidError::Argument(format!( + "namespace_index {} out of bounds for common.poly_commits {}", + namespace_index, + common.poly_commits.len() + ))); + } + + let (primefield_bytes_len, ..) = compile_time_checks::(); + let start = namespace_index * self.payload_chunk_size * primefield_bytes_len; + + // check args: `namespace_index` in bounds for `payload`. + if start >= payload.as_slice().len() { + return Err(VidError::Argument(format!( + "namespace_index {} out of bounds for payload {}", + namespace_index, + payload.as_slice().len() + ))); + } + + // check args: `common` consistent with `commit` + let rebuilt_commit = { + let mut hasher = H::new(); + for poly_commit in common.poly_commits.iter() { + // TODO compiler bug? `as` should not be needed here! + (poly_commit as &P::Commitment) + .serialize_uncompressed(&mut hasher) + .map_err(vid)?; + } + hasher.finalize() + }; + if rebuilt_commit != *commit { + return Err(VidError::Argument( + "common inconsistent with commit".to_string(), + )); + } + + // rebuild the `namespace_index`th poly commit, check against `common` + let poly_commit = { + let elems_iter = bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start..].iter()) + .map(|elem| *elem.borrow()) + .take(self.payload_chunk_size); + + // TODO TEMPORARY: use FFT to encode polynomials in eval form + // Remove these FFTs after we get KZG in eval form + // https://github.com/EspressoSystems/jellyfish/issues/339 + let mut coeffs: Vec<_> = elems_iter.collect(); + self.eval_domain.fft_in_place(&mut coeffs); + + let poly = P::Polynomial::from_coefficients_vec(coeffs); + P::commit(&self.ck, &poly).map_err(vid)? + }; + if poly_commit != common.poly_commits[namespace_index] { + return Ok(Err(())); + } + + Ok(Ok(())) } } + +#[cfg(test)] +mod tests {} diff --git a/utilities/src/conversion.rs b/utilities/src/conversion.rs index 079e6b3fb..0dab92f6a 100644 --- a/utilities/src/conversion.rs +++ b/utilities/src/conversion.rs @@ -273,7 +273,9 @@ where /// # Panics /// /// Panics under the conditions listed at [`bytes_to_field_elements`]. -fn compile_time_checks() -> (usize, usize, usize) { +/// +/// TODO remove `pub`, move this somewhere else. +pub fn compile_time_checks() -> (usize, usize, usize) { assert!( F::BasePrimeField::MODULUS_BIT_SIZE > 64, "base prime field modulus bit len {} too small to hold a u64", From f146fd2ad02f3f81edd6cfeabba521e9f68bb6e5 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 25 Oct 2023 18:26:42 -0400 Subject: [PATCH 07/54] port prove_namespace test to advz/namespaces.rs --- primitives/src/vid/advz.rs | 70 ++-------------------------- primitives/src/vid/advz/namespace.rs | 43 ++++++++++++++++- 2 files changed, 47 insertions(+), 66 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index d93a25d56..0023fb882 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -615,7 +615,6 @@ mod tests { rand::{CryptoRng, RngCore}, vec, }; - use digest::{generic_array::ArrayLength, OutputSizeUser}; use sha2::Sha256; #[test] @@ -799,71 +798,12 @@ mod tests { } } - fn prove_namespace_generic() - where - E: Pairing, - H: Digest + DynDigest + Default + Clone + Write, - <::OutputSize as ArrayLength>::ArrayType: Copy, - { - // play with these items - let (payload_chunk_size, num_storage_nodes) = (4, 6); - let num_polys = 4; - - // more items as a function of the above - let payload_elems_len = num_polys * payload_chunk_size; - let payload_bytes_len = payload_elems_len * modulus_byte_len::(); - let mut rng = jf_utils::test_rng(); - let payload_bytes = init_random_payload(payload_bytes_len, &mut rng); - let srs = init_srs(payload_elems_len, &mut rng); - - let advz = Advz::::new(payload_chunk_size, num_storage_nodes, srs).unwrap(); - let d = advz.disperse(&payload_bytes).unwrap(); - - // TEST: verify "namespaces" (each namespace is a polynomial) - // This test is currently trivial: we simply repeat the commit computation. - // In the future there will be a proper API that can be tested meaningfully. - - // encode payload as field elements, partition into polynomials, compute - // commitments, compare against VID common data - let elems_iter = bytes_to_field::<_, E::ScalarField>(payload_bytes); - for (coeffs_iter, poly_commit) in elems_iter - .chunks(payload_chunk_size) - .into_iter() - .zip(d.common.poly_commits.iter()) - { - let mut coeffs: Vec<_> = coeffs_iter.collect(); - advz.eval_domain.fft_in_place(&mut coeffs); - - let poly = as PolynomialCommitmentScheme>::Polynomial::from_coefficients_vec(coeffs); - let my_poly_commit = UnivariateKzgPCS::::commit(&advz.ck, &poly).unwrap(); - assert_eq!(my_poly_commit, *poly_commit); - } - - // compute payload commitment and verify - let commit = { - let mut hasher = H::new(); - for poly_commit in d.common.poly_commits.iter() { - // TODO compiler bug? `as` should not be needed here! - (poly_commit as & as PolynomialCommitmentScheme>::Commitment) - .serialize_uncompressed(&mut hasher) - .unwrap(); - } - hasher.finalize() - }; - assert_eq!(commit, d.commit); - } - - #[test] - fn prove_namespace() { - prove_namespace_generic::(); - } - /// Routine initialization tasks. /// /// Returns the following tuple: /// 1. An initialized [`Advz`] instance. /// 2. A `Vec` filled with random bytes. - fn avdz_init() -> (Advz, Payload) { + pub(super) fn avdz_init() -> (Advz, Payload) { let (payload_chunk_size, num_storage_nodes) = (4, 6); let mut rng = jf_utils::test_rng(); let srs = init_srs(payload_chunk_size, &mut rng); @@ -873,11 +813,11 @@ mod tests { } /// Convenience wrapper to assert [`VidError::Argument`] return value. - fn assert_arg_err(res: VidResult, msg: &str) { + pub(super) fn assert_arg_err(res: VidResult, msg: &str) { assert!(matches!(res, Err(Argument(_))), "{}", msg); } - fn init_random_payload(len: usize, rng: &mut R) -> Payload + pub(super) fn init_random_payload(len: usize, rng: &mut R) -> Payload where R: RngCore + CryptoRng, { @@ -886,7 +826,7 @@ mod tests { Payload::from_vec(bytes_random) } - fn init_srs(num_coeffs: usize, rng: &mut R) -> UnivariateUniversalParams + pub(super) fn init_srs(num_coeffs: usize, rng: &mut R) -> UnivariateUniversalParams where E: Pairing, R: RngCore + CryptoRng, @@ -895,7 +835,7 @@ mod tests { .unwrap() } - fn modulus_byte_len() -> usize + pub(super) fn modulus_byte_len() -> usize where E: Pairing, { diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 59b61075e..4232028bf 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -123,4 +123,45 @@ where } #[cfg(test)] -mod tests {} +mod tests { + use crate::vid::{ + advz::{tests::*, *}, + namespace::Namespacer, + }; + use ark_bls12_381::Bls12_381; + use digest::{generic_array::ArrayLength, OutputSizeUser}; + use sha2::Sha256; + + fn prove_namespace_generic() + where + E: Pairing, + H: Digest + DynDigest + Default + Clone + Write, + <::OutputSize as ArrayLength>::ArrayType: Copy, + { + // play with these items + let (payload_chunk_size, num_storage_nodes) = (4, 6); + let num_polys = 4; + + // more items as a function of the above + let payload_elems_len = num_polys * payload_chunk_size; + let payload_bytes_len = payload_elems_len * modulus_byte_len::(); + let mut rng = jf_utils::test_rng(); + let payload = init_random_payload(payload_bytes_len, &mut rng); + let srs = init_srs(payload_elems_len, &mut rng); + + let advz = Advz::::new(payload_chunk_size, num_storage_nodes, srs).unwrap(); + let d = advz.disperse(&payload).unwrap(); + + // TEST: verify "namespaces" (each namespace is a polynomial) + for namespace_index in 0..d.common.poly_commits.len() { + advz.namespace_verify(&payload, namespace_index, &d.commit, &d.common) + .unwrap() + .unwrap(); + } + } + + #[test] + fn prove_namespace() { + prove_namespace_generic::(); + } +} From af98cb603cb9550a7c2ecc4390a97bfd9afe527a Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Thu, 26 Oct 2023 17:22:08 -0400 Subject: [PATCH 08/54] wip: untested implementation of data_proof --- primitives/src/vid/advz/namespace.rs | 135 +++++++++++++++++++++++++-- primitives/src/vid/advz/payload.rs | 2 +- primitives/src/vid/namespace.rs | 8 +- 3 files changed, 131 insertions(+), 14 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 4232028bf..82ce58dde 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -33,15 +33,89 @@ where V::MembershipProof: Sync + Debug, V::Index: From, { - type DataProof = (); + // TODO should be P::Proof, not Vec + // https://github.com/EspressoSystems/jellyfish/issues/387 + type DataProof = Vec; fn data_proof( &self, - _payload: &Self::Payload, - _start: usize, - _len: usize, + payload: &Self::Payload, + start: usize, + len: usize, ) -> crate::vid::VidResult { - todo!() + // check args: `len` must be positive + // if len == 0 { + // return Err(VidError::Argument( + // "request for zero-length data proof".to_string(), + // )); + // } + + // check args: `start`, `len` in bounds for `payload` + if start + len >= payload.as_slice().len() { + return Err(VidError::Argument(format!( + "start {} + len {} out of bounds for payload {}", + start, + len, + payload.as_slice().len() + ))); + } + + let (start_elem, len_elem) = self.range_byte_to_elem(start, len); + let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); + let start_namespace_byte = self.index_poly_to_byte(start_namespace); + + // check args: + // TODO TEMPORARY: forbid requests that span multiple polynomials + if len_namespace != 1 { + return Err(VidError::Argument(format!( + "request spans {} polynomials, expect 1", + len_namespace + ))); + } + + // grab the `start_namespace`th polynomial + // TODO refactor copied code + let polynomial = { + let elems_iter = bytes_to_field::<_, P::Evaluation>( + payload.as_slice()[start_namespace_byte..].iter(), + ) + .map(|elem| *elem.borrow()) + .take(self.payload_chunk_size); + + // TODO TEMPORARY: use FFT to encode polynomials in eval form + // Remove these FFTs after we get KZG in eval form + // https://github.com/EspressoSystems/jellyfish/issues/339 + let mut coeffs: Vec<_> = elems_iter.collect(); + self.eval_domain.fft_in_place(&mut coeffs); + + P::Polynomial::from_coefficients_vec(coeffs) + }; + + // prepare the list of input points + let points: Vec<_> = { + let offset = start_elem - self.index_byte_to_elem(start_namespace_byte); + self.eval_domain + .elements() + .skip(offset) + .take(len_elem) + .collect() + }; + + let (proofs, evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; + + // sanity check: evals == data + // TODO move this to a test? + { + let start_elem_byte = self.index_elem_to_byte(start_elem); + let data_elems: Vec<_> = + bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_elem_byte..].iter()) + .map(|elem| *elem.borrow()) + .take(len_elem) + .collect(); + assert_eq!(data_elems, evals); + } + + Ok(proofs) } fn data_verify( @@ -49,7 +123,7 @@ where _payload: &Self::Payload, _start: usize, _len: usize, - _proof: Self::DataProof, + _proof: &Self::DataProof, ) -> crate::vid::VidResult> { todo!() } @@ -70,8 +144,7 @@ where ))); } - let (primefield_bytes_len, ..) = compile_time_checks::(); - let start = namespace_index * self.payload_chunk_size * primefield_bytes_len; + let start = self.index_poly_to_byte(namespace_index); // check args: `namespace_index` in bounds for `payload`. if start >= payload.as_slice().len() { @@ -122,6 +195,52 @@ where } } +impl GenericAdvz +where + // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 + P: UnivariatePCS::Evaluation>, + P::Evaluation: PrimeField, + P::Polynomial: DenseUVPolynomial, + P::Commitment: From + AsRef, + T: AffineRepr, + H: Digest + DynDigest + Default + Clone + Write, + V: MerkleTreeScheme>, + V::MembershipProof: Sync + Debug, + V::Index: From, +{ + // lots of index manipulation. + // with infinite dev time we should implement type-safe indices to preclude + // index-misuse bugs. fn index_byte_to_poly(&self, index: usize) -> usize { + // self.index_poly_to_byte(self.index_elem_to_poly(self. + // index_byte_to_elem(index))) } + fn range_byte_to_elem(&self, start: usize, len: usize) -> (usize, usize) { + let (primefield_bytes_len, ..) = compile_time_checks::(); + let elem_start = start / primefield_bytes_len; + let elem_end = (start + len - 1) / primefield_bytes_len; + (elem_start, elem_end - elem_start + 1) + } + fn range_elem_to_poly(&self, start: usize, len: usize) -> (usize, usize) { + let poly_start = start / self.payload_chunk_size; + let poly_end = (start + len - 1) / self.payload_chunk_size; + (poly_start, poly_end - poly_start + 1) + } + fn index_byte_to_elem(&self, index: usize) -> usize { + let (primefield_bytes_len, ..) = compile_time_checks::(); + index / primefield_bytes_len // round down + } + fn index_elem_to_byte(&self, index: usize) -> usize { + let (primefield_bytes_len, ..) = compile_time_checks::(); + index * primefield_bytes_len + } + // fn index_elem_to_poly(&self, index: usize) -> usize { + // index / self.payload_chunk_size // round down + // } + fn index_poly_to_byte(&self, index: usize) -> usize { + let (primefield_bytes_len, ..) = compile_time_checks::(); + index * self.payload_chunk_size * primefield_bytes_len + } +} + #[cfg(test)] mod tests { use crate::vid::{ diff --git a/primitives/src/vid/advz/payload.rs b/primitives/src/vid/advz/payload.rs index 457d63a28..ebb3c0ec7 100644 --- a/primitives/src/vid/advz/payload.rs +++ b/primitives/src/vid/advz/payload.rs @@ -10,7 +10,7 @@ use ark_std::{ /// payload #[derive(Debug, PartialEq)] pub struct Payload { - // TODO store as Vec instead? + // TODO store as Vec instead? Or Vec? payload: Vec, } diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index 86e44460d..a4f2f33c6 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -13,8 +13,7 @@ pub trait Namespacer: VidScheme { /// data proof type DataProof; - /// Compute a proof for `payload` for data index range `start` to `start + - /// len`. + /// Compute a proof for `payload` for data index range `start..start+len`. fn data_proof( &self, payload: &Self::Payload, @@ -22,14 +21,13 @@ pub trait Namespacer: VidScheme { len: usize, ) -> VidResult; - /// Verify a proof for `payload` for data index range `start` to `start + - /// len`. + /// Verify a proof for `payload` for data index range `start..start+len`. fn data_verify( &self, payload: &Self::Payload, start: usize, len: usize, - proof: Self::DataProof, + proof: &Self::DataProof, ) -> VidResult>; /// Verify the `payload` namespace indexed by `namespace_index` against From 75011f60006e869f7fa03bca790f0b96f2118f19 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 27 Oct 2023 14:14:02 -0400 Subject: [PATCH 09/54] fix: use ifft instead of fft, debugging code for posterity --- primitives/src/vid/advz/namespace.rs | 80 ++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 9 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 82ce58dde..dee303a86 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -18,7 +18,7 @@ use crate::{ alloc::string::ToString, vid::{namespace::Namespacer, vid, VidError}, }; -use ark_std::{borrow::Borrow, format}; +use ark_std::{borrow::Borrow, format, println}; impl Namespacer for GenericAdvz where @@ -64,6 +64,11 @@ where let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); let start_namespace_byte = self.index_poly_to_byte(start_namespace); + // println!("(start, len): ({}, {})\n(start_elem, len_elem): ({}, {})\n(start_namespace, len_namespace): ({}, {})", start, len, start_elem, len_elem, start_namespace, len_namespace); + // let payload_elems: Vec<_> = + // bytes_to_field::<_, P::Evaluation>(payload.as_slice().iter()).collect(); + // println!("payload {} elems: {:?}", payload_elems.len(), payload_elems); + // check args: // TODO TEMPORARY: forbid requests that span multiple polynomials if len_namespace != 1 { @@ -76,24 +81,40 @@ where // grab the `start_namespace`th polynomial // TODO refactor copied code let polynomial = { - let elems_iter = bytes_to_field::<_, P::Evaluation>( + let mut coeffs: Vec<_> = bytes_to_field::<_, P::Evaluation>( payload.as_slice()[start_namespace_byte..].iter(), ) - .map(|elem| *elem.borrow()) - .take(self.payload_chunk_size); + .take(self.payload_chunk_size) + .collect(); // TODO TEMPORARY: use FFT to encode polynomials in eval form // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 - let mut coeffs: Vec<_> = elems_iter.collect(); - self.eval_domain.fft_in_place(&mut coeffs); + self.eval_domain.ifft_in_place(&mut coeffs); P::Polynomial::from_coefficients_vec(coeffs) }; + // debug + // { + // let points: Vec<_> = self + // .eval_domain + // .elements() + // .take(self.payload_chunk_size) + // .collect(); + // let (_proofs, evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; + // println!( + // "all {} evals (should equal namespace elems): {:?}", + // evals.len(), + // evals + // ); + // } + // prepare the list of input points + // TODO perf: can't avoid use of `skip` let points: Vec<_> = { let offset = start_elem - self.index_byte_to_elem(start_namespace_byte); + println!("points offset {}", offset); self.eval_domain .elements() .skip(offset) @@ -105,8 +126,10 @@ where // sanity check: evals == data // TODO move this to a test? + println!("evals: {:?}", evals); { let start_elem_byte = self.index_elem_to_byte(start_elem); + println!("start_elem_byte {}", start_elem_byte); let data_elems: Vec<_> = bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_elem_byte..].iter()) .map(|elem| *elem.borrow()) @@ -248,10 +271,13 @@ mod tests { namespace::Namespacer, }; use ark_bls12_381::Bls12_381; + use ark_poly::Polynomial; + use ark_std::UniformRand; use digest::{generic_array::ArrayLength, OutputSizeUser}; + use jf_utils::test_rng; use sha2::Sha256; - fn prove_namespace_generic() + fn namespace_generic() where E: Pairing, H: Digest + DynDigest + Default + Clone + Write, @@ -277,10 +303,46 @@ mod tests { .unwrap() .unwrap(); } + + // TEST: prove a data range + let start = ((num_polys / 2) * payload_chunk_size) * modulus_byte_len::() + 13; + let len = 21; + let _ = advz.data_proof(&payload, start, len).unwrap(); } #[test] - fn prove_namespace() { - prove_namespace_generic::(); + fn namespace() { + namespace_generic::(); + } + + #[test] + fn polynomial_debug() { + let mut rng = test_rng(); + let srs = init_srs(4, &mut rng); + let (ck, _vk) = + as UnivariatePCS>::trim_fft_size(srs, 3).unwrap(); + + let payload = vec![::ScalarField::rand(&mut rng); 4]; + // println!("payload {:?}", payload); + + let eval_domain = + Radix2EvaluationDomain::<::ScalarField>::new(4).unwrap(); + let coeffs = eval_domain.ifft(&payload); + let polynomial = as PolynomialCommitmentScheme>::Polynomial::from_coefficients_vec(coeffs); + + let points: Vec<_> = eval_domain.elements().collect(); + + let manual_evals: Vec<_> = points.iter().map(|p| polynomial.evaluate(p)).collect(); + assert_eq!(manual_evals, payload); + + let (_proofs, evals) = + as PolynomialCommitmentScheme>::multi_open( + &ck, + &polynomial, + &points, + ) + .unwrap(); + + assert_eq!(evals, payload); } } From efc81c5ce354c5063785f3d44779ae30e2849bb4 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 27 Oct 2023 14:15:21 -0400 Subject: [PATCH 10/54] remove debugging code --- primitives/src/vid/advz/namespace.rs | 59 +--------------------------- 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index dee303a86..6ea831d82 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -18,7 +18,7 @@ use crate::{ alloc::string::ToString, vid::{namespace::Namespacer, vid, VidError}, }; -use ark_std::{borrow::Borrow, format, println}; +use ark_std::{borrow::Borrow, format}; impl Namespacer for GenericAdvz where @@ -64,11 +64,6 @@ where let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); let start_namespace_byte = self.index_poly_to_byte(start_namespace); - // println!("(start, len): ({}, {})\n(start_elem, len_elem): ({}, {})\n(start_namespace, len_namespace): ({}, {})", start, len, start_elem, len_elem, start_namespace, len_namespace); - // let payload_elems: Vec<_> = - // bytes_to_field::<_, P::Evaluation>(payload.as_slice().iter()).collect(); - // println!("payload {} elems: {:?}", payload_elems.len(), payload_elems); - // check args: // TODO TEMPORARY: forbid requests that span multiple polynomials if len_namespace != 1 { @@ -95,26 +90,10 @@ where P::Polynomial::from_coefficients_vec(coeffs) }; - // debug - // { - // let points: Vec<_> = self - // .eval_domain - // .elements() - // .take(self.payload_chunk_size) - // .collect(); - // let (_proofs, evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; - // println!( - // "all {} evals (should equal namespace elems): {:?}", - // evals.len(), - // evals - // ); - // } - // prepare the list of input points // TODO perf: can't avoid use of `skip` let points: Vec<_> = { let offset = start_elem - self.index_byte_to_elem(start_namespace_byte); - println!("points offset {}", offset); self.eval_domain .elements() .skip(offset) @@ -126,10 +105,8 @@ where // sanity check: evals == data // TODO move this to a test? - println!("evals: {:?}", evals); { let start_elem_byte = self.index_elem_to_byte(start_elem); - println!("start_elem_byte {}", start_elem_byte); let data_elems: Vec<_> = bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_elem_byte..].iter()) .map(|elem| *elem.borrow()) @@ -271,10 +248,7 @@ mod tests { namespace::Namespacer, }; use ark_bls12_381::Bls12_381; - use ark_poly::Polynomial; - use ark_std::UniformRand; use digest::{generic_array::ArrayLength, OutputSizeUser}; - use jf_utils::test_rng; use sha2::Sha256; fn namespace_generic() @@ -314,35 +288,4 @@ mod tests { fn namespace() { namespace_generic::(); } - - #[test] - fn polynomial_debug() { - let mut rng = test_rng(); - let srs = init_srs(4, &mut rng); - let (ck, _vk) = - as UnivariatePCS>::trim_fft_size(srs, 3).unwrap(); - - let payload = vec![::ScalarField::rand(&mut rng); 4]; - // println!("payload {:?}", payload); - - let eval_domain = - Radix2EvaluationDomain::<::ScalarField>::new(4).unwrap(); - let coeffs = eval_domain.ifft(&payload); - let polynomial = as PolynomialCommitmentScheme>::Polynomial::from_coefficients_vec(coeffs); - - let points: Vec<_> = eval_domain.elements().collect(); - - let manual_evals: Vec<_> = points.iter().map(|p| polynomial.evaluate(p)).collect(); - assert_eq!(manual_evals, payload); - - let (_proofs, evals) = - as PolynomialCommitmentScheme>::multi_open( - &ck, - &polynomial, - &points, - ) - .unwrap(); - - assert_eq!(evals, payload); - } } From 2e65705a98b7f512fbfa2abae33031a497d47cb7 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 27 Oct 2023 14:50:37 -0400 Subject: [PATCH 11/54] fix bug in modulus_byte_len, should refactor later --- primitives/src/vid/advz.rs | 4 +++- primitives/src/vid/advz/namespace.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 0023fb882..1cd85c835 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -839,7 +839,9 @@ mod tests { where E: Pairing, { + // TODO should equal compile_time_checks.0 + // refactor copied code usize::try_from((< as PolynomialCommitmentScheme>::Evaluation as Field>::BasePrimeField - ::MODULUS_BIT_SIZE - 7)/8 + 1).unwrap() + ::MODULUS_BIT_SIZE - 1)/8 ).unwrap() } } diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 6ea831d82..acb67b7b7 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -272,7 +272,8 @@ mod tests { let d = advz.disperse(&payload).unwrap(); // TEST: verify "namespaces" (each namespace is a polynomial) - for namespace_index in 0..d.common.poly_commits.len() { + assert_eq!(num_polys, d.common.poly_commits.len()); + for namespace_index in 0..num_polys { advz.namespace_verify(&payload, namespace_index, &d.commit, &d.common) .unwrap() .unwrap(); From b8f1505d1aaf3ab884bc86bb748c2604bb25fb63 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 27 Oct 2023 16:17:21 -0400 Subject: [PATCH 12/54] test for data_proof --- primitives/src/vid/advz/namespace.rs | 91 ++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index acb67b7b7..73d94d540 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -51,7 +51,7 @@ where // } // check args: `start`, `len` in bounds for `payload` - if start + len >= payload.as_slice().len() { + if start + len > payload.as_slice().len() { return Err(VidError::Argument(format!( "start {} + len {} out of bounds for payload {}", start, @@ -66,7 +66,7 @@ where // check args: // TODO TEMPORARY: forbid requests that span multiple polynomials - if len_namespace != 1 { + if len_namespace > 1 { return Err(VidError::Argument(format!( "request spans {} polynomials, expect 1", len_namespace @@ -214,15 +214,10 @@ where // self.index_poly_to_byte(self.index_elem_to_poly(self. // index_byte_to_elem(index))) } fn range_byte_to_elem(&self, start: usize, len: usize) -> (usize, usize) { - let (primefield_bytes_len, ..) = compile_time_checks::(); - let elem_start = start / primefield_bytes_len; - let elem_end = (start + len - 1) / primefield_bytes_len; - (elem_start, elem_end - elem_start + 1) + range_conversion(start, len, compile_time_checks::().0) } fn range_elem_to_poly(&self, start: usize, len: usize) -> (usize, usize) { - let poly_start = start / self.payload_chunk_size; - let poly_end = (start + len - 1) / self.payload_chunk_size; - (poly_start, poly_end - poly_start + 1) + range_conversion(start, len, self.payload_chunk_size) } fn index_byte_to_elem(&self, index: usize) -> usize { let (primefield_bytes_len, ..) = compile_time_checks::(); @@ -241,6 +236,18 @@ where } } +fn range_conversion(start: usize, len: usize, denominator: usize) -> (usize, usize) { + let new_start = start / denominator; + + // underflow occurs if len is 0, so handle this case separately + if len == 0 { + return (new_start, 0); + } + + let new_end = (start + len - 1) / denominator; + (new_start, new_end - new_start + 1) +} + #[cfg(test)] mod tests { use crate::vid::{ @@ -248,6 +255,7 @@ mod tests { namespace::Namespacer, }; use ark_bls12_381::Bls12_381; + use ark_std::rand::Rng; use digest::{generic_array::ArrayLength, OutputSizeUser}; use sha2::Sha256; @@ -279,10 +287,67 @@ mod tests { .unwrap(); } - // TEST: prove a data range - let start = ((num_polys / 2) * payload_chunk_size) * modulus_byte_len::() + 13; - let len = 21; - let _ = advz.data_proof(&payload, start, len).unwrap(); + // TEST: prove data ranges for this paylaod + let namespace_bytes_len = payload_chunk_size * modulus_byte_len::(); + let edge_cases = { + let mut edge_cases = Vec::new(); + for namespace in 0..num_polys { + let random_offset = rng.gen_range(0..namespace_bytes_len); + let random_start = random_offset + (namespace * namespace_bytes_len); + + // len edge cases + edge_cases.push((namespace, random_start, 0)); + edge_cases.push((namespace, random_start, 1)); + edge_cases.push(( + namespace, + random_start, + namespace_bytes_len - random_offset - 1, + )); + edge_cases.push((namespace, random_start, namespace_bytes_len - random_offset)); + + // start edge cases + edge_cases.push((namespace, 0, rng.gen_range(0..namespace_bytes_len))); + edge_cases.push((namespace, 1, rng.gen_range(0..namespace_bytes_len - 1))); + edge_cases.push((namespace, namespace_bytes_len - 2, rng.gen_range(0..1))); + edge_cases.push((namespace, namespace_bytes_len - 1, 0)); + } + edge_cases + }; + let random_cases = { + let num_cases = edge_cases.len(); + let mut random_cases = Vec::with_capacity(num_cases); + for _ in 0..num_cases { + let namespace = rng.gen_range(0..num_polys); + let offset = rng.gen_range(0..namespace_bytes_len); + let start = offset + (namespace * namespace_bytes_len); + let len = rng.gen_range(0..namespace_bytes_len - offset); + random_cases.push((namespace, start, len)); + } + random_cases + }; + + for (i, range) in edge_cases.iter().enumerate() { + println!( + "{}/{} edge case: namespace {}, start {}, len {}", + i, + edge_cases.len(), + range.0, + range.1, + range.2 + ); + let _ = advz.data_proof(&payload, range.1, range.2).unwrap(); + } + for (i, range) in random_cases.iter().enumerate() { + println!( + "{}/{} random case: namespace {}, start {}, len {}", + i, + random_cases.len(), + range.0, + range.1, + range.2 + ); + let _ = advz.data_proof(&payload, range.1, range.2).unwrap(); + } } #[test] From 29187f5e4baaa3ef5bcd6f849e558bcd5f08eaf7 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 30 Oct 2023 16:05:53 -0400 Subject: [PATCH 13/54] impl data_verify, fix bug fft -> ifft --- primitives/src/vid/advz.rs | 6 +- primitives/src/vid/advz/namespace.rs | 135 +++++++++++++++++++-------- primitives/src/vid/namespace.rs | 16 +++- 3 files changed, 112 insertions(+), 45 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 1cd85c835..180328529 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -208,7 +208,7 @@ where // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 let mut coeffs: Vec<_> = coeffs_iter.collect(); - self.eval_domain.fft_in_place(&mut coeffs); + self.eval_domain.ifft_in_place(&mut coeffs); let poly = DenseUVPolynomial::from_coefficients_vec(coeffs); let commitment = P::commit(&self.ck, &poly).map_err(vid)?; @@ -232,7 +232,7 @@ where // https://github.com/EspressoSystems/jellyfish/issues/339 let mut coeffs: Vec<_> = coeffs_iter.collect(); let pre_fft_len = coeffs.len(); - self.eval_domain.fft_in_place(&mut coeffs); + self.eval_domain.ifft_in_place(&mut coeffs); // sanity check: the fft did not resize coeffs. // If pre_fft_len != self.payload_chunk_size then we must be in the final chunk. @@ -459,7 +459,7 @@ where // TODO TEMPORARY: use FFT to encode polynomials in eval form // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 - self.eval_domain.ifft_in_place(&mut coeffs); + self.eval_domain.fft_in_place(&mut coeffs); result.append(&mut coeffs); } diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 73d94d540..c85f2bd33 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -60,6 +60,7 @@ where ))); } + // index conversion let (start_elem, len_elem) = self.range_byte_to_elem(start, len); let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); let start_namespace_byte = self.index_poly_to_byte(start_namespace); @@ -101,31 +102,88 @@ where .collect() }; - let (proofs, evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; - - // sanity check: evals == data - // TODO move this to a test? - { - let start_elem_byte = self.index_elem_to_byte(start_elem); - let data_elems: Vec<_> = - bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_elem_byte..].iter()) - .map(|elem| *elem.borrow()) - .take(len_elem) - .collect(); - assert_eq!(data_elems, evals); - } - + let (proofs, _evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; Ok(proofs) } fn data_verify( &self, - _payload: &Self::Payload, - _start: usize, - _len: usize, - _proof: &Self::DataProof, + payload: &Self::Payload, + start: usize, + len: usize, + commit: &Self::Commit, + common: &Self::Common, + proof: &Self::DataProof, ) -> crate::vid::VidResult> { - todo!() + // check args: `start`, `len` in bounds for `payload` + if start + len > payload.as_slice().len() { + return Err(VidError::Argument(format!( + "start {} + len {} out of bounds for payload {}", + start, + len, + payload.as_slice().len() + ))); + } + + // check args: `common` consistent with `commit` + // TODO refactor copied code + let rebuilt_commit = { + let mut hasher = H::new(); + for poly_commit in common.poly_commits.iter() { + // TODO compiler bug? `as` should not be needed here! + (poly_commit as &P::Commitment) + .serialize_uncompressed(&mut hasher) + .map_err(vid)?; + } + hasher.finalize() + }; + if rebuilt_commit != *commit { + return Err(VidError::Argument( + "common inconsistent with commit".to_string(), + )); + } + + // index conversion + let (start_elem, len_elem) = self.range_byte_to_elem(start, len); + let (start_namespace, _len_namespace) = self.range_elem_to_poly(start_elem, len_elem); + let start_namespace_byte = self.index_poly_to_byte(start_namespace); + + // prepare list of data elems + let start_elem_byte = self.index_elem_to_byte(start_elem); + let data_elems: Vec<_> = + bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_elem_byte..].iter()) + .take(len_elem) + .collect(); + + // prepare list of input points + // TODO perf: can't avoid use of `skip` + let points: Vec<_> = { + let offset = start_elem - self.index_byte_to_elem(start_namespace_byte); + self.eval_domain + .elements() + .skip(offset) + .take(len_elem) + .collect() + }; + + // verify proof + // TODO naive verify for multi_open + // https://github.com/EspressoSystems/jellyfish/issues/387 + if data_elems.len() != proof.len() { + return Err(VidError::Argument(format!( + "data len {} differs from proof len {}", + data_elems.len(), + proof.len() + ))); + } + assert_eq!(data_elems.len(), points.len()); // sanity + let poly_commit = &common.poly_commits[start_namespace]; + for (point, (elem, pf)) in points.iter().zip(data_elems.iter().zip(proof.iter())) { + if !P::verify(&self.vk, poly_commit, point, elem, pf).map_err(vid)? { + return Ok(Err(())); + } + } + Ok(Ok(())) } fn namespace_verify( @@ -182,7 +240,7 @@ where // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 let mut coeffs: Vec<_> = elems_iter.collect(); - self.eval_domain.fft_in_place(&mut coeffs); + self.eval_domain.ifft_in_place(&mut coeffs); let poly = P::Polynomial::from_coefficients_vec(coeffs); P::commit(&self.ck, &poly).map_err(vid)? @@ -214,10 +272,13 @@ where // self.index_poly_to_byte(self.index_elem_to_poly(self. // index_byte_to_elem(index))) } fn range_byte_to_elem(&self, start: usize, len: usize) -> (usize, usize) { - range_conversion(start, len, compile_time_checks::().0) + range_coarsen(start, len, compile_time_checks::().0) + } + fn _range_elem_to_byte(&self, start: usize, len: usize) -> (usize, usize) { + _range_refine(start, len, compile_time_checks::().0) } fn range_elem_to_poly(&self, start: usize, len: usize) -> (usize, usize) { - range_conversion(start, len, self.payload_chunk_size) + range_coarsen(start, len, self.payload_chunk_size) } fn index_byte_to_elem(&self, index: usize) -> usize { let (primefield_bytes_len, ..) = compile_time_checks::(); @@ -236,7 +297,7 @@ where } } -fn range_conversion(start: usize, len: usize, denominator: usize) -> (usize, usize) { +fn range_coarsen(start: usize, len: usize, denominator: usize) -> (usize, usize) { let new_start = start / denominator; // underflow occurs if len is 0, so handle this case separately @@ -248,6 +309,10 @@ fn range_conversion(start: usize, len: usize, denominator: usize) -> (usize, usi (new_start, new_end - new_start + 1) } +fn _range_refine(start: usize, len: usize, multiplier: usize) -> (usize, usize) { + (start * multiplier, len * multiplier) +} + #[cfg(test)] mod tests { use crate::vid::{ @@ -288,6 +353,8 @@ mod tests { } // TEST: prove data ranges for this paylaod + // it takes too long to test all combos of (namespace, start, len) + // so do some edge cases and random cases let namespace_bytes_len = payload_chunk_size * modulus_byte_len::(); let edge_cases = { let mut edge_cases = Vec::new(); @@ -326,27 +393,19 @@ mod tests { random_cases }; - for (i, range) in edge_cases.iter().enumerate() { + for (i, range) in edge_cases.iter().chain(random_cases.iter()).enumerate() { println!( - "{}/{} edge case: namespace {}, start {}, len {}", + "case {}/{}: namespace {}, start {}, len {}", i, - edge_cases.len(), + edge_cases.len() + random_cases.len(), range.0, range.1, range.2 ); - let _ = advz.data_proof(&payload, range.1, range.2).unwrap(); - } - for (i, range) in random_cases.iter().enumerate() { - println!( - "{}/{} random case: namespace {}, start {}, len {}", - i, - random_cases.len(), - range.0, - range.1, - range.2 - ); - let _ = advz.data_proof(&payload, range.1, range.2).unwrap(); + let proof = advz.data_proof(&payload, range.1, range.2).unwrap(); + advz.data_verify(&payload, range.1, range.2, &d.commit, &d.common, &proof) + .unwrap() + .unwrap(); } } diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index a4f2f33c6..abbb796e4 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -13,7 +13,9 @@ pub trait Namespacer: VidScheme { /// data proof type DataProof; - /// Compute a proof for `payload` for data index range `start..start+len`. + /// Compute a proof for `payload` for data index range `start..start+len-1`. + /// + /// See TODO in [`namespace_verify`] on `payload`. fn data_proof( &self, payload: &Self::Payload, @@ -21,20 +23,26 @@ pub trait Namespacer: VidScheme { len: usize, ) -> VidResult; - /// Verify a proof for `payload` for data index range `start..start+len`. + /// Verify a proof for `payload` for data index range `start..start+len-1`. + /// + /// See TODO in [`namespace_verify`] on `payload`. fn data_verify( &self, payload: &Self::Payload, start: usize, len: usize, + commit: &Self::Commit, + common: &Self::Common, proof: &Self::DataProof, ) -> VidResult>; /// Verify the `payload` namespace indexed by `namespace_index` against /// `commit`, `common`. /// - /// TODO: Seems ugly to include `common` in this API but [`advz`] impl needs - /// it. + /// TODO: We prefer not to include the whole `payload`. But the namespace + /// proof needs a few payload bytes from outside the namespace. In the + /// future `payload` should be replaced by a payload subset that includes + /// only the bytes needed to verify a namespace. fn namespace_verify( &self, payload: &Self::Payload, From 2cc8acb2aff048801796bdc276832eb1c5b5fc6a Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 30 Oct 2023 17:01:22 -0400 Subject: [PATCH 14/54] refactor polynomial construction --- primitives/src/vid/advz.rs | 47 ++++++++++++++-------------- primitives/src/vid/advz/namespace.rs | 42 ++++++------------------- 2 files changed, 33 insertions(+), 56 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 180328529..c02b6e953 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -204,13 +204,7 @@ where let mut hasher = H::new(); let elems_iter = bytes_to_field::<_, P::Evaluation>(payload); for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { - // TODO TEMPORARY: use FFT to encode polynomials in eval form - // Remove these FFTs after we get KZG in eval form - // https://github.com/EspressoSystems/jellyfish/issues/339 - let mut coeffs: Vec<_> = coeffs_iter.collect(); - self.eval_domain.ifft_in_place(&mut coeffs); - - let poly = DenseUVPolynomial::from_coefficients_vec(coeffs); + let poly = self.polynomial(coeffs_iter); let commitment = P::commit(&self.ck, &poly).map_err(vid)?; commitment .serialize_uncompressed(&mut hasher) @@ -227,22 +221,7 @@ where let elems_iter = bytes_to_field::<_, P::Evaluation>(payload).map(|elem| *elem.borrow()); let mut polys = Vec::new(); for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { - // TODO TEMPORARY: use FFT to encode polynomials in eval form - // Remove these FFTs after we get KZG in eval form - // https://github.com/EspressoSystems/jellyfish/issues/339 - let mut coeffs: Vec<_> = coeffs_iter.collect(); - let pre_fft_len = coeffs.len(); - self.eval_domain.ifft_in_place(&mut coeffs); - - // sanity check: the fft did not resize coeffs. - // If pre_fft_len != self.payload_chunk_size then we must be in the final chunk. - // In that case coeffs.len() could be anything, so there's nothing to sanity - // check. - if pre_fft_len == self.payload_chunk_size { - assert_eq!(coeffs.len(), pre_fft_len); - } - - polys.push(DenseUVPolynomial::from_coefficients_vec(coeffs)); + polys.push(self.polynomial(coeffs_iter)); } // evaluate polynomials @@ -510,6 +489,28 @@ where .ok_or_else(|| anyhow!("hash_to_field output is empty")) .map_err(vid)?) } + + fn polynomial(&self, coeffs: I) -> P::Polynomial + where + I: Iterator, + { + // TODO TEMPORARY: use FFT to encode polynomials in eval form + // Remove these FFTs after we get KZG in eval form + // https://github.com/EspressoSystems/jellyfish/issues/339 + let mut coeffs_vec: Vec<_> = coeffs.collect(); + let pre_fft_len = coeffs_vec.len(); + self.eval_domain.ifft_in_place(&mut coeffs_vec); + + // sanity check: the fft did not resize coeffs. + // If pre_fft_len != self.payload_chunk_size then we were not given the correct + // number of coeffs. In that case coeffs.len() could be anything, so + // there's nothing to sanity check. + if pre_fft_len == self.payload_chunk_size { + assert_eq!(coeffs_vec.len(), pre_fft_len); + } + + DenseUVPolynomial::from_coefficients_vec(coeffs_vec) + } } // `From` impls for `VidError` diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index c85f2bd33..ad0a5d510 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -18,7 +18,7 @@ use crate::{ alloc::string::ToString, vid::{namespace::Namespacer, vid, VidError}, }; -use ark_std::{borrow::Borrow, format}; +use ark_std::format; impl Namespacer for GenericAdvz where @@ -43,13 +43,6 @@ where start: usize, len: usize, ) -> crate::vid::VidResult { - // check args: `len` must be positive - // if len == 0 { - // return Err(VidError::Argument( - // "request for zero-length data proof".to_string(), - // )); - // } - // check args: `start`, `len` in bounds for `payload` if start + len > payload.as_slice().len() { return Err(VidError::Argument(format!( @@ -76,20 +69,10 @@ where // grab the `start_namespace`th polynomial // TODO refactor copied code - let polynomial = { - let mut coeffs: Vec<_> = bytes_to_field::<_, P::Evaluation>( - payload.as_slice()[start_namespace_byte..].iter(), - ) - .take(self.payload_chunk_size) - .collect(); - - // TODO TEMPORARY: use FFT to encode polynomials in eval form - // Remove these FFTs after we get KZG in eval form - // https://github.com/EspressoSystems/jellyfish/issues/339 - self.eval_domain.ifft_in_place(&mut coeffs); - - P::Polynomial::from_coefficients_vec(coeffs) - }; + let polynomial = self.polynomial( + bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_namespace_byte..].iter()) + .take(self.payload_chunk_size), + ); // prepare the list of input points // TODO perf: can't avoid use of `skip` @@ -232,17 +215,10 @@ where // rebuild the `namespace_index`th poly commit, check against `common` let poly_commit = { - let elems_iter = bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start..].iter()) - .map(|elem| *elem.borrow()) - .take(self.payload_chunk_size); - - // TODO TEMPORARY: use FFT to encode polynomials in eval form - // Remove these FFTs after we get KZG in eval form - // https://github.com/EspressoSystems/jellyfish/issues/339 - let mut coeffs: Vec<_> = elems_iter.collect(); - self.eval_domain.ifft_in_place(&mut coeffs); - - let poly = P::Polynomial::from_coefficients_vec(coeffs); + let poly = self.polynomial( + bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start..].iter()) + .take(self.payload_chunk_size), + ); P::commit(&self.ck, &poly).map_err(vid)? }; if poly_commit != common.poly_commits[namespace_index] { From 26bb9ec8407cbcb5c4958f395e89aceb44a7bcd8 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 30 Oct 2023 18:10:31 -0400 Subject: [PATCH 15/54] refactor computation of block hash from polynomial commitments --- primitives/src/vid/advz.rs | 39 ++++++++++++++++------------ primitives/src/vid/advz/namespace.rs | 27 ++----------------- 2 files changed, 25 insertions(+), 41 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index c02b6e953..d16314518 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -201,6 +201,10 @@ where type Payload = Payload; fn commit_only(&self, payload: &Self::Payload) -> VidResult { + // Can't use `Self::poly_commits_hash()`` here because `P::commit()`` returns + // `Result`` instead of `P::Commitment`. + // There's probably an idiomatic way to do this using eg. + // itertools::process_results() but the code is unreadable. let mut hasher = H::new(); let elems_iter = bytes_to_field::<_, P::Evaluation>(payload); for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { @@ -277,17 +281,7 @@ where bytes_len, }; - let commit = { - let mut hasher = H::new(); - for poly_commit in common.poly_commits.iter() { - // TODO compiler bug? `as` should not be needed here! - (poly_commit as &P::Commitment) - .serialize_uncompressed(&mut hasher) - .map_err(vid)?; - } - hasher.finalize() - }; - + let commit = Self::poly_commits_hash(common.poly_commits.iter())?; let pseudorandom_scalar = Self::pseudorandom_scalar(&common)?; // Compute aggregate polynomial @@ -464,11 +458,9 @@ where { fn pseudorandom_scalar(common: &::Common) -> VidResult { let mut hasher = H::new(); - for poly_commit in common.poly_commits.iter() { - poly_commit - .serialize_uncompressed(&mut hasher) - .map_err(vid)?; - } + Self::poly_commits_hash(common.poly_commits.iter())? + .serialize_uncompressed(&mut hasher) + .map_err(vid)?; common .all_evals_digest .serialize_uncompressed(&mut hasher) @@ -511,6 +503,21 @@ where DenseUVPolynomial::from_coefficients_vec(coeffs_vec) } + + fn poly_commits_hash(poly_commits: I) -> VidResult<::Commit> + where + I: Iterator, + I::Item: Borrow, + { + let mut hasher = H::new(); + for poly_commit in poly_commits { + poly_commit + .borrow() + .serialize_uncompressed(&mut hasher) + .map_err(vid)?; + } + Ok(hasher.finalize()) + } } // `From` impls for `VidError` diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index ad0a5d510..de6ede307 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -7,7 +7,6 @@ //! Implementation of [`Namespacer`] for [`Advz`]. use ark_poly::EvaluationDomain; -use ark_serialize::CanonicalSerialize; use jf_utils::{bytes_to_field, compile_time_checks}; use super::{ @@ -68,7 +67,6 @@ where } // grab the `start_namespace`th polynomial - // TODO refactor copied code let polynomial = self.polynomial( bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_namespace_byte..].iter()) .take(self.payload_chunk_size), @@ -109,18 +107,7 @@ where } // check args: `common` consistent with `commit` - // TODO refactor copied code - let rebuilt_commit = { - let mut hasher = H::new(); - for poly_commit in common.poly_commits.iter() { - // TODO compiler bug? `as` should not be needed here! - (poly_commit as &P::Commitment) - .serialize_uncompressed(&mut hasher) - .map_err(vid)?; - } - hasher.finalize() - }; - if rebuilt_commit != *commit { + if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { return Err(VidError::Argument( "common inconsistent with commit".to_string(), )); @@ -197,17 +184,7 @@ where } // check args: `common` consistent with `commit` - let rebuilt_commit = { - let mut hasher = H::new(); - for poly_commit in common.poly_commits.iter() { - // TODO compiler bug? `as` should not be needed here! - (poly_commit as &P::Commitment) - .serialize_uncompressed(&mut hasher) - .map_err(vid)?; - } - hasher.finalize() - }; - if rebuilt_commit != *commit { + if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { return Err(VidError::Argument( "common inconsistent with commit".to_string(), )); From 24534041b1f89d9bee6e08708af64c7fd7ae694e Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 30 Oct 2023 18:15:17 -0400 Subject: [PATCH 16/54] delete superfluous modulus_byte_len --- primitives/src/vid/advz.rs | 10 ---------- primitives/src/vid/advz/namespace.rs | 5 +++-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index d16314518..30dab63e4 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -842,14 +842,4 @@ mod tests { UnivariateKzgPCS::gen_srs_for_testing(rng, checked_fft_size(num_coeffs - 1).unwrap()) .unwrap() } - - pub(super) fn modulus_byte_len() -> usize - where - E: Pairing, - { - // TODO should equal compile_time_checks.0 - // refactor copied code - usize::try_from((< as PolynomialCommitmentScheme>::Evaluation as Field>::BasePrimeField - ::MODULUS_BIT_SIZE - 1)/8 ).unwrap() - } } diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index de6ede307..82329dd2b 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -275,6 +275,7 @@ mod tests { use ark_bls12_381::Bls12_381; use ark_std::rand::Rng; use digest::{generic_array::ArrayLength, OutputSizeUser}; + use jf_utils::compile_time_checks; use sha2::Sha256; fn namespace_generic() @@ -289,7 +290,7 @@ mod tests { // more items as a function of the above let payload_elems_len = num_polys * payload_chunk_size; - let payload_bytes_len = payload_elems_len * modulus_byte_len::(); + let payload_bytes_len = payload_elems_len * compile_time_checks::().0; let mut rng = jf_utils::test_rng(); let payload = init_random_payload(payload_bytes_len, &mut rng); let srs = init_srs(payload_elems_len, &mut rng); @@ -308,7 +309,7 @@ mod tests { // TEST: prove data ranges for this paylaod // it takes too long to test all combos of (namespace, start, len) // so do some edge cases and random cases - let namespace_bytes_len = payload_chunk_size * modulus_byte_len::(); + let namespace_bytes_len = payload_chunk_size * compile_time_checks::().0; let edge_cases = { let mut edge_cases = Vec::new(); for namespace in 0..num_polys { From c753718b087cba96309fadf1de073a7e4e704200 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 30 Oct 2023 19:19:04 -0400 Subject: [PATCH 17/54] tidy index conversion utils --- primitives/src/vid/advz/namespace.rs | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 82329dd2b..336c6028b 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -221,32 +221,21 @@ where { // lots of index manipulation. // with infinite dev time we should implement type-safe indices to preclude - // index-misuse bugs. fn index_byte_to_poly(&self, index: usize) -> usize { - // self.index_poly_to_byte(self.index_elem_to_poly(self. - // index_byte_to_elem(index))) } + // index-misuse bugs. fn range_byte_to_elem(&self, start: usize, len: usize) -> (usize, usize) { range_coarsen(start, len, compile_time_checks::().0) } - fn _range_elem_to_byte(&self, start: usize, len: usize) -> (usize, usize) { - _range_refine(start, len, compile_time_checks::().0) - } fn range_elem_to_poly(&self, start: usize, len: usize) -> (usize, usize) { range_coarsen(start, len, self.payload_chunk_size) } fn index_byte_to_elem(&self, index: usize) -> usize { - let (primefield_bytes_len, ..) = compile_time_checks::(); - index / primefield_bytes_len // round down + index / compile_time_checks::().0 // round down } fn index_elem_to_byte(&self, index: usize) -> usize { - let (primefield_bytes_len, ..) = compile_time_checks::(); - index * primefield_bytes_len + index * compile_time_checks::().0 } - // fn index_elem_to_poly(&self, index: usize) -> usize { - // index / self.payload_chunk_size // round down - // } fn index_poly_to_byte(&self, index: usize) -> usize { - let (primefield_bytes_len, ..) = compile_time_checks::(); - index * self.payload_chunk_size * primefield_bytes_len + index * self.payload_chunk_size * compile_time_checks::().0 } } @@ -262,10 +251,6 @@ fn range_coarsen(start: usize, len: usize, denominator: usize) -> (usize, usize) (new_start, new_end - new_start + 1) } -fn _range_refine(start: usize, len: usize, multiplier: usize) -> (usize, usize) { - (start * multiplier, len * multiplier) -} - #[cfg(test)] mod tests { use crate::vid::{ From 08a536eed5ec573b76a278d4fecef2010b2c3769 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 30 Oct 2023 19:21:57 -0400 Subject: [PATCH 18/54] tidy --- primitives/src/vid/advz/payload.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/primitives/src/vid/advz/payload.rs b/primitives/src/vid/advz/payload.rs index ebb3c0ec7..a7626bc15 100644 --- a/primitives/src/vid/advz/payload.rs +++ b/primitives/src/vid/advz/payload.rs @@ -1,8 +1,6 @@ //! paylod module doc -// use ark_ff::PrimeField; use ark_std::{ - // borrow::Borrow, slice::Iter, vec::{IntoIter, Vec}, }; @@ -15,17 +13,6 @@ pub struct Payload { } impl Payload { - /// from_iter - // pub fn from_iter(payload: I) -> Self - // where - // I: IntoIterator, - // I::Item: Borrow, - // { - // Self { - // payload: payload.into_iter().map(|b| *b.borrow()).collect(), - // } - // } - /// from_vec pub fn from_vec(payload: Vec) -> Self { Self { payload } @@ -40,13 +27,6 @@ impl Payload { pub fn iter(&self) -> Iter<'_, u8> { self.payload.iter() } - - // fn field_iter(&self) -> impl Iterator - // where - // F: PrimeField, - // { - // todo!() - // } } // delegation boilerplate From 7d3636129942df69a4e867353ce287d682e15658 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 31 Oct 2023 09:55:08 -0400 Subject: [PATCH 19/54] fix test build (my vscode settings has std feature enabled by default, github CI does not) --- primitives/src/vid/advz/namespace.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 336c6028b..97269dcd5 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -258,7 +258,7 @@ mod tests { namespace::Namespacer, }; use ark_bls12_381::Bls12_381; - use ark_std::rand::Rng; + use ark_std::{println, rand::Rng}; use digest::{generic_array::ArrayLength, OutputSizeUser}; use jf_utils::compile_time_checks; use sha2::Sha256; From 293ed1a880815f333748a8009b225039615bb5ba Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 31 Oct 2023 09:59:41 -0400 Subject: [PATCH 20/54] update changelog (is this always worthwhile?) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a3d40eaa..e5390b63f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and follow [semantic versioning](https://semver.org/) for our releases. - [#343](https://github.com/EspressoSystems/jellyfish/pull/343) Rescue parameter for `ark_bn254::Fq` - [#362](https://github.com/EspressoSystems/jellyfish/pull/362) Derive Eq, Hash at a bunch of places - [#381](https://github.com/EspressoSystems/jellyfish/pull/381) VID take iterator instead of slice +- [#389](https://github.com/EspressoSystems/jellyfish/pull/389) Hello-world namespace support for ADVZ VID scheme ### Changed From 96e8f878d633121aa89859adec2a3e6d653f7fb3 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 31 Oct 2023 10:41:29 -0400 Subject: [PATCH 21/54] remove broken rustdoc links --- primitives/src/vid.rs | 2 +- primitives/src/vid/advz/namespace.rs | 2 +- primitives/src/vid/namespace.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index efaa8bef7..9f79ce6c4 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -43,7 +43,7 @@ pub trait VidScheme { /// Recover payload from shares. /// Do not verify shares or check recovered payload against anything. /// - /// TODO: return type [`Payload`]? + /// TODO: return type `Payload`? fn recover_payload( &self, shares: &[Self::Share], diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 97269dcd5..f888189cb 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -4,7 +4,7 @@ // You should have received a copy of the MIT License // along with the Jellyfish library. If not, see . -//! Implementation of [`Namespacer`] for [`Advz`]. +//! Implementation of [`Namespacer`] for `Advz`. use ark_poly::EvaluationDomain; use jf_utils::{bytes_to_field, compile_time_checks}; diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index abbb796e4..461760a4b 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -15,7 +15,7 @@ pub trait Namespacer: VidScheme { /// Compute a proof for `payload` for data index range `start..start+len-1`. /// - /// See TODO in [`namespace_verify`] on `payload`. + /// See TODO in `namespace_verify` on `payload`. fn data_proof( &self, payload: &Self::Payload, @@ -25,7 +25,7 @@ pub trait Namespacer: VidScheme { /// Verify a proof for `payload` for data index range `start..start+len-1`. /// - /// See TODO in [`namespace_verify`] on `payload`. + /// See TODO in `namespace_verify` on `payload`. fn data_verify( &self, payload: &Self::Payload, From 4dd89f4899ece3fa4ab510cd06772b3dfc337704 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 1 Nov 2023 15:14:17 -0400 Subject: [PATCH 22/54] add timer data to VID disperse, remove it temmporarily from KZG to reduce noise in output --- primitives/src/pcs/univariate_kzg/mod.rs | 14 ++++---- primitives/src/vid/advz.rs | 45 +++++++++++++++++++++++- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/primitives/src/pcs/univariate_kzg/mod.rs b/primitives/src/pcs/univariate_kzg/mod.rs index 3a0a1cadc..8247ac6ce 100644 --- a/primitives/src/pcs/univariate_kzg/mod.rs +++ b/primitives/src/pcs/univariate_kzg/mod.rs @@ -92,8 +92,8 @@ impl PolynomialCommitmentScheme for UnivariateKzgPCS { poly: &Self::Polynomial, ) -> Result { let prover_param = prover_param.borrow(); - let commit_time = - start_timer!(|| format!("Committing to polynomial of degree {} ", poly.degree())); + // let commit_time = + // start_timer!(|| format!("Committing to polynomial of degree {} ", poly.degree())); if poly.degree() > prover_param.powers_of_g.len() { return Err(PCSError::InvalidParameters(format!( @@ -105,15 +105,15 @@ impl PolynomialCommitmentScheme for UnivariateKzgPCS { let (num_leading_zeros, plain_coeffs) = skip_leading_zeros_and_convert_to_bigints(poly); - let msm_time = start_timer!(|| "MSM to compute commitment to plaintext poly"); + // let msm_time = start_timer!(|| "MSM to compute commitment to plaintext poly"); let commitment = E::G1::msm_bigint( &prover_param.powers_of_g[num_leading_zeros..], &plain_coeffs, ) .into_affine(); - end_timer!(msm_time); + // end_timer!(msm_time); - end_timer!(commit_time); + // end_timer!(commit_time); Ok(Commitment(commitment)) } @@ -405,9 +405,9 @@ fn skip_leading_zeros_and_convert_to_bigints(p: &[F]) -> Vec { - let to_bigint_time = start_timer!(|| "Converting polynomial coeffs to bigints"); + // let to_bigint_time = start_timer!(|| "Converting polynomial coeffs to bigints"); let coeffs = p.iter().map(|s| s.into_bigint()).collect::>(); - end_timer!(to_bigint_time); + // end_timer!(to_bigint_time); coeffs } diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 30dab63e4..b93a51d54 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -28,11 +28,12 @@ use ark_poly::{DenseUVPolynomial, EvaluationDomain, Radix2EvaluationDomain}; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Write}; use ark_std::{ borrow::Borrow, + end_timer, fmt::Debug, format, marker::PhantomData, ops::{Add, Mul}, - vec, + start_timer, vec, vec::Vec, Zero, }; @@ -218,17 +219,28 @@ where } fn disperse(&self, payload: &Self::Payload) -> VidResult> { + let disperse_time = start_timer!(|| format!( + "VID disperse {} chunks to {} nodes", + self.payload_chunk_size, self.num_storage_nodes + )); let bytes_len = payload.as_slice().len(); // partition payload into polynomial coefficients // and count `elems_len` for later + let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let elems_iter = bytes_to_field::<_, P::Evaluation>(payload).map(|elem| *elem.borrow()); let mut polys = Vec::new(); for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { polys.push(self.polynomial(coeffs_iter)); } + end_timer!(bytes_to_polys_time); // evaluate polynomials + let all_storage_node_evals_timer = start_timer!(|| format!( + "compute all storage node evals for {} polynomials of degree {}", + polys.len(), + self.payload_chunk_size + )); let all_storage_node_evals = { let mut all_storage_node_evals = vec![Vec::with_capacity(polys.len()); self.num_storage_nodes]; @@ -253,9 +265,12 @@ where all_storage_node_evals }; + end_timer!(all_storage_node_evals_timer); // vector commitment to polynomial evaluations // TODO why do I need to compute the height of the merkle tree? + let all_evals_commit_timer = + start_timer!(|| "compute merkle root of all storage node evals"); let height: usize = all_storage_node_evals .len() .checked_ilog(V::ARITY) @@ -270,7 +285,9 @@ where .expect("num_storage_nodes log base arity should fit into usize"); let height = height + 1; // avoid fully qualified syntax for try_into() let all_evals_commit = V::from_elems(height, &all_storage_node_evals).map_err(vid)?; + end_timer!(all_evals_commit_timer); + let common_timer = start_timer!(|| format!("compute {} KZG commitments", polys.len())); let common = Common { poly_commits: polys .iter() @@ -280,7 +297,11 @@ where all_evals_digest: all_evals_commit.commitment().digest(), bytes_len, }; + end_timer!(common_timer); + let misc_timer = start_timer!(|| { + "should be fast: compute polynomial commitments hash, pseudorandom scalar, aggregate polynomial" + }); let commit = Self::poly_commits_hash(common.poly_commits.iter())?; let pseudorandom_scalar = Self::pseudorandom_scalar(&common)?; @@ -290,7 +311,12 @@ where // and whose input point is the pseudorandom scalar. let aggregate_poly = polynomial_eval(polys.iter().map(PolynomialMultiplier), pseudorandom_scalar); + end_timer!(misc_timer); + let agg_proofs_timer = start_timer!(|| format!( + "compute aggregate proofs for {} storage nodes", + self.num_storage_nodes + )); let aggregate_proofs = P::multi_open_rou_proofs( &self.ck, &aggregate_poly, @@ -298,7 +324,9 @@ where &self.multi_open_domain, ) .map_err(vid)?; + end_timer!(agg_proofs_timer); + let assemblage_timer = start_timer!(|| "assemble shares for dispersal"); let shares = all_storage_node_evals .into_iter() .zip(aggregate_proofs) @@ -316,7 +344,9 @@ where }) }) .collect::>()?; + end_timer!(assemblage_timer); + end_timer!(disperse_time); Ok(VidDisperse { shares, common, @@ -625,6 +655,19 @@ mod tests { }; use sha2::Sha256; + #[test] + fn disperse_timer() { + // run with 'print-trace' feature to see timer output + let (payload_chunk_size, num_storage_nodes) = (256, 512); + let mut rng = jf_utils::test_rng(); + let srs = init_srs(payload_chunk_size, &mut rng); + let advz = + Advz::::new(payload_chunk_size, num_storage_nodes, srs).unwrap(); + let payload_random = init_random_payload(1 << 20, &mut rng); + + let _ = advz.disperse(&payload_random); + } + #[test] fn sad_path_verify_share_corrupt_share() { let (advz, bytes_random) = avdz_init(); From 58feac5576364bd704b2622020c46c224780380e Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Thu, 2 Nov 2023 14:15:09 -0400 Subject: [PATCH 23/54] remove namespace_verify, add chunk_prove, chunk_verify --- primitives/src/pcs/univariate_kzg/mod.rs | 9 +- primitives/src/vid/advz.rs | 5 +- primitives/src/vid/advz/namespace.rs | 132 ++++++++++++++++++----- primitives/src/vid/namespace.rs | 21 +++- 4 files changed, 131 insertions(+), 36 deletions(-) diff --git a/primitives/src/pcs/univariate_kzg/mod.rs b/primitives/src/pcs/univariate_kzg/mod.rs index 8247ac6ce..1b0cddb16 100644 --- a/primitives/src/pcs/univariate_kzg/mod.rs +++ b/primitives/src/pcs/univariate_kzg/mod.rs @@ -93,7 +93,8 @@ impl PolynomialCommitmentScheme for UnivariateKzgPCS { ) -> Result { let prover_param = prover_param.borrow(); // let commit_time = - // start_timer!(|| format!("Committing to polynomial of degree {} ", poly.degree())); + // start_timer!(|| format!("Committing to polynomial of degree {} ", + // poly.degree())); if poly.degree() > prover_param.powers_of_g.len() { return Err(PCSError::InvalidParameters(format!( @@ -105,7 +106,8 @@ impl PolynomialCommitmentScheme for UnivariateKzgPCS { let (num_leading_zeros, plain_coeffs) = skip_leading_zeros_and_convert_to_bigints(poly); - // let msm_time = start_timer!(|| "MSM to compute commitment to plaintext poly"); + // let msm_time = start_timer!(|| "MSM to compute commitment to plaintext + // poly"); let commitment = E::G1::msm_bigint( &prover_param.powers_of_g[num_leading_zeros..], &plain_coeffs, @@ -405,7 +407,8 @@ fn skip_leading_zeros_and_convert_to_bigints(p: &[F]) -> Vec { - // let to_bigint_time = start_timer!(|| "Converting polynomial coeffs to bigints"); + // let to_bigint_time = start_timer!(|| "Converting polynomial coeffs to + // bigints"); let coeffs = p.iter().map(|s| s.into_bigint()).collect::>(); // end_timer!(to_bigint_time); coeffs diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index b93a51d54..b4fa0b78e 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -514,12 +514,13 @@ where fn polynomial(&self, coeffs: I) -> P::Polynomial where - I: Iterator, + I: Iterator, + I::Item: Borrow, { // TODO TEMPORARY: use FFT to encode polynomials in eval form // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 - let mut coeffs_vec: Vec<_> = coeffs.collect(); + let mut coeffs_vec: Vec<_> = coeffs.map(|c| *c.borrow()).collect(); let pre_fft_len = coeffs_vec.len(); self.eval_domain.ifft_in_place(&mut coeffs_vec); diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index f888189cb..01a8f04db 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -36,6 +36,8 @@ where // https://github.com/EspressoSystems/jellyfish/issues/387 type DataProof = Vec; + type ChunkProof = ChunkProof; + fn data_proof( &self, payload: &Self::Payload, @@ -156,30 +158,78 @@ where Ok(Ok(())) } - fn namespace_verify( + fn chunk_proof( &self, payload: &Self::Payload, - namespace_index: usize, + start: usize, + len: usize, + ) -> crate::vid::VidResult { + // check args: `start`, `len` in bounds for `payload` + if start + len > payload.as_slice().len() { + return Err(VidError::Argument(format!( + "start {} + len {} out of bounds for payload {}", + start, + len, + payload.as_slice().len() + ))); + } + + // index conversion + let (start_elem, len_elem) = self.range_byte_to_elem(start, len); + let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); + let start_namespace_byte = self.index_poly_to_byte(start_namespace); + let offset_elem = start_elem - self.index_byte_to_elem(start_namespace_byte); + + // check args: + // TODO TEMPORARY: forbid requests that span multiple polynomials + if len_namespace > 1 { + return Err(VidError::Argument(format!( + "request spans {} polynomials, expect 1", + len_namespace + ))); + } + + // return the prefix and suffix elems + let mut elems_iter = + bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_namespace_byte..].iter()) + .take(self.payload_chunk_size); + let prefix: Vec<_> = elems_iter.by_ref().take(offset_elem).collect(); + let suffix: Vec<_> = elems_iter.skip(len_elem).collect(); + + Ok(ChunkProof { prefix, suffix }) + } + + fn chunk_verify( + &self, + payload: &Self::Payload, + start: usize, + len: usize, commit: &Self::Commit, common: &Self::Common, + proof: &Self::ChunkProof, ) -> crate::vid::VidResult> { - // check args: `namespace_index`` in bounds for `common`. - if namespace_index >= common.poly_commits.len() { + // check args: `start`, `len` in bounds for `payload` + if start + len > payload.as_slice().len() { return Err(VidError::Argument(format!( - "namespace_index {} out of bounds for common.poly_commits {}", - namespace_index, - common.poly_commits.len() + "start {} + len {} out of bounds for payload {}", + start, + len, + payload.as_slice().len() ))); } - let start = self.index_poly_to_byte(namespace_index); + // index conversion + let (start_elem, len_elem) = self.range_byte_to_elem(start, len); + let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); + let start_elem_byte = self.index_elem_to_byte(start_elem); + // let offset_elem = start_elem - self.index_byte_to_elem(start_namespace_byte); - // check args: `namespace_index` in bounds for `payload`. - if start >= payload.as_slice().len() { + // check args: + // TODO TEMPORARY: forbid requests that span multiple polynomials + if len_namespace > 1 { return Err(VidError::Argument(format!( - "namespace_index {} out of bounds for payload {}", - namespace_index, - payload.as_slice().len() + "request spans {} polynomials, expect 1", + len_namespace ))); } @@ -193,12 +243,21 @@ where // rebuild the `namespace_index`th poly commit, check against `common` let poly_commit = { let poly = self.polynomial( - bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start..].iter()) - .take(self.payload_chunk_size), + proof + .prefix + .iter() + .cloned() + .chain( + bytes_to_field::<_, P::Evaluation>( + payload.as_slice()[start_elem_byte..].iter(), + ) + .take(self.payload_chunk_size), + ) + .chain(proof.suffix.iter().cloned()), ); P::commit(&self.ck, &poly).map_err(vid)? }; - if poly_commit != common.poly_commits[namespace_index] { + if poly_commit != common.poly_commits[start_namespace] { return Ok(Err(())); } @@ -206,6 +265,12 @@ where } } +/// doc +pub struct ChunkProof { + prefix: Vec, + suffix: Vec, +} + impl GenericAdvz where // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 @@ -283,14 +348,6 @@ mod tests { let advz = Advz::::new(payload_chunk_size, num_storage_nodes, srs).unwrap(); let d = advz.disperse(&payload).unwrap(); - // TEST: verify "namespaces" (each namespace is a polynomial) - assert_eq!(num_polys, d.common.poly_commits.len()); - for namespace_index in 0..num_polys { - advz.namespace_verify(&payload, namespace_index, &d.commit, &d.common) - .unwrap() - .unwrap(); - } - // TEST: prove data ranges for this paylaod // it takes too long to test all combos of (namespace, start, len) // so do some edge cases and random cases @@ -341,10 +398,29 @@ mod tests { range.1, range.2 ); - let proof = advz.data_proof(&payload, range.1, range.2).unwrap(); - advz.data_verify(&payload, range.1, range.2, &d.commit, &d.common, &proof) - .unwrap() - .unwrap(); + let data_proof = advz.data_proof(&payload, range.1, range.2).unwrap(); + advz.data_verify( + &payload, + range.1, + range.2, + &d.commit, + &d.common, + &data_proof, + ) + .unwrap() + .unwrap(); + + let chunk_proof = advz.chunk_proof(&payload, range.1, range.2).unwrap(); + advz.chunk_verify( + &payload, + range.1, + range.2, + &d.commit, + &d.common, + &chunk_proof, + ) + .unwrap() + .unwrap(); } } diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index 461760a4b..794aed854 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -5,6 +5,8 @@ // along with the Jellyfish library. If not, see . //! Trait for namespace functionality in Verifiable Information Retrieval (VID). +//! +//! TODO should this trait even exist? It's very implementation-dependent. use super::{VidResult, VidScheme}; @@ -13,9 +15,12 @@ pub trait Namespacer: VidScheme { /// data proof type DataProof; + /// chunk proof + type ChunkProof; + /// Compute a proof for `payload` for data index range `start..start+len-1`. /// - /// See TODO in `namespace_verify` on `payload`. + /// TODO explain how this differs from `chunk_proof` fn data_proof( &self, payload: &Self::Payload, @@ -36,6 +41,14 @@ pub trait Namespacer: VidScheme { proof: &Self::DataProof, ) -> VidResult>; + /// Compute a proof for `payload` for data index range `start..start+len-1`. + fn chunk_proof( + &self, + payload: &Self::Payload, + start: usize, + len: usize, + ) -> VidResult; + /// Verify the `payload` namespace indexed by `namespace_index` against /// `commit`, `common`. /// @@ -43,11 +56,13 @@ pub trait Namespacer: VidScheme { /// proof needs a few payload bytes from outside the namespace. In the /// future `payload` should be replaced by a payload subset that includes /// only the bytes needed to verify a namespace. - fn namespace_verify( + fn chunk_verify( &self, payload: &Self::Payload, - namespace_index: usize, + start: usize, + len: usize, commit: &Self::Commit, common: &Self::Common, + proof: &Self::ChunkProof, ) -> VidResult>; } From daabb287dde2891ec37bc1c48d1cedd3fc2a832b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Thu, 2 Nov 2023 17:18:14 -0400 Subject: [PATCH 24/54] wip: stubs --- primitives/src/vid/advz/namespace.rs | 54 ++++++++++++++++++++++++---- primitives/src/vid/namespace.rs | 31 ++++++++++++++++ 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 01a8f04db..04ea20986 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -11,13 +11,13 @@ use jf_utils::{bytes_to_field, compile_time_checks}; use super::{ AffineRepr, Debug, DenseUVPolynomial, Digest, DynDigest, GenericAdvz, MerkleTreeScheme, - PolynomialCommitmentScheme, PrimeField, UnivariatePCS, Vec, Write, + PolynomialCommitmentScheme, PrimeField, UnivariatePCS, Vec, VidResult, Write, }; use crate::{ alloc::string::ToString, vid::{namespace::Namespacer, vid, VidError}, }; -use ark_std::format; +use ark_std::{format, ops::Range}; impl Namespacer for GenericAdvz where @@ -37,13 +37,14 @@ where type DataProof = Vec; type ChunkProof = ChunkProof; + type ChunkProof2 = ChunkProof2; fn data_proof( &self, payload: &Self::Payload, start: usize, len: usize, - ) -> crate::vid::VidResult { + ) -> VidResult { // check args: `start`, `len` in bounds for `payload` if start + len > payload.as_slice().len() { return Err(VidError::Argument(format!( @@ -97,7 +98,7 @@ where commit: &Self::Commit, common: &Self::Common, proof: &Self::DataProof, - ) -> crate::vid::VidResult> { + ) -> VidResult> { // check args: `start`, `len` in bounds for `payload` if start + len > payload.as_slice().len() { return Err(VidError::Argument(format!( @@ -163,7 +164,7 @@ where payload: &Self::Payload, start: usize, len: usize, - ) -> crate::vid::VidResult { + ) -> VidResult { // check args: `start`, `len` in bounds for `payload` if start + len > payload.as_slice().len() { return Err(VidError::Argument(format!( @@ -199,6 +200,32 @@ where Ok(ChunkProof { prefix, suffix }) } + fn chunk_proof2(&self, _payload: B, _range: Range) -> VidResult + where + B: AsRef<[u8]>, + { + Ok(ChunkProof2 { + _prefix_elems: Vec::new(), + _suffix_elems: Vec::new(), + _prefix_bytes: Vec::new(), + _suffix_bytes: Vec::new(), + }) + } + + fn chunk_verify2( + &self, + _chunk: B, + _commit: &Self::Commit, + _common: &Self::Common, + proof: &Self::ChunkProof2, + ) -> VidResult> + where + B: AsRef<[u8]>, + { + let _ = proof; + Ok(Ok(())) + } + fn chunk_verify( &self, payload: &Self::Payload, @@ -207,7 +234,7 @@ where commit: &Self::Commit, common: &Self::Common, proof: &Self::ChunkProof, - ) -> crate::vid::VidResult> { + ) -> VidResult> { // check args: `start`, `len` in bounds for `payload` if start + len > payload.as_slice().len() { return Err(VidError::Argument(format!( @@ -271,6 +298,14 @@ pub struct ChunkProof { suffix: Vec, } +/// doc +pub struct ChunkProof2 { + _prefix_elems: Vec, + _suffix_elems: Vec, + _prefix_bytes: Vec, + _suffix_bytes: Vec, +} + impl GenericAdvz where // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 @@ -293,6 +328,9 @@ where fn range_elem_to_poly(&self, start: usize, len: usize) -> (usize, usize) { range_coarsen(start, len, self.payload_chunk_size) } + // fn range_elem_to_byte(&self, start: usize, len: usize) -> (usize, usize) { + // range_refine(start, len, compile_time_checks::().0) + // } fn index_byte_to_elem(&self, index: usize) -> usize { index / compile_time_checks::().0 // round down } @@ -316,6 +354,10 @@ fn range_coarsen(start: usize, len: usize, denominator: usize) -> (usize, usize) (new_start, new_end - new_start + 1) } +// fn range_refine(start: usize, len: usize, multiplier: usize) -> (usize, usize) { +// (start * multiplier, len * multiplier) +// } + #[cfg(test)] mod tests { use crate::vid::{ diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index 794aed854..a4eee84ff 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -9,6 +9,18 @@ //! TODO should this trait even exist? It's very implementation-dependent. use super::{VidResult, VidScheme}; +use ark_std::ops::Range; + +// pub trait Namespacer2

: VidScheme { + +// fn data_proof( +// &self, +// payload: &[u8], +// start: usize, +// len: usize, +// ) -> VidResult; + +// } /// Namespace functionality for [`VidScheme`]. pub trait Namespacer: VidScheme { @@ -18,6 +30,9 @@ pub trait Namespacer: VidScheme { /// chunk proof type ChunkProof; + /// doc + type ChunkProof2; + /// Compute a proof for `payload` for data index range `start..start+len-1`. /// /// TODO explain how this differs from `chunk_proof` @@ -49,6 +64,22 @@ pub trait Namespacer: VidScheme { len: usize, ) -> VidResult; + /// doc + fn chunk_proof2(&self, payload: B, range: Range) -> VidResult + where + B: AsRef<[u8]>; + + /// doc + fn chunk_verify2( + &self, + chunk: B, + commit: &Self::Commit, + common: &Self::Common, + proof: &Self::ChunkProof2, + ) -> VidResult> + where + B: AsRef<[u8]>; + /// Verify the `payload` namespace indexed by `namespace_index` against /// `commit`, `common`. /// From cc5e3c612fcb9c64ee72ed2b5a8f21d0bc935247 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 3 Nov 2023 12:26:59 -0400 Subject: [PATCH 25/54] new chunk_proof_2 use Range instead of start, len --- primitives/src/vid/advz/namespace.rs | 95 ++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 04ea20986..698b33646 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -200,15 +200,47 @@ where Ok(ChunkProof { prefix, suffix }) } - fn chunk_proof2(&self, _payload: B, _range: Range) -> VidResult + fn chunk_proof2(&self, payload: B, range: Range) -> VidResult where B: AsRef<[u8]>, { + let payload = payload.as_ref(); + + // check args: `range` in bounds for `payload` + if range.start >= payload.len() || range.end > payload.len() { + return Err(VidError::Argument(format!( + "range ({}..{}) out of bounds for payload len {}", + range.start, + range.end, + payload.len() + ))); + } + + // index conversion + let range_elem = self.range_byte_to_elem2(&range); + let range_poly = self.range_elem_to_poly2(&range_elem); + let start_namespace_byte = self.index_poly_to_byte(range_poly.start); + let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); + + // check args: + // TODO TEMPORARY: forbid requests that span multiple polynomials + if range_poly.len() > 1 { + return Err(VidError::Argument(format!( + "request spans {} polynomials, expect 1", + range_poly.len() + ))); + } + + // return the prefix and suffix elems + let mut elems_iter = + bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) + .take(self.payload_chunk_size); + let prefix: Vec<_> = elems_iter.by_ref().take(offset_elem).collect(); + let suffix: Vec<_> = elems_iter.skip(range_elem.len()).collect(); + Ok(ChunkProof2 { - _prefix_elems: Vec::new(), - _suffix_elems: Vec::new(), - _prefix_bytes: Vec::new(), - _suffix_bytes: Vec::new(), + _prefix_elems: prefix, + _suffix_elems: suffix, }) } @@ -302,8 +334,8 @@ pub struct ChunkProof { pub struct ChunkProof2 { _prefix_elems: Vec, _suffix_elems: Vec, - _prefix_bytes: Vec, - _suffix_bytes: Vec, + // _prefix_bytes: Vec, + // _suffix_bytes: Vec, } impl GenericAdvz @@ -332,16 +364,46 @@ where // range_refine(start, len, compile_time_checks::().0) // } fn index_byte_to_elem(&self, index: usize) -> usize { - index / compile_time_checks::().0 // round down + index_coarsen(index, compile_time_checks::().0) } fn index_elem_to_byte(&self, index: usize) -> usize { - index * compile_time_checks::().0 + index_refine(index, compile_time_checks::().0) } fn index_poly_to_byte(&self, index: usize) -> usize { - index * self.payload_chunk_size * compile_time_checks::().0 + index_refine( + index, + self.payload_chunk_size * compile_time_checks::().0, + ) + } + fn range_byte_to_elem2(&self, range: &Range) -> Range { + range_coarsen2(range, compile_time_checks::().0) + } + fn range_elem_to_poly2(&self, range: &Range) -> Range { + range_coarsen2(range, self.payload_chunk_size) + } +} + +fn range_coarsen2(range: &Range, denominator: usize) -> Range { + let new_start = index_coarsen(range.start, denominator); + let new_end = if range.end <= range.start { + 0 + } else { + index_coarsen(range.end - 1, denominator) + 1 + }; + Range { + start: new_start, + end: new_end, } } +fn index_coarsen(index: usize, denominator: usize) -> usize { + index / denominator +} + +fn index_refine(index: usize, multiplier: usize) -> usize { + index * multiplier +} + fn range_coarsen(start: usize, len: usize, denominator: usize) -> (usize, usize) { let new_start = start / denominator; @@ -365,7 +427,7 @@ mod tests { namespace::Namespacer, }; use ark_bls12_381::Bls12_381; - use ark_std::{println, rand::Rng}; + use ark_std::{ops::Range, println, rand::Rng}; use digest::{generic_array::ArrayLength, OutputSizeUser}; use jf_utils::compile_time_checks; use sha2::Sha256; @@ -453,6 +515,17 @@ mod tests { .unwrap(); let chunk_proof = advz.chunk_proof(&payload, range.1, range.2).unwrap(); + let chunk_proof2 = advz + .chunk_proof2( + payload.as_slice(), + Range { + start: range.1, + end: range.1 + range.2, + }, + ) + .unwrap(); + assert_eq!(chunk_proof.prefix, chunk_proof2._prefix_elems); + assert_eq!(chunk_proof.suffix, chunk_proof2._suffix_elems); advz.chunk_verify( &payload, range.1, From 7054ea4246c6861feb7a62ace7792b1555aaffc7 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 3 Nov 2023 16:29:55 -0400 Subject: [PATCH 26/54] wip ugly debugging for index gymnastics, forbid 0-length data --- primitives/src/vid/advz/namespace.rs | 187 ++++++++++++++++++++++++--- 1 file changed, 169 insertions(+), 18 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 698b33646..b8b8cdafa 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -17,6 +17,7 @@ use crate::{ alloc::string::ToString, vid::{namespace::Namespacer, vid, VidError}, }; +use ark_std::println; use ark_std::{format, ops::Range}; impl Namespacer for GenericAdvz @@ -204,6 +205,14 @@ where where B: AsRef<[u8]>, { + // check args: `range` nonempty + if range.is_empty() { + return Err(VidError::Argument(format!( + "empty range ({}..{})", + range.start, range.end + ))); + } + let payload = payload.as_ref(); // check args: `range` in bounds for `payload` @@ -221,6 +230,7 @@ where let range_poly = self.range_elem_to_poly2(&range_elem); let start_namespace_byte = self.index_poly_to_byte(range_poly.start); let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); + let range_elem_byte = self.range_elem_to_byte2(&range_elem); // check args: // TODO TEMPORARY: forbid requests that span multiple polynomials @@ -236,25 +246,136 @@ where bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) .take(self.payload_chunk_size); let prefix: Vec<_> = elems_iter.by_ref().take(offset_elem).collect(); - let suffix: Vec<_> = elems_iter.skip(range_elem.len()).collect(); + let _elems: Vec<_> = elems_iter.by_ref().take(range_elem.len()).collect(); + // let suffix: Vec<_> = elems_iter.skip(range_elem.len()).collect(); + let suffix: Vec<_> = elems_iter.collect(); + + println!( + "proof:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", + &payload[range_elem_byte.start..range.start], + &payload[range.start..range.end], + &payload[range.end..ark_std::cmp::max(range.end, range_elem_byte.end)] + ); + println!( + "proof poly elems:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", + prefix, _elems, suffix + ); + let poly_elems: Vec<_> = + bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) + .take(self.payload_chunk_size) + .collect(); + println!( + "proof poly elems direct from payload bytes {:?}", + poly_elems + ); Ok(ChunkProof2 { - _prefix_elems: prefix, - _suffix_elems: suffix, + prefix_elems: prefix, + suffix_elems: suffix, + prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), + suffix_bytes: payload[range.end..ark_std::cmp::max(range.end, range_elem_byte.end)] + .to_vec(), + chunk_range: range, }) } fn chunk_verify2( &self, - _chunk: B, - _commit: &Self::Commit, - _common: &Self::Common, + chunk: B, + commit: &Self::Commit, + common: &Self::Common, proof: &Self::ChunkProof2, ) -> VidResult> where B: AsRef<[u8]>, { - let _ = proof; + let chunk = chunk.as_ref(); + + // check args: `chunk` len consistent with `proof` + if chunk.len() != proof.chunk_range.len() { + return Err(VidError::Argument(format!( + "chunk length {} inconsistent with proof length {}", + chunk.len(), + proof.chunk_range.len() + ))); + } + + // index conversion + let range_elem = self.range_byte_to_elem2(&proof.chunk_range); + let range_poly = self.range_elem_to_poly2(&range_elem); + + // check args: + // TODO TEMPORARY: forbid requests that span multiple polynomials + if range_poly.len() > 1 { + return Err(VidError::Argument(format!( + "request spans {} polynomials, expect 1", + range_poly.len() + ))); + } + + // check args: `common` consistent with `commit` + if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { + return Err(VidError::Argument( + "common inconsistent with commit".to_string(), + )); + } + + println!( + "verify:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", + proof.prefix_bytes, chunk, proof.suffix_bytes + ); + let _elems: Vec<_> = bytes_to_field::<_, P::Evaluation>( + proof + .prefix_bytes + .iter() + .chain(chunk) + .chain(proof.suffix_bytes.iter()), + ) + .collect(); + println!( + "verify poly elems:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", + proof.prefix_elems, _elems, proof.suffix_elems + ); + let poly_elems: Vec<_> = proof + .prefix_elems + .iter() + .cloned() + .chain(bytes_to_field::<_, P::Evaluation>( + proof + .prefix_bytes + .iter() + .chain(chunk) + .chain(proof.suffix_bytes.iter()), + )) + .chain(proof.suffix_elems.iter().cloned()) + .collect(); + println!("verify poly elems chained {:?}", poly_elems); + + // rebuild the poly commit, check against `common` + let poly_commit = { + let poly = self.polynomial( + proof + .prefix_elems + .iter() + .cloned() + .chain( + bytes_to_field::<_, P::Evaluation>( + proof + .prefix_bytes + .iter() + .chain(chunk) + .chain(proof.suffix_bytes.iter()), + ) + .take(self.payload_chunk_size), // TODO delete this + ) + .chain(proof.suffix_elems.iter().cloned()), + ); + P::commit(&self.ck, &poly).map_err(vid)? + }; + if poly_commit != common.poly_commits[range_poly.start] { + return Ok(Err(())); + } + Ok(Ok(())) } @@ -332,10 +453,11 @@ pub struct ChunkProof { /// doc pub struct ChunkProof2 { - _prefix_elems: Vec, - _suffix_elems: Vec, - // _prefix_bytes: Vec, - // _suffix_bytes: Vec, + prefix_elems: Vec, + suffix_elems: Vec, + prefix_bytes: Vec, + suffix_bytes: Vec, + chunk_range: Range, } impl GenericAdvz @@ -378,6 +500,15 @@ where fn range_byte_to_elem2(&self, range: &Range) -> Range { range_coarsen2(range, compile_time_checks::().0) } + fn range_elem_to_byte2(&self, range: &Range) -> Range { + range_refine2(range, compile_time_checks::().0) + } + // fn range_poly_to_byte2(&self, range: &Range) -> Range { + // range_refine2( + // range, + // self.payload_chunk_size * compile_time_checks::().0, + // ) + // } fn range_elem_to_poly2(&self, range: &Range) -> Range { range_coarsen2(range, self.payload_chunk_size) } @@ -386,7 +517,7 @@ where fn range_coarsen2(range: &Range, denominator: usize) -> Range { let new_start = index_coarsen(range.start, denominator); let new_end = if range.end <= range.start { - 0 + new_start } else { index_coarsen(range.end - 1, denominator) + 1 }; @@ -396,6 +527,19 @@ fn range_coarsen2(range: &Range, denominator: usize) -> Range { } } +fn range_refine2(range: &Range, multiplier: usize) -> Range { + let new_start = index_refine(range.start, multiplier); + let new_end = if range.end <= range.start { + new_start + } else { + index_refine(range.end, multiplier) + }; + Range { + start: new_start, + end: new_end, + } +} + fn index_coarsen(index: usize, denominator: usize) -> usize { index / denominator } @@ -463,7 +607,6 @@ mod tests { let random_start = random_offset + (namespace * namespace_bytes_len); // len edge cases - edge_cases.push((namespace, random_start, 0)); edge_cases.push((namespace, random_start, 1)); edge_cases.push(( namespace, @@ -475,8 +618,8 @@ mod tests { // start edge cases edge_cases.push((namespace, 0, rng.gen_range(0..namespace_bytes_len))); edge_cases.push((namespace, 1, rng.gen_range(0..namespace_bytes_len - 1))); - edge_cases.push((namespace, namespace_bytes_len - 2, rng.gen_range(0..1))); - edge_cases.push((namespace, namespace_bytes_len - 1, 0)); + edge_cases.push((namespace, namespace_bytes_len - 2, 1)); + edge_cases.push((namespace, namespace_bytes_len - 1, 1)); } edge_cases }; @@ -487,7 +630,7 @@ mod tests { let namespace = rng.gen_range(0..num_polys); let offset = rng.gen_range(0..namespace_bytes_len); let start = offset + (namespace * namespace_bytes_len); - let len = rng.gen_range(0..namespace_bytes_len - offset); + let len = rng.gen_range(1..namespace_bytes_len - offset); random_cases.push((namespace, start, len)); } random_cases @@ -524,8 +667,8 @@ mod tests { }, ) .unwrap(); - assert_eq!(chunk_proof.prefix, chunk_proof2._prefix_elems); - assert_eq!(chunk_proof.suffix, chunk_proof2._suffix_elems); + assert_eq!(chunk_proof.prefix, chunk_proof2.prefix_elems); + assert_eq!(chunk_proof.suffix, chunk_proof2.suffix_elems); advz.chunk_verify( &payload, range.1, @@ -536,6 +679,14 @@ mod tests { ) .unwrap() .unwrap(); + advz.chunk_verify2( + &payload.as_slice()[range.1..range.1 + range.2], + &d.commit, + &d.common, + &chunk_proof2, + ) + .unwrap() + .unwrap(); } } From 76998f790aa6cc048cdab426e52baff58114dd2e Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 3 Nov 2023 16:51:50 -0400 Subject: [PATCH 27/54] tighten checks in chunk_xxx2() --- primitives/src/vid/advz/namespace.rs | 55 ++++++++++++---------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index b8b8cdafa..2c8bb5ee9 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -216,7 +216,7 @@ where let payload = payload.as_ref(); // check args: `range` in bounds for `payload` - if range.start >= payload.len() || range.end > payload.len() { + if range.end > payload.len() { return Err(VidError::Argument(format!( "range ({}..{}) out of bounds for payload len {}", range.start, @@ -234,7 +234,7 @@ where // check args: // TODO TEMPORARY: forbid requests that span multiple polynomials - if range_poly.len() > 1 { + if range_poly.len() != 1 { return Err(VidError::Argument(format!( "request spans {} polynomials, expect 1", range_poly.len() @@ -254,7 +254,7 @@ where "proof:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", &payload[range_elem_byte.start..range.start], &payload[range.start..range.end], - &payload[range.end..ark_std::cmp::max(range.end, range_elem_byte.end)] + &payload[range.end..range_elem_byte.end] ); println!( "proof poly elems:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", @@ -273,8 +273,7 @@ where prefix_elems: prefix, suffix_elems: suffix, prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), - suffix_bytes: payload[range.end..ark_std::cmp::max(range.end, range_elem_byte.end)] - .to_vec(), + suffix_bytes: payload[range.end..range_elem_byte.end].to_vec(), chunk_range: range, }) } @@ -291,6 +290,11 @@ where { let chunk = chunk.as_ref(); + // check args: `chunk` nonempty + if chunk.is_empty() { + return Err(VidError::Argument("empty chunk".to_string())); + } + // check args: `chunk` len consistent with `proof` if chunk.len() != proof.chunk_range.len() { return Err(VidError::Argument(format!( @@ -306,7 +310,7 @@ where // check args: // TODO TEMPORARY: forbid requests that span multiple polynomials - if range_poly.len() > 1 { + if range_poly.len() != 1 { return Err(VidError::Argument(format!( "request spans {} polynomials, expect 1", range_poly.len() @@ -358,16 +362,13 @@ where .prefix_elems .iter() .cloned() - .chain( - bytes_to_field::<_, P::Evaluation>( - proof - .prefix_bytes - .iter() - .chain(chunk) - .chain(proof.suffix_bytes.iter()), - ) - .take(self.payload_chunk_size), // TODO delete this - ) + .chain(bytes_to_field::<_, P::Evaluation>( + proof + .prefix_bytes + .iter() + .chain(chunk) + .chain(proof.suffix_bytes.iter()), + )) .chain(proof.suffix_elems.iter().cloned()), ); P::commit(&self.ck, &poly).map_err(vid)? @@ -515,28 +516,18 @@ where } fn range_coarsen2(range: &Range, denominator: usize) -> Range { - let new_start = index_coarsen(range.start, denominator); - let new_end = if range.end <= range.start { - new_start - } else { - index_coarsen(range.end - 1, denominator) + 1 - }; + assert!(!range.is_empty(), "{:?}", range); Range { - start: new_start, - end: new_end, + start: index_coarsen(range.start, denominator), + end: index_coarsen(range.end - 1, denominator) + 1, } } fn range_refine2(range: &Range, multiplier: usize) -> Range { - let new_start = index_refine(range.start, multiplier); - let new_end = if range.end <= range.start { - new_start - } else { - index_refine(range.end, multiplier) - }; + assert!(!range.is_empty(), "{:?}", range); Range { - start: new_start, - end: new_end, + start: index_refine(range.start, multiplier), + end: index_refine(range.end, multiplier), } } From 373602b7d0b1804b01ee1ec77eede715af817ae4 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 3 Nov 2023 18:03:56 -0400 Subject: [PATCH 28/54] tidy test --- primitives/src/vid/advz/namespace.rs | 241 ++++++++++++--------------- 1 file changed, 102 insertions(+), 139 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 2c8bb5ee9..0aa303a74 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -17,7 +17,7 @@ use crate::{ alloc::string::ToString, vid::{namespace::Namespacer, vid, VidError}, }; -use ark_std::println; +// use ark_std::println; use ark_std::{format, ops::Range}; impl Namespacer for GenericAdvz @@ -241,33 +241,12 @@ where ))); } - // return the prefix and suffix elems + // compute the prefix and suffix elems let mut elems_iter = bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) .take(self.payload_chunk_size); let prefix: Vec<_> = elems_iter.by_ref().take(offset_elem).collect(); - let _elems: Vec<_> = elems_iter.by_ref().take(range_elem.len()).collect(); - // let suffix: Vec<_> = elems_iter.skip(range_elem.len()).collect(); - let suffix: Vec<_> = elems_iter.collect(); - - println!( - "proof:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", - &payload[range_elem_byte.start..range.start], - &payload[range.start..range.end], - &payload[range.end..range_elem_byte.end] - ); - println!( - "proof poly elems:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", - prefix, _elems, suffix - ); - let poly_elems: Vec<_> = - bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) - .take(self.payload_chunk_size) - .collect(); - println!( - "proof poly elems direct from payload bytes {:?}", - poly_elems - ); + let suffix: Vec<_> = elems_iter.skip(range_elem.len()).collect(); Ok(ChunkProof2 { prefix_elems: prefix, @@ -305,8 +284,7 @@ where } // index conversion - let range_elem = self.range_byte_to_elem2(&proof.chunk_range); - let range_poly = self.range_elem_to_poly2(&range_elem); + let range_poly = self.range_byte_to_poly2(&proof.chunk_range); // check args: // TODO TEMPORARY: forbid requests that span multiple polynomials @@ -324,37 +302,6 @@ where )); } - println!( - "verify:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", - proof.prefix_bytes, chunk, proof.suffix_bytes - ); - let _elems: Vec<_> = bytes_to_field::<_, P::Evaluation>( - proof - .prefix_bytes - .iter() - .chain(chunk) - .chain(proof.suffix_bytes.iter()), - ) - .collect(); - println!( - "verify poly elems:\n\tprefix {:?}\n\tchunk {:?}\n\tsuffix {:?}", - proof.prefix_elems, _elems, proof.suffix_elems - ); - let poly_elems: Vec<_> = proof - .prefix_elems - .iter() - .cloned() - .chain(bytes_to_field::<_, P::Evaluation>( - proof - .prefix_bytes - .iter() - .chain(chunk) - .chain(proof.suffix_bytes.iter()), - )) - .chain(proof.suffix_elems.iter().cloned()) - .collect(); - println!("verify poly elems chained {:?}", poly_elems); - // rebuild the poly commit, check against `common` let poly_commit = { let poly = self.polynomial( @@ -513,6 +460,12 @@ where fn range_elem_to_poly2(&self, range: &Range) -> Range { range_coarsen2(range, self.payload_chunk_size) } + fn range_byte_to_poly2(&self, range: &Range) -> Range { + range_coarsen2( + range, + self.payload_chunk_size * compile_time_checks::().0, + ) + } } fn range_coarsen2(range: &Range, denominator: usize) -> Range { @@ -580,6 +533,7 @@ mod tests { // more items as a function of the above let payload_elems_len = num_polys * payload_chunk_size; let payload_bytes_len = payload_elems_len * compile_time_checks::().0; + let poly_bytes_len = payload_chunk_size * compile_time_checks::().0; let mut rng = jf_utils::test_rng(); let payload = init_random_payload(payload_bytes_len, &mut rng); let srs = init_srs(payload_elems_len, &mut rng); @@ -588,96 +542,105 @@ mod tests { let d = advz.disperse(&payload).unwrap(); // TEST: prove data ranges for this paylaod - // it takes too long to test all combos of (namespace, start, len) + // it takes too long to test all combos of (polynomial, start, len) // so do some edge cases and random cases - let namespace_bytes_len = payload_chunk_size * compile_time_checks::().0; - let edge_cases = { - let mut edge_cases = Vec::new(); - for namespace in 0..num_polys { - let random_offset = rng.gen_range(0..namespace_bytes_len); - let random_start = random_offset + (namespace * namespace_bytes_len); - - // len edge cases - edge_cases.push((namespace, random_start, 1)); - edge_cases.push(( - namespace, - random_start, - namespace_bytes_len - random_offset - 1, - )); - edge_cases.push((namespace, random_start, namespace_bytes_len - random_offset)); - - // start edge cases - edge_cases.push((namespace, 0, rng.gen_range(0..namespace_bytes_len))); - edge_cases.push((namespace, 1, rng.gen_range(0..namespace_bytes_len - 1))); - edge_cases.push((namespace, namespace_bytes_len - 2, 1)); - edge_cases.push((namespace, namespace_bytes_len - 1, 1)); - } - edge_cases - }; + let edge_cases = vec![ + Range { start: 0, end: 1 }, + Range { start: 0, end: 2 }, + Range { + start: 0, + end: poly_bytes_len - 1, + }, + Range { + start: 0, + end: poly_bytes_len, + }, + Range { start: 1, end: 2 }, + Range { start: 1, end: 3 }, + Range { + start: 1, + end: poly_bytes_len - 1, + }, + Range { + start: 1, + end: poly_bytes_len, + }, + Range { + start: poly_bytes_len - 2, + end: poly_bytes_len - 1, + }, + Range { + start: poly_bytes_len - 2, + end: poly_bytes_len, + }, + Range { + start: poly_bytes_len - 1, + end: poly_bytes_len, + }, + ]; let random_cases = { let num_cases = edge_cases.len(); let mut random_cases = Vec::with_capacity(num_cases); for _ in 0..num_cases { - let namespace = rng.gen_range(0..num_polys); - let offset = rng.gen_range(0..namespace_bytes_len); - let start = offset + (namespace * namespace_bytes_len); - let len = rng.gen_range(1..namespace_bytes_len - offset); - random_cases.push((namespace, start, len)); + let start = rng.gen_range(0..poly_bytes_len - 1); + let end = rng.gen_range(start + 1..poly_bytes_len); + random_cases.push(Range { start, end }); } random_cases }; - - for (i, range) in edge_cases.iter().chain(random_cases.iter()).enumerate() { - println!( - "case {}/{}: namespace {}, start {}, len {}", - i, - edge_cases.len() + random_cases.len(), - range.0, - range.1, - range.2 - ); - let data_proof = advz.data_proof(&payload, range.1, range.2).unwrap(); - advz.data_verify( - &payload, - range.1, - range.2, - &d.commit, - &d.common, - &data_proof, - ) - .unwrap() - .unwrap(); - - let chunk_proof = advz.chunk_proof(&payload, range.1, range.2).unwrap(); - let chunk_proof2 = advz - .chunk_proof2( - payload.as_slice(), - Range { - start: range.1, - end: range.1 + range.2, - }, - ) - .unwrap(); - assert_eq!(chunk_proof.prefix, chunk_proof2.prefix_elems); - assert_eq!(chunk_proof.suffix, chunk_proof2.suffix_elems); - advz.chunk_verify( - &payload, - range.1, - range.2, - &d.commit, - &d.common, - &chunk_proof, - ) - .unwrap() - .unwrap(); - advz.chunk_verify2( - &payload.as_slice()[range.1..range.1 + range.2], - &d.commit, - &d.common, - &chunk_proof2, - ) - .unwrap() - .unwrap(); + let all_cases = [(edge_cases, "edge"), (random_cases, "random")]; + + for poly in 0..num_polys { + let poly_offset = poly * poly_bytes_len; + + for cases in all_cases.iter() { + for range in cases.0.iter() { + let range = Range { + start: range.start + poly_offset, + end: range.end + poly_offset, + }; + println!("{} case: {:?}", cases.1, range); + + let data_proof = advz.data_proof(&payload, range.start, range.len()).unwrap(); + advz.data_verify( + &payload, + range.start, + range.len(), + &d.commit, + &d.common, + &data_proof, + ) + .unwrap() + .unwrap(); + + let chunk_proof = advz + .chunk_proof(&payload, range.start, range.len()) + .unwrap(); + let chunk_proof2 = advz + .chunk_proof2(payload.as_slice(), range.clone()) + .unwrap(); + assert_eq!(chunk_proof.prefix, chunk_proof2.prefix_elems); + assert_eq!(chunk_proof.suffix, chunk_proof2.suffix_elems); + advz.chunk_verify( + &payload, + range.start, + range.len(), + &d.commit, + &d.common, + &chunk_proof, + ) + .unwrap() + .unwrap(); + advz.chunk_verify2( + &payload.as_slice()[range.clone()], + &d.commit, + &d.common, + &chunk_proof2, + ) + .unwrap() + .unwrap(); + } + } } } From 59376a6ca7a0cbb3bd85cd266a98a1a9c74f5b30 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 3 Nov 2023 21:22:29 -0400 Subject: [PATCH 29/54] wip data_xxx2 methods, untested --- primitives/src/vid/advz/namespace.rs | 178 +++++++++++++++++++++++++++ primitives/src/vid/namespace.rs | 19 +++ 2 files changed, 197 insertions(+) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 0aa303a74..637ae79d8 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -36,6 +36,7 @@ where // TODO should be P::Proof, not Vec // https://github.com/EspressoSystems/jellyfish/issues/387 type DataProof = Vec; + type DataProof2 = DataProof2; type ChunkProof = ChunkProof; type ChunkProof2 = ChunkProof2; @@ -91,6 +92,175 @@ where Ok(proofs) } + fn data_proof2(&self, payload: B, range: Range) -> VidResult + where + B: AsRef<[u8]>, + { + // TODO refactor copied arg check code + + // check args: `range` nonempty + if range.is_empty() { + return Err(VidError::Argument(format!( + "empty range ({}..{})", + range.start, range.end + ))); + } + + let payload = payload.as_ref(); + + // check args: `range` in bounds for `payload` + if range.end > payload.len() { + return Err(VidError::Argument(format!( + "range ({}..{}) out of bounds for payload len {}", + range.start, + range.end, + payload.len() + ))); + } + + // index conversion + let range_elem = self.range_byte_to_elem2(&range); + let range_poly = self.range_elem_to_poly2(&range_elem); + let start_namespace_byte = self.index_poly_to_byte(range_poly.start); + let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); + let range_elem_byte = self.range_elem_to_byte2(&range_elem); + + // check args: + // TODO TEMPORARY: forbid requests that span multiple polynomials + if range_poly.len() != 1 { + return Err(VidError::Argument(format!( + "request spans {} polynomials, expect 1", + range_poly.len() + ))); + } + + // grab the `start_namespace`th polynomial + let polynomial = self.polynomial( + bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) + .take(self.payload_chunk_size), + ); + + // prepare the list of input points + // TODO perf: can't avoid use of `skip` + let points: Vec<_> = { + self.eval_domain + .elements() + .skip(offset_elem) + .take(range_elem.len()) + .collect() + }; + + let (proofs, _evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; + + Ok(DataProof2 { + proofs, + // TODO refactor copied code for prefix/suffix bytes + prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), + suffix_bytes: payload[range.end..range_elem_byte.end].to_vec(), + chunk_range: range, + }) + } + + fn data_verify2( + &self, + chunk: B, + commit: &Self::Commit, + common: &Self::Common, + proof: &Self::DataProof2, + ) -> VidResult> + where + B: AsRef<[u8]>, + { + let chunk = chunk.as_ref(); + + // TODO refactor copied arg check code + + // check args: `chunk` nonempty + if chunk.is_empty() { + return Err(VidError::Argument("empty chunk".to_string())); + } + + // check args: `chunk` len consistent with `proof` + if chunk.len() != proof.chunk_range.len() { + return Err(VidError::Argument(format!( + "chunk length {} inconsistent with proof length {}", + chunk.len(), + proof.chunk_range.len() + ))); + } + + // index conversion + + // let (start_elem, len_elem) = self.range_byte_to_elem(start, len); + // let (start_namespace, _len_namespace) = self.range_elem_to_poly(start_elem, len_elem); + // let start_namespace_byte = self.index_poly_to_byte(start_namespace); + + let range_elem = self.range_byte_to_elem2(&proof.chunk_range); + let range_poly = self.range_elem_to_poly2(&range_elem); + let start_namespace_byte = self.index_poly_to_byte(range_poly.start); + let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); + + // check args: + // TODO TEMPORARY: forbid requests that span multiple polynomials + if range_poly.len() != 1 { + return Err(VidError::Argument(format!( + "request spans {} polynomials, expect 1", + range_poly.len() + ))); + } + + // check args: `common` consistent with `commit` + if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { + return Err(VidError::Argument( + "common inconsistent with commit".to_string(), + )); + } + + // prepare list of data elems + // TODO refactor copied code + let data_elems: Vec<_> = bytes_to_field::<_, P::Evaluation>( + proof + .prefix_bytes + .iter() + .chain(chunk) + .chain(proof.suffix_bytes.iter()), + ) + .collect(); + + // prepare list of input points + // TODO perf: can't avoid use of `skip` + // TODO refactor copied code + let points: Vec<_> = { + self.eval_domain + .elements() + .skip(offset_elem) + .take(range_elem.len()) + .collect() + }; + + // verify proof + // TODO naive verify for multi_open + // https://github.com/EspressoSystems/jellyfish/issues/387 + if data_elems.len() != proof.proofs.len() { + return Err(VidError::Argument(format!( + "data len {} differs from proof len {}", + data_elems.len(), + proof.proofs.len() + ))); + } + assert_eq!(data_elems.len(), points.len()); // sanity + let poly_commit = &common.poly_commits[range_poly.start]; + for (point, (elem, pf)) in points + .iter() + .zip(data_elems.iter().zip(proof.proofs.iter())) + { + if !P::verify(&self.vk, poly_commit, point, elem, pf).map_err(vid)? { + return Ok(Err(())); + } + } + Ok(Ok(())) + } + fn data_verify( &self, payload: &Self::Payload, @@ -393,6 +563,14 @@ where } } +///doc +pub struct DataProof2

{ + proofs: Vec

, + prefix_bytes: Vec, + suffix_bytes: Vec, + chunk_range: Range, +} + /// doc pub struct ChunkProof { prefix: Vec, diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index a4eee84ff..47d6b0b4a 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -27,6 +27,9 @@ pub trait Namespacer: VidScheme { /// data proof type DataProof; + ///doc + type DataProof2; + /// chunk proof type ChunkProof; @@ -43,6 +46,22 @@ pub trait Namespacer: VidScheme { len: usize, ) -> VidResult; + ///doc + fn data_proof2(&self, payload: B, range: Range) -> VidResult + where + B: AsRef<[u8]>; + + /// doc + fn data_verify2( + &self, + chunk: B, + commit: &Self::Commit, + common: &Self::Common, + proof: &Self::DataProof2, + ) -> VidResult> + where + B: AsRef<[u8]>; + /// Verify a proof for `payload` for data index range `start..start+len-1`. /// /// See TODO in `namespace_verify` on `payload`. From 9bc5d33346d20a2125587e6b3d9a05ebe5b17519 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Sun, 5 Nov 2023 13:41:35 -0500 Subject: [PATCH 30/54] test pass for data_verify2 (yay) --- primitives/src/vid/advz/namespace.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 637ae79d8..9196c0a1b 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -766,7 +766,7 @@ mod tests { } random_cases }; - let all_cases = [(edge_cases, "edge"), (random_cases, "random")]; + let all_cases = [(edge_cases, "edge"), (random_cases, "rand")]; for poly in 0..num_polys { let poly_offset = poly * poly_bytes_len; @@ -777,7 +777,7 @@ mod tests { start: range.start + poly_offset, end: range.end + poly_offset, }; - println!("{} case: {:?}", cases.1, range); + println!("poly {} {} case: {:?}", poly, cases.1, range); let data_proof = advz.data_proof(&payload, range.start, range.len()).unwrap(); advz.data_verify( @@ -791,6 +791,16 @@ mod tests { .unwrap() .unwrap(); + let data_proof2 = advz.data_proof2(payload.as_slice(), range.clone()).unwrap(); + advz.data_verify2( + &payload.as_slice()[range.clone()], + &d.commit, + &d.common, + &data_proof2, + ) + .unwrap() + .unwrap(); + let chunk_proof = advz .chunk_proof(&payload, range.start, range.len()) .unwrap(); From 0ea565ce4477871993b36cc94db61f8ee8be6b55 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Sun, 5 Nov 2023 13:57:29 -0500 Subject: [PATCH 31/54] remove old [data|chunk]_[proof|verify] methods --- primitives/src/vid/advz/namespace.rs | 288 +-------------------------- primitives/src/vid/namespace.rs | 54 ----- 2 files changed, 1 insertion(+), 341 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 9196c0a1b..8800b91b8 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -35,63 +35,10 @@ where { // TODO should be P::Proof, not Vec // https://github.com/EspressoSystems/jellyfish/issues/387 - type DataProof = Vec; type DataProof2 = DataProof2; - type ChunkProof = ChunkProof; type ChunkProof2 = ChunkProof2; - fn data_proof( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - ) -> VidResult { - // check args: `start`, `len` in bounds for `payload` - if start + len > payload.as_slice().len() { - return Err(VidError::Argument(format!( - "start {} + len {} out of bounds for payload {}", - start, - len, - payload.as_slice().len() - ))); - } - - // index conversion - let (start_elem, len_elem) = self.range_byte_to_elem(start, len); - let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); - let start_namespace_byte = self.index_poly_to_byte(start_namespace); - - // check args: - // TODO TEMPORARY: forbid requests that span multiple polynomials - if len_namespace > 1 { - return Err(VidError::Argument(format!( - "request spans {} polynomials, expect 1", - len_namespace - ))); - } - - // grab the `start_namespace`th polynomial - let polynomial = self.polynomial( - bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_namespace_byte..].iter()) - .take(self.payload_chunk_size), - ); - - // prepare the list of input points - // TODO perf: can't avoid use of `skip` - let points: Vec<_> = { - let offset = start_elem - self.index_byte_to_elem(start_namespace_byte); - self.eval_domain - .elements() - .skip(offset) - .take(len_elem) - .collect() - }; - - let (proofs, _evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; - Ok(proofs) - } - fn data_proof2(&self, payload: B, range: Range) -> VidResult where B: AsRef<[u8]>, @@ -261,116 +208,6 @@ where Ok(Ok(())) } - fn data_verify( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - commit: &Self::Commit, - common: &Self::Common, - proof: &Self::DataProof, - ) -> VidResult> { - // check args: `start`, `len` in bounds for `payload` - if start + len > payload.as_slice().len() { - return Err(VidError::Argument(format!( - "start {} + len {} out of bounds for payload {}", - start, - len, - payload.as_slice().len() - ))); - } - - // check args: `common` consistent with `commit` - if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { - return Err(VidError::Argument( - "common inconsistent with commit".to_string(), - )); - } - - // index conversion - let (start_elem, len_elem) = self.range_byte_to_elem(start, len); - let (start_namespace, _len_namespace) = self.range_elem_to_poly(start_elem, len_elem); - let start_namespace_byte = self.index_poly_to_byte(start_namespace); - - // prepare list of data elems - let start_elem_byte = self.index_elem_to_byte(start_elem); - let data_elems: Vec<_> = - bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_elem_byte..].iter()) - .take(len_elem) - .collect(); - - // prepare list of input points - // TODO perf: can't avoid use of `skip` - let points: Vec<_> = { - let offset = start_elem - self.index_byte_to_elem(start_namespace_byte); - self.eval_domain - .elements() - .skip(offset) - .take(len_elem) - .collect() - }; - - // verify proof - // TODO naive verify for multi_open - // https://github.com/EspressoSystems/jellyfish/issues/387 - if data_elems.len() != proof.len() { - return Err(VidError::Argument(format!( - "data len {} differs from proof len {}", - data_elems.len(), - proof.len() - ))); - } - assert_eq!(data_elems.len(), points.len()); // sanity - let poly_commit = &common.poly_commits[start_namespace]; - for (point, (elem, pf)) in points.iter().zip(data_elems.iter().zip(proof.iter())) { - if !P::verify(&self.vk, poly_commit, point, elem, pf).map_err(vid)? { - return Ok(Err(())); - } - } - Ok(Ok(())) - } - - fn chunk_proof( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - ) -> VidResult { - // check args: `start`, `len` in bounds for `payload` - if start + len > payload.as_slice().len() { - return Err(VidError::Argument(format!( - "start {} + len {} out of bounds for payload {}", - start, - len, - payload.as_slice().len() - ))); - } - - // index conversion - let (start_elem, len_elem) = self.range_byte_to_elem(start, len); - let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); - let start_namespace_byte = self.index_poly_to_byte(start_namespace); - let offset_elem = start_elem - self.index_byte_to_elem(start_namespace_byte); - - // check args: - // TODO TEMPORARY: forbid requests that span multiple polynomials - if len_namespace > 1 { - return Err(VidError::Argument(format!( - "request spans {} polynomials, expect 1", - len_namespace - ))); - } - - // return the prefix and suffix elems - let mut elems_iter = - bytes_to_field::<_, P::Evaluation>(payload.as_slice()[start_namespace_byte..].iter()) - .take(self.payload_chunk_size); - let prefix: Vec<_> = elems_iter.by_ref().take(offset_elem).collect(); - let suffix: Vec<_> = elems_iter.skip(len_elem).collect(); - - Ok(ChunkProof { prefix, suffix }) - } - fn chunk_proof2(&self, payload: B, range: Range) -> VidResult where B: AsRef<[u8]>, @@ -496,71 +333,6 @@ where Ok(Ok(())) } - - fn chunk_verify( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - commit: &Self::Commit, - common: &Self::Common, - proof: &Self::ChunkProof, - ) -> VidResult> { - // check args: `start`, `len` in bounds for `payload` - if start + len > payload.as_slice().len() { - return Err(VidError::Argument(format!( - "start {} + len {} out of bounds for payload {}", - start, - len, - payload.as_slice().len() - ))); - } - - // index conversion - let (start_elem, len_elem) = self.range_byte_to_elem(start, len); - let (start_namespace, len_namespace) = self.range_elem_to_poly(start_elem, len_elem); - let start_elem_byte = self.index_elem_to_byte(start_elem); - // let offset_elem = start_elem - self.index_byte_to_elem(start_namespace_byte); - - // check args: - // TODO TEMPORARY: forbid requests that span multiple polynomials - if len_namespace > 1 { - return Err(VidError::Argument(format!( - "request spans {} polynomials, expect 1", - len_namespace - ))); - } - - // check args: `common` consistent with `commit` - if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { - return Err(VidError::Argument( - "common inconsistent with commit".to_string(), - )); - } - - // rebuild the `namespace_index`th poly commit, check against `common` - let poly_commit = { - let poly = self.polynomial( - proof - .prefix - .iter() - .cloned() - .chain( - bytes_to_field::<_, P::Evaluation>( - payload.as_slice()[start_elem_byte..].iter(), - ) - .take(self.payload_chunk_size), - ) - .chain(proof.suffix.iter().cloned()), - ); - P::commit(&self.ck, &poly).map_err(vid)? - }; - if poly_commit != common.poly_commits[start_namespace] { - return Ok(Err(())); - } - - Ok(Ok(())) - } } ///doc @@ -571,12 +343,6 @@ pub struct DataProof2

{ chunk_range: Range, } -/// doc -pub struct ChunkProof { - prefix: Vec, - suffix: Vec, -} - /// doc pub struct ChunkProof2 { prefix_elems: Vec, @@ -602,19 +368,10 @@ where // lots of index manipulation. // with infinite dev time we should implement type-safe indices to preclude // index-misuse bugs. - fn range_byte_to_elem(&self, start: usize, len: usize) -> (usize, usize) { - range_coarsen(start, len, compile_time_checks::().0) - } - fn range_elem_to_poly(&self, start: usize, len: usize) -> (usize, usize) { - range_coarsen(start, len, self.payload_chunk_size) - } - // fn range_elem_to_byte(&self, start: usize, len: usize) -> (usize, usize) { - // range_refine(start, len, compile_time_checks::().0) - // } fn index_byte_to_elem(&self, index: usize) -> usize { index_coarsen(index, compile_time_checks::().0) } - fn index_elem_to_byte(&self, index: usize) -> usize { + fn _index_elem_to_byte(&self, index: usize) -> usize { index_refine(index, compile_time_checks::().0) } fn index_poly_to_byte(&self, index: usize) -> usize { @@ -670,22 +427,6 @@ fn index_refine(index: usize, multiplier: usize) -> usize { index * multiplier } -fn range_coarsen(start: usize, len: usize, denominator: usize) -> (usize, usize) { - let new_start = start / denominator; - - // underflow occurs if len is 0, so handle this case separately - if len == 0 { - return (new_start, 0); - } - - let new_end = (start + len - 1) / denominator; - (new_start, new_end - new_start + 1) -} - -// fn range_refine(start: usize, len: usize, multiplier: usize) -> (usize, usize) { -// (start * multiplier, len * multiplier) -// } - #[cfg(test)] mod tests { use crate::vid::{ @@ -779,18 +520,6 @@ mod tests { }; println!("poly {} {} case: {:?}", poly, cases.1, range); - let data_proof = advz.data_proof(&payload, range.start, range.len()).unwrap(); - advz.data_verify( - &payload, - range.start, - range.len(), - &d.commit, - &d.common, - &data_proof, - ) - .unwrap() - .unwrap(); - let data_proof2 = advz.data_proof2(payload.as_slice(), range.clone()).unwrap(); advz.data_verify2( &payload.as_slice()[range.clone()], @@ -801,24 +530,9 @@ mod tests { .unwrap() .unwrap(); - let chunk_proof = advz - .chunk_proof(&payload, range.start, range.len()) - .unwrap(); let chunk_proof2 = advz .chunk_proof2(payload.as_slice(), range.clone()) .unwrap(); - assert_eq!(chunk_proof.prefix, chunk_proof2.prefix_elems); - assert_eq!(chunk_proof.suffix, chunk_proof2.suffix_elems); - advz.chunk_verify( - &payload, - range.start, - range.len(), - &d.commit, - &d.common, - &chunk_proof, - ) - .unwrap() - .unwrap(); advz.chunk_verify2( &payload.as_slice()[range.clone()], &d.commit, diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index 47d6b0b4a..f0e2b3a2f 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -24,28 +24,12 @@ use ark_std::ops::Range; /// Namespace functionality for [`VidScheme`]. pub trait Namespacer: VidScheme { - /// data proof - type DataProof; - ///doc type DataProof2; - /// chunk proof - type ChunkProof; - /// doc type ChunkProof2; - /// Compute a proof for `payload` for data index range `start..start+len-1`. - /// - /// TODO explain how this differs from `chunk_proof` - fn data_proof( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - ) -> VidResult; - ///doc fn data_proof2(&self, payload: B, range: Range) -> VidResult where @@ -62,27 +46,6 @@ pub trait Namespacer: VidScheme { where B: AsRef<[u8]>; - /// Verify a proof for `payload` for data index range `start..start+len-1`. - /// - /// See TODO in `namespace_verify` on `payload`. - fn data_verify( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - commit: &Self::Commit, - common: &Self::Common, - proof: &Self::DataProof, - ) -> VidResult>; - - /// Compute a proof for `payload` for data index range `start..start+len-1`. - fn chunk_proof( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - ) -> VidResult; - /// doc fn chunk_proof2(&self, payload: B, range: Range) -> VidResult where @@ -98,21 +61,4 @@ pub trait Namespacer: VidScheme { ) -> VidResult> where B: AsRef<[u8]>; - - /// Verify the `payload` namespace indexed by `namespace_index` against - /// `commit`, `common`. - /// - /// TODO: We prefer not to include the whole `payload`. But the namespace - /// proof needs a few payload bytes from outside the namespace. In the - /// future `payload` should be replaced by a payload subset that includes - /// only the bytes needed to verify a namespace. - fn chunk_verify( - &self, - payload: &Self::Payload, - start: usize, - len: usize, - commit: &Self::Commit, - common: &Self::Common, - proof: &Self::ChunkProof, - ) -> VidResult>; } From 054ae180d33d4745e3c80ec01b3e07fa65f7908b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Sun, 5 Nov 2023 14:25:59 -0500 Subject: [PATCH 32/54] remove unneeded Payload struct --- primitives/src/vid.rs | 19 ++++------- primitives/src/vid/advz.rs | 52 ++++++++++++++++-------------- primitives/src/vid/advz/payload.rs | 50 ---------------------------- primitives/tests/advz.rs | 3 +- primitives/tests/vid/mod.rs | 9 +++--- 5 files changed, 40 insertions(+), 93 deletions(-) delete mode 100644 primitives/src/vid/advz/payload.rs diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index 9f79ce6c4..35b26b5cb 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -13,9 +13,6 @@ use serde::{Deserialize, Serialize}; /// VID: Verifiable Information Dispersal pub trait VidScheme { - /// Payload - type Payload: Debug + PartialEq; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 - /// Payload commitment. type Commit: Clone + Debug + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 @@ -26,10 +23,14 @@ pub trait VidScheme { type Common: CanonicalSerialize + CanonicalDeserialize + Clone + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 /// Compute a payload commitment - fn commit_only(&self, payload: &Self::Payload) -> VidResult; + fn commit_only(&self, payload: B) -> VidResult + where + B: AsRef<[u8]>; /// Compute shares to send to the storage nodes - fn disperse(&self, payload: &Self::Payload) -> VidResult>; + fn disperse(&self, payload: B) -> VidResult> + where + B: AsRef<[u8]>; /// Verify a share. Used by both storage node and retrieval client. /// Why is return type a nested `Result`? See @@ -42,13 +43,7 @@ pub trait VidScheme { /// Recover payload from shares. /// Do not verify shares or check recovered payload against anything. - /// - /// TODO: return type `Payload`? - fn recover_payload( - &self, - shares: &[Self::Share], - common: &Self::Common, - ) -> VidResult; + fn recover_payload(&self, shares: &[Self::Share], common: &Self::Common) -> VidResult>; } /// Convenience struct to aggregate disperse data. diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index b4fa0b78e..c7e939d77 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -8,7 +8,6 @@ //! //! `advz` named for the authors Alhaddad-Duan-Varia-Zhang. -use self::payload::Payload; use super::{vid, VidDisperse, VidError, VidResult, VidScheme}; use crate::{ merkle_tree::{hasher::HasherMerkleTree, MerkleCommitment, MerkleTreeScheme}, @@ -44,7 +43,6 @@ use jf_utils::{bytes_to_field, canonical, field_to_bytes}; use serde::{Deserialize, Serialize}; pub mod namespace; -pub mod payload; /// The [ADVZ VID scheme](https://eprint.iacr.org/2021/1500), a concrete impl for [`VidScheme`]. /// @@ -199,9 +197,13 @@ where type Commit = Output; type Share = Share; type Common = Common; - type Payload = Payload; - fn commit_only(&self, payload: &Self::Payload) -> VidResult { + fn commit_only(&self, payload: B) -> VidResult + where + B: AsRef<[u8]>, + { + let payload = payload.as_ref(); + // Can't use `Self::poly_commits_hash()`` here because `P::commit()`` returns // `Result`` instead of `P::Commitment`. // There's probably an idiomatic way to do this using eg. @@ -218,17 +220,21 @@ where Ok(hasher.finalize()) } - fn disperse(&self, payload: &Self::Payload) -> VidResult> { + fn disperse(&self, payload: B) -> VidResult> + where + B: AsRef<[u8]>, + { + let payload = payload.as_ref(); + let payload_len = payload.len(); let disperse_time = start_timer!(|| format!( - "VID disperse {} chunks to {} nodes", - self.payload_chunk_size, self.num_storage_nodes + "VID disperse {} payload bytes to {} nodes", + payload_len, self.num_storage_nodes )); - let bytes_len = payload.as_slice().len(); // partition payload into polynomial coefficients // and count `elems_len` for later let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let elems_iter = bytes_to_field::<_, P::Evaluation>(payload).map(|elem| *elem.borrow()); + let elems_iter = bytes_to_field::<_, P::Evaluation>(payload); let mut polys = Vec::new(); for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { polys.push(self.polynomial(coeffs_iter)); @@ -295,7 +301,7 @@ where .collect::>() .map_err(vid)?, all_evals_digest: all_evals_commit.commitment().digest(), - bytes_len, + bytes_len: payload_len, }; end_timer!(common_timer); @@ -416,11 +422,7 @@ where .ok_or(())) } - fn recover_payload( - &self, - shares: &[Self::Share], - common: &Self::Common, - ) -> VidResult { + fn recover_payload(&self, shares: &[Self::Share], common: &Self::Common) -> VidResult> { if shares.len() < self.payload_chunk_size { return Err(VidError::Argument(format!( "not enough shares {}, expected at least {}", @@ -449,8 +451,8 @@ where ))); } - let result_len = num_polys * self.payload_chunk_size; - let mut result = Vec::with_capacity(result_len); + let elems_capacity = num_polys * self.payload_chunk_size; + let mut elems = Vec::with_capacity(elems_capacity); for i in 0..num_polys { let mut coeffs = reed_solomon_erasure_decode_rou( shares.iter().map(|s| (s.index, s.evals[i])), @@ -464,13 +466,13 @@ where // https://github.com/EspressoSystems/jellyfish/issues/339 self.eval_domain.fft_in_place(&mut coeffs); - result.append(&mut coeffs); + elems.append(&mut coeffs); } - assert_eq!(result.len(), result_len); + assert_eq!(elems.len(), elems_capacity); - let mut result_bytes: Vec<_> = field_to_bytes(result).collect(); - result_bytes.truncate(common.bytes_len); - Ok(Payload::from_vec(result_bytes)) + let mut payload: Vec<_> = field_to_bytes(elems).collect(); + payload.truncate(common.bytes_len); + Ok(payload) } } @@ -855,7 +857,7 @@ mod tests { /// Returns the following tuple: /// 1. An initialized [`Advz`] instance. /// 2. A `Vec` filled with random bytes. - pub(super) fn avdz_init() -> (Advz, Payload) { + pub(super) fn avdz_init() -> (Advz, Vec) { let (payload_chunk_size, num_storage_nodes) = (4, 6); let mut rng = jf_utils::test_rng(); let srs = init_srs(payload_chunk_size, &mut rng); @@ -869,13 +871,13 @@ mod tests { assert!(matches!(res, Err(Argument(_))), "{}", msg); } - pub(super) fn init_random_payload(len: usize, rng: &mut R) -> Payload + pub(super) fn init_random_payload(len: usize, rng: &mut R) -> Vec where R: RngCore + CryptoRng, { let mut bytes_random = vec![0u8; len]; rng.fill_bytes(&mut bytes_random); - Payload::from_vec(bytes_random) + bytes_random } pub(super) fn init_srs(num_coeffs: usize, rng: &mut R) -> UnivariateUniversalParams diff --git a/primitives/src/vid/advz/payload.rs b/primitives/src/vid/advz/payload.rs deleted file mode 100644 index a7626bc15..000000000 --- a/primitives/src/vid/advz/payload.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! paylod module doc - -use ark_std::{ - slice::Iter, - vec::{IntoIter, Vec}, -}; - -/// payload -#[derive(Debug, PartialEq)] -pub struct Payload { - // TODO store as Vec instead? Or Vec? - payload: Vec, -} - -impl Payload { - /// from_vec - pub fn from_vec(payload: Vec) -> Self { - Self { payload } - } - - /// as_slice - pub fn as_slice(&self) -> &[u8] { - &self.payload - } - - /// iter - pub fn iter(&self) -> Iter<'_, u8> { - self.payload.iter() - } -} - -// delegation boilerplate -impl<'a> IntoIterator for &'a Payload { - type Item = &'a u8; - type IntoIter = Iter<'a, u8>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -// delegation boilerplate -impl IntoIterator for Payload { - type Item = u8; - type IntoIter = IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.payload.into_iter() - } -} diff --git a/primitives/tests/advz.rs b/primitives/tests/advz.rs index eff1f731c..d442e09e0 100644 --- a/primitives/tests/advz.rs +++ b/primitives/tests/advz.rs @@ -3,7 +3,7 @@ use ark_bls12_381::Bls12_381; use ark_ff::{Field, PrimeField}; use jf_primitives::{ pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}, - vid::advz::{payload::Payload, Advz}, + vid::advz::Advz, }; use sha2::Sha256; @@ -34,7 +34,6 @@ fn round_trip() { |payload_chunk_size, num_storage_nodes| { Advz::::new(payload_chunk_size, num_storage_nodes, &srs).unwrap() }, - |bytes| Payload::from_vec(bytes), &vid_sizes, &payload_byte_lens, &mut rng, diff --git a/primitives/tests/vid/mod.rs b/primitives/tests/vid/mod.rs index 0e0096284..9affdfbce 100644 --- a/primitives/tests/vid/mod.rs +++ b/primitives/tests/vid/mod.rs @@ -13,7 +13,6 @@ use ark_std::{ /// pub fn round_trip( vid_factory: impl Fn(usize, usize) -> V, - payload_factory: impl Fn(Vec) -> V::Payload, vid_sizes: &[(usize, usize)], payload_byte_lens: &[usize], rng: &mut R, @@ -30,9 +29,11 @@ pub fn round_trip( payload_chunk_size, num_storage_nodes, len ); - let mut bytes_random = vec![0u8; len]; - rng.fill_bytes(&mut bytes_random); - let bytes_random = payload_factory(bytes_random); + let bytes_random = { + let mut bytes_random = vec![0u8; len]; + rng.fill_bytes(&mut bytes_random); + bytes_random + }; let disperse = vid.disperse(&bytes_random).unwrap(); let (mut shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); From 6c1f1618ba90aa4de29c8d329ef726ac3b160442 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Sun, 5 Nov 2023 15:07:55 -0500 Subject: [PATCH 33/54] replace Namespacer with 2 impls for PayloadProver --- primitives/src/vid/advz/namespace.rs | 90 ++++++++++++++++------------ primitives/src/vid/namespace.rs | 44 ++------------ 2 files changed, 58 insertions(+), 76 deletions(-) diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/namespace.rs index 8800b91b8..6073d69ab 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/namespace.rs @@ -4,7 +4,7 @@ // You should have received a copy of the MIT License // along with the Jellyfish library. If not, see . -//! Implementation of [`Namespacer`] for `Advz`. +//! Implementations of [`PayloadProver`] for `Advz`. use ark_poly::EvaluationDomain; use jf_utils::{bytes_to_field, compile_time_checks}; @@ -15,12 +15,12 @@ use super::{ }; use crate::{ alloc::string::ToString, - vid::{namespace::Namespacer, vid, VidError}, + vid::{namespace::PayloadProver, vid, VidError}, }; // use ark_std::println; use ark_std::{format, ops::Range}; -impl Namespacer for GenericAdvz +impl PayloadProver> for GenericAdvz where // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 P: UnivariatePCS::Evaluation>, @@ -33,13 +33,7 @@ where V::MembershipProof: Sync + Debug, V::Index: From, { - // TODO should be P::Proof, not Vec - // https://github.com/EspressoSystems/jellyfish/issues/387 - type DataProof2 = DataProof2; - - type ChunkProof2 = ChunkProof2; - - fn data_proof2(&self, payload: B, range: Range) -> VidResult + fn payload_proof(&self, payload: B, range: Range) -> VidResult> where B: AsRef<[u8]>, { @@ -99,7 +93,7 @@ where let (proofs, _evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; - Ok(DataProof2 { + Ok(Proof { proofs, // TODO refactor copied code for prefix/suffix bytes prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), @@ -108,12 +102,12 @@ where }) } - fn data_verify2( + fn payload_verify( &self, chunk: B, commit: &Self::Commit, common: &Self::Common, - proof: &Self::DataProof2, + proof: &Proof, ) -> VidResult> where B: AsRef<[u8]>, @@ -207,8 +201,34 @@ where } Ok(Ok(())) } +} + +/// KZG batch proofs and accompanying metadata. +pub struct Proof

{ + proofs: Vec

, + prefix_bytes: Vec, + suffix_bytes: Vec, + chunk_range: Range, +} - fn chunk_proof2(&self, payload: B, range: Range) -> VidResult +impl PayloadProver> for GenericAdvz +where + // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 + P: UnivariatePCS::Evaluation>, + P::Evaluation: PrimeField, + P::Polynomial: DenseUVPolynomial, + P::Commitment: From + AsRef, + T: AffineRepr, + H: Digest + DynDigest + Default + Clone + Write, + V: MerkleTreeScheme>, + V::MembershipProof: Sync + Debug, + V::Index: From, +{ + fn payload_proof( + &self, + payload: B, + range: Range, + ) -> VidResult> where B: AsRef<[u8]>, { @@ -255,7 +275,7 @@ where let prefix: Vec<_> = elems_iter.by_ref().take(offset_elem).collect(); let suffix: Vec<_> = elems_iter.skip(range_elem.len()).collect(); - Ok(ChunkProof2 { + Ok(CommitRecovery { prefix_elems: prefix, suffix_elems: suffix, prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), @@ -264,12 +284,12 @@ where }) } - fn chunk_verify2( + fn payload_verify( &self, chunk: B, commit: &Self::Commit, common: &Self::Common, - proof: &Self::ChunkProof2, + proof: &CommitRecovery, ) -> VidResult> where B: AsRef<[u8]>, @@ -335,16 +355,8 @@ where } } -///doc -pub struct DataProof2

{ - proofs: Vec

, - prefix_bytes: Vec, - suffix_bytes: Vec, - chunk_range: Range, -} - -/// doc -pub struct ChunkProof2 { +/// Metadata needed to recover a KZG commitment +pub struct CommitRecovery { prefix_elems: Vec, suffix_elems: Vec, prefix_bytes: Vec, @@ -430,8 +442,12 @@ fn index_refine(index: usize, multiplier: usize) -> usize { #[cfg(test)] mod tests { use crate::vid::{ - advz::{tests::*, *}, - namespace::Namespacer, + advz::{ + namespace::{CommitRecovery, Proof}, + tests::*, + *, + }, + namespace::PayloadProver, }; use ark_bls12_381::Bls12_381; use ark_std::{ops::Range, println, rand::Rng}; @@ -520,9 +536,10 @@ mod tests { }; println!("poly {} {} case: {:?}", poly, cases.1, range); - let data_proof2 = advz.data_proof2(payload.as_slice(), range.clone()).unwrap(); - advz.data_verify2( - &payload.as_slice()[range.clone()], + let data_proof2: Proof<_> = + advz.payload_proof(&payload, range.clone()).unwrap(); + advz.payload_verify( + &payload[range.clone()], &d.commit, &d.common, &data_proof2, @@ -530,11 +547,10 @@ mod tests { .unwrap() .unwrap(); - let chunk_proof2 = advz - .chunk_proof2(payload.as_slice(), range.clone()) - .unwrap(); - advz.chunk_verify2( - &payload.as_slice()[range.clone()], + let chunk_proof2: CommitRecovery<_> = + advz.payload_proof(&payload, range.clone()).unwrap(); + advz.payload_verify( + &payload[range.clone()], &d.commit, &d.common, &chunk_proof2, diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/namespace.rs index f0e2b3a2f..f6718c42e 100644 --- a/primitives/src/vid/namespace.rs +++ b/primitives/src/vid/namespace.rs @@ -11,53 +11,19 @@ use super::{VidResult, VidScheme}; use ark_std::ops::Range; -// pub trait Namespacer2

: VidScheme { - -// fn data_proof( -// &self, -// payload: &[u8], -// start: usize, -// len: usize, -// ) -> VidResult; - -// } - -/// Namespace functionality for [`VidScheme`]. -pub trait Namespacer: VidScheme { +/// Payload proof functionality for [`VidScheme`]. +pub trait PayloadProver: VidScheme { ///doc - type DataProof2; - - /// doc - type ChunkProof2; - - ///doc - fn data_proof2(&self, payload: B, range: Range) -> VidResult + fn payload_proof(&self, payload: B, range: Range) -> VidResult where B: AsRef<[u8]>; - - /// doc - fn data_verify2( - &self, - chunk: B, - commit: &Self::Commit, - common: &Self::Common, - proof: &Self::DataProof2, - ) -> VidResult> - where - B: AsRef<[u8]>; - - /// doc - fn chunk_proof2(&self, payload: B, range: Range) -> VidResult - where - B: AsRef<[u8]>; - /// doc - fn chunk_verify2( + fn payload_verify( &self, chunk: B, commit: &Self::Commit, common: &Self::Common, - proof: &Self::ChunkProof2, + proof: &PROOF, ) -> VidResult> where B: AsRef<[u8]>; From 4089612e8c40960fb4230e8147d067848bd330aa Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Sun, 5 Nov 2023 15:13:32 -0500 Subject: [PATCH 34/54] rename files: namespace.rs -> payload_prover.rs --- primitives/src/vid.rs | 2 +- primitives/src/vid/advz.rs | 2 +- primitives/src/vid/advz/{namespace.rs => payload_prover.rs} | 6 +++--- primitives/src/vid/{namespace.rs => payload_prover.rs} | 0 4 files changed, 5 insertions(+), 5 deletions(-) rename primitives/src/vid/advz/{namespace.rs => payload_prover.rs} (99%) rename primitives/src/vid/{namespace.rs => payload_prover.rs} (100%) diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index 35b26b5cb..17175eb71 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -73,7 +73,7 @@ pub struct VidDisperse { pub commit: V::Commit, } -pub mod namespace; +pub mod payload_prover; pub mod advz; // instantiation of `VidScheme` diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index c7e939d77..a2744d737 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -42,7 +42,7 @@ use itertools::Itertools; use jf_utils::{bytes_to_field, canonical, field_to_bytes}; use serde::{Deserialize, Serialize}; -pub mod namespace; +pub mod payload_prover; /// The [ADVZ VID scheme](https://eprint.iacr.org/2021/1500), a concrete impl for [`VidScheme`]. /// diff --git a/primitives/src/vid/advz/namespace.rs b/primitives/src/vid/advz/payload_prover.rs similarity index 99% rename from primitives/src/vid/advz/namespace.rs rename to primitives/src/vid/advz/payload_prover.rs index 6073d69ab..0d4666c51 100644 --- a/primitives/src/vid/advz/namespace.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -15,7 +15,7 @@ use super::{ }; use crate::{ alloc::string::ToString, - vid::{namespace::PayloadProver, vid, VidError}, + vid::{payload_prover::PayloadProver, vid, VidError}, }; // use ark_std::println; use ark_std::{format, ops::Range}; @@ -443,11 +443,11 @@ fn index_refine(index: usize, multiplier: usize) -> usize { mod tests { use crate::vid::{ advz::{ - namespace::{CommitRecovery, Proof}, + payload_prover::{CommitRecovery, Proof}, tests::*, *, }, - namespace::PayloadProver, + payload_prover::PayloadProver, }; use ark_bls12_381::Bls12_381; use ark_std::{ops::Range, println, rand::Rng}; diff --git a/primitives/src/vid/namespace.rs b/primitives/src/vid/payload_prover.rs similarity index 100% rename from primitives/src/vid/namespace.rs rename to primitives/src/vid/payload_prover.rs From 0ae66d10b158cb90e93dfc464afae5a9c98dc9be Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 09:27:49 -0500 Subject: [PATCH 35/54] refactor common arg-checking code --- primitives/src/vid/advz/payload_prover.rs | 190 ++++++++-------------- 1 file changed, 68 insertions(+), 122 deletions(-) diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 0d4666c51..4d194f6d3 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -15,7 +15,7 @@ use super::{ }; use crate::{ alloc::string::ToString, - vid::{payload_prover::PayloadProver, vid, VidError}, + vid::{payload_prover::PayloadProver, vid, VidError, VidScheme}, }; // use ark_std::println; use ark_std::{format, ops::Range}; @@ -37,27 +37,8 @@ where where B: AsRef<[u8]>, { - // TODO refactor copied arg check code - - // check args: `range` nonempty - if range.is_empty() { - return Err(VidError::Argument(format!( - "empty range ({}..{})", - range.start, range.end - ))); - } - let payload = payload.as_ref(); - - // check args: `range` in bounds for `payload` - if range.end > payload.len() { - return Err(VidError::Argument(format!( - "range ({}..{}) out of bounds for payload len {}", - range.start, - range.end, - payload.len() - ))); - } + check_range_nonempty_and_inside_payload(payload, &range)?; // index conversion let range_elem = self.range_byte_to_elem2(&range); @@ -66,14 +47,7 @@ where let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); let range_elem_byte = self.range_elem_to_byte2(&range_elem); - // check args: - // TODO TEMPORARY: forbid requests that span multiple polynomials - if range_poly.len() != 1 { - return Err(VidError::Argument(format!( - "request spans {} polynomials, expect 1", - range_poly.len() - ))); - } + check_range_poly(&range_poly)?; // grab the `start_namespace`th polynomial let polynomial = self.polynomial( @@ -82,7 +56,7 @@ where ); // prepare the list of input points - // TODO perf: can't avoid use of `skip` + // perf: can't avoid use of `skip` let points: Vec<_> = { self.eval_domain .elements() @@ -95,7 +69,6 @@ where Ok(Proof { proofs, - // TODO refactor copied code for prefix/suffix bytes prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), suffix_bytes: payload[range.end..range_elem_byte.end].to_vec(), chunk_range: range, @@ -113,49 +86,16 @@ where B: AsRef<[u8]>, { let chunk = chunk.as_ref(); - - // TODO refactor copied arg check code - - // check args: `chunk` nonempty - if chunk.is_empty() { - return Err(VidError::Argument("empty chunk".to_string())); - } - - // check args: `chunk` len consistent with `proof` - if chunk.len() != proof.chunk_range.len() { - return Err(VidError::Argument(format!( - "chunk length {} inconsistent with proof length {}", - chunk.len(), - proof.chunk_range.len() - ))); - } + check_chunk_proof_consistency(&chunk, proof.chunk_range.len())?; // index conversion - - // let (start_elem, len_elem) = self.range_byte_to_elem(start, len); - // let (start_namespace, _len_namespace) = self.range_elem_to_poly(start_elem, len_elem); - // let start_namespace_byte = self.index_poly_to_byte(start_namespace); - let range_elem = self.range_byte_to_elem2(&proof.chunk_range); let range_poly = self.range_elem_to_poly2(&range_elem); let start_namespace_byte = self.index_poly_to_byte(range_poly.start); let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); - // check args: - // TODO TEMPORARY: forbid requests that span multiple polynomials - if range_poly.len() != 1 { - return Err(VidError::Argument(format!( - "request spans {} polynomials, expect 1", - range_poly.len() - ))); - } - - // check args: `common` consistent with `commit` - if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { - return Err(VidError::Argument( - "common inconsistent with commit".to_string(), - )); - } + check_range_poly(&range_poly)?; + Self::check_common_commit_consistency(&common, &commit)?; // prepare list of data elems // TODO refactor copied code @@ -232,25 +172,8 @@ where where B: AsRef<[u8]>, { - // check args: `range` nonempty - if range.is_empty() { - return Err(VidError::Argument(format!( - "empty range ({}..{})", - range.start, range.end - ))); - } - let payload = payload.as_ref(); - - // check args: `range` in bounds for `payload` - if range.end > payload.len() { - return Err(VidError::Argument(format!( - "range ({}..{}) out of bounds for payload len {}", - range.start, - range.end, - payload.len() - ))); - } + check_range_nonempty_and_inside_payload(payload, &range)?; // index conversion let range_elem = self.range_byte_to_elem2(&range); @@ -259,14 +182,7 @@ where let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); let range_elem_byte = self.range_elem_to_byte2(&range_elem); - // check args: - // TODO TEMPORARY: forbid requests that span multiple polynomials - if range_poly.len() != 1 { - return Err(VidError::Argument(format!( - "request spans {} polynomials, expect 1", - range_poly.len() - ))); - } + check_range_poly(&range_poly)?; // compute the prefix and suffix elems let mut elems_iter = @@ -295,39 +211,13 @@ where B: AsRef<[u8]>, { let chunk = chunk.as_ref(); - - // check args: `chunk` nonempty - if chunk.is_empty() { - return Err(VidError::Argument("empty chunk".to_string())); - } - - // check args: `chunk` len consistent with `proof` - if chunk.len() != proof.chunk_range.len() { - return Err(VidError::Argument(format!( - "chunk length {} inconsistent with proof length {}", - chunk.len(), - proof.chunk_range.len() - ))); - } + check_chunk_proof_consistency(&chunk, proof.chunk_range.len())?; // index conversion let range_poly = self.range_byte_to_poly2(&proof.chunk_range); - // check args: - // TODO TEMPORARY: forbid requests that span multiple polynomials - if range_poly.len() != 1 { - return Err(VidError::Argument(format!( - "request spans {} polynomials, expect 1", - range_poly.len() - ))); - } - - // check args: `common` consistent with `commit` - if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { - return Err(VidError::Argument( - "common inconsistent with commit".to_string(), - )); - } + check_range_poly(&range_poly)?; + Self::check_common_commit_consistency(&common, &commit)?; // rebuild the poly commit, check against `common` let poly_commit = { @@ -413,6 +303,18 @@ where self.payload_chunk_size * compile_time_checks::().0, ) } + + fn check_common_commit_consistency( + common: &::Common, + commit: &::Commit, + ) -> VidResult<()> { + if *commit != Self::poly_commits_hash(common.poly_commits.iter())? { + return Err(VidError::Argument( + "common inconsistent with commit".to_string(), + )); + } + Ok(()) + } } fn range_coarsen2(range: &Range, denominator: usize) -> Range { @@ -439,6 +341,50 @@ fn index_refine(index: usize, multiplier: usize) -> usize { index * multiplier } +fn check_range_nonempty_and_inside_payload(payload: &[u8], range: &Range) -> VidResult<()> { + if range.is_empty() { + return Err(VidError::Argument(format!( + "empty range ({}..{})", + range.start, range.end + ))); + } + if range.end > payload.len() { + return Err(VidError::Argument(format!( + "range ({}..{}) out of bounds for payload len {}", + range.start, + range.end, + payload.len() + ))); + } + Ok(()) +} + +fn check_range_poly(range_poly: &Range) -> VidResult<()> { + // TODO TEMPORARY: forbid requests that span multiple polynomials + if range_poly.len() != 1 { + return Err(VidError::Argument(format!( + "request spans {} polynomials, expect 1", + range_poly.len() + ))); + } + Ok(()) +} + +fn check_chunk_proof_consistency(chunk: &[u8], proof_chunk_len: usize) -> VidResult<()> { + if chunk.is_empty() { + return Err(VidError::Argument("empty chunk".to_string())); + } + + if chunk.len() != proof_chunk_len { + return Err(VidError::Argument(format!( + "chunk length {} inconsistent with proof length {}", + chunk.len(), + proof_chunk_len + ))); + } + Ok(()) +} + #[cfg(test)] mod tests { use crate::vid::{ From 7b314987a3d870a0abcff6c33aa867423483ffea Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 09:35:02 -0500 Subject: [PATCH 36/54] clean up TODOs --- primitives/src/vid/advz/payload_prover.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 4d194f6d3..19186b875 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -55,7 +55,7 @@ where .take(self.payload_chunk_size), ); - // prepare the list of input points + // prepare list of input points // perf: can't avoid use of `skip` let points: Vec<_> = { self.eval_domain @@ -98,7 +98,6 @@ where Self::check_common_commit_consistency(&common, &commit)?; // prepare list of data elems - // TODO refactor copied code let data_elems: Vec<_> = bytes_to_field::<_, P::Evaluation>( proof .prefix_bytes @@ -109,8 +108,7 @@ where .collect(); // prepare list of input points - // TODO perf: can't avoid use of `skip` - // TODO refactor copied code + // perf: can't avoid use of `skip` let points: Vec<_> = { self.eval_domain .elements() @@ -120,8 +118,7 @@ where }; // verify proof - // TODO naive verify for multi_open - // https://github.com/EspressoSystems/jellyfish/issues/387 + // TODO naive verify for multi_open https://github.com/EspressoSystems/jellyfish/issues/387 if data_elems.len() != proof.proofs.len() { return Err(VidError::Argument(format!( "data len {} differs from proof len {}", From 6ab15f3f0bef6f600937e8c70bf15d5504d1a199 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 09:38:08 -0500 Subject: [PATCH 37/54] fix bench --- primitives/benches/advz.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/primitives/benches/advz.rs b/primitives/benches/advz.rs index d5e0dc4b6..935c747c7 100644 --- a/primitives/benches/advz.rs +++ b/primitives/benches/advz.rs @@ -23,10 +23,7 @@ mod feature_gated { use digest::{crypto_common::generic_array::ArrayLength, Digest, DynDigest, OutputSizeUser}; use jf_primitives::{ pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}, - vid::{ - advz::{payload::Payload, Advz}, - VidScheme, - }, + vid::{advz::Advz, VidScheme}, }; use sha2::Sha256; @@ -64,9 +61,11 @@ mod feature_gated { // run all benches for each payload_byte_lens for len in payload_byte_lens { // random payload data - let mut payload_bytes = vec![0u8; len]; - rng.fill_bytes(&mut payload_bytes); - let payload_bytes = Payload::from_vec(payload_bytes); + let payload_bytes = { + let mut payload_bytes = vec![0u8; len]; + rng.fill_bytes(&mut payload_bytes); + payload_bytes + }; let benchmark_group_name = |op_name| format!("advz_{}_{}_{}KB", pairing_name, op_name, len / KB); From 5b19e06ee207a1ec5c0636cb7efd7a5d529d3389 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 10:24:03 -0500 Subject: [PATCH 38/54] tidy --- primitives/src/vid/advz/payload_prover.rs | 80 ++++++++++------------- primitives/src/vid/payload_prover.rs | 9 ++- 2 files changed, 38 insertions(+), 51 deletions(-) diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 19186b875..f4b195c5f 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -5,6 +5,10 @@ // along with the Jellyfish library. If not, see . //! Implementations of [`PayloadProver`] for `Advz`. +//! +//! Two implementations: +//! 1. `PROOF = `[`Proof`]: KZG batch proof for the data. Useful for small sub-slices of `payload` such as an individual transaction within a block. Not snark-friendly because it requires a pairing. +//! 2. `PROOF = `[`CommitRecovery`]: Rebuild the KZG commitment. Useful for larger sub-slices of `payload` such as a complete namespace. Snark-friendly because it does not require a pairing. use ark_poly::EvaluationDomain; use jf_utils::{bytes_to_field, compile_time_checks}; @@ -17,7 +21,6 @@ use crate::{ alloc::string::ToString, vid::{payload_prover::PayloadProver, vid, VidError, VidScheme}, }; -// use ark_std::println; use ark_std::{format, ops::Range}; impl PayloadProver> for GenericAdvz @@ -41,15 +44,15 @@ where check_range_nonempty_and_inside_payload(payload, &range)?; // index conversion - let range_elem = self.range_byte_to_elem2(&range); - let range_poly = self.range_elem_to_poly2(&range_elem); + let range_elem = self.range_byte_to_elem(&range); + let range_poly = self.range_elem_to_poly(&range_elem); let start_namespace_byte = self.index_poly_to_byte(range_poly.start); let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); - let range_elem_byte = self.range_elem_to_byte2(&range_elem); + let range_elem_byte = self.range_elem_to_byte(&range_elem); check_range_poly(&range_poly)?; - // grab the `start_namespace`th polynomial + // grab the polynomial that contains `range` let polynomial = self.polynomial( bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) .take(self.payload_chunk_size), @@ -89,8 +92,8 @@ where check_chunk_proof_consistency(&chunk, proof.chunk_range.len())?; // index conversion - let range_elem = self.range_byte_to_elem2(&proof.chunk_range); - let range_poly = self.range_elem_to_poly2(&range_elem); + let range_elem = self.range_byte_to_elem(&proof.chunk_range); + let range_poly = self.range_elem_to_poly(&range_elem); let start_namespace_byte = self.index_poly_to_byte(range_poly.start); let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); @@ -141,6 +144,8 @@ where } /// KZG batch proofs and accompanying metadata. +/// +/// TODO use batch proof instead of `Vec

` https://github.com/EspressoSystems/jellyfish/issues/387 pub struct Proof

{ proofs: Vec

, prefix_bytes: Vec, @@ -173,11 +178,11 @@ where check_range_nonempty_and_inside_payload(payload, &range)?; // index conversion - let range_elem = self.range_byte_to_elem2(&range); - let range_poly = self.range_elem_to_poly2(&range_elem); + let range_elem = self.range_byte_to_elem(&range); + let range_poly = self.range_elem_to_poly(&range_elem); let start_namespace_byte = self.index_poly_to_byte(range_poly.start); let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); - let range_elem_byte = self.range_elem_to_byte2(&range_elem); + let range_elem_byte = self.range_elem_to_byte(&range_elem); check_range_poly(&range_poly)?; @@ -211,7 +216,7 @@ where check_chunk_proof_consistency(&chunk, proof.chunk_range.len())?; // index conversion - let range_poly = self.range_byte_to_poly2(&proof.chunk_range); + let range_poly = self.range_byte_to_poly(&proof.chunk_range); check_range_poly(&range_poly)?; Self::check_common_commit_consistency(&common, &commit)?; @@ -242,7 +247,7 @@ where } } -/// Metadata needed to recover a KZG commitment +/// Metadata needed to recover a KZG commitment. pub struct CommitRecovery { prefix_elems: Vec, suffix_elems: Vec, @@ -264,38 +269,27 @@ where V::MembershipProof: Sync + Debug, V::Index: From, { - // lots of index manipulation. - // with infinite dev time we should implement type-safe indices to preclude - // index-misuse bugs. + // lots of index manipulation fn index_byte_to_elem(&self, index: usize) -> usize { index_coarsen(index, compile_time_checks::().0) } - fn _index_elem_to_byte(&self, index: usize) -> usize { - index_refine(index, compile_time_checks::().0) - } fn index_poly_to_byte(&self, index: usize) -> usize { index_refine( index, self.payload_chunk_size * compile_time_checks::().0, ) } - fn range_byte_to_elem2(&self, range: &Range) -> Range { - range_coarsen2(range, compile_time_checks::().0) + fn range_byte_to_elem(&self, range: &Range) -> Range { + range_coarsen(range, compile_time_checks::().0) } - fn range_elem_to_byte2(&self, range: &Range) -> Range { - range_refine2(range, compile_time_checks::().0) + fn range_elem_to_byte(&self, range: &Range) -> Range { + range_refine(range, compile_time_checks::().0) } - // fn range_poly_to_byte2(&self, range: &Range) -> Range { - // range_refine2( - // range, - // self.payload_chunk_size * compile_time_checks::().0, - // ) - // } - fn range_elem_to_poly2(&self, range: &Range) -> Range { - range_coarsen2(range, self.payload_chunk_size) + fn range_elem_to_poly(&self, range: &Range) -> Range { + range_coarsen(range, self.payload_chunk_size) } - fn range_byte_to_poly2(&self, range: &Range) -> Range { - range_coarsen2( + fn range_byte_to_poly(&self, range: &Range) -> Range { + range_coarsen( range, self.payload_chunk_size * compile_time_checks::().0, ) @@ -314,7 +308,7 @@ where } } -fn range_coarsen2(range: &Range, denominator: usize) -> Range { +fn range_coarsen(range: &Range, denominator: usize) -> Range { assert!(!range.is_empty(), "{:?}", range); Range { start: index_coarsen(range.start, denominator), @@ -322,7 +316,7 @@ fn range_coarsen2(range: &Range, denominator: usize) -> Range { } } -fn range_refine2(range: &Range, multiplier: usize) -> Range { +fn range_refine(range: &Range, multiplier: usize) -> Range { assert!(!range.is_empty(), "{:?}", range); Range { start: index_refine(range.start, multiplier), @@ -479,24 +473,18 @@ mod tests { }; println!("poly {} {} case: {:?}", poly, cases.1, range); - let data_proof2: Proof<_> = - advz.payload_proof(&payload, range.clone()).unwrap(); - advz.payload_verify( - &payload[range.clone()], - &d.commit, - &d.common, - &data_proof2, - ) - .unwrap() - .unwrap(); + let data_proof: Proof<_> = advz.payload_proof(&payload, range.clone()).unwrap(); + advz.payload_verify(&payload[range.clone()], &d.commit, &d.common, &data_proof) + .unwrap() + .unwrap(); - let chunk_proof2: CommitRecovery<_> = + let chunk_proof: CommitRecovery<_> = advz.payload_proof(&payload, range.clone()).unwrap(); advz.payload_verify( &payload[range.clone()], &d.commit, &d.common, - &chunk_proof2, + &chunk_proof, ) .unwrap() .unwrap(); diff --git a/primitives/src/vid/payload_prover.rs b/primitives/src/vid/payload_prover.rs index f6718c42e..91d60d3ea 100644 --- a/primitives/src/vid/payload_prover.rs +++ b/primitives/src/vid/payload_prover.rs @@ -4,20 +4,19 @@ // You should have received a copy of the MIT License // along with the Jellyfish library. If not, see . -//! Trait for namespace functionality in Verifiable Information Retrieval (VID). -//! -//! TODO should this trait even exist? It's very implementation-dependent. +//! Trait for additional functionality in Verifiable Information Retrieval (VID) to make and verify a proof of correctness of an arbitrary sub-slice of data from a payload. use super::{VidResult, VidScheme}; use ark_std::ops::Range; /// Payload proof functionality for [`VidScheme`]. pub trait PayloadProver: VidScheme { - ///doc + /// Compute a proof for a sub-slice of payload data. fn payload_proof(&self, payload: B, range: Range) -> VidResult where B: AsRef<[u8]>; - /// doc + + /// Verify a proof made by `payload_proof`. fn payload_verify( &self, chunk: B, From cd6e3414c44403dcc5f31cee8ac01bbb3ec0bac3 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 10:39:44 -0500 Subject: [PATCH 39/54] coeffs_iter -> evals_iter as per https://github.com/EspressoSystems/jellyfish/pull/389#discussion_r1377912432 --- primitives/src/vid/advz.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index a2744d737..cd14ebed5 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -210,8 +210,8 @@ where // itertools::process_results() but the code is unreadable. let mut hasher = H::new(); let elems_iter = bytes_to_field::<_, P::Evaluation>(payload); - for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { - let poly = self.polynomial(coeffs_iter); + for evals_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { + let poly = self.polynomial(evals_iter); let commitment = P::commit(&self.ck, &poly).map_err(vid)?; commitment .serialize_uncompressed(&mut hasher) @@ -236,8 +236,8 @@ where let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let elems_iter = bytes_to_field::<_, P::Evaluation>(payload); let mut polys = Vec::new(); - for coeffs_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { - polys.push(self.polynomial(coeffs_iter)); + for evals_iter in elems_iter.chunks(self.payload_chunk_size).into_iter() { + polys.push(self.polynomial(evals_iter)); } end_timer!(bytes_to_polys_time); From 8f2240b3f9d01c29f77142c8b2dd8698c0be79fe Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 12:46:06 -0500 Subject: [PATCH 40/54] address https://github.com/EspressoSystems/jellyfish/pull/389#discussion_r1377920295 --- primitives/src/vid/advz.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index cd14ebed5..9ac5f4d66 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -309,7 +309,7 @@ where "should be fast: compute polynomial commitments hash, pseudorandom scalar, aggregate polynomial" }); let commit = Self::poly_commits_hash(common.poly_commits.iter())?; - let pseudorandom_scalar = Self::pseudorandom_scalar(&common)?; + let pseudorandom_scalar = Self::pseudorandom_scalar(&common, &commit)?; // Compute aggregate polynomial // as a pseudorandom linear combo of polynomials @@ -389,7 +389,8 @@ where return Ok(Err(())); } - let pseudorandom_scalar = Self::pseudorandom_scalar(common)?; + let commit = Self::poly_commits_hash(common.poly_commits.iter())?; + let pseudorandom_scalar = Self::pseudorandom_scalar(common, &commit)?; // Compute aggregate polynomial [commitment|evaluation] // as a pseudorandom linear combo of [commitments|evaluations] @@ -488,11 +489,12 @@ where V::MembershipProof: Sync + Debug, /* TODO https://github.com/EspressoSystems/jellyfish/issues/253 */ V::Index: From, { - fn pseudorandom_scalar(common: &::Common) -> VidResult { + fn pseudorandom_scalar( + common: &::Common, + commit: &::Commit, + ) -> VidResult { let mut hasher = H::new(); - Self::poly_commits_hash(common.poly_commits.iter())? - .serialize_uncompressed(&mut hasher) - .map_err(vid)?; + commit.serialize_uncompressed(&mut hasher).map_err(vid)?; common .all_evals_digest .serialize_uncompressed(&mut hasher) From ff764b4becbf95f32b1bf4b01d1c93f7a915207d Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 13:08:21 -0500 Subject: [PATCH 41/54] add commit arg to verify_share() --- primitives/src/vid.rs | 8 +++++-- primitives/src/vid/advz.rs | 48 ++++++++++++++++++++++++------------- primitives/tests/vid/mod.rs | 2 +- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index 17175eb71..ecce4fceb 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -38,8 +38,12 @@ pub trait VidScheme { /// - VidResult::Err in case of actual error /// - VidResult::Ok(Result::Err) if verification fails /// - VidResult::Ok(Result::Ok) if verification succeeds - fn verify_share(&self, share: &Self::Share, common: &Self::Common) - -> VidResult>; + fn verify_share( + &self, + share: &Self::Share, + common: &Self::Common, + commit: &Self::Commit, + ) -> VidResult>; /// Recover payload from shares. /// Do not verify shares or check recovered payload against anything. diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 9ac5f4d66..273c46c67 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -10,6 +10,7 @@ use super::{vid, VidDisperse, VidError, VidResult, VidScheme}; use crate::{ + alloc::string::ToString, merkle_tree::{hasher::HasherMerkleTree, MerkleCommitment, MerkleTreeScheme}, pcs::{ prelude::UnivariateKzgPCS, PolynomialCommitmentScheme, StructuredReferenceString, @@ -364,6 +365,7 @@ where &self, share: &Self::Share, common: &Self::Common, + commit: &Self::Commit, ) -> VidResult> { // check arguments if share.evals.len() != common.poly_commits.len() { @@ -377,6 +379,14 @@ where return Ok(Err(())); // not an arg error } + // check `common` against `commit` + let commit_rebuilt = Self::poly_commits_hash(common.poly_commits.iter())?; + if commit_rebuilt != *commit { + return Err(VidError::Argument( + "commit inconsistent with common".to_string(), + )); + } + // verify eval proof if V::verify( common.all_evals_digest, @@ -389,7 +399,6 @@ where return Ok(Err(())); } - let commit = Self::poly_commits_hash(common.poly_commits.iter())?; let pseudorandom_scalar = Self::pseudorandom_scalar(common, &commit)?; // Compute aggregate polynomial [commitment|evaluation] @@ -677,7 +686,7 @@ mod tests { fn sad_path_verify_share_corrupt_share() { let (advz, bytes_random) = avdz_init(); let disperse = advz.disperse(&bytes_random).unwrap(); - let (shares, common) = (disperse.shares, disperse.common); + let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); for (i, share) in shares.iter().enumerate() { // missing share eval @@ -687,7 +696,7 @@ mod tests { ..share.clone() }; assert_arg_err( - advz.verify_share(&share_missing_eval, &common), + advz.verify_share(&share_missing_eval, &common, &commit), "1 missing share should be arg error", ); } @@ -696,7 +705,7 @@ mod tests { { let mut share_bad_eval = share.clone(); share_bad_eval.evals[0].double_in_place(); - advz.verify_share(&share_bad_eval, &common) + advz.verify_share(&share_bad_eval, &common, &commit) .unwrap() .expect_err("bad share value should fail verification"); } @@ -707,7 +716,7 @@ mod tests { index: (share.index + 1) % advz.num_storage_nodes, ..share.clone() }; - advz.verify_share(&share_bad_index, &common) + advz.verify_share(&share_bad_index, &common, &commit) .unwrap() .expect_err("bad share index should fail verification"); } @@ -718,7 +727,7 @@ mod tests { index: share.index + advz.num_storage_nodes, ..share.clone() }; - advz.verify_share(&share_bad_index, &common) + advz.verify_share(&share_bad_index, &common, &commit) .unwrap() .expect_err("bad share index should fail verification"); } @@ -732,7 +741,7 @@ mod tests { evals_proof: shares[(i + 1) % shares.len()].evals_proof.clone(), ..share.clone() }; - advz.verify_share(&share_bad_evals_proof, &common) + advz.verify_share(&share_bad_evals_proof, &common, &commit) .unwrap() .expect_err("bad share evals proof should fail verification"); } @@ -743,7 +752,7 @@ mod tests { fn sad_path_verify_share_corrupt_commit() { let (advz, bytes_random) = avdz_init(); let disperse = advz.disperse(&bytes_random).unwrap(); - let (shares, common) = (disperse.shares, disperse.common); + let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); // missing commit let common_missing_item = Common { @@ -751,7 +760,7 @@ mod tests { ..common.clone() }; assert_arg_err( - advz.verify_share(&shares[0], &common_missing_item), + advz.verify_share(&shares[0], &common_missing_item, &commit), "1 missing commit should be arg error", ); @@ -761,9 +770,10 @@ mod tests { corrupted.poly_commits[0] = ::G1Affine::zero().into(); corrupted }; - advz.verify_share(&shares[0], &common_1_poly_corruption) - .unwrap() - .expect_err("1 corrupt poly_commit should fail verification"); + assert_arg_err( + advz.verify_share(&shares[0], &common_1_poly_corruption, &commit), + "corrupted commit should be arg error", + ); // 1 corrupt commit, all_evals_digest let common_1_digest_corruption = { @@ -779,7 +789,7 @@ mod tests { .expect("digest deserialization should succeed"); corrupted }; - advz.verify_share(&shares[0], &common_1_digest_corruption) + advz.verify_share(&shares[0], &common_1_digest_corruption, &commit) .unwrap() .expect_err("1 corrupt all_evals_digest should fail verification"); } @@ -788,19 +798,25 @@ mod tests { fn sad_path_verify_share_corrupt_share_and_commit() { let (advz, bytes_random) = avdz_init(); let disperse = advz.disperse(&bytes_random).unwrap(); - let (mut shares, mut common) = (disperse.shares, disperse.common); + let (mut shares, mut common, commit) = (disperse.shares, disperse.common, disperse.commit); common.poly_commits.pop(); shares[0].evals.pop(); // equal nonzero lengths for common, share - advz.verify_share(&shares[0], &common).unwrap().unwrap_err(); + assert_arg_err( + advz.verify_share(&shares[0], &common, &commit), + "common inconsistent with commit should be arg error", + ); common.poly_commits.clear(); shares[0].evals.clear(); // zero length for common, share - advz.verify_share(&shares[0], &common).unwrap().unwrap_err(); + assert_arg_err( + advz.verify_share(&shares[0], &common, &commit), + "expect arg error for common inconsistent with commit", + ); } #[test] diff --git a/primitives/tests/vid/mod.rs b/primitives/tests/vid/mod.rs index 9affdfbce..2fc0df056 100644 --- a/primitives/tests/vid/mod.rs +++ b/primitives/tests/vid/mod.rs @@ -41,7 +41,7 @@ pub fn round_trip( assert_eq!(commit, vid.commit_only(&bytes_random).unwrap()); for share in shares.iter() { - vid.verify_share(share, &common).unwrap().unwrap(); + vid.verify_share(share, &common, &commit).unwrap().unwrap(); } // sample a random subset of shares with size payload_chunk_size From 89c9feac7c1fdeaaaf1f9a0139f2cbffc350f7ee Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 13:15:01 -0500 Subject: [PATCH 42/54] add issue comment for https://github.com/EspressoSystems/jellyfish/pull/389/files/0d9f70443d98ac78a6c398ff0e7a9cf3ba05592c#r1377950064 --- primitives/src/vid/advz/payload_prover.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index f4b195c5f..5c648f0e0 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -53,6 +53,7 @@ where check_range_poly(&range_poly)?; // grab the polynomial that contains `range` + // TODO allow precomputation: https://github.com/EspressoSystems/jellyfish/issues/397 let polynomial = self.polynomial( bytes_to_field::<_, P::Evaluation>(payload[start_namespace_byte..].iter()) .take(self.payload_chunk_size), From 93e396ebed1421458e2647d6461932dffb44680d Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 13:18:53 -0500 Subject: [PATCH 43/54] clippy and fmt --- primitives/src/vid/advz.rs | 2 +- primitives/src/vid/advz/payload_prover.rs | 16 ++++++++++------ primitives/src/vid/payload_prover.rs | 4 +++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 273c46c67..805ea3dec 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -399,7 +399,7 @@ where return Ok(Err(())); } - let pseudorandom_scalar = Self::pseudorandom_scalar(common, &commit)?; + let pseudorandom_scalar = Self::pseudorandom_scalar(common, commit)?; // Compute aggregate polynomial [commitment|evaluation] // as a pseudorandom linear combo of [commitments|evaluations] diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 5c648f0e0..89ccb58b4 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -7,8 +7,12 @@ //! Implementations of [`PayloadProver`] for `Advz`. //! //! Two implementations: -//! 1. `PROOF = `[`Proof`]: KZG batch proof for the data. Useful for small sub-slices of `payload` such as an individual transaction within a block. Not snark-friendly because it requires a pairing. -//! 2. `PROOF = `[`CommitRecovery`]: Rebuild the KZG commitment. Useful for larger sub-slices of `payload` such as a complete namespace. Snark-friendly because it does not require a pairing. +//! 1. `PROOF = `[`Proof`]: KZG batch proof for the data. Useful for small +//! sub-slices of `payload` such as an individual transaction within a block. +//! Not snark-friendly because it requires a pairing. +//! 2. `PROOF = `[`CommitRecovery`]: Rebuild the KZG commitment. Useful for +//! larger sub-slices of `payload` such as a complete namespace. +//! Snark-friendly because it does not require a pairing. use ark_poly::EvaluationDomain; use jf_utils::{bytes_to_field, compile_time_checks}; @@ -90,7 +94,7 @@ where B: AsRef<[u8]>, { let chunk = chunk.as_ref(); - check_chunk_proof_consistency(&chunk, proof.chunk_range.len())?; + check_chunk_proof_consistency(chunk, proof.chunk_range.len())?; // index conversion let range_elem = self.range_byte_to_elem(&proof.chunk_range); @@ -99,7 +103,7 @@ where let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); check_range_poly(&range_poly)?; - Self::check_common_commit_consistency(&common, &commit)?; + Self::check_common_commit_consistency(common, commit)?; // prepare list of data elems let data_elems: Vec<_> = bytes_to_field::<_, P::Evaluation>( @@ -214,13 +218,13 @@ where B: AsRef<[u8]>, { let chunk = chunk.as_ref(); - check_chunk_proof_consistency(&chunk, proof.chunk_range.len())?; + check_chunk_proof_consistency(chunk, proof.chunk_range.len())?; // index conversion let range_poly = self.range_byte_to_poly(&proof.chunk_range); check_range_poly(&range_poly)?; - Self::check_common_commit_consistency(&common, &commit)?; + Self::check_common_commit_consistency(common, commit)?; // rebuild the poly commit, check against `common` let poly_commit = { diff --git a/primitives/src/vid/payload_prover.rs b/primitives/src/vid/payload_prover.rs index 91d60d3ea..b7e7424ca 100644 --- a/primitives/src/vid/payload_prover.rs +++ b/primitives/src/vid/payload_prover.rs @@ -4,7 +4,9 @@ // You should have received a copy of the MIT License // along with the Jellyfish library. If not, see . -//! Trait for additional functionality in Verifiable Information Retrieval (VID) to make and verify a proof of correctness of an arbitrary sub-slice of data from a payload. +//! Trait for additional functionality in Verifiable Information Retrieval (VID) +//! to make and verify a proof of correctness of an arbitrary sub-slice of data +//! from a payload. use super::{VidResult, VidScheme}; use ark_std::ops::Range; From 254c2e56d57674d17ab4030befb4562da7aa04fc Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 13:31:02 -0500 Subject: [PATCH 44/54] clarify rustdoc --- primitives/src/vid/payload_prover.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/primitives/src/vid/payload_prover.rs b/primitives/src/vid/payload_prover.rs index b7e7424ca..33f4e28be 100644 --- a/primitives/src/vid/payload_prover.rs +++ b/primitives/src/vid/payload_prover.rs @@ -19,6 +19,10 @@ pub trait PayloadProver: VidScheme { B: AsRef<[u8]>; /// Verify a proof made by `payload_proof`. + /// + /// `chunk` is the payload sub-slice for which a proof was generated via + /// `payload_proof` using `range`. In other words, `chunk` should equal + /// `payload[range.start..range.end]`. fn payload_verify( &self, chunk: B, From beeaea9add4d58b20bb8469c02736fc39bda83f8 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 13:32:55 -0500 Subject: [PATCH 45/54] fix bench --- primitives/benches/advz.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/primitives/benches/advz.rs b/primitives/benches/advz.rs index 935c747c7..d700d3b84 100644 --- a/primitives/benches/advz.rs +++ b/primitives/benches/advz.rs @@ -106,13 +106,17 @@ mod feature_gated { for (poly_degree, num_storage_nodes) in vid_sizes_iter.clone() { let advz = Advz::::new(poly_degree, num_storage_nodes, &srs).unwrap(); let disperse = advz.disperse(&payload_bytes).unwrap(); - let (shares, common) = (disperse.shares, disperse.common); + let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); grp.bench_with_input( BenchmarkId::from_parameter(num_storage_nodes), &num_storage_nodes, |b, _| { // verify only the 0th share - b.iter(|| advz.verify_share(&shares[0], &common).unwrap().unwrap()); + b.iter(|| { + advz.verify_share(&shares[0], &common, &commit) + .unwrap() + .unwrap() + }); }, ); } From 39d16e89e36f9b02bb4d373dc180391a1b0686cc Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 13:56:37 -0500 Subject: [PATCH 46/54] move bytes_to_field, etc from conversion.rs to new bytes_to_field.rs --- primitives/src/vid/advz.rs | 4 +- primitives/src/vid/advz/bytes_to_field.rs | 224 ++++++++++++++++++++++ primitives/src/vid/advz/payload_prover.rs | 6 +- utilities/src/conversion.rs | 212 +------------------- 4 files changed, 232 insertions(+), 214 deletions(-) create mode 100644 primitives/src/vid/advz/bytes_to_field.rs diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 805ea3dec..2f94701dc 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -37,12 +37,14 @@ use ark_std::{ vec::Vec, Zero, }; +use bytes_to_field::{bytes_to_field, field_to_bytes}; use derivative::Derivative; use digest::{crypto_common::Output, Digest, DynDigest}; use itertools::Itertools; -use jf_utils::{bytes_to_field, canonical, field_to_bytes}; +use jf_utils::canonical; use serde::{Deserialize, Serialize}; +mod bytes_to_field; pub mod payload_prover; /// The [ADVZ VID scheme](https://eprint.iacr.org/2021/1500), a concrete impl for [`VidScheme`]. diff --git a/primitives/src/vid/advz/bytes_to_field.rs b/primitives/src/vid/advz/bytes_to_field.rs new file mode 100644 index 000000000..402b844d7 --- /dev/null +++ b/primitives/src/vid/advz/bytes_to_field.rs @@ -0,0 +1,224 @@ +use ark_ff::{BigInteger, Field, PrimeField}; +use ark_std::{ + borrow::Borrow, + iter::Take, + marker::PhantomData, + vec::{IntoIter, Vec}, +}; +use jf_utils::compile_time_checks; + +/// Deterministic, infallible, invertible iterator adaptor to convert from +/// arbitrary bytes to field elements. +/// +/// The final field element is padded with zero bytes as needed. +/// +/// # Example +/// +/// ``` +/// # use jf_utils::bytes_to_field; +/// # use ark_ed_on_bn254::Fr as Fr254; +/// let bytes = [1, 2, 3]; +/// let mut elems_iter = bytes_to_field::<_, Fr254>(bytes); +/// assert_eq!(elems_iter.next(), Some(Fr254::from(197121u64))); +/// assert_eq!(elems_iter.next(), None); +/// ``` +/// +/// # Panics +/// +/// Panics only under conditions that should be checkable at compile time: +/// +/// - The [`PrimeField`] modulus bit length is too small to hold a `u64`. +/// - The [`PrimeField`] byte length is too large to fit inside a `usize`. +/// +/// If any of the above conditions holds then this function *always* panics. +pub fn bytes_to_field(bytes: I) -> impl Iterator +where + F: PrimeField, + I: IntoIterator, + I::Item: Borrow, +{ + BytesToField::new(bytes.into_iter()) +} + +/// Deterministic, infallible inverse of [`bytes_to_field`]. +/// +/// The composition of [`field_to_bytes`] with [`bytes_to_field`] might contain +/// extra zero bytes. +/// +/// # Example +/// +/// ``` +/// # use jf_utils::{bytes_to_field, field_to_bytes}; +/// # use ark_ed_on_bn254::Fr as Fr254; +/// let bytes = [1, 2, 3]; +/// let mut bytes_iter = field_to_bytes(bytes_to_field::<_, Fr254>(bytes)); +/// assert_eq!(bytes_iter.next(), Some(1)); +/// assert_eq!(bytes_iter.next(), Some(2)); +/// assert_eq!(bytes_iter.next(), Some(3)); +/// for _ in 0..28 { +/// assert_eq!(bytes_iter.next(), Some(0)); +/// } +/// assert_eq!(bytes_iter.next(), None); +/// ``` +/// +/// ## Panics +/// +/// Panics under the conditions listed at [`bytes_to_field`]. +pub fn field_to_bytes(elems: I) -> impl Iterator +where + F: PrimeField, + I: IntoIterator, + I::Item: Borrow, +{ + FieldToBytes::new(elems.into_iter()) +} + +struct BytesToField { + bytes_iter: I, + primefield_bytes_len: usize, + _phantom: PhantomData, +} + +impl BytesToField +where + F: Field, +{ + fn new(bytes_iter: I) -> Self { + let (primefield_bytes_len, ..) = compile_time_checks::(); + Self { + bytes_iter, + primefield_bytes_len, + _phantom: PhantomData, + } + } +} + +impl Iterator for BytesToField +where + I: Iterator, + I::Item: Borrow, + F: PrimeField, +{ + type Item = F; + + fn next(&mut self) -> Option { + let mut elem_bytes = Vec::with_capacity(self.primefield_bytes_len); + for _ in 0..elem_bytes.capacity() { + if let Some(byte) = self.bytes_iter.next() { + elem_bytes.push(*byte.borrow()); + } else { + break; + } + } + if elem_bytes.is_empty() { + None + } else { + Some(F::from_le_bytes_mod_order(&elem_bytes)) + } + } +} + +struct FieldToBytes { + elems_iter: I, + bytes_iter: Take>, + primefield_bytes_len: usize, + _phantom: PhantomData, +} + +impl FieldToBytes +where + F: Field, +{ + fn new(elems_iter: I) -> Self { + let (primefield_bytes_len, ..) = compile_time_checks::(); + Self { + elems_iter, + bytes_iter: Vec::new().into_iter().take(0), + primefield_bytes_len, + _phantom: PhantomData, + } + } +} + +impl Iterator for FieldToBytes +where + I: Iterator, + I::Item: Borrow, + F: PrimeField, +{ + type Item = u8; + + fn next(&mut self) -> Option { + if let Some(byte) = self.bytes_iter.next() { + return Some(byte); + } + if let Some(elem) = self.elems_iter.next() { + self.bytes_iter = elem + .borrow() + .into_bigint() + .to_bytes_le() + .into_iter() + .take(self.primefield_bytes_len); + return self.bytes_iter.next(); + } + None + } +} + +#[cfg(test)] +mod tests { + use super::{bytes_to_field, field_to_bytes, PrimeField, Vec}; + use ark_ed_on_bls12_377::Fr as Fr377; + use ark_ed_on_bls12_381::Fr as Fr381; + use ark_ed_on_bn254::Fr as Fr254; + use ark_std::rand::RngCore; + + fn bytes_to_field_iter() { + let byte_lens = [0, 1, 2, 16, 31, 32, 33, 48, 65, 100, 200, 5000]; + + let max_len = *byte_lens.iter().max().unwrap(); + let mut bytes = Vec::with_capacity(max_len); + // TODO pre-allocate space for elems, owned, borrowed + let mut rng = jf_utils::test_rng(); + + for len in byte_lens { + // fill bytes with random bytes and trailing zeros + bytes.resize(len, 0); + rng.fill_bytes(&mut bytes); + + // round trip, owned: + // bytes as Iterator, elems as Iterator + let owned: Vec<_> = field_to_bytes(bytes_to_field::<_, F>(bytes.clone())) + .take(bytes.len()) + .collect(); + assert_eq!(owned, bytes); + + // round trip, borrowed: + // bytes as Iterator, elems as Iterator + let elems: Vec<_> = bytes_to_field::<_, F>(bytes.iter()).collect(); + let borrowed: Vec<_> = field_to_bytes::<_, F>(elems.iter()) + .take(bytes.len()) + .collect(); + assert_eq!(borrowed, bytes); + } + + // empty input -> empty output + let bytes = Vec::new(); + assert!(bytes.iter().next().is_none()); + let mut elems_iter = bytes_to_field::<_, F>(bytes.iter()); + assert!(elems_iter.next().is_none()); + + // 1-item input -> 1-item output + let bytes = [42u8; 1]; + let mut elems_iter = bytes_to_field::<_, F>(bytes.iter()); + assert_eq!(elems_iter.next().unwrap(), F::from(42u64)); + assert!(elems_iter.next().is_none()); + } + + #[test] + fn test_bytes_field_elems_iter() { + bytes_to_field_iter::(); + bytes_to_field_iter::(); + bytes_to_field_iter::(); + } +} diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 89ccb58b4..57a69155a 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -15,11 +15,11 @@ //! Snark-friendly because it does not require a pairing. use ark_poly::EvaluationDomain; -use jf_utils::{bytes_to_field, compile_time_checks}; +use jf_utils::compile_time_checks; use super::{ - AffineRepr, Debug, DenseUVPolynomial, Digest, DynDigest, GenericAdvz, MerkleTreeScheme, - PolynomialCommitmentScheme, PrimeField, UnivariatePCS, Vec, VidResult, Write, + bytes_to_field, AffineRepr, Debug, DenseUVPolynomial, Digest, DynDigest, GenericAdvz, + MerkleTreeScheme, PolynomialCommitmentScheme, PrimeField, UnivariatePCS, Vec, VidResult, Write, }; use crate::{ alloc::string::ToString, diff --git a/utilities/src/conversion.rs b/utilities/src/conversion.rs index 0dab92f6a..4e6782f23 100644 --- a/utilities/src/conversion.rs +++ b/utilities/src/conversion.rs @@ -9,10 +9,9 @@ use ark_ff::{BigInteger, Field, PrimeField}; use ark_std::{ borrow::Borrow, cmp::min, - iter::{once, repeat, Take}, - marker::PhantomData, + iter::{once, repeat}, mem, - vec::{IntoIter, Vec}, + vec::Vec, }; use sha2::{Digest, Sha512}; @@ -292,164 +291,6 @@ pub fn compile_time_checks() -> (usize, usize, usize) { (primefield_bytes_len, extension_degree, field_bytes_len) } -/// Deterministic, infallible, invertible iterator adaptor to convert from -/// arbitrary bytes to field elements. -/// -/// The final field element is padded with zero bytes as needed. -/// -/// # Example -/// -/// ``` -/// # use jf_utils::bytes_to_field; -/// # use ark_ed_on_bn254::Fr as Fr254; -/// let bytes = [1, 2, 3]; -/// let mut elems_iter = bytes_to_field::<_, Fr254>(bytes); -/// assert_eq!(elems_iter.next(), Some(Fr254::from(197121u64))); -/// assert_eq!(elems_iter.next(), None); -/// ``` -/// -/// # Panics -/// -/// Panics only under conditions that should be checkable at compile time: -/// -/// - The [`PrimeField`] modulus bit length is too small to hold a `u64`. -/// - The [`PrimeField`] byte length is too large to fit inside a `usize`. -/// -/// If any of the above conditions holds then this function *always* panics. -pub fn bytes_to_field(bytes: I) -> impl Iterator -where - F: PrimeField, - I: IntoIterator, - I::Item: Borrow, -{ - BytesToField::new(bytes.into_iter()) -} - -/// Deterministic, infallible inverse of [`bytes_to_field`]. -/// -/// The composition of [`field_to_bytes`] with [`bytes_to_field`] might contain -/// extra zero bytes. -/// -/// # Example -/// -/// ``` -/// # use jf_utils::{bytes_to_field, field_to_bytes}; -/// # use ark_ed_on_bn254::Fr as Fr254; -/// let bytes = [1, 2, 3]; -/// let mut bytes_iter = field_to_bytes(bytes_to_field::<_, Fr254>(bytes)); -/// assert_eq!(bytes_iter.next(), Some(1)); -/// assert_eq!(bytes_iter.next(), Some(2)); -/// assert_eq!(bytes_iter.next(), Some(3)); -/// for _ in 0..28 { -/// assert_eq!(bytes_iter.next(), Some(0)); -/// } -/// assert_eq!(bytes_iter.next(), None); -/// ``` -/// -/// ## Panics -/// -/// Panics under the conditions listed at [`bytes_to_field`]. -pub fn field_to_bytes(elems: I) -> impl Iterator -where - F: PrimeField, - I: IntoIterator, - I::Item: Borrow, -{ - FieldToBytes::new(elems.into_iter()) -} - -struct BytesToField { - bytes_iter: I, - primefield_bytes_len: usize, - _phantom: PhantomData, -} - -impl BytesToField -where - F: Field, -{ - fn new(bytes_iter: I) -> Self { - let (primefield_bytes_len, ..) = compile_time_checks::(); - Self { - bytes_iter, - primefield_bytes_len, - _phantom: PhantomData, - } - } -} - -impl Iterator for BytesToField -where - I: Iterator, - I::Item: Borrow, - F: PrimeField, -{ - type Item = F; - - fn next(&mut self) -> Option { - let mut elem_bytes = Vec::with_capacity(self.primefield_bytes_len); - for _ in 0..elem_bytes.capacity() { - if let Some(byte) = self.bytes_iter.next() { - elem_bytes.push(*byte.borrow()); - } else { - break; - } - } - if elem_bytes.is_empty() { - None - } else { - Some(F::from_le_bytes_mod_order(&elem_bytes)) - } - } -} - -struct FieldToBytes { - elems_iter: I, - bytes_iter: Take>, - primefield_bytes_len: usize, - _phantom: PhantomData, -} - -impl FieldToBytes -where - F: Field, -{ - fn new(elems_iter: I) -> Self { - let (primefield_bytes_len, ..) = compile_time_checks::(); - Self { - elems_iter, - bytes_iter: Vec::new().into_iter().take(0), - primefield_bytes_len, - _phantom: PhantomData, - } - } -} - -impl Iterator for FieldToBytes -where - I: Iterator, - I::Item: Borrow, - F: PrimeField, -{ - type Item = u8; - - fn next(&mut self) -> Option { - if let Some(byte) = self.bytes_iter.next() { - return Some(byte); - } - if let Some(elem) = self.elems_iter.next() { - self.bytes_iter = elem - .borrow() - .into_bigint() - .to_bytes_le() - .into_iter() - .take(self.primefield_bytes_len); - return self.bytes_iter.next(); - } - None - } -} - #[cfg(test)] mod tests { use crate::test_rng; @@ -525,48 +366,6 @@ mod tests { } } - fn bytes_to_field_iter() { - let byte_lens = [0, 1, 2, 16, 31, 32, 33, 48, 65, 100, 200, 5000]; - - let max_len = *byte_lens.iter().max().unwrap(); - let mut bytes = Vec::with_capacity(max_len); - // TODO pre-allocate space for elems, owned, borrowed - let mut rng = test_rng(); - - for len in byte_lens { - // fill bytes with random bytes and trailing zeros - bytes.resize(len, 0); - rng.fill_bytes(&mut bytes); - - // round trip, owned: - // bytes as Iterator, elems as Iterator - let owned: Vec<_> = field_to_bytes(bytes_to_field::<_, F>(bytes.clone())) - .take(bytes.len()) - .collect(); - assert_eq!(owned, bytes); - - // round trip, borrowed: - // bytes as Iterator, elems as Iterator - let elems: Vec<_> = bytes_to_field::<_, F>(bytes.iter()).collect(); - let borrowed: Vec<_> = field_to_bytes::<_, F>(elems.iter()) - .take(bytes.len()) - .collect(); - assert_eq!(borrowed, bytes); - } - - // empty input -> empty output - let bytes = Vec::new(); - assert!(bytes.iter().next().is_none()); - let mut elems_iter = bytes_to_field::<_, F>(bytes.iter()); - assert!(elems_iter.next().is_none()); - - // 1-item input -> 1-item output - let bytes = [42u8; 1]; - let mut elems_iter = bytes_to_field::<_, F>(bytes.iter()); - assert_eq!(elems_iter.next().unwrap(), F::from(42u64)); - assert!(elems_iter.next().is_none()); - } - #[test] fn test_bytes_field_elems() { bytes_field_elems::(); @@ -576,11 +375,4 @@ mod tests { bytes_field_elems::(); bytes_field_elems::(); } - - #[test] - fn test_bytes_field_elems_iter() { - bytes_to_field_iter::(); - bytes_to_field_iter::(); - bytes_to_field_iter::(); - } } From 4b3a80602c5773346d6a2163092a00ad7696a659 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 14:24:34 -0500 Subject: [PATCH 47/54] replace compile_time_checks with new elem_byte_capacity owned by advz --- primitives/src/vid/advz/bytes_to_field.rs | 32 ++++++++++++++--------- primitives/src/vid/advz/payload_prover.rs | 28 ++++++++++---------- utilities/src/conversion.rs | 4 +-- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/primitives/src/vid/advz/bytes_to_field.rs b/primitives/src/vid/advz/bytes_to_field.rs index 402b844d7..8e9ade6e0 100644 --- a/primitives/src/vid/advz/bytes_to_field.rs +++ b/primitives/src/vid/advz/bytes_to_field.rs @@ -1,11 +1,10 @@ -use ark_ff::{BigInteger, Field, PrimeField}; +use ark_ff::{BigInteger, PrimeField}; use ark_std::{ borrow::Borrow, iter::Take, marker::PhantomData, vec::{IntoIter, Vec}, }; -use jf_utils::compile_time_checks; /// Deterministic, infallible, invertible iterator adaptor to convert from /// arbitrary bytes to field elements. @@ -75,19 +74,18 @@ where struct BytesToField { bytes_iter: I, - primefield_bytes_len: usize, + elem_byte_capacity: usize, _phantom: PhantomData, } impl BytesToField where - F: Field, + F: PrimeField, { fn new(bytes_iter: I) -> Self { - let (primefield_bytes_len, ..) = compile_time_checks::(); Self { bytes_iter, - primefield_bytes_len, + elem_byte_capacity: elem_byte_capacity::(), _phantom: PhantomData, } } @@ -102,7 +100,7 @@ where type Item = F; fn next(&mut self) -> Option { - let mut elem_bytes = Vec::with_capacity(self.primefield_bytes_len); + let mut elem_bytes = Vec::with_capacity(self.elem_byte_capacity); for _ in 0..elem_bytes.capacity() { if let Some(byte) = self.bytes_iter.next() { elem_bytes.push(*byte.borrow()); @@ -121,20 +119,19 @@ where struct FieldToBytes { elems_iter: I, bytes_iter: Take>, - primefield_bytes_len: usize, + elem_byte_capacity: usize, _phantom: PhantomData, } impl FieldToBytes where - F: Field, + F: PrimeField, { fn new(elems_iter: I) -> Self { - let (primefield_bytes_len, ..) = compile_time_checks::(); Self { elems_iter, bytes_iter: Vec::new().into_iter().take(0), - primefield_bytes_len, + elem_byte_capacity: elem_byte_capacity::(), _phantom: PhantomData, } } @@ -158,13 +155,24 @@ where .into_bigint() .to_bytes_le() .into_iter() - .take(self.primefield_bytes_len); + .take(self.elem_byte_capacity); return self.bytes_iter.next(); } None } } +/// Return the number of bytes that can be encoded into a generic [`PrimeField`] parameter. +/// +/// Returns the byte length of the [`PrimeField`] modulus minus 1. +/// +/// It should be possible to do all this at compile time but I don't know how. +/// Want to panic on overflow, so use checked arithetic and type conversion. +pub fn elem_byte_capacity() -> usize { + usize::try_from((F::MODULUS_BIT_SIZE - 1) / 8) + .expect("prime field modulus byte len should fit into usize") +} + #[cfg(test)] mod tests { use super::{bytes_to_field, field_to_bytes, PrimeField, Vec}; diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 57a69155a..57a8e89ae 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -15,11 +15,11 @@ //! Snark-friendly because it does not require a pairing. use ark_poly::EvaluationDomain; -use jf_utils::compile_time_checks; use super::{ - bytes_to_field, AffineRepr, Debug, DenseUVPolynomial, Digest, DynDigest, GenericAdvz, - MerkleTreeScheme, PolynomialCommitmentScheme, PrimeField, UnivariatePCS, Vec, VidResult, Write, + bytes_to_field, bytes_to_field::elem_byte_capacity, AffineRepr, Debug, DenseUVPolynomial, + Digest, DynDigest, GenericAdvz, MerkleTreeScheme, PolynomialCommitmentScheme, PrimeField, + UnivariatePCS, Vec, VidResult, Write, }; use crate::{ alloc::string::ToString, @@ -276,19 +276,19 @@ where { // lots of index manipulation fn index_byte_to_elem(&self, index: usize) -> usize { - index_coarsen(index, compile_time_checks::().0) + index_coarsen(index, elem_byte_capacity::()) } fn index_poly_to_byte(&self, index: usize) -> usize { index_refine( index, - self.payload_chunk_size * compile_time_checks::().0, + self.payload_chunk_size * elem_byte_capacity::(), ) } fn range_byte_to_elem(&self, range: &Range) -> Range { - range_coarsen(range, compile_time_checks::().0) + range_coarsen(range, elem_byte_capacity::()) } fn range_elem_to_byte(&self, range: &Range) -> Range { - range_refine(range, compile_time_checks::().0) + range_refine(range, elem_byte_capacity::()) } fn range_elem_to_poly(&self, range: &Range) -> Range { range_coarsen(range, self.payload_chunk_size) @@ -296,7 +296,7 @@ where fn range_byte_to_poly(&self, range: &Range) -> Range { range_coarsen( range, - self.payload_chunk_size * compile_time_checks::().0, + self.payload_chunk_size * elem_byte_capacity::(), ) } @@ -385,6 +385,7 @@ fn check_chunk_proof_consistency(chunk: &[u8], proof_chunk_len: usize) -> VidRes mod tests { use crate::vid::{ advz::{ + bytes_to_field::elem_byte_capacity, payload_prover::{CommitRecovery, Proof}, tests::*, *, @@ -394,10 +395,9 @@ mod tests { use ark_bls12_381::Bls12_381; use ark_std::{ops::Range, println, rand::Rng}; use digest::{generic_array::ArrayLength, OutputSizeUser}; - use jf_utils::compile_time_checks; use sha2::Sha256; - fn namespace_generic() + fn correctness_generic() where E: Pairing, H: Digest + DynDigest + Default + Clone + Write, @@ -409,8 +409,8 @@ mod tests { // more items as a function of the above let payload_elems_len = num_polys * payload_chunk_size; - let payload_bytes_len = payload_elems_len * compile_time_checks::().0; - let poly_bytes_len = payload_chunk_size * compile_time_checks::().0; + let payload_bytes_len = payload_elems_len * elem_byte_capacity::(); + let poly_bytes_len = payload_chunk_size * elem_byte_capacity::(); let mut rng = jf_utils::test_rng(); let payload = init_random_payload(payload_bytes_len, &mut rng); let srs = init_srs(payload_elems_len, &mut rng); @@ -499,7 +499,7 @@ mod tests { } #[test] - fn namespace() { - namespace_generic::(); + fn correctness() { + correctness_generic::(); } } diff --git a/utilities/src/conversion.rs b/utilities/src/conversion.rs index 4e6782f23..f394fcd91 100644 --- a/utilities/src/conversion.rs +++ b/utilities/src/conversion.rs @@ -272,9 +272,7 @@ where /// # Panics /// /// Panics under the conditions listed at [`bytes_to_field_elements`]. -/// -/// TODO remove `pub`, move this somewhere else. -pub fn compile_time_checks() -> (usize, usize, usize) { +fn compile_time_checks() -> (usize, usize, usize) { assert!( F::BasePrimeField::MODULUS_BIT_SIZE > 64, "base prime field modulus bit len {} too small to hold a u64", From 7dcb84f7eed2ce9ffc968ee13a8c33936a05b4a0 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 14:33:55 -0500 Subject: [PATCH 48/54] fmt and doctest --- primitives/src/vid/advz/bytes_to_field.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/primitives/src/vid/advz/bytes_to_field.rs b/primitives/src/vid/advz/bytes_to_field.rs index 8e9ade6e0..f12e0d7af 100644 --- a/primitives/src/vid/advz/bytes_to_field.rs +++ b/primitives/src/vid/advz/bytes_to_field.rs @@ -13,8 +13,9 @@ use ark_std::{ /// /// # Example /// -/// ``` -/// # use jf_utils::bytes_to_field; +/// [doctest ignored because it's a private module.] +/// ```ignore +/// # use jf_primitives::vid::advz::{bytes_to_field}; /// # use ark_ed_on_bn254::Fr as Fr254; /// let bytes = [1, 2, 3]; /// let mut elems_iter = bytes_to_field::<_, Fr254>(bytes); @@ -46,8 +47,9 @@ where /// /// # Example /// -/// ``` -/// # use jf_utils::{bytes_to_field, field_to_bytes}; +/// [doctest ignored because it's a private module.] +/// ```ignore +/// # use jf_primitives::vid::advz::{bytes_to_field, field_to_bytes}; /// # use ark_ed_on_bn254::Fr as Fr254; /// let bytes = [1, 2, 3]; /// let mut bytes_iter = field_to_bytes(bytes_to_field::<_, Fr254>(bytes)); @@ -162,7 +164,8 @@ where } } -/// Return the number of bytes that can be encoded into a generic [`PrimeField`] parameter. +/// Return the number of bytes that can be encoded into a generic [`PrimeField`] +/// parameter. /// /// Returns the byte length of the [`PrimeField`] modulus minus 1. /// From a0678336d0061f4f6fb2406401a5cc2179191c2e Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 15:26:09 -0500 Subject: [PATCH 49/54] timer tidy as per https://github.com/EspressoSystems/jellyfish/pull/389#discussion_r1383869564 --- primitives/Cargo.toml | 3 +++ primitives/src/pcs/univariate_kzg/mod.rs | 31 ++++++++++++++++-------- primitives/src/vid/advz.rs | 4 --- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml index 6fd804592..f31422696 100644 --- a/primitives/Cargo.toml +++ b/primitives/Cargo.toml @@ -112,6 +112,9 @@ std = [ "jf-relation/std", ] print-trace = ["ark-std/print-trace"] +kzg-print-trace = [ + "print-trace", +] # leave disabled to reduce pollution in downstream users of KZG (such as VID) parallel = [ "ark-ff/parallel", "ark-ec/parallel", diff --git a/primitives/src/pcs/univariate_kzg/mod.rs b/primitives/src/pcs/univariate_kzg/mod.rs index 1b0cddb16..8ab7ff7ec 100644 --- a/primitives/src/pcs/univariate_kzg/mod.rs +++ b/primitives/src/pcs/univariate_kzg/mod.rs @@ -92,9 +92,10 @@ impl PolynomialCommitmentScheme for UnivariateKzgPCS { poly: &Self::Polynomial, ) -> Result { let prover_param = prover_param.borrow(); - // let commit_time = - // start_timer!(|| format!("Committing to polynomial of degree {} ", - // poly.degree())); + + #[cfg(feature = "kzg-print-trace")] + let commit_time = + start_timer!(|| format!("Committing to polynomial of degree {} ", poly.degree())); if poly.degree() > prover_param.powers_of_g.len() { return Err(PCSError::InvalidParameters(format!( @@ -106,16 +107,21 @@ impl PolynomialCommitmentScheme for UnivariateKzgPCS { let (num_leading_zeros, plain_coeffs) = skip_leading_zeros_and_convert_to_bigints(poly); - // let msm_time = start_timer!(|| "MSM to compute commitment to plaintext - // poly"); + #[cfg(feature = "kzg-print-trace")] + let msm_time = start_timer!(|| "MSM to compute commitment to plaintext + poly"); + let commitment = E::G1::msm_bigint( &prover_param.powers_of_g[num_leading_zeros..], &plain_coeffs, ) .into_affine(); - // end_timer!(msm_time); - // end_timer!(commit_time); + #[cfg(feature = "kzg-print-trace")] + end_timer!(msm_time); + + #[cfg(feature = "kzg-print-trace")] + end_timer!(commit_time); Ok(Commitment(commitment)) } @@ -407,10 +413,15 @@ fn skip_leading_zeros_and_convert_to_bigints(p: &[F]) -> Vec { - // let to_bigint_time = start_timer!(|| "Converting polynomial coeffs to - // bigints"); + #[cfg(feature = "kzg-print-trace")] + let to_bigint_time = start_timer!(|| "Converting polynomial coeffs to + bigints"); + let coeffs = p.iter().map(|s| s.into_bigint()).collect::>(); - // end_timer!(to_bigint_time); + + #[cfg(feature = "kzg-print-trace")] + end_timer!(to_bigint_time); + coeffs } diff --git a/primitives/src/vid/advz.rs b/primitives/src/vid/advz.rs index 2f94701dc..e9a3151c7 100644 --- a/primitives/src/vid/advz.rs +++ b/primitives/src/vid/advz.rs @@ -308,9 +308,6 @@ where }; end_timer!(common_timer); - let misc_timer = start_timer!(|| { - "should be fast: compute polynomial commitments hash, pseudorandom scalar, aggregate polynomial" - }); let commit = Self::poly_commits_hash(common.poly_commits.iter())?; let pseudorandom_scalar = Self::pseudorandom_scalar(&common, &commit)?; @@ -320,7 +317,6 @@ where // and whose input point is the pseudorandom scalar. let aggregate_poly = polynomial_eval(polys.iter().map(PolynomialMultiplier), pseudorandom_scalar); - end_timer!(misc_timer); let agg_proofs_timer = start_timer!(|| format!( "compute aggregate proofs for {} storage nodes", From b7e1480b3a8615b6a0294ecb3efdb68e9703fc58 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 15:31:22 -0500 Subject: [PATCH 50/54] fix rustdoc --- primitives/src/vid/advz/payload_prover.rs | 2 +- relation/src/gadgets/emulated.rs | 2 +- utilities/src/conversion.rs | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 57a8e89ae..fe8f775fb 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -150,7 +150,7 @@ where /// KZG batch proofs and accompanying metadata. /// -/// TODO use batch proof instead of `Vec

` https://github.com/EspressoSystems/jellyfish/issues/387 +/// TODO use batch proof instead of `Vec

` pub struct Proof

{ proofs: Vec

, prefix_bytes: Vec, diff --git a/relation/src/gadgets/emulated.rs b/relation/src/gadgets/emulated.rs index 95cb9c433..ec77bedc0 100644 --- a/relation/src/gadgets/emulated.rs +++ b/relation/src/gadgets/emulated.rs @@ -19,7 +19,7 @@ use core::marker::PhantomData; use itertools::izip; use num_bigint::BigUint; -/// Parameters needed for emulating field operations over [`F`]. +/// Parameters needed for emulating field operations over [`PrimeField`]. pub trait EmulationConfig: PrimeField { /// Log2 of the other CRT modulus is 2^T. const T: usize; diff --git a/utilities/src/conversion.rs b/utilities/src/conversion.rs index f394fcd91..67fdee4e5 100644 --- a/utilities/src/conversion.rs +++ b/utilities/src/conversion.rs @@ -108,10 +108,10 @@ where /// `u64`. /// - Partition `bytes` into chunks of length P, where P is the field /// characteristic byte length minus 1. -/// - Convert each chunk into [`BasePrimeField`] via -/// [`from_le_bytes_mod_order`]. Reduction modulo the field characteristic is +/// - Convert each chunk into [`Field::BasePrimeField`] via +/// [`PrimeField::from_le_bytes_mod_order`]. Reduction modulo the field characteristic is /// guaranteed not to occur because chunk byte length is sufficiently small. -/// - Collect [`BasePrimeField`] elements into [`Field`] elements and append to +/// - Collect [`Field::BasePrimeField`] elements into [`Field`] elements and append to /// result. /// - If `bytes` is empty then result is empty. /// @@ -119,8 +119,8 @@ where /// /// Panics only under conditions that should be checkable at compile time: /// -/// - The [`BasePrimeField`] modulus bit length is too small to hold a `u64`. -/// - The byte length of a single [`BasePrimeField`] element fails to fit inside +/// - The [`Field::BasePrimeField`] modulus bit length is too small to hold a `u64`. +/// - The byte length of a single [`Field::BasePrimeField`] element fails to fit inside /// a `usize`. /// - The extension degree of the [`Field`] fails to fit inside a `usize`. /// - The byte length of a [`Field`] element fails to fit inside a `usize`. From 68453c67e11d12503b46f82c7a69cb344d4d5e2b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 15:32:02 -0500 Subject: [PATCH 51/54] fmt --- utilities/src/conversion.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/utilities/src/conversion.rs b/utilities/src/conversion.rs index 67fdee4e5..194149871 100644 --- a/utilities/src/conversion.rs +++ b/utilities/src/conversion.rs @@ -109,19 +109,21 @@ where /// - Partition `bytes` into chunks of length P, where P is the field /// characteristic byte length minus 1. /// - Convert each chunk into [`Field::BasePrimeField`] via -/// [`PrimeField::from_le_bytes_mod_order`]. Reduction modulo the field characteristic is -/// guaranteed not to occur because chunk byte length is sufficiently small. -/// - Collect [`Field::BasePrimeField`] elements into [`Field`] elements and append to -/// result. +/// [`PrimeField::from_le_bytes_mod_order`]. Reduction modulo the field +/// characteristic is guaranteed not to occur because chunk byte length is +/// sufficiently small. +/// - Collect [`Field::BasePrimeField`] elements into [`Field`] elements and +/// append to result. /// - If `bytes` is empty then result is empty. /// /// # Panics /// /// Panics only under conditions that should be checkable at compile time: /// -/// - The [`Field::BasePrimeField`] modulus bit length is too small to hold a `u64`. -/// - The byte length of a single [`Field::BasePrimeField`] element fails to fit inside -/// a `usize`. +/// - The [`Field::BasePrimeField`] modulus bit length is too small to hold a +/// `u64`. +/// - The byte length of a single [`Field::BasePrimeField`] element fails to fit +/// inside a `usize`. /// - The extension degree of the [`Field`] fails to fit inside a `usize`. /// - The byte length of a [`Field`] element fails to fit inside a `usize`. /// From f399a4fbc7903b3b9d1c39df86ae7e248b6cd893 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 19:52:32 -0500 Subject: [PATCH 52/54] new Statement type to simplify args for payload_verify as per https://github.com/EspressoSystems/jellyfish/pull/389#discussion_r1383921182 --- primitives/src/vid.rs | 13 ++- primitives/src/vid/advz/payload_prover.rs | 106 +++++++++++----------- primitives/src/vid/payload_prover.rs | 53 +++++++++-- 3 files changed, 108 insertions(+), 64 deletions(-) diff --git a/primitives/src/vid.rs b/primitives/src/vid.rs index ecce4fceb..d3e7750b8 100644 --- a/primitives/src/vid.rs +++ b/primitives/src/vid.rs @@ -14,13 +14,20 @@ use serde::{Deserialize, Serialize}; /// VID: Verifiable Information Dispersal pub trait VidScheme { /// Payload commitment. - type Commit: Clone + Debug + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + type Commit: Clone + Debug + Eq + PartialEq + Hash + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 /// Share-specific data sent to a storage node. - type Share: Clone + Debug + Eq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + type Share: Clone + Debug + Eq + PartialEq + Hash + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 /// Common data sent to all storage nodes. - type Common: CanonicalSerialize + CanonicalDeserialize + Clone + Eq + PartialEq + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 + type Common: CanonicalSerialize + + CanonicalDeserialize + + Clone + + Debug + + Eq + + PartialEq + + Hash + + Sync; // TODO https://github.com/EspressoSystems/jellyfish/issues/253 /// Compute a payload commitment fn commit_only(&self, payload: B) -> VidResult diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index fe8f775fb..894c8335c 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -23,7 +23,10 @@ use super::{ }; use crate::{ alloc::string::ToString, - vid::{payload_prover::PayloadProver, vid, VidError, VidScheme}, + vid::{ + payload_prover::{PayloadProver, Statement}, + vid, VidError, VidScheme, + }, }; use ark_std::{format, ops::Range}; @@ -83,18 +86,12 @@ where }) } - fn payload_verify( + fn payload_verify( &self, - chunk: B, - commit: &Self::Commit, - common: &Self::Common, + stmt: Statement, proof: &Proof, - ) -> VidResult> - where - B: AsRef<[u8]>, - { - let chunk = chunk.as_ref(); - check_chunk_proof_consistency(chunk, proof.chunk_range.len())?; + ) -> VidResult> { + Self::check_stmt_proof_consistency(&stmt, &proof.chunk_range)?; // index conversion let range_elem = self.range_byte_to_elem(&proof.chunk_range); @@ -103,14 +100,14 @@ where let offset_elem = range_elem.start - self.index_byte_to_elem(start_namespace_byte); check_range_poly(&range_poly)?; - Self::check_common_commit_consistency(common, commit)?; + Self::check_common_commit_consistency(stmt.common, stmt.commit)?; // prepare list of data elems let data_elems: Vec<_> = bytes_to_field::<_, P::Evaluation>( proof .prefix_bytes .iter() - .chain(chunk) + .chain(stmt.payload_subslice) .chain(proof.suffix_bytes.iter()), ) .collect(); @@ -135,7 +132,7 @@ where ))); } assert_eq!(data_elems.len(), points.len()); // sanity - let poly_commit = &common.poly_commits[range_poly.start]; + let poly_commit = &stmt.common.poly_commits[range_poly.start]; for (point, (elem, pf)) in points .iter() .zip(data_elems.iter().zip(proof.proofs.iter())) @@ -207,24 +204,18 @@ where }) } - fn payload_verify( + fn payload_verify( &self, - chunk: B, - commit: &Self::Commit, - common: &Self::Common, + stmt: Statement, proof: &CommitRecovery, - ) -> VidResult> - where - B: AsRef<[u8]>, - { - let chunk = chunk.as_ref(); - check_chunk_proof_consistency(chunk, proof.chunk_range.len())?; + ) -> VidResult> { + Self::check_stmt_proof_consistency(&stmt, &proof.chunk_range)?; // index conversion let range_poly = self.range_byte_to_poly(&proof.chunk_range); check_range_poly(&range_poly)?; - Self::check_common_commit_consistency(common, commit)?; + Self::check_common_commit_consistency(stmt.common, stmt.commit)?; // rebuild the poly commit, check against `common` let poly_commit = { @@ -237,14 +228,14 @@ where proof .prefix_bytes .iter() - .chain(chunk) + .chain(stmt.payload_subslice) .chain(proof.suffix_bytes.iter()), )) .chain(proof.suffix_elems.iter().cloned()), ); P::commit(&self.ck, &poly).map_err(vid)? }; - if poly_commit != common.poly_commits[range_poly.start] { + if poly_commit != stmt.common.poly_commits[range_poly.start] { return Ok(Err(())); } @@ -311,6 +302,32 @@ where } Ok(()) } + + fn check_stmt_proof_consistency( + stmt: &Statement, + proof_range: &Range, + ) -> VidResult<()> { + if stmt.range.is_empty() { + return Err(VidError::Argument(format!( + "empty range ({},{})", + stmt.range.start, stmt.range.end + ))); + } + if stmt.payload_subslice.len() != stmt.range.len() { + return Err(VidError::Argument(format!( + "payload_subslice length {} inconsistent with range length {}", + stmt.payload_subslice.len(), + stmt.range.len() + ))); + } + if stmt.range != *proof_range { + return Err(VidError::Argument(format!( + "statement range ({},{}) differs from proof range ({},{})", + stmt.range.start, stmt.range.end, proof_range.start, proof_range.end, + ))); + } + Ok(()) + } } fn range_coarsen(range: &Range, denominator: usize) -> Range { @@ -366,27 +383,12 @@ fn check_range_poly(range_poly: &Range) -> VidResult<()> { Ok(()) } -fn check_chunk_proof_consistency(chunk: &[u8], proof_chunk_len: usize) -> VidResult<()> { - if chunk.is_empty() { - return Err(VidError::Argument("empty chunk".to_string())); - } - - if chunk.len() != proof_chunk_len { - return Err(VidError::Argument(format!( - "chunk length {} inconsistent with proof length {}", - chunk.len(), - proof_chunk_len - ))); - } - Ok(()) -} - #[cfg(test)] mod tests { use crate::vid::{ advz::{ bytes_to_field::elem_byte_capacity, - payload_prover::{CommitRecovery, Proof}, + payload_prover::{CommitRecovery, Proof, Statement}, tests::*, *, }, @@ -478,21 +480,21 @@ mod tests { }; println!("poly {} {} case: {:?}", poly, cases.1, range); + let stmt = Statement { + payload_subslice: &payload[range.clone()], + range: range.clone(), + commit: &d.commit, + common: &d.common, + }; + let data_proof: Proof<_> = advz.payload_proof(&payload, range.clone()).unwrap(); - advz.payload_verify(&payload[range.clone()], &d.commit, &d.common, &data_proof) + advz.payload_verify(stmt.clone(), &data_proof) .unwrap() .unwrap(); let chunk_proof: CommitRecovery<_> = advz.payload_proof(&payload, range.clone()).unwrap(); - advz.payload_verify( - &payload[range.clone()], - &d.commit, - &d.common, - &chunk_proof, - ) - .unwrap() - .unwrap(); + advz.payload_verify(stmt, &chunk_proof).unwrap().unwrap(); } } } diff --git a/primitives/src/vid/payload_prover.rs b/primitives/src/vid/payload_prover.rs index 33f4e28be..5125669fd 100644 --- a/primitives/src/vid/payload_prover.rs +++ b/primitives/src/vid/payload_prover.rs @@ -23,13 +23,48 @@ pub trait PayloadProver: VidScheme { /// `chunk` is the payload sub-slice for which a proof was generated via /// `payload_proof` using `range`. In other words, `chunk` should equal /// `payload[range.start..range.end]`. - fn payload_verify( - &self, - chunk: B, - commit: &Self::Commit, - common: &Self::Common, - proof: &PROOF, - ) -> VidResult> - where - B: AsRef<[u8]>; + fn payload_verify(&self, stmt: Statement, proof: &PROOF) -> VidResult>; +} + +/// A convenience struct to reduce the list of arguments to [`PayloadProver::payload_verify`]. +/// It's the statement proved by [`PayloadProver::payload_proof`]. +/// +/// # Why the `?Sized` bound? +/// Rust hates you: +// TODO: figure out how to derive basic things like Clone, Debug, etc. +// Nothing works with the combo of both type parameter `V` and lifetime 'a. +// #[derive(Derivative)] +// #[derivative( +// Clone(bound = "V::Common: Clone, V::Commit:Clone"), +// // Debug(bound = "for<'b> &'b V::Common: ark_std::fmt::Debug, for<'b> &'b V::Commit: ark_std::fmt::Debug"), +// // Eq(bound = ""), +// // Hash(bound = ""), +// // PartialEq(bound = "") +// )] +pub struct Statement<'a, V> +where + V: VidScheme + ?Sized, +{ + /// The subslice `payload[range.start..range.end]` from a call to [`PayloadProver::payload_proof`]. + pub payload_subslice: &'a [u8], + /// The range used to make [`Self::payload_subslice`]. + pub range: Range, + /// VID commitment against which the proof will be checked. + pub commit: &'a V::Commit, + /// VID data against which the proof will be checked. + pub common: &'a V::Common, +} + +impl<'a, V> Clone for Statement<'a, V> +where + V: VidScheme, +{ + fn clone(&self) -> Self { + Self { + payload_subslice: self.payload_subslice, + range: self.range.clone(), + commit: self.commit, + common: self.common, + } + } } From 4805bfb1eaf7dd44904e04092afb0c219c4ffd4b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 19:54:07 -0500 Subject: [PATCH 53/54] fmt --- primitives/src/vid/payload_prover.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/primitives/src/vid/payload_prover.rs b/primitives/src/vid/payload_prover.rs index 5125669fd..d6218d4aa 100644 --- a/primitives/src/vid/payload_prover.rs +++ b/primitives/src/vid/payload_prover.rs @@ -26,8 +26,9 @@ pub trait PayloadProver: VidScheme { fn payload_verify(&self, stmt: Statement, proof: &PROOF) -> VidResult>; } -/// A convenience struct to reduce the list of arguments to [`PayloadProver::payload_verify`]. -/// It's the statement proved by [`PayloadProver::payload_proof`]. +/// A convenience struct to reduce the list of arguments to +/// [`PayloadProver::payload_verify`]. It's the statement proved by +/// [`PayloadProver::payload_proof`]. /// /// # Why the `?Sized` bound? /// Rust hates you: @@ -36,7 +37,8 @@ pub trait PayloadProver: VidScheme { // #[derive(Derivative)] // #[derivative( // Clone(bound = "V::Common: Clone, V::Commit:Clone"), -// // Debug(bound = "for<'b> &'b V::Common: ark_std::fmt::Debug, for<'b> &'b V::Commit: ark_std::fmt::Debug"), +// // Debug(bound = "for<'b> &'b V::Common: ark_std::fmt::Debug, for<'b> &'b +// V::Commit: ark_std::fmt::Debug"), // // Eq(bound = ""), // // Hash(bound = ""), // // PartialEq(bound = "") @@ -45,7 +47,8 @@ pub struct Statement<'a, V> where V: VidScheme + ?Sized, { - /// The subslice `payload[range.start..range.end]` from a call to [`PayloadProver::payload_proof`]. + /// The subslice `payload[range.start..range.end]` from a call to + /// [`PayloadProver::payload_proof`]. pub payload_subslice: &'a [u8], /// The range used to make [`Self::payload_subslice`]. pub range: Range, From eb8fde47d2e88a7dd27c2e8f88c9df6e75f2564d Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 6 Nov 2023 20:20:46 -0500 Subject: [PATCH 54/54] rename proof types, tidy, comment --- primitives/src/vid/advz/payload_prover.rs | 88 +++++++++++++---------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/primitives/src/vid/advz/payload_prover.rs b/primitives/src/vid/advz/payload_prover.rs index 894c8335c..18e98de23 100644 --- a/primitives/src/vid/advz/payload_prover.rs +++ b/primitives/src/vid/advz/payload_prover.rs @@ -7,12 +7,13 @@ //! Implementations of [`PayloadProver`] for `Advz`. //! //! Two implementations: -//! 1. `PROOF = `[`Proof`]: KZG batch proof for the data. Useful for small -//! sub-slices of `payload` such as an individual transaction within a block. -//! Not snark-friendly because it requires a pairing. -//! 2. `PROOF = `[`CommitRecovery`]: Rebuild the KZG commitment. Useful for -//! larger sub-slices of `payload` such as a complete namespace. -//! Snark-friendly because it does not require a pairing. +//! 1. `PROOF = `[`SmallRangeProof`]: Useful for small sub-slices of `payload` +//! such as an individual transaction within a block. Not snark-friendly +//! because it requires a pairing. Consists of metadata required to verify a +//! KZG batch proof. +//! 2. `PROOF = `[`LargeRangeProof`]: Useful for large sub-slices of `payload` +//! such as a complete namespace. Snark-friendly because it does not require +//! a pairing. Consists of metadata required to rebuild a KZG commitment. use ark_poly::EvaluationDomain; @@ -30,7 +31,30 @@ use crate::{ }; use ark_std::{format, ops::Range}; -impl PayloadProver> for GenericAdvz +/// A proof intended for use on small payload subslices. +/// +/// KZG batch proofs and accompanying metadata. +/// +/// TODO use batch proof instead of `Vec

` +pub struct SmallRangeProof

{ + proofs: Vec

, + prefix_bytes: Vec, + suffix_bytes: Vec, + chunk_range: Range, +} + +/// A proof intended for use on large payload subslices. +/// +/// Metadata needed to recover a KZG commitment. +pub struct LargeRangeProof { + prefix_elems: Vec, + suffix_elems: Vec, + prefix_bytes: Vec, + suffix_bytes: Vec, + chunk_range: Range, +} + +impl PayloadProver> for GenericAdvz where // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 P: UnivariatePCS::Evaluation>, @@ -43,7 +67,11 @@ where V::MembershipProof: Sync + Debug, V::Index: From, { - fn payload_proof(&self, payload: B, range: Range) -> VidResult> + fn payload_proof( + &self, + payload: B, + range: Range, + ) -> VidResult> where B: AsRef<[u8]>, { @@ -78,7 +106,7 @@ where let (proofs, _evals) = P::multi_open(&self.ck, &polynomial, &points).map_err(vid)?; - Ok(Proof { + Ok(SmallRangeProof { proofs, prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), suffix_bytes: payload[range.end..range_elem_byte.end].to_vec(), @@ -89,7 +117,7 @@ where fn payload_verify( &self, stmt: Statement, - proof: &Proof, + proof: &SmallRangeProof, ) -> VidResult> { Self::check_stmt_proof_consistency(&stmt, &proof.chunk_range)?; @@ -145,17 +173,7 @@ where } } -/// KZG batch proofs and accompanying metadata. -/// -/// TODO use batch proof instead of `Vec

` -pub struct Proof

{ - proofs: Vec

, - prefix_bytes: Vec, - suffix_bytes: Vec, - chunk_range: Range, -} - -impl PayloadProver> for GenericAdvz +impl PayloadProver> for GenericAdvz where // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 P: UnivariatePCS::Evaluation>, @@ -172,7 +190,7 @@ where &self, payload: B, range: Range, - ) -> VidResult> + ) -> VidResult> where B: AsRef<[u8]>, { @@ -195,7 +213,7 @@ where let prefix: Vec<_> = elems_iter.by_ref().take(offset_elem).collect(); let suffix: Vec<_> = elems_iter.skip(range_elem.len()).collect(); - Ok(CommitRecovery { + Ok(LargeRangeProof { prefix_elems: prefix, suffix_elems: suffix, prefix_bytes: payload[range_elem_byte.start..range.start].to_vec(), @@ -207,7 +225,7 @@ where fn payload_verify( &self, stmt: Statement, - proof: &CommitRecovery, + proof: &LargeRangeProof, ) -> VidResult> { Self::check_stmt_proof_consistency(&stmt, &proof.chunk_range)?; @@ -243,15 +261,6 @@ where } } -/// Metadata needed to recover a KZG commitment. -pub struct CommitRecovery { - prefix_elems: Vec, - suffix_elems: Vec, - prefix_bytes: Vec, - suffix_bytes: Vec, - chunk_range: Range, -} - impl GenericAdvz where // TODO ugly trait bounds https://github.com/EspressoSystems/jellyfish/issues/253 @@ -388,7 +397,7 @@ mod tests { use crate::vid::{ advz::{ bytes_to_field::elem_byte_capacity, - payload_prover::{CommitRecovery, Proof, Statement}, + payload_prover::{LargeRangeProof, SmallRangeProof, Statement}, tests::*, *, }, @@ -487,14 +496,17 @@ mod tests { common: &d.common, }; - let data_proof: Proof<_> = advz.payload_proof(&payload, range.clone()).unwrap(); - advz.payload_verify(stmt.clone(), &data_proof) + let small_range_proof: SmallRangeProof<_> = + advz.payload_proof(&payload, range.clone()).unwrap(); + advz.payload_verify(stmt.clone(), &small_range_proof) .unwrap() .unwrap(); - let chunk_proof: CommitRecovery<_> = + let large_range_proof: LargeRangeProof<_> = advz.payload_proof(&payload, range.clone()).unwrap(); - advz.payload_verify(stmt, &chunk_proof).unwrap().unwrap(); + advz.payload_verify(stmt, &large_range_proof) + .unwrap() + .unwrap(); } } }