Merge branch 'main' into refactor-ident

Cargo.toml

@@ -58,11 +58,12 @@ serde_json = { version = "1", default-features = false, features = ["alloc"] }
 sha2 = "0.10.8"
 snafu = { version = "0.8.4", default-features = false }
 sqlparser = { version = "0.45.0", default-features = false }
+sysinfo = { version = "0.33" }
 tiny-keccak = { version = "2.0.2", features = [ "keccak" ] }
 tempfile = "3.13.0"
 tracing = { version = "0.1.36", default-features = false }
 tracing-opentelemetry = { version = "0.22.0" }
-tracing-subscriber = { version = "0.3.0" }
+tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
 wasm-bindgen = { version = "0.2.92" }
 zerocopy = { version = "0.7.34" }
 

crates/proof-of-sql/Cargo.toml

@@ -51,6 +51,7 @@ serde_json = { workspace = true }
 sha2 = { workspace = true }
 snafu = { workspace = true }
 sqlparser = { workspace = true, features = ["serde"] }
+sysinfo = {workspace = true }
 tiny-keccak = { workspace = true }
 tracing = { workspace = true, features = ["attributes"] }
 zerocopy = { workspace = true }

crates/proof-of-sql/benches/README.md

@@ -20,6 +20,15 @@ To run benchmarks with Jaeger, you need to do the following
     docker kill jaeger
     ```
 
+### Memory logging (optional)
+
+Jaeger benchmarks default to logging any traces at `DEBUG` level and above. Memory consumption is logged at `TRACE` level. In order to capture memory consumption in the Jaeger benchmarks, add `RUST_LOG=trace` to the command.
+
+Example
+```
+RUST_LOG=trace cargo bench -p proof-of-sql --bench jaeger_benches DynamicDory
+```
+
 ## Criterion benchmarking
 
 To run benchmarks with Criterion, you need to do the following

crates/proof-of-sql/benches/jaeger_benches.rs

@@ -24,14 +24,21 @@ const SIZE: usize = 1_000_000;
 #[allow(clippy::items_after_statements)]
 fn main() {
     init_backend();
-    use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+
+    use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
+
     let tracer = opentelemetry_jaeger::new_agent_pipeline()
         .with_service_name("benches")
         .install_simple()
         .unwrap();
+
     let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer);
+
+    let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("DEBUG"));
+
     tracing_subscriber::registry()
         .with(opentelemetry)
+        .with(filter)
         .try_init()
         .unwrap();
 

crates/proof-of-sql/src/base/polynomial/evaluation_vector.rs

@@ -1,4 +1,4 @@
-use crate::base::if_rayon;
+use crate::{base::if_rayon, utils::log};
 use core::{
     cmp,
     ops::{Mul, MulAssign, Sub, SubAssign},
@@ -43,6 +43,8 @@ pub fn compute_evaluation_vector<F>(v: &mut [F], point: &[F])
 where
     F: One + Sub<Output = F> + MulAssign + SubAssign + Mul<Output = F> + Send + Sync + Copy,
 {
+    log::log_memory_usage("Start");
+
     assert!(v.len() <= (1 << point.len()));
     if point.is_empty() || v.is_empty() {
         // v is guaranteed to be at most length 1 by the assert!.
@@ -62,4 +64,6 @@ where
         };
         compute_evaluation_vector_impl(left, right, *p);
     }
+
+    log::log_memory_usage("End");
 }

crates/proof-of-sql/src/proof_primitive/dory/blitzar_metadata_table.rs

@@ -7,6 +7,7 @@ use crate::{
             full_width_of_row, index_from_row_and_column, matrix_size, row_and_column_from_index,
         },
     },
+    utils::log,
 };
 use alloc::{vec, vec::Vec};
 use ark_ec::CurveGroup;
@@ -61,7 +62,9 @@ pub fn signed_commits(
         return vec![];
     }
 
-    if_rayon!(
+    log::log_memory_usage("Start");
+
+    let res = if_rayon!(
         all_sub_commits.par_chunks_exact(committable_columns.len() * 2),
         all_sub_commits.chunks_exact(committable_columns.len() * 2)
     )
@@ -80,7 +83,11 @@ pub fn signed_commits(
         })
         .collect::<Vec<_>>()
     })
-    .collect()
+    .collect();
+
+    log::log_memory_usage("End");
+
+    res
 }
 
 /// Copies the column data to the scalar row slice.
@@ -151,6 +158,8 @@ pub fn create_blitzar_metadata_tables(
         return (vec![], vec![], vec![]);
     }
 
+    log::log_memory_usage("Start");
+
     // Keep track of the lengths of the columns to handled signed data columns.
     let ones_columns_lengths = committable_columns
         .iter()
@@ -218,7 +227,7 @@ pub fn create_blitzar_metadata_tables(
     let mut blitzar_scalars = vec![0u8; num_scalar_rows * num_scalar_columns];
 
     // Populate the scalars array.
-    let span = span!(Level::INFO, "pack_blitzar_scalars").entered();
+    let span = span!(Level::DEBUG, "pack_blitzar_scalars").entered();
     if !blitzar_scalars.is_empty() {
         if_rayon!(
             blitzar_scalars.par_chunks_exact_mut(num_scalar_columns),
@@ -269,6 +278,8 @@ pub fn create_blitzar_metadata_tables(
     }
     span.exit();
 
+    log::log_memory_usage("End");
+
     (
         blitzar_output_bit_table,
         blitzar_output_length_table,

crates/proof-of-sql/src/proof_primitive/dory/deferred_msm.rs

@@ -1,3 +1,4 @@
+use crate::utils::log;
 use alloc::{vec, vec::Vec};
 use ark_ec::VariableBaseMSM;
 use core::ops::{Add, AddAssign, Mul, MulAssign};
@@ -24,12 +25,18 @@ impl<G, F: One> DeferredMSM<G, F> {
     /// Collapse/compute the MSM into a single group element
     #[tracing::instrument(name = "DeferredMSM::compute", level = "debug", skip_all)]
     pub fn compute<V: VariableBaseMSM<MulBase = G, ScalarField = F>>(self) -> V {
+        log::log_memory_usage("Start");
+
         let (bases, scalars): (Vec<_>, Vec<_>) = self
             .pairs
             .into_iter()
             .map(|(gt, f)| (gt, f.unwrap_or(F::one())))
             .unzip();
-        V::msm_unchecked(&bases, &scalars)
+        let res = V::msm_unchecked(&bases, &scalars);
+
+        log::log_memory_usage("End");
+
+        res
     }
 }
 

crates/proof-of-sql/src/proof_primitive/dory/dory_commitment_evaluation_proof.rs

@@ -5,7 +5,10 @@ use super::{
     extended_dory_reduce_helper::extended_dory_reduce_verify_fold_s_vecs, DeferredGT,
     DoryCommitment, DoryMessages, DoryProverPublicSetup, DoryScalar, DoryVerifierPublicSetup, F,
 };
-use crate::base::{commitment::CommitmentEvaluationProof, proof::Transcript};
+use crate::{
+    base::{commitment::CommitmentEvaluationProof, proof::Transcript},
+    utils::log,
+};
 use snafu::Snafu;
 
 /// The `CommitmentEvaluationProof` for the Dory PCS.
@@ -40,6 +43,8 @@ impl CommitmentEvaluationProof for DoryEvaluationProof {
         generators_offset: u64,
         setup: &Self::ProverPublicSetup<'_>,
     ) -> Self {
+        log::log_memory_usage("Start");
+
         // Dory PCS Logic
         if generators_offset != 0 {
             // TODO: support offsets other than 0.
@@ -59,6 +64,9 @@ impl CommitmentEvaluationProof for DoryEvaluationProof {
         let mut messages = DoryMessages::default();
         let extended_state = eval_vmv_re_prove(&mut messages, transcript, state, prover_setup);
         extended_dory_inner_product_prove(&mut messages, transcript, extended_state, prover_setup);
+
+        log::log_memory_usage("End");
+
         messages
     }
 
@@ -78,6 +86,8 @@ impl CommitmentEvaluationProof for DoryEvaluationProof {
         _table_length: usize,
         setup: &Self::VerifierPublicSetup<'_>,
     ) -> Result<(), Self::Error> {
+        log::log_memory_usage("Start");
+
         let a_commit = DeferredGT::new(
             commit_batch.iter().map(|c| c.0),
             batching_factors.iter().map(|f| f.0),
@@ -115,6 +125,9 @@ impl CommitmentEvaluationProof for DoryEvaluationProof {
         ) {
             Err(DoryError::VerificationError)?;
         }
+
+        log::log_memory_usage("End");
+
         Ok(())
     }
 }

crates/proof-of-sql/src/proof_primitive/dory/dory_commitment_helper_cpu.rs

@@ -1,5 +1,5 @@
 use super::{pairings, DoryCommitment, DoryProverPublicSetup, DoryScalar, G1Projective};
-use crate::base::commitment::CommittableColumn;
+use crate::{base::commitment::CommittableColumn, utils::log};
 use alloc::vec::Vec;
 use ark_ec::VariableBaseMSM;
 use core::iter::once;
@@ -21,6 +21,8 @@ where
     &'a T: Into<DoryScalar>,
     T: Sync,
 {
+    log::log_memory_usage("Start");
+
     // Compute offsets for the matrix.
     let num_columns = 1 << setup.sigma();
     let first_row_offset = offset % num_columns;
@@ -46,11 +48,15 @@ where
     });
 
     // Compute the commitment for the entire matrix.
-    DoryCommitment(pairings::multi_pairing(
+    let res = DoryCommitment(pairings::multi_pairing(
         once(first_row_commit).chain(remaining_row_commits),
         &setup.prover_setup().Gamma_2.last().unwrap()
             [rows_offset..(rows_offset + remaining_row_count + 1)],
-    ))
+    ));
+
+    log::log_memory_usage("End");
+
+    res
 }
 
 fn compute_dory_commitment(

crates/proof-of-sql/src/proof_primitive/dory/dory_commitment_helper_gpu.rs

@@ -1,5 +1,8 @@
 use super::{pack_scalars, pairings, DoryCommitment, DoryProverPublicSetup, G1Affine};
-use crate::base::{commitment::CommittableColumn, if_rayon, slice_ops::slice_cast};
+use crate::{
+    base::{commitment::CommittableColumn, if_rayon, slice_ops::slice_cast},
+    utils::log,
+};
 use blitzar::compute::ElementP2;
 #[cfg(feature = "rayon")]
 use rayon::prelude::*;
@@ -20,6 +23,8 @@ fn compute_dory_commitments_packed_impl(
     offset: usize,
     setup: &DoryProverPublicSetup,
 ) -> Vec<DoryCommitment> {
+    log::log_memory_usage("Start");
+
     // Make sure that the committable columns are not empty.
     if committable_columns.is_empty() {
         return vec![];
@@ -84,7 +89,7 @@ fn compute_dory_commitments_packed_impl(
         .collect();
 
     // Compute the Dory commitments using multi pairing of sub-commits.
-    let span = span!(Level::INFO, "multi_pairing").entered();
+    let span = span!(Level::DEBUG, "multi_pairing").entered();
     let dc: Vec<DoryCommitment> = if_rayon!(
         cumulative_sub_commit_sums.par_iter(),
         cumulative_sub_commit_sums.iter()
@@ -100,6 +105,8 @@ fn compute_dory_commitments_packed_impl(
     .collect();
     span.exit();
 
+    log::log_memory_usage("End");
+
     dc
 }
 

crates/proof-of-sql/src/proof_primitive/dory/dory_reduce_helper.rs

@@ -2,7 +2,7 @@ use super::{
     pairings::{multi_pairing_2, multi_pairing_4},
     DeferredGT, ProverSetup, ProverState, VerifierSetup, VerifierState, F, GT,
 };
-use crate::base::if_rayon;
+use crate::{base::if_rayon, utils::log};
 #[cfg(feature = "rayon")]
 use rayon::{
     iter::IndexedParallelIterator,
@@ -24,6 +24,8 @@ pub fn dory_reduce_prove_compute_Ds(
     setup: &ProverSetup,
     half_n: usize,
 ) -> (GT, GT, GT, GT) {
+    log::log_memory_usage("Start");
+
     let (v_1L, v_1R) = state.v1.split_at(half_n);
     let (v_2L, v_2R) = state.v2.split_at(half_n);
     let (D_1L, D_1R, D_2L, D_2R) = multi_pairing_4(
@@ -32,6 +34,9 @@ pub fn dory_reduce_prove_compute_Ds(
         (setup.Gamma_1[state.nu - 1], v_2L),
         (setup.Gamma_1[state.nu - 1], v_2R),
     );
+
+    log::log_memory_usage("End");
+
     (D_1L, D_1R, D_2L, D_2R)
 }
 /// From the Dory-Reduce algorithm in section 3.2 of https://eprint.iacr.org/2020/1274.pdf.
@@ -45,12 +50,16 @@ pub fn dory_reduce_prove_mutate_v_vecs(
     setup: &ProverSetup,
     (beta, beta_inv): (F, F),
 ) {
+    log::log_memory_usage("Start");
+
     if_rayon!(state.v1.par_iter_mut(), state.v1.iter_mut())
         .zip(setup.Gamma_1[state.nu])
         .for_each(|(v, &g)| *v = (*v + g * beta).into());
     if_rayon!(state.v2.par_iter_mut(), state.v2.iter_mut())
         .zip(setup.Gamma_2[state.nu])
         .for_each(|(v, &g)| *v = (*v + g * beta_inv).into());
+
+    log::log_memory_usage("End");
 }
 /// From the Dory-Reduce algorithm in section 3.2 of https://eprint.iacr.org/2020/1274.pdf.
 ///
@@ -59,9 +68,14 @@ pub fn dory_reduce_prove_mutate_v_vecs(
 /// * C_minus = <v_1R, v_2L>
 #[tracing::instrument(level = "debug", skip_all)]
 pub fn dory_reduce_prove_compute_Cs(state: &ProverState, half_n: usize) -> (GT, GT) {
+    log::log_memory_usage("Start");
+
     let (v_1L, v_1R) = state.v1.split_at(half_n);
     let (v_2L, v_2R) = state.v2.split_at(half_n);
     let (C_plus, C_minus) = multi_pairing_2((v_1L, v_2R), (v_1R, v_2L));
+
+    log::log_memory_usage("End");
+
     (C_plus, C_minus)
 }
 
@@ -76,6 +90,8 @@ pub fn dory_reduce_prove_fold_v_vecs(
     (alpha, alpha_inv): (F, F),
     half_n: usize,
 ) {
+    log::log_memory_usage("Start");
+
     let (v_1L, v_1R) = state.v1.split_at_mut(half_n);
     let (v_2L, v_2R) = state.v2.split_at_mut(half_n);
     if_rayon!(v_1L.par_iter_mut(), v_1L.iter_mut())
@@ -86,6 +102,8 @@ pub fn dory_reduce_prove_fold_v_vecs(
         .for_each(|(v_L, v_R)| *v_L = (*v_L * alpha_inv + v_R).into());
     state.v1.truncate(half_n);
     state.v2.truncate(half_n);
+
+    log::log_memory_usage("End");
 }
 /// From the Dory-Reduce algorithm in section 3.2 of <https://eprint.iacr.org/2020/1274.pdf>.
 ///