From 9e39d7488405c42baa995d46ce16f0054f34bdd5 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Wed, 1 Nov 2023 21:20:22 +0800 Subject: [PATCH 1/4] Use halo2 `mv-lookup` branch. --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbda9765..220c8a81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,7 +1000,7 @@ source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#6 dependencies = [ "ethers-core", "ethers-signers", - "halo2_proofs 0.2.0 (git+https://github.com/scroll-tech/halo2.git?branch=develop)", + "halo2_proofs 0.2.0 (git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup)", "hex", "itertools", "lazy_static", @@ -1295,7 +1295,7 @@ source = "git+https://github.com/scroll-tech/halo2-lib?tag=v0.1.5#70588177930400 dependencies = [ "ff", "halo2_proofs 0.2.0 (git+https://github.com/axiom-crypto/halo2.git?tag=v2023_01_17)", - "halo2_proofs 0.2.0 (git+https://github.com/scroll-tech/halo2.git?branch=develop)", + "halo2_proofs 0.2.0 (git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup)", "itertools", "num-bigint", "num-integer", @@ -1341,7 +1341,7 @@ dependencies = [ [[package]] name = "halo2_proofs" version = "0.2.0" -source = "git+https://github.com/scroll-tech/halo2.git?branch=develop#9bf3562083dd9bed8a19f651b52bc810f5e2235f" +source = "git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup#6f1e6785fcf7db5a26ea3d2807c8dd76a5826786" dependencies = [ "ark-std 0.3.0", "blake2b_simd", @@ -2142,7 +2142,7 @@ version = "0.1.0" source = "git+https://github.com/scroll-tech/poseidon-circuit.git?branch=scroll-dev-0408#89e9c0456f0fa00d49cd7920c6b437c66b48db17" dependencies = [ "bitvec 1.0.1", - "halo2_proofs 0.2.0 (git+https://github.com/scroll-tech/halo2.git?branch=develop)", + "halo2_proofs 0.2.0 (git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup)", "lazy_static", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index a0fd816e..6cc07dea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ inherits = "release" debug = true [patch."https://github.com/privacy-scaling-explorations/halo2.git"] -halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" } +halo2_proofs = { git = "https://github.com/geometryresearch/halo2.git", branch = "develop-mvlookup" } [patch."https://github.com/privacy-scaling-explorations/poseidon.git"] poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "scroll-dev-0220" } [patch."https://github.com/privacy-scaling-explorations/halo2curves.git"] From a7251c592381716afde24e9898c908f57a2cf2af Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Thu, 2 Nov 2023 11:52:36 +0800 Subject: [PATCH 2/4] Replace to mv-lookup in verifier. --- snark-verifier/src/system/halo2.rs | 134 ++++++++++++---------------- snark-verifier/src/util.rs | 2 +- snark-verifier/src/util/protocol.rs | 8 +- 3 files changed, 64 insertions(+), 80 deletions(-) diff --git a/snark-verifier/src/system/halo2.rs b/snark-verifier/src/system/halo2.rs index d6a68460..1b424faf 100644 --- a/snark-verifier/src/system/halo2.rs +++ b/snark-verifier/src/system/halo2.rs @@ -1,4 +1,5 @@ //! [`halo2_proofs`] proof system +//! Reference use crate::{ halo2_proofs::{ plonk::{ @@ -12,6 +13,7 @@ use crate::{ }, util::{ arithmetic::{root_of_unity, CurveAffine, Domain, FieldExt, Rotation}, + izip, protocol::{ CommonPolynomial, Expression, InstanceCommittingKey, Query, QuotientPolynomial, }, @@ -116,7 +118,7 @@ pub fn compile<'a, C: CurveAffine, P: Params<'a, C>>( .chain(polynomials.random_query()) .chain(polynomials.permutation_fixed_queries()) .chain((0..num_proof).flat_map(move |t| polynomials.permutation_z_queries::(t))) - .chain((0..num_proof).flat_map(move |t| polynomials.lookup_queries::(t))) + .chain((0..num_proof).flat_map(move |t| polynomials.lookup_queries(t))) .collect(); let queries = (0..num_proof) @@ -125,7 +127,7 @@ pub fn compile<'a, C: CurveAffine, P: Params<'a, C>>( .chain(polynomials.instance_queries(t)) .chain(polynomials.advice_queries(t)) .chain(polynomials.permutation_z_queries::(t)) - .chain(polynomials.lookup_queries::(t)) + .chain(polynomials.lookup_queries(t)) }) .chain(polynomials.fixed_queries()) .chain(polynomials.permutation_fixed_queries()) @@ -180,10 +182,10 @@ struct Polynomials<'a, F: FieldExt> { num_challenge: Vec, advice_index: Vec, challenge_index: Vec, - num_lookup_permuted: usize, + num_lookup_m: usize, permutation_chunk_size: usize, num_permutation_z: usize, - num_lookup_z: usize, + num_lookup_phi: usize, } impl<'a, F: FieldExt> Polynomials<'a, F> { @@ -236,13 +238,13 @@ impl<'a, F: FieldExt> Polynomials<'a, F> { num_challenge, advice_index, challenge_index, - num_lookup_permuted: 2 * cs.lookups().len(), + num_lookup_m: cs.lookups().len(), permutation_chunk_size, num_permutation_z: Integer::div_ceil( &cs.permutation().get_columns().len(), &permutation_chunk_size, ), - num_lookup_z: cs.lookups().len(), + num_lookup_phi: cs.lookups().len(), } } @@ -258,8 +260,8 @@ impl<'a, F: FieldExt> Polynomials<'a, F> { iter::empty() .chain(self.num_advice.clone().iter().map(|num| self.num_proof * num)) .chain([ - self.num_proof * self.num_lookup_permuted, - self.num_proof * (self.num_permutation_z + self.num_lookup_z) + self.zk as usize, + self.num_proof * self.num_lookup_m, + self.num_proof * (self.num_permutation_z + self.num_lookup_phi) + self.zk as usize, ]) .collect() } @@ -376,40 +378,17 @@ impl<'a, F: FieldExt> Polynomials<'a, F> { } } - fn lookup_poly(&'a self, t: usize, i: usize) -> (usize, usize, usize) { - let permuted_offset = self.cs_witness_offset(); - let z_offset = permuted_offset - + self.num_witness()[self.num_advice.len()] - + self.num_proof * self.num_permutation_z; - let z = z_offset + t * self.num_lookup_z + i; - let permuted_input = permuted_offset + 2 * (t * self.num_lookup_z + i); - let permuted_table = permuted_input + 1; - (z, permuted_input, permuted_table) + fn lookup_poly(&'a self, t: usize, i: usize) -> (usize, usize) { + let m = self.cs_witness_offset() + t * self.num_lookup_m + i; + let phi = + m + self.num_witness()[self.num_advice.len()] + self.num_proof * self.num_permutation_z; + (m, phi) } - fn lookup_queries( - &'a self, - t: usize, - ) -> impl IntoIterator + 'a { - (0..self.num_lookup_z).flat_map(move |i| { - let (z, permuted_input, permuted_table) = self.lookup_poly(t, i); - if EVAL { - [ - Query::new(z, 0), - Query::new(z, 1), - Query::new(permuted_input, 0), - Query::new(permuted_input, -1), - Query::new(permuted_table, 0), - ] - } else { - [ - Query::new(z, 0), - Query::new(permuted_input, 0), - Query::new(permuted_table, 0), - Query::new(permuted_input, -1), - Query::new(z, 1), - ] - } + fn lookup_queries(&'a self, t: usize) -> impl IntoIterator + 'a { + (0..self.num_lookup_phi).flat_map(move |i| { + let (m, phi) = self.lookup_poly(t, i); + [Query::new(phi, 0), Query::new(phi, 1), Query::new(m, 0)] }) } @@ -598,22 +577,18 @@ impl<'a, F: FieldExt> Polynomials<'a, F> { } fn lookup_constraints(&'a self, t: usize) -> impl IntoIterator> + 'a { - let one = &Expression::Constant(F::one()); let l_0 = &Expression::::CommonPolynomial(CommonPolynomial::Lagrange(0)); let l_last = &self.l_last(); let l_active = &self.l_active(); let beta = &self.beta(); - let gamma = &self.gamma(); - let polys = (0..self.num_lookup_z) + let polys = (0..self.num_lookup_phi) .map(|i| { - let (z, permuted_input, permuted_table) = self.lookup_poly(t, i); + let (m, phi) = self.lookup_poly(t, i); ( - Expression::::Polynomial(Query::new(z, 0)), - Expression::::Polynomial(Query::new(z, 1)), - Expression::::Polynomial(Query::new(permuted_input, 0)), - Expression::::Polynomial(Query::new(permuted_input, -1)), - Expression::::Polynomial(Query::new(permuted_table, 0)), + Expression::::Polynomial(Query::new(phi, 0)), + Expression::::Polynomial(Query::new(phi, 1)), + Expression::::Polynomial(Query::new(m, 0)), ) }) .collect_vec(); @@ -629,35 +604,38 @@ impl<'a, F: FieldExt> Polynomials<'a, F> { .lookups() .iter() .zip(polys.iter()) - .flat_map( - |( - lookup, - (z, z_omega, permuted_input, permuted_input_omega_inv, permuted_table), - )| { - let input = compress(lookup.input_expressions()); - let table = compress(lookup.table_expressions()); - iter::empty() - .chain(Some(l_0 * (one - z))) - .chain(self.zk.then(|| l_last * (z * z - z))) - .chain(Some(if self.zk { - l_active - * (z_omega * (permuted_input + beta) * (permuted_table + gamma) - - z * (input + beta) * (table + gamma)) - } else { - z_omega * (permuted_input + beta) * (permuted_table + gamma) - - z * (input + beta) * (table + gamma) - })) - .chain(self.zk.then(|| l_0 * (permuted_input - permuted_table))) - .chain(Some(if self.zk { - l_active - * (permuted_input - permuted_table) - * (permuted_input - permuted_input_omega_inv) - } else { - (permuted_input - permuted_table) - * (permuted_input - permuted_input_omega_inv) - })) - }, - ) + .flat_map(|(lookup, (phi, phi_omega, m))| { + let inputs = lookup + .input_expressions() + .iter() + .map(|expressions| compress(expressions) + beta) + .collect_vec(); + let table = &(compress(lookup.table_expressions()) + beta); + iter::empty().chain(Some(l_0 * phi)).chain(self.zk.then(|| l_last * phi)).chain( + Some(if self.zk { + let input_prod = &inputs.iter().cloned().product::>(); + let lhs = table * input_prod * (phi_omega - phi); + let rhs = (inputs.len() > 1) + .then(|| { + (0..inputs.len()) + .map(|i| { + izip!(0.., &inputs) + .filter_map(|(j, input)| (i != j).then_some(input)) + .cloned() + .product() + }) + .sum::>() + * table + }) + .unwrap_or_else(|| table.clone()) + - m * input_prod; + + l_active * (lhs - rhs) + } else { + unimplemented!() + }), + ) + }) .collect_vec() } diff --git a/snark-verifier/src/util.rs b/snark-verifier/src/util.rs index a7792590..959cf701 100644 --- a/snark-verifier/src/util.rs +++ b/snark-verifier/src/util.rs @@ -7,7 +7,7 @@ pub mod poly; pub mod protocol; pub mod transcript; -pub(crate) use itertools::Itertools; +pub(crate) use itertools::{izip, Itertools}; #[cfg(feature = "parallel")] pub(crate) use rayon::current_num_threads; diff --git a/snark-verifier/src/util/protocol.rs b/snark-verifier/src/util/protocol.rs index e6e698cd..3c9b46e0 100644 --- a/snark-verifier/src/util/protocol.rs +++ b/snark-verifier/src/util/protocol.rs @@ -14,7 +14,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, iter::{ - Sum, {self}, + Product, Sum, {self}, }, ops::{Add, Mul, Neg, Sub}, }; @@ -346,6 +346,12 @@ impl Sum for Expression { } } +impl Product for Expression { + fn product>(iter: I) -> Self { + iter.reduce(|acc, item| acc * item).unwrap_or_else(|| Expression::Constant(F::one())) + } +} + impl One for Expression { fn one() -> Self { Expression::Constant(F::one()) From 5822e1f243b93307b00beae8142a4c606255694b Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Tue, 7 Nov 2023 17:01:47 +0800 Subject: [PATCH 3/4] Update unit test for single-layer-aggregation. --- snark-verifier-sdk/src/tests/mod.rs | 14 ++++++++++++-- .../src/tests/single_layer_aggregation.rs | 6 +++--- snark-verifier-sdk/src/tests/test_circuit_1.rs | 11 +++++++++++ snark-verifier-sdk/src/tests/test_circuit_2.rs | 11 +++++++++++ snark-verifier/src/system/halo2.rs | 2 +- 5 files changed, 38 insertions(+), 6 deletions(-) diff --git a/snark-verifier-sdk/src/tests/mod.rs b/snark-verifier-sdk/src/tests/mod.rs index 22d0737c..1ed39dd7 100644 --- a/snark-verifier-sdk/src/tests/mod.rs +++ b/snark-verifier-sdk/src/tests/mod.rs @@ -1,7 +1,7 @@ use halo2_base::halo2_proofs; use halo2_proofs::{ halo2curves::bn256::Fr, - plonk::{Advice, Column, ConstraintSystem, Fixed, Instance}, + plonk::{Advice, Column, ConstraintSystem, Fixed, Instance, TableColumn}, poly::Rotation, }; use test_circuit_1::TestCircuit1; @@ -25,6 +25,7 @@ pub struct StandardPlonkConfig { constant: Column, #[allow(dead_code)] instance: Column, + table: TableColumn, } impl StandardPlonkConfig { @@ -32,6 +33,7 @@ impl StandardPlonkConfig { let [a, b, c] = [(); 3].map(|_| meta.advice_column()); let [q_a, q_b, q_c, q_ab, constant] = [(); 5].map(|_| meta.fixed_column()); let instance = meta.instance_column(); + let table = meta.lookup_table_column(); [a, b, c].map(|column| meta.enable_equality(column)); @@ -53,6 +55,14 @@ impl StandardPlonkConfig { }, ); - StandardPlonkConfig { a, b, c, q_a, q_b, q_c, q_ab, constant, instance } + // Lookup for multiple times to test mv-lookup. + (0..5).for_each(|_| { + meta.lookup("lookup a", |meta| { + let a = meta.query_advice(a, Rotation::cur()); + vec![(a, table)] + }) + }); + + StandardPlonkConfig { a, b, c, q_a, q_b, q_c, q_ab, constant, instance, table } } } diff --git a/snark-verifier-sdk/src/tests/single_layer_aggregation.rs b/snark-verifier-sdk/src/tests/single_layer_aggregation.rs index c937d667..36f3d25c 100644 --- a/snark-verifier-sdk/src/tests/single_layer_aggregation.rs +++ b/snark-verifier-sdk/src/tests/single_layer_aggregation.rs @@ -18,7 +18,7 @@ use std::path::Path; fn test_shplonk_then_sphplonk_with_evm_verification() { std::env::set_var("VERIFY_CONFIG", "./configs/example_evm_accumulator.config"); let k = 8; - let k_agg = 21; + let k_agg = 24; let mut rng = test_rng(); let params_outer = gen_srs(k_agg); @@ -46,7 +46,7 @@ fn test_shplonk_then_sphplonk_with_evm_verification() { let snarks_2 = gen_snark_shplonk( ¶ms_inner, &pk_inner_2, - circuit_1.clone(), + circuit_2.clone(), &mut rng, Some(Path::new("data/inner_2.snark")), ); @@ -54,7 +54,7 @@ fn test_shplonk_then_sphplonk_with_evm_verification() { // Proof for circuit 2 let circuit_3 = TestCircuit2::rand(&mut rng); - let pk_inner_3 = gen_pk(¶ms_inner, &circuit_1, Some(Path::new("data/inner_3.pkey"))); + let pk_inner_3 = gen_pk(¶ms_inner, &circuit_3, Some(Path::new("data/inner_3.pkey"))); let snarks_3 = gen_snark_shplonk( ¶ms_inner, &pk_inner_3, diff --git a/snark-verifier-sdk/src/tests/test_circuit_1.rs b/snark-verifier-sdk/src/tests/test_circuit_1.rs index f2e1f7ac..c5a429e0 100644 --- a/snark-verifier-sdk/src/tests/test_circuit_1.rs +++ b/snark-verifier-sdk/src/tests/test_circuit_1.rs @@ -66,6 +66,17 @@ impl Circuit for TestCircuit1 { a.copy_advice(|| "", &mut region, config.b, 3)?; a.copy_advice(|| "", &mut region, config.c, 4)?; + Ok(()) + }, + )?; + + layouter.assign_table( + || "lookup table for values of `a`", + |mut table| { + for (i, value) in [self.0, Fr::zero(), Fr::one(), -Fr::from(5)].iter().enumerate() { + table.assign_cell(|| "", config.table, i, || Value::known(value))?; + } + Ok(()) }, ) diff --git a/snark-verifier-sdk/src/tests/test_circuit_2.rs b/snark-verifier-sdk/src/tests/test_circuit_2.rs index 687cbcc5..c1953eca 100644 --- a/snark-verifier-sdk/src/tests/test_circuit_2.rs +++ b/snark-verifier-sdk/src/tests/test_circuit_2.rs @@ -61,6 +61,17 @@ impl Circuit for TestCircuit2 { a.copy_advice(|| "", &mut region, config.b, 3)?; a.copy_advice(|| "", &mut region, config.c, 4)?; + Ok(()) + }, + )?; + + layouter.assign_table( + || "lookup table for values of `a`", + |mut table| { + for (i, value) in [self.0, Fr::zero(), Fr::one(), -Fr::from(5)].iter().enumerate() { + table.assign_cell(|| "", config.table, i, || Value::known(value))?; + } + Ok(()) }, ) diff --git a/snark-verifier/src/system/halo2.rs b/snark-verifier/src/system/halo2.rs index 1b424faf..311861b4 100644 --- a/snark-verifier/src/system/halo2.rs +++ b/snark-verifier/src/system/halo2.rs @@ -1,5 +1,5 @@ //! [`halo2_proofs`] proof system -//! Reference +//! Reference for mv-lookup use crate::{ halo2_proofs::{ plonk::{ From 22e337494d94178bd4fe4734bade73089bf64da9 Mon Sep 17 00:00:00 2001 From: Steven Gu Date: Tue, 14 Nov 2023 08:19:34 +0800 Subject: [PATCH 4/4] Update `halo2` to `develop-rc` branch. --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 220c8a81..7d3301af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,7 +1000,7 @@ source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#6 dependencies = [ "ethers-core", "ethers-signers", - "halo2_proofs 0.2.0 (git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup)", + "halo2_proofs 0.2.0 (git+https://github.com/scroll-tech/halo2.git?branch=develop-rc)", "hex", "itertools", "lazy_static", @@ -1295,7 +1295,7 @@ source = "git+https://github.com/scroll-tech/halo2-lib?tag=v0.1.5#70588177930400 dependencies = [ "ff", "halo2_proofs 0.2.0 (git+https://github.com/axiom-crypto/halo2.git?tag=v2023_01_17)", - "halo2_proofs 0.2.0 (git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup)", + "halo2_proofs 0.2.0 (git+https://github.com/scroll-tech/halo2.git?branch=develop-rc)", "itertools", "num-bigint", "num-integer", @@ -1341,7 +1341,7 @@ dependencies = [ [[package]] name = "halo2_proofs" version = "0.2.0" -source = "git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup#6f1e6785fcf7db5a26ea3d2807c8dd76a5826786" +source = "git+https://github.com/scroll-tech/halo2.git?branch=develop-rc#d7b4a5b076aeadc42a59f4323f9e456e737c96a6" dependencies = [ "ark-std 0.3.0", "blake2b_simd", @@ -2142,7 +2142,7 @@ version = "0.1.0" source = "git+https://github.com/scroll-tech/poseidon-circuit.git?branch=scroll-dev-0408#89e9c0456f0fa00d49cd7920c6b437c66b48db17" dependencies = [ "bitvec 1.0.1", - "halo2_proofs 0.2.0 (git+https://github.com/geometryresearch/halo2.git?branch=develop-mvlookup)", + "halo2_proofs 0.2.0 (git+https://github.com/scroll-tech/halo2.git?branch=develop-rc)", "lazy_static", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index 6cc07dea..47e520d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ inherits = "release" debug = true [patch."https://github.com/privacy-scaling-explorations/halo2.git"] -halo2_proofs = { git = "https://github.com/geometryresearch/halo2.git", branch = "develop-mvlookup" } +halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop-rc" } [patch."https://github.com/privacy-scaling-explorations/poseidon.git"] poseidon = { git = "https://github.com/scroll-tech/poseidon.git", branch = "scroll-dev-0220" } [patch."https://github.com/privacy-scaling-explorations/halo2curves.git"]