Update to latest upstream (#13)

* feat: expose `transcript_repr` of `VerifyingKey` and reduce the trait constraint (privacy-scaling-explorations#200) * Synchronize with upstream (privacy-scaling-explorations#199) * refactor: add default impl for `SyncDeps` for backward compatability * feat: pick changes from zcash#728 and changes of flag `test-dev-graph` * feat: pick changes from zcash#622 * feat: pick changes about mod `circuit` and mod `dev` * feat: pick rest changes of `halo2_proofs` * fix: when `--no-default-features` * ci: sync from upstream, and deduplicate jobs when push to `main`, and remove always failing job `codecov`. * fix: make `commit_zk` runnable when `--no-default-features` * fix: put remaining multicore code behind feature flag * style: run cargo fmt --------- Co-authored-by: Han <[email protected]>
axiom-crypto · Sep 17, 2023 · f0baec1 · f0baec1
1 parent e1a04ce
commit f0baec1
Show file tree

Hide file tree

Showing 68 changed files with 2,034 additions and 1,726 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,14 +1,25 @@
 name: CI checks
 
-on: [pull_request, push]
+on:
+  merge_group:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   test:
-    name: Test on ${{ matrix.os }}
+    name: Test on ${{ matrix.os }} with ${{ matrix.feature_set }} features
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        feature_set: [basic, all]
         os: [ubuntu-latest]
+        include:
+          - feature_set: basic
+            features: batch,dev-graph,gadget-traces
+          - feature_set: all
+            features: batch,dev-graph,gadget-traces,multicore,test-dev-graph,thread-safe-region,sanity-checks,circuit-params
 
     steps:
       - uses: actions/checkout@v3
@@ -19,7 +30,67 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --verbose --release --all --all-features
+          args: --verbose --release --workspace --no-default-features --features "${{ matrix.features }}"
+
+  build:
+    name: Build target ${{ matrix.target }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - wasm32-unknown-unknown
+          - wasm32-wasi
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: false
+      - name: Add target
+        run: rustup target add ${{ matrix.target }}
+      - name: cargo build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --no-default-features --features batch,dev-graph,gadget-traces --target ${{ matrix.target }}
+
+  bitrot:
+    name: Bitrot check
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: false
+      # Build benchmarks to prevent bitrot
+      - name: Build benchmarks
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --benches --examples --all-features
+
+  doc-links:
+    name: Intra-doc links
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          override: false
+      - name: cargo fetch
+        uses: actions-rs/cargo@v1
+        with:
+          command: fetch
+
+      # Ensure intra-documentation links all resolve correctly
+      # Requires #![deny(intra_doc_link_resolution_failure)] in crates.
+      - name: Check intra-doc links
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --all --document-private-items
 
   example:
     name: Examples on ubuntu

diff --git a/.github/workflows/ci_main.yml b/.github/workflows/ci_main.yml
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@
 
 ## Minimum Supported Rust Version
 
-Requires Rust **1.56.1** or higher.
+Requires Rust **1.65.0** or higher.
 
 Minimum supported Rust version can be changed in the future, but it will be done with a
 minor version bump.
@@ -14,6 +14,10 @@ minor version bump.
 `halo2` currently uses [rayon](https://github.com/rayon-rs/rayon) for parallel computation.
 The `RAYON_NUM_THREADS` environment variable can be used to set the number of threads.
 
+You can disable `rayon` by disabling the `"multicore"` feature.
+Warning! Halo2 will lose access to parallelism if you disable the `"multicore"` feature.
+This will significantly degrade performance.
+
 ## License
 
 Licensed under either of

diff --git a/halo2/Cargo.toml b/halo2/Cargo.toml
@@ -19,7 +19,7 @@ all-features = true
 rustdoc-args = ["--cfg", "docsrs", "--html-in-header", "katex-header.html"]
 
 [dependencies]
-halo2_proofs = { version = "0.2", path = "../halo2_proofs" }
+halo2_proofs = { version = "0.2", path = "../halo2_proofs", default-features = false }
 
 [lib]
 bench = false
diff --git a/halo2_proofs/Cargo.toml b/halo2_proofs/Cargo.toml
@@ -50,7 +50,6 @@ harness = false
 
 [dependencies]
 backtrace = { version = "0.3", optional = true }
-rayon = "1.7"
 crossbeam = "0.8"
 ff = "0.13"
 group = "0.13"
@@ -63,11 +62,15 @@ blake2b_simd = "1"
 rustc-hash = "1.1"
 sha3 = "0.10"
 ark-std = { version = "0.3.0", features = ["print-trace"], optional = true }
+maybe-rayon = { version = "0.1.0", default-features = false }
 
 # Developer tooling dependencies
-plotters = { version = "0.3.0", optional = true }
+plotters = { version = "0.3.0", default-features = false, optional = true }
 tabbycat = { version = "0.1", features = ["attributes"], optional = true }
 
+# Legacy circuit compatibility
+halo2_legacy_pdqsort = { version = "0.1.0", optional = true }
+
 [dev-dependencies]
 assert_matches = "1.5"
 criterion = "0.3"
@@ -80,8 +83,15 @@ rand_chacha = "0.3.1"
 getrandom = { version = "0.2", features = ["js"] }
 
 [features]
-default = ["batch", "circuit-params"]
+default = ["batch", "multicore", "circuit-params"]
+multicore = ["maybe-rayon/threads"]
 dev-graph = ["plotters", "tabbycat"]
+test-dev-graph = [
+    "dev-graph",
+    "plotters/bitmap_backend",
+    "plotters/bitmap_encoder",
+    "plotters/ttf",
+]
 gadget-traces = ["backtrace"]
 # thread-safe-region = []
 sanity-checks = []
@@ -100,4 +110,4 @@ name = "serialization"
 name = "shuffle"
 
 [[example]]
-name = "shuffle_api"
+name = "shuffle_api"
diff --git a/halo2_proofs/README.md b/halo2_proofs/README.md
@@ -4,7 +4,7 @@
 
 ## Minimum Supported Rust Version
 
-Requires Rust **1.56.1** or higher.
+Requires Rust **1.65.0** or higher.
 
 Minimum supported Rust version can be changed in the future, but it will be done with a
 minor version bump.
@@ -15,6 +15,10 @@ minor version bump.
 computation. The `RAYON_NUM_THREADS` environment variable can be used to set the number of
 threads.
 
+You can disable `rayon` by disabling the `"multicore"` feature.
+Warning! Halo2 will lose access to parallelism if you disable the `"multicore"` feature.
+This will significantly degrade performance.
+
 ## License
 
 Licensed under either of

diff --git a/halo2_proofs/benches/commit_zk.rs b/halo2_proofs/benches/commit_zk.rs
@@ -2,12 +2,20 @@ extern crate criterion;
 
 use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
 use group::ff::Field;
-use halo2_proofs::*;
+use halo2_proofs::arithmetic::parallelize;
 use halo2curves::pasta::pallas::Scalar;
 use rand_chacha::rand_core::RngCore;
 use rand_chacha::ChaCha20Rng;
 use rand_core::SeedableRng;
-use rayon::{current_num_threads, prelude::*};
+use std::{collections::HashMap, iter};
+
+#[cfg(feature = "multicore")]
+use maybe_rayon::current_num_threads;
+
+#[cfg(not(feature = "multicore"))]
+fn current_num_threads() -> usize {
+    1
+}
 
 fn rand_poly_serial(mut rng: ChaCha20Rng, domain: usize) -> Vec<Scalar> {
     // Sample a random polynomial of degree n - 1
@@ -21,25 +29,33 @@ fn rand_poly_serial(mut rng: ChaCha20Rng, domain: usize) -> Vec<Scalar> {
 
 fn rand_poly_par(mut rng: ChaCha20Rng, domain: usize) -> Vec<Scalar> {
     // Sample a random polynomial of degree n - 1
-    let n_threads = current_num_threads();
-    let n = 1usize << domain;
-    let n_chunks = n_threads + usize::from(n % n_threads != 0);
-    let mut rand_vec = vec![Scalar::zero(); n];
+    let n = 1usize << domain as usize;
+    let mut random_poly = vec![Scalar::ZERO; n];
 
-    let mut thread_seeds: Vec<ChaCha20Rng> = (0..n_chunks)
-        .map(|_| {
+    let num_threads = current_num_threads();
+    let chunk_size = n / num_threads;
+    let thread_seeds = (0..)
+        .step_by(chunk_size + 1)
+        .take(n % num_threads)
+        .chain(
+            (chunk_size != 0)
+                .then(|| ((n % num_threads) * (chunk_size + 1)..).step_by(chunk_size))
+                .into_iter()
+                .flatten(),
+        )
+        .take(num_threads)
+        .zip(iter::repeat_with(|| {
             let mut seed = [0u8; 32];
             rng.fill_bytes(&mut seed);
             ChaCha20Rng::from_seed(seed)
-        })
-        .collect();
+        }))
+        .collect::<HashMap<_, _>>();
 
-    thread_seeds
-        .par_iter_mut()
-        .zip_eq(rand_vec.par_chunks_mut(n / n_threads))
-        .for_each(|(mut rng, chunk)| chunk.iter_mut().for_each(|v| *v = Scalar::random(&mut rng)));
-
-    rand_vec
+    parallelize(&mut random_poly, |chunk, offset| {
+        let mut rng = thread_seeds[&offset].clone();
+        chunk.iter_mut().for_each(|v| *v = Scalar::random(&mut rng));
+    });
+    random_poly
 }
 
 fn bench_commit(c: &mut Criterion) {

diff --git a/halo2_proofs/src/arithmetic.rs b/halo2_proofs/src/arithmetic.rs
@@ -56,7 +56,7 @@ fn multiexp_serial<C: CurveAffine>(coeffs: &[C::Scalar], bases: &[C], acc: &mut
 
         let mut tmp = u64::from_le_bytes(v);
         tmp >>= skip_bits - (skip_bytes * 8);
-        tmp = tmp % (1 << c);
+        tmp %= 1 << c;
 
         tmp as usize
     }
@@ -141,7 +141,7 @@ fn multiexp_serial<C: CurveAffine>(coeffs: &[C::Scalar], bases: &[C], acc: &mut
         let mut running_sum = C::Curve::identity();
         for exp in buckets.into_iter().take(max_bits).rev() {
             running_sum = exp.add(running_sum);
-            *acc = *acc + &running_sum;
+            *acc += &running_sum;
         }
     }
 }
@@ -304,7 +304,7 @@ pub fn recursive_butterfly_arithmetic<Scalar: Field, G: FftGroup<Scalar>>(
         a[1] -= &t;
     } else {
         let (left, right) = a.split_at_mut(n / 2);
-        rayon::join(
+        multicore::join(
             || recursive_butterfly_arithmetic(left, n / 2, twiddle_chunk * 2, twiddles),
             || recursive_butterfly_arithmetic(right, n / 2, twiddle_chunk * 2, twiddles),
         );