Merge branch 'main' into arithmetic-bench-fix

akoshelev · Aug 17, 2023 · 17af08e · 17af08e
2 parents 94f3121 + e45f74e
commit 17af08e
Show file tree

Hide file tree

Showing 182 changed files with 30,859 additions and 12,075 deletions.
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -15,13 +15,15 @@ on:
       - "benches/**/*"
       - "tests/**/*"
 
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: -D warnings
+  RUSTDOCFLAGS: -D warnings
+
 jobs:
   basic:
     name: Basic Checks
     env:
-      CARGO_TERM_COLOR: always
-      RUSTFLAGS: -D warnings
-      RUSTDOCFLAGS: -D warnings
       CARGO_INCREMENTAL: 0
 
     runs-on: ubuntu-latest
@@ -44,7 +46,7 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
 
       - name: Check formatting
-        run: cargo fmt --all -- --check --config imports_granularity=Crate
+        run: cargo fmt --all -- --check --config "imports_granularity=Crate,group_imports=StdExternalCrate"
 
       - name: Clippy
         if: ${{ success() || failure() }}
@@ -64,12 +66,13 @@ jobs:
       - name: Run Web Tests
         run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate"
 
+      - name: Run compact gate tests
+        run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+
   extra:
     name: Additional Builds and Concurrency Tests
     env:
-      CARGO_TERM_COLOR: always
       RUSTFLAGS: -D warnings -C target-cpu=native
-      RUSTDOCFLAGS: -D warnings
 
     runs-on: ubuntu-latest
 
@@ -113,3 +116,28 @@ jobs:
 
       - name: Run compact gate tests
         run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+
+  aggregate:
+    name: Build and test aggregate circuit
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy,rustfmt
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
+
+      - name: Web Tests
+        run: cargo test --no-default-features --features "aggregate-circuit cli web-app real-world-infra test-fixture descriptive-gate"
+
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -34,5 +34,5 @@ jobs:
         context: .
         file: ./docker/ci/helper.Dockerfile
         push: true
-        tags: ghcr.io/${{ github.actor }}/ipa-helper:latest
+        tags: ghcr.io/${{ github.repository }}/ipa-helper:latest
         platforms: linux/amd64
diff --git a/0001-CI-split-extra-step-into-two.patch b/0001-CI-split-extra-step-into-two.patch
@@ -0,0 +1,88 @@
+From ba6c503fbe930a9298d83c04bab15f329faf13ae Mon Sep 17 00:00:00 2001
+From: Alex Koshelev <[email protected]>
+Date: Fri, 11 Aug 2023 14:25:03 -0700
+Subject: [PATCH] CI: split extra step into two
+
+This is an attempt to increase parallelism on CI. Those builds take ~4 mins and running tests take another 5-6 mins. Splitting this step should make CI run faster.
+---
+ .github/workflows/check.yml | 42 ++++++++++++++++++++++++++++---------
+ 1 file changed, 32 insertions(+), 10 deletions(-)
+
+diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
+index 54f869ac..9a8e1886 100644
+--- a/.github/workflows/check.yml
++++ b/.github/workflows/check.yml
+@@ -15,13 +15,15 @@ on:
+       - "benches/**/*"
+       - "tests/**/*"
+
++env:
++  CARGO_TERM_COLOR: always
++  RUSTFLAGS: -D warnings
++  RUSTDOCFLAGS: -D warnings
++
+ jobs:
+   basic:
+     name: Basic Checks
+     env:
+-      CARGO_TERM_COLOR: always
+-      RUSTFLAGS: -D warnings
+-      RUSTDOCFLAGS: -D warnings
+       CARGO_INCREMENTAL: 0
+
+     runs-on: ubuntu-latest
+@@ -64,12 +66,13 @@ jobs:
+       - name: Run Web Tests
+         run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate"
+
+-  extra:
+-    name: Additional Builds and Concurrency Tests
++      - name: Run compact gate tests
++        run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
++
++  extra-builds:
++    name: Additional Builds
+     env:
+-      CARGO_TERM_COLOR: always
+       RUSTFLAGS: -D warnings -C target-cpu=native
+-      RUSTDOCFLAGS: -D warnings
+
+     runs-on: ubuntu-latest
+
+@@ -102,11 +105,30 @@ jobs:
+       - name: Build concurrency tests (debug mode)
+         run: cargo build --features shuttle
+
++  benches-and-fuzzy:
++    name: Run benchmarks and concurrency tests
++
++    runs-on: ubuntu-latest
++
++    steps:
++      - uses: actions/checkout@v3
++
++      - uses: dtolnay/rust-toolchain@stable
++        with:
++          components: clippy,rustfmt
++
++      - uses: actions/cache@v3
++        with:
++          path: |
++            ~/.cargo/bin/
++            ~/.cargo/registry/index/
++            ~/.cargo/registry/cache/
++            ~/.cargo/git/db/
++            target/
++          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
++
+       - name: Run concurrency tests
+         run: cargo test --release --features shuttle
+
+       - name: Run IPA bench
+         run: cargo bench --bench oneshot_ipa --no-default-features --features "enable-benches descriptive-gate"
+-
+-      - name: Run compact gate tests
+-        run: cargo test --no-default-features --features "cli web-app real-world-infra test-fixture compact-gate"
+-- 
+2.31.1
+
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,12 +5,17 @@ rust-version = "1.64.0"
 edition = "2021"
 
 [features]
-# by default remove all TRACE, DEBUG spans from release builds
-default = ["web-app", "in-memory-infra", "tracing/max_level_trace", "tracing/release_max_level_info", "descriptive-gate"]
+default = [
+    "web-app",
+    "in-memory-infra",
+    # by default remove all TRACE, DEBUG spans from release builds
+    "tracing/max_level_trace",
+    "tracing/release_max_level_info",
+    "descriptive-gate",
+    "aggregate-circuit"
+]
 cli = ["comfy-table", "clap"]
 enable-serde = ["serde", "serde_json"]
-protocol-perf-testing = ["no-prss"]
-no-prss = []
 disable-metrics = []
 # TODO move web-app to a separate crate. It adds a lot of build time to people who mostly write protocols
 # TODO Consider moving out benches as well
@@ -33,6 +38,10 @@ step-trace = ["descriptive-gate"]
 descriptive-gate = []
 compact-gate = []
 
+# Standalone aggregation protocol. We use IPA infra for communication
+# but it has nothing to do with IPA.
+aggregate-circuit = []
+
 [dependencies]
 ipa-macros = { version = "*", path = "./ipa-macros" }
 aes = "0.8"
@@ -55,7 +64,7 @@ generic-array = "0.14.7"
 hex = { version = "0.4", features = ["serde"] }
 hkdf = "0.12.3"
 hpke = { version = "0.10.0", default-features = false, features = ["std", "x25519-dalek"] }
-hyper = { version = "0.14.26", optional = true, features = ["client", "h2", "stream"] }
+hyper = { version = "0.14.26", optional = true, features = ["client", "h2", "stream", "runtime"] }
 hyper-rustls = { version = "0.24.0", optional = true, features = ["http2"] }
 iai = { version = "0.1.1", optional = true }
 metrics = "0.21.0"
@@ -66,12 +75,14 @@ pin-project = "1.0"
 rand = "0.8"
 rand_core = "0.6"
 rcgen = { version = "0.10", optional = true }
+rustls = "0.21.5"
 rustls-pemfile = { version = "1", optional = true }
 # TODO consider using zerocopy or serde_bytes or in-house serialization
 serde = { version = "1.0", optional = true, features = ["derive"] }
 serde_json = { version = "1.0", optional = true }
 sha2 = "0.10.6"
 shuttle-crate = { package = "shuttle", version = "0.6.1", optional = true }
+strum = {version = "0.25", features = ["derive"] }
 thiserror = "1.0"
 time = { version = "0.3", optional = true }
 tinyvec = "1.6"

diff --git a/benches/oneshot/arithmetic_circuit.rs b/benches/oneshot/arithmetic_circuit.rs
@@ -1,6 +1,7 @@
+use std::time::Instant;
+
 use clap::Parser;
 use ipa::{ff::Fp31, secret_sharing::SharedValue, test_fixture::circuit};
-use std::time::Instant;
 
 #[derive(Debug, Parser)]
 pub struct CircuitArgs {

diff --git a/benches/oneshot/ipa.rs b/benches/oneshot/ipa.rs
@@ -1,3 +1,8 @@
+use std::{
+    num::{NonZeroU32, NonZeroUsize},
+    time::Instant,
+};
+
 use clap::Parser;
 use ipa::{
     error::Error,
@@ -8,11 +13,7 @@ use ipa::{
         EventGenerator, EventGeneratorConfig, TestWorld, TestWorldConfig,
     },
 };
-use rand::{rngs::StdRng, thread_rng, Rng, SeedableRng};
-use std::{
-    num::{NonZeroU32, NonZeroUsize},
-    time::Instant,
-};
+use rand::{random, rngs::StdRng, SeedableRng};
 use tokio::runtime::Builder;
 
 #[cfg(all(not(target_env = "msvc"), not(feature = "dhat-heap")))]
@@ -102,7 +103,7 @@ async fn run(args: Args) -> Result<(), Error> {
         ..TestWorldConfig::default()
     };
 
-    let seed = args.random_seed.unwrap_or_else(|| thread_rng().gen());
+    let seed = args.random_seed.unwrap_or_else(|| random());
     tracing::trace!(
         "Using random seed: {seed} for {q} records",
         q = args.query_size
@@ -120,8 +121,12 @@ async fn run(args: Args) -> Result<(), Error> {
     .take(args.query_size)
     .collect::<Vec<_>>();
 
-    let expected_results =
-        ipa_in_the_clear(&raw_data, args.per_user_cap, args.attribution_window());
+    let expected_results = ipa_in_the_clear(
+        &raw_data,
+        args.per_user_cap,
+        args.attribution_window(),
+        args.breakdown_keys,
+    );
 
     let world = TestWorld::new_with(config.clone());
     tracing::trace!("Preparation complete in {:?}", _prep_time.elapsed());

diff --git a/benches/oneshot/sort.rs b/benches/oneshot/sort.rs
@@ -1,18 +1,21 @@
+use std::time::Instant;
+
+use futures::stream::iter as stream_iter;
 use ipa::{
     error::Error,
-    ff::{Field, Fp32BitPrime, GaloisField, Gf40Bit},
+    ff::{Field, Fp32BitPrime, GaloisField},
     helpers::GatewayConfig,
     protocol::{
-        context::{Context, Validator},
-        modulus_conversion::{convert_all_bits, convert_all_bits_local},
+        context::{validator::SemiHonest as SemiHonestValidator, Validator},
         sort::generate_permutation_opt::generate_permutation_opt,
         MatchKey,
     },
-    secret_sharing::SharedValue,
-    test_fixture::{join3, Reconstruct, Runner, TestWorld, TestWorldConfig},
+    secret_sharing::{
+        replicated::semi_honest::AdditiveShare as Replicated, IntoShares, SharedValue,
+    },
+    test_fixture::{join3, Reconstruct, TestWorld, TestWorldConfig},
 };
 use rand::Rng;
-use std::time::Instant;
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 3)]
 async fn main() -> Result<(), Error> {
@@ -34,27 +37,14 @@ async fn main() -> Result<(), Error> {
         match_keys.push(rng.gen::<MatchKey>());
     }
 
-    let converted_shares = world
-        .semi_honest(
-            match_keys.clone().into_iter(),
-            |ctx, match_key| async move {
-                convert_all_bits::<BenchField, _, _>(
-                    &ctx,
-                    &convert_all_bits_local(ctx.role(), match_key.into_iter()),
-                    Gf40Bit::BITS,
-                    NUM_MULTI_BITS,
-                )
-                .await
-                .unwrap()
-            },
-        )
-        .await;
+    let [s0, s1, s2] = match_keys.iter().cloned().share_with(&mut rng);
 
     let start = Instant::now();
-    let [(v0, r0), (v1, r1), (v2, r2)] = join3(
-        generate_permutation_opt(ctx0, converted_shares[0].iter()),
-        generate_permutation_opt(ctx1, converted_shares[1].iter()),
-        generate_permutation_opt(ctx2, converted_shares[2].iter()),
+    let [(v0, r0), (v1, r1), (v2, r2)]: [(SemiHonestValidator<'_, BenchField>, Vec<Replicated<_>>);
+        3] = join3(
+        generate_permutation_opt(ctx0, stream_iter(s0), NUM_MULTI_BITS, MatchKey::BITS),
+        generate_permutation_opt(ctx1, stream_iter(s1), NUM_MULTI_BITS, MatchKey::BITS),
+        generate_permutation_opt(ctx2, stream_iter(s2), NUM_MULTI_BITS, MatchKey::BITS),
     )
     .await;
     let result = join3(v0.validate(r0), v1.validate(r1), v2.validate(r2)).await;

diff --git a/docker/helper.Dockerfile b/docker/helper.Dockerfile
@@ -18,7 +18,7 @@ ARG IDENTITY
 ARG HOSTNAME
 ARG SOURCES_DIR
 
-RUN apt-get update && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
 COPY --from=builder ${SOURCES_DIR}/target/release/helper $HELPER_BIN_PATH
 
 # generate certificate/private key for TLS

diff --git a/ipa-macros/Cargo.toml b/ipa-macros/Cargo.toml
@@ -16,7 +16,7 @@ path = "tests/mod.rs"
 required-features = ["trybuild"]
 
 [dependencies]
-syn = { version = "2.0.15", features = ["extra-traits"] }
+syn = { version = "2.0.15", features = ["full"] }
 quote = "1.0.27"
 
 [dev-dependencies]