From 849ec548ade6455981537ac26336f5544239ce3b Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Thu, 27 Jul 2023 11:35:53 +0800 Subject: [PATCH 1/6] collect step tracing each time a step is narrowed --- src/protocol/step/descriptive.rs | 2 ++ src/telemetry/mod.rs | 7 +++++++ src/telemetry/step_stats.rs | 11 +++++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/protocol/step/descriptive.rs b/src/protocol/step/descriptive.rs index c681df59a..3a285ffa1 100644 --- a/src/protocol/step/descriptive.rs +++ b/src/protocol/step/descriptive.rs @@ -1,4 +1,5 @@ use super::{Step, StepNarrow}; +use crate::telemetry::{labels::STEP, metrics::STEP_NARROWED}; use std::fmt::{Debug, Display, Formatter}; /// A descriptive representation of a unique step in protocol execution. @@ -53,6 +54,7 @@ impl StepNarrow for Descriptive { id += [std::any::type_name::(), "::"].concat().as_ref(); } id += step.as_ref(); + metrics::increment_counter!(STEP_NARROWED, STEP => id.clone()); Self { id } } diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index bb4ece965..95a8ae0a6 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -16,6 +16,7 @@ pub mod metrics { pub const BYTES_SENT: &str = "bytes.sent"; pub const INDEXED_PRSS_GENERATED: &str = "i.prss.gen"; pub const SEQUENTIAL_PRSS_GENERATED: &str = "s.prss.gen"; + pub const STEP_NARROWED: &str = "step.narrowed"; #[cfg(feature = "web-app")] pub mod web { @@ -98,5 +99,11 @@ pub mod metrics { Unit::Count, "Number of times PRSS is used as CPRNG to generate a random value" ); + + describe_counter!( + STEP_NARROWED, + Unit::Count, + "Trace steps each time they are narrowed" + ); } } diff --git a/src/telemetry/step_stats.rs b/src/telemetry/step_stats.rs index ae99b3587..5fc4090e7 100644 --- a/src/telemetry/step_stats.rs +++ b/src/telemetry/step_stats.rs @@ -3,7 +3,9 @@ use crate::telemetry::{ labels, - metrics::{BYTES_SENT, INDEXED_PRSS_GENERATED, RECORDS_SENT, SEQUENTIAL_PRSS_GENERATED}, + metrics::{ + BYTES_SENT, INDEXED_PRSS_GENERATED, RECORDS_SENT, SEQUENTIAL_PRSS_GENERATED, STEP_NARROWED, + }, stats::Metrics, }; use std::{ @@ -38,18 +40,19 @@ impl CsvExporter for Metrics { if self.print_header { writeln!( w, - "Step,Records sent,Bytes sent,Indexed PRSS,Sequential PRSS" + "Step,Records sent,Bytes sent,Indexed PRSS,Sequential PRSS,Step narrowed" )?; } for (step, stats) in steps_stats.all_steps() { writeln!( w, - "{},{},{},{},{}", + "{},{},{},{},{},{}", step, stats.get(RECORDS_SENT), stats.get(BYTES_SENT), stats.get(INDEXED_PRSS_GENERATED), - stats.get(SEQUENTIAL_PRSS_GENERATED) + stats.get(SEQUENTIAL_PRSS_GENERATED), + stats.get(STEP_NARROWED), )?; } From 69795bf9abc7807cdf446083d0f919674234514f Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Wed, 2 Aug 2023 14:45:24 +0800 Subject: [PATCH 2/6] hide step tracing --- src/protocol/step/descriptive.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/protocol/step/descriptive.rs b/src/protocol/step/descriptive.rs index 3a285ffa1..5d42ddcd8 100644 --- a/src/protocol/step/descriptive.rs +++ b/src/protocol/step/descriptive.rs @@ -1,4 +1,5 @@ use super::{Step, StepNarrow}; +#[cfg(feature = "step-trace")] use crate::telemetry::{labels::STEP, metrics::STEP_NARROWED}; use std::fmt::{Debug, Display, Formatter}; @@ -54,7 +55,10 @@ impl StepNarrow for Descriptive { id += [std::any::type_name::(), "::"].concat().as_ref(); } id += step.as_ref(); - metrics::increment_counter!(STEP_NARROWED, STEP => id.clone()); + #[cfg(feature = "step-trace")] + { + metrics::increment_counter!(STEP_NARROWED, STEP => id.clone()); + } Self { id } } From 0003ddaa09d4f9497175424456b756a0ddaba784 Mon Sep 17 00:00:00 2001 From: Taiki Yamaguchi Date: Thu, 3 Aug 2023 10:44:17 +0800 Subject: [PATCH 3/6] better counter description --- src/telemetry/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index 95a8ae0a6..8f90befbb 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -103,7 +103,7 @@ pub mod metrics { describe_counter!( STEP_NARROWED, Unit::Count, - "Trace steps each time they are narrowed" + "Number of times the step is narrowed" ); } } From df0f01ebfd4b636655caf573cf733dbee9fdb1ea Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Thu, 10 Aug 2023 10:01:38 -0700 Subject: [PATCH 4/6] Use strings in Compact gate Display and Debug impl These are mostly used to print errors, so allocations don't happen often Otherwise, errors look like this ``` InfraError(ReceiveError { source: H1, step: "12287", inner: EndOfStream { record_id: RecordId(999424) } }) ``` --- src/protocol/step/compact.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/protocol/step/compact.rs b/src/protocol/step/compact.rs index 5ba8d9acf..5314a11bb 100644 --- a/src/protocol/step/compact.rs +++ b/src/protocol/step/compact.rs @@ -20,13 +20,13 @@ impl From<&str> for Compact { impl Display for Compact { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) + write!(f, "{}", self.as_ref()) } } impl Debug for Compact { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "step={}", self.0) + write!(f, "gate[{}]={}", self.0, self.as_ref()) } } From 885fdadeae00a3dfe53d4b53ce439edca418f0d9 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Fri, 11 Aug 2023 11:29:56 -0700 Subject: [PATCH 5/6] Change pre-commit and CI hook to reflect new formatting rules --- .github/workflows/check.yml | 2 +- pre-commit | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b15244baa..54f869acc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -44,7 +44,7 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} - name: Check formatting - run: cargo fmt --all -- --check --config imports_granularity=Crate + run: cargo fmt --all -- --check --config "imports_granularity=Crate,group_imports=StdExternalCrate" - name: Clippy if: ${{ success() || failure() }} diff --git a/pre-commit b/pre-commit index ccb0418d5..e5846d02e 100755 --- a/pre-commit +++ b/pre-commit @@ -53,9 +53,11 @@ stashfile=$(mktemp .pre-commit.stashXXXXXX) trap 'set +e;git stash pop -q; rm -f "$stashfile"; restore_merge_files' EXIT save_merge_files git stash push -k -u -q -m "pre-commit stash" -if ! errors=($(cargo fmt -- --check --config imports_granularity=Crate -l)); then + +fmtconfig="imports_granularity=Crate,group_imports=StdExternalCrate" +if ! errors=($(cargo fmt --all -- --check --config "$fmtconfig" -l)); then echo "Formatting errors found." - echo "Run \`cargo fmt -- --config imports_granularity=Crate\` to fix the following files:" + echo "Run \`cargo fmt --all -- --config \"$fmtconfig\"\` to fix the following files:" for err in "${errors[@]}"; do echo " $err" done @@ -102,4 +104,4 @@ check "Sort benchmark" \ check "ipa-macros tests" \ pushd ipa-macros && cargo test --features="trybuild" && popd - \ No newline at end of file + From e3d974848542131ad298ef590335a8dd3bb903f9 Mon Sep 17 00:00:00 2001 From: Alex Koshelev Date: Fri, 11 Aug 2023 11:30:40 -0700 Subject: [PATCH 6/6] Fix imports according to formatting rules --- benches/oneshot/arithmetic_circuit.rs | 3 +- benches/oneshot/ipa.rs | 9 +-- benches/oneshot/sort.rs | 3 +- ipa-macros/src/derive_gate/mod.rs | 3 +- ipa-macros/src/parser.rs | 6 +- ipa-macros/src/tree.rs | 3 +- src/bin/helper.rs | 15 ++-- src/bin/ipa_bench/cmd.rs | 10 +-- src/bin/ipa_bench/config.rs | 3 +- src/bin/ipa_bench/gen_events.rs | 20 +++--- src/bin/ipa_bench/models.rs | 8 +-- src/bin/ipa_bench/sample.rs | 6 +- src/bin/report_collector.rs | 36 ++++------ src/bin/test_mpc.rs | 3 +- src/chunkscan.rs | 8 ++- src/cli/clientconf.rs | 12 ++-- src/cli/ipa_output.rs | 3 +- src/cli/keygen.rs | 14 ++-- src/cli/metric_collector.rs | 6 +- src/cli/noise.rs | 7 +- src/cli/playbook/input.rs | 6 +- src/cli/playbook/ipa.rs | 24 ++++--- src/cli/playbook/mod.rs | 16 +++-- src/cli/playbook/multiply.rs | 10 +-- src/cli/test_setup.rs | 18 ++--- src/cli/verbosity.rs | 12 ++-- src/config.rs | 33 ++++----- src/error.rs | 4 +- src/exact.rs | 8 ++- src/ff/field.rs | 6 +- src/ff/galois_field.rs | 14 ++-- src/ff/mod.rs | 5 +- src/ff/prime_field.rs | 8 ++- src/helpers/buffers/mod.rs | 3 +- src/helpers/buffers/ordering_mpsc.rs | 41 ++++++----- src/helpers/buffers/ordering_sender.rs | 44 ++++++------ src/helpers/buffers/unordered_receiver.rs | 28 ++++---- src/helpers/error.rs | 5 +- src/helpers/gateway/mod.rs | 10 +-- src/helpers/gateway/receive.rs | 8 ++- src/helpers/gateway/send.rs | 7 +- src/helpers/mod.rs | 39 +++++------ src/helpers/prss_protocol.rs | 9 ++- src/helpers/transport/callbacks.rs | 3 +- src/helpers/transport/in_memory/mod.rs | 4 +- src/helpers/transport/in_memory/transport.rs | 47 +++++++------ src/helpers/transport/mod.rs | 8 ++- src/helpers/transport/query.rs | 12 ++-- src/helpers/transport/receive.rs | 14 ++-- src/helpers/transport/stream/axum_body.rs | 11 +-- src/helpers/transport/stream/box_body.rs | 7 +- src/helpers/transport/stream/collection.rs | 14 ++-- src/helpers/transport/stream/input.rs | 33 +++++---- src/helpers/transport/stream/mod.rs | 7 +- src/hpke/mod.rs | 17 +++-- src/hpke/registry.rs | 11 +-- src/lib.rs | 3 +- src/net/client/mod.rs | 52 +++++++------- src/net/error.rs | 3 +- src/net/http_serde.rs | 68 +++++++++++-------- src/net/server/handlers/echo.rs | 6 +- src/net/server/handlers/mod.rs | 3 +- src/net/server/handlers/query/create.rs | 20 +++--- src/net/server/handlers/query/input.rs | 10 +-- src/net/server/handlers/query/mod.rs | 15 ++-- src/net/server/handlers/query/prepare.rs | 10 +-- src/net/server/handlers/query/results.rs | 10 +-- src/net/server/handlers/query/status.rs | 10 +-- src/net/server/handlers/query/step.rs | 10 +-- src/net/server/mod.rs | 56 +++++++-------- src/net/test.rs | 18 ++--- src/net/transport.rs | 38 ++++++----- src/protocol/attribution/accumulate_credit.rs | 11 +-- src/protocol/attribution/aggregate_credit.rs | 7 +- .../attribution/apply_attribution_window.rs | 17 +++-- src/protocol/attribution/credit_capping.rs | 16 +++-- src/protocol/attribution/input.rs | 12 ++-- src/protocol/attribution/mod.rs | 16 +++-- src/protocol/basics/check_zero.rs | 8 ++- src/protocol/basics/mul/malicious.rs | 10 +-- src/protocol/basics/mul/mod.rs | 3 +- src/protocol/basics/mul/semi_honest.rs | 6 +- src/protocol/basics/mul/sparse.rs | 8 ++- src/protocol/basics/reshare.rs | 10 +-- src/protocol/basics/reveal.rs | 26 +++---- src/protocol/basics/share_known_value.rs | 3 +- .../basics/sum_of_product/malicious.rs | 10 +-- src/protocol/basics/sum_of_product/mod.rs | 3 +- src/protocol/boolean/add_constant.rs | 5 +- src/protocol/boolean/bit_decomposition.rs | 8 ++- src/protocol/boolean/bitwise_equal.rs | 11 +-- .../boolean/bitwise_less_than_prime.rs | 13 ++-- src/protocol/boolean/comparison.rs | 10 +-- src/protocol/boolean/generate_random_bits.rs | 3 +- src/protocol/boolean/mod.rs | 3 +- src/protocol/boolean/or.rs | 3 +- src/protocol/boolean/random_bits_generator.rs | 15 ++-- src/protocol/boolean/solved_bits.rs | 16 +++-- src/protocol/boolean/xor.rs | 3 +- src/protocol/context/malicious.rs | 17 ++--- src/protocol/context/mod.rs | 36 +++++----- src/protocol/context/prss.rs | 3 +- src/protocol/context/semi_honest.rs | 16 ++--- src/protocol/context/upgrade.rs | 12 ++-- src/protocol/context/validator.rs | 23 ++++--- src/protocol/dp/distributions.rs | 10 +-- src/protocol/dp/insecure.rs | 11 +-- src/protocol/ipa/mod.rs | 40 ++++++----- src/protocol/mod.rs | 9 +-- .../modulus_conversion/convert_shares.rs | 26 ++++--- src/protocol/prss/crypto.rs | 13 ++-- src/protocol/prss/mod.rs | 20 +++--- src/protocol/sort/apply.rs | 3 +- src/protocol/sort/apply_sort/mod.rs | 3 +- src/protocol/sort/apply_sort/shuffle.rs | 6 +- src/protocol/sort/bit_permutation.rs | 6 +- src/protocol/sort/compose.rs | 6 +- src/protocol/sort/generate_permutation.rs | 13 ++-- src/protocol/sort/generate_permutation_opt.rs | 14 ++-- src/protocol/sort/mod.rs | 8 ++- src/protocol/sort/multi_bit_permutation.rs | 3 +- src/protocol/sort/secureapplyinv.rs | 6 +- src/protocol/sort/shuffle.rs | 15 ++-- src/protocol/step/compact.rs | 6 +- src/protocol/step/descriptive.rs | 3 +- src/query/completion.rs | 12 ++-- src/query/executor.rs | 41 +++++------ src/query/mod.rs | 5 +- src/query/processor.rs | 30 ++++---- src/query/runner/ipa.rs | 21 +++--- src/query/runner/mod.rs | 6 +- src/query/runner/test_multiply.rs | 8 ++- src/query/state.rs | 20 +++--- src/report.rs | 29 ++++---- src/secret_sharing/decomposed.rs | 3 +- src/secret_sharing/mod.rs | 11 +-- .../replicated/malicious/additive_share.rs | 26 +++---- .../replicated/semi_honest/additive_share.rs | 14 ++-- src/secret_sharing/scheme.rs | 3 +- src/seq_join.rs | 28 ++++---- src/telemetry/stats.rs | 4 +- src/telemetry/step_stats.rs | 11 +-- src/test_fixture/app.rs | 15 ++-- src/test_fixture/circuit.rs | 6 +- src/test_fixture/event_gen.rs | 12 ++-- src/test_fixture/input/sharing.rs | 6 +- src/test_fixture/logging.rs | 3 +- src/test_fixture/metrics.rs | 11 +-- src/test_fixture/mod.rs | 22 +++--- src/test_fixture/sharing.rs | 3 +- src/test_fixture/world.rs | 14 ++-- tests/common/mod.rs | 13 ++-- tests/common/tempdir.rs | 1 + tests/compact_gate.rs | 3 +- tests/helper_networks.rs | 3 +- 155 files changed, 1146 insertions(+), 878 deletions(-) diff --git a/benches/oneshot/arithmetic_circuit.rs b/benches/oneshot/arithmetic_circuit.rs index f23941e10..757672033 100644 --- a/benches/oneshot/arithmetic_circuit.rs +++ b/benches/oneshot/arithmetic_circuit.rs @@ -1,6 +1,7 @@ +use std::time::Instant; + use clap::Parser; use ipa::{ff::Fp31, secret_sharing::SharedValue, test_fixture::circuit}; -use std::time::Instant; #[derive(Debug, Parser)] pub struct CircuitArgs { diff --git a/benches/oneshot/ipa.rs b/benches/oneshot/ipa.rs index 976eb763e..577bdacd3 100644 --- a/benches/oneshot/ipa.rs +++ b/benches/oneshot/ipa.rs @@ -1,3 +1,8 @@ +use std::{ + num::{NonZeroU32, NonZeroUsize}, + time::Instant, +}; + use clap::Parser; use ipa::{ error::Error, @@ -9,10 +14,6 @@ use ipa::{ }, }; use rand::{rngs::StdRng, thread_rng, Rng, SeedableRng}; -use std::{ - num::{NonZeroU32, NonZeroUsize}, - time::Instant, -}; use tokio::runtime::Builder; #[cfg(all(not(target_env = "msvc"), not(feature = "dhat-heap")))] diff --git a/benches/oneshot/sort.rs b/benches/oneshot/sort.rs index f28a18b7b..4561a4b74 100644 --- a/benches/oneshot/sort.rs +++ b/benches/oneshot/sort.rs @@ -1,3 +1,5 @@ +use std::time::Instant; + use futures::stream::iter as stream_iter; use ipa::{ error::Error, @@ -14,7 +16,6 @@ use ipa::{ test_fixture::{join3, Reconstruct, TestWorld, TestWorldConfig}, }; use rand::Rng; -use std::time::Instant; #[tokio::main(flavor = "multi_thread", worker_threads = 3)] async fn main() -> Result<(), Error> { diff --git a/ipa-macros/src/derive_gate/mod.rs b/ipa-macros/src/derive_gate/mod.rs index 14749e07e..f1623c809 100644 --- a/ipa-macros/src/derive_gate/mod.rs +++ b/ipa-macros/src/derive_gate/mod.rs @@ -1,8 +1,9 @@ -use crate::parser::{group_by_modules, ipa_state_transition_map, module_string_to_ast}; use proc_macro::TokenStream; use quote::quote; use syn::{parse_macro_input, DeriveInput}; +use crate::parser::{group_by_modules, ipa_state_transition_map, module_string_to_ast}; + // Procedural macro to derive the Step and StepNarrow traits and generate a memory-efficient gate. // // The goal is to generate a state transition graph and the corresponding `StepNarrow` implementations diff --git a/ipa-macros/src/parser.rs b/ipa-macros/src/parser.rs index 2668e19aa..c256d7366 100644 --- a/ipa-macros/src/parser.rs +++ b/ipa-macros/src/parser.rs @@ -1,11 +1,13 @@ -use crate::tree::Node; -use quote::format_ident; use std::{ collections::{HashMap, VecDeque}, io::Read, path::PathBuf, }; +use quote::format_ident; + +use crate::tree::Node; + const TARGET_CRATE: &str = "ipa"; const STEPS_FILE_PATH: &str = "/../src/protocol/step/"; #[cfg(not(feature = "trybuild"))] diff --git a/ipa-macros/src/tree.rs b/ipa-macros/src/tree.rs index 7909a18c8..a90c0eae3 100644 --- a/ipa-macros/src/tree.rs +++ b/ipa-macros/src/tree.rs @@ -83,9 +83,10 @@ impl Deref for InnerNode { #[cfg(test)] mod tests { - use crate::tree::Node; use std::rc::{Rc, Weak}; + use crate::tree::Node; + #[derive(Debug)] struct TestData(u8); diff --git a/src/bin/helper.rs b/src/bin/helper.rs index f985724e7..229150f84 100644 --- a/src/bin/helper.rs +++ b/src/bin/helper.rs @@ -1,3 +1,11 @@ +use std::{ + fs, + net::TcpListener, + os::fd::{FromRawFd, RawFd}, + path::{Path, PathBuf}, + process, +}; + use clap::{self, Parser, Subcommand}; use hyper::http::uri::Scheme; use ipa::{ @@ -10,13 +18,6 @@ use ipa::{ net::{ClientIdentity, HttpTransport, MpcHelperClient}, AppSetup, }; -use std::{ - fs, - net::TcpListener, - os::fd::{FromRawFd, RawFd}, - path::{Path, PathBuf}, - process, -}; use tracing::{error, info}; #[cfg(not(target_env = "msvc"))] diff --git a/src/bin/ipa_bench/cmd.rs b/src/bin/ipa_bench/cmd.rs index e5d6d2554..982a8d1f7 100644 --- a/src/bin/ipa_bench/cmd.rs +++ b/src/bin/ipa_bench/cmd.rs @@ -1,15 +1,17 @@ -use crate::{gen_events::generate_events, sample::Sample}; -use clap::Parser; -use ipa::cli::Verbosity; -use rand::{rngs::StdRng, SeedableRng}; use std::{ fs::File, io, path::{Path, PathBuf}, process, }; + +use clap::Parser; +use ipa::cli::Verbosity; +use rand::{rngs::StdRng, SeedableRng}; use tracing::{debug, error, info}; +use crate::{gen_events::generate_events, sample::Sample}; + const DEFAULT_EVENT_GEN_COUNT: u32 = 100_000; #[derive(Debug, Parser)] diff --git a/src/bin/ipa_bench/config.rs b/src/bin/ipa_bench/config.rs index 76dc80564..21bc41fa1 100644 --- a/src/bin/ipa_bench/config.rs +++ b/src/bin/ipa_bench/config.rs @@ -1,6 +1,7 @@ -use serde::{Deserialize, Serialize}; use std::ops::Range; +use serde::{Deserialize, Serialize}; + #[cfg(feature = "enable-serde")] #[derive(Serialize, Deserialize, Debug)] pub struct WeightedIndex { diff --git a/src/bin/ipa_bench/gen_events.rs b/src/bin/ipa_bench/gen_events.rs index adaff2087..0af9ef8e4 100644 --- a/src/bin/ipa_bench/gen_events.rs +++ b/src/bin/ipa_bench/gen_events.rs @@ -1,15 +1,17 @@ -use crate::{ - models::{Epoch, Event, EventTimestamp, GenericReport, MatchKey, Number}, - sample::Sample, -}; +use std::io; + use bitvec::view::BitViewSized; use rand::{ distributions::{Bernoulli, Distribution}, CryptoRng, Rng, RngCore, }; -use std::io; use tracing::{debug, info, trace}; +use crate::{ + models::{Epoch, Event, EventTimestamp, GenericReport, MatchKey, Number}, + sample::Sample, +}; + // 0x1E. https://datatracker.ietf.org/doc/html/rfc7464 const RECORD_SEPARATOR: u8 = 30; @@ -170,14 +172,16 @@ fn add_event_timestamps(rhs: EventTimestamp, lhs: EventTimestamp) -> EventTimest #[cfg(all(test, unit_test))] mod tests { - use super::{gen_reports, generate_events, EventTimestamp, GenericReport}; - use crate::{gen_events::add_event_timestamps, models::Epoch, sample::Sample}; - use rand::{rngs::StdRng, SeedableRng}; use std::{ cmp::Ordering, io::{Cursor, Write}, }; + use rand::{rngs::StdRng, SeedableRng}; + + use super::{gen_reports, generate_events, EventTimestamp, GenericReport}; + use crate::{gen_events::add_event_timestamps, models::Epoch, sample::Sample}; + const DATA: &str = r#" { "devices_per_user": [ diff --git a/src/bin/ipa_bench/models.rs b/src/bin/ipa_bench/models.rs index 2876c4f5b..cade92aaa 100644 --- a/src/bin/ipa_bench/models.rs +++ b/src/bin/ipa_bench/models.rs @@ -1,11 +1,12 @@ -use rand::{CryptoRng, Rng, RngCore}; -use serde::{Deserialize, Serialize}; use std::{ fmt::{Debug, Formatter}, io::{Error as IoError, ErrorKind as IoErrorKind}, ops::Range, }; +use rand::{CryptoRng, Rng, RngCore}; +use serde::{Deserialize, Serialize}; + // Type aliases to indicate whether the parameter should be encrypted, secret shared, etc. // Underlying types are temporalily assigned for PoC. pub type CipherText = Vec; @@ -318,9 +319,8 @@ impl Debug for TriggerFanoutQuery { #[cfg(all(test, unit_test))] mod tests { - use crate::models::Epoch; - use super::EventTimestamp; + use crate::models::Epoch; #[test] fn event_timestamp_new() { diff --git a/src/bin/ipa_bench/sample.rs b/src/bin/ipa_bench/sample.rs index 5b22174d8..8024ebe2d 100644 --- a/src/bin/ipa_bench/sample.rs +++ b/src/bin/ipa_bench/sample.rs @@ -1,9 +1,11 @@ -use crate::config::Config; +use std::time::Duration; + use rand::{ distributions::{Distribution, WeightedIndex}, CryptoRng, Rng, RngCore, }; -use std::time::Duration; + +use crate::config::Config; pub struct Sample<'a> { config: &'a Config, diff --git a/src/bin/report_collector.rs b/src/bin/report_collector.rs index d4c817b9f..b7a0629d4 100644 --- a/src/bin/report_collector.rs +++ b/src/bin/report_collector.rs @@ -1,13 +1,26 @@ +use std::{ + borrow::Cow, + error::Error, + fmt::Debug, + fs::{File, OpenOptions}, + io, + io::{stdout, Write}, + ops::Deref, + path::{Path, PathBuf}, +}; + use clap::{Parser, Subcommand}; +use comfy_table::{Cell, Table}; use hyper::http::uri::Scheme; use ipa::{ cli::{ + noise::{apply, ApplyDpArgs}, playbook::{make_clients, playbook_ipa, validate, InputSource}, - Verbosity, + CsvSerializer, IpaQueryResult, Verbosity, }, config::NetworkConfig, ff::{FieldType, Fp32BitPrime}, - helpers::query::{IpaQueryConfig, QueryConfig, QueryType}, + helpers::query::{IpaQueryConfig, QueryConfig, QuerySize, QueryType}, hpke::{KeyRegistry, PublicKeyOnly}, net::MpcHelperClient, protocol::{BreakdownKey, MatchKey}, @@ -17,27 +30,8 @@ use ipa::{ EventGenerator, EventGeneratorConfig, }, }; - -use comfy_table::{Cell, Table}; -use ipa::{ - cli::{ - noise::{apply, ApplyDpArgs}, - CsvSerializer, IpaQueryResult, - }, - helpers::query::QuerySize, -}; use rand::{distributions::Alphanumeric, rngs::StdRng, thread_rng, Rng}; use rand_core::SeedableRng; -use std::{ - borrow::Cow, - error::Error, - fmt::Debug, - fs::{File, OpenOptions}, - io, - io::{stdout, Write}, - ops::Deref, - path::{Path, PathBuf}, -}; #[derive(Debug, Parser)] #[clap(name = "rc", about = "Report Collector CLI")] diff --git a/src/bin/test_mpc.rs b/src/bin/test_mpc.rs index 34309d52a..edee8d82d 100644 --- a/src/bin/test_mpc.rs +++ b/src/bin/test_mpc.rs @@ -1,3 +1,5 @@ +use std::{error::Error, fmt::Debug, ops::Add, path::PathBuf}; + use clap::{Parser, Subcommand}; use generic_array::ArrayLength; use hyper::http::uri::Scheme; @@ -11,7 +13,6 @@ use ipa::{ net::MpcHelperClient, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, }; -use std::{error::Error, fmt::Debug, ops::Add, path::PathBuf}; #[derive(Debug, Parser)] #[clap( diff --git a/src/chunkscan.rs b/src/chunkscan.rs index 8898e1cce..35591a91e 100644 --- a/src/chunkscan.rs +++ b/src/chunkscan.rs @@ -1,14 +1,16 @@ -use crate::error::BoxError; -use futures::{ready, Stream}; -use pin_project::pin_project; use std::{ future::Future, mem, pin::Pin, task::{Context, Poll}, }; + +use futures::{ready, Stream}; +use pin_project::pin_project; use tracing::error; +use crate::error::BoxError; + /// A variant of stream transform that combines semantic of `StreamExt::chunks` and `StreamExt::scan`. /// Consumes the input stream and keeps accumulating items in the internal buffer until it reaches /// `capacity` elements. Then the elements are moved to the `f` function that must produce a future diff --git a/src/cli/clientconf.rs b/src/cli/clientconf.rs index 3ff05bd72..32c6dc193 100644 --- a/src/cli/clientconf.rs +++ b/src/cli/clientconf.rs @@ -1,13 +1,15 @@ +use std::{fs, fs::File, io::Write, iter::zip, path::PathBuf}; + +use clap::Args; +use config::Map; +use hpke::Serializable as _; +use toml::{Table, Value}; + use crate::{ cli::paths::PathExt, config::{ClientConfig, HpkeClientConfig, NetworkConfig, PeerConfig}, error::BoxError, }; -use clap::Args; -use config::Map; -use hpke::Serializable as _; -use std::{fs, fs::File, io::Write, iter::zip, path::PathBuf}; -use toml::{Table, Value}; #[derive(Debug, Args)] #[clap(about = "Generate client config for 3 MPC helper parties")] diff --git a/src/cli/ipa_output.rs b/src/cli/ipa_output.rs index 6d75fd4ee..466d324d0 100644 --- a/src/cli/ipa_output.rs +++ b/src/cli/ipa_output.rs @@ -1,6 +1,7 @@ -use crate::helpers::query::{IpaQueryConfig, QuerySize}; use std::time::Duration; +use crate::helpers::query::{IpaQueryConfig, QuerySize}; + #[derive(Debug)] #[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] pub struct QueryResult { diff --git a/src/cli/keygen.rs b/src/cli/keygen.rs index fc1b89b60..a32c30ac3 100644 --- a/src/cli/keygen.rs +++ b/src/cli/keygen.rs @@ -1,4 +1,9 @@ -use crate::{error::BoxError, hpke::KeyPair}; +use std::{ + fs::File, + io::{self, Write}, + path::{Path, PathBuf}, +}; + use clap::Args; use rand::{thread_rng, Rng}; use rand_core::CryptoRng; @@ -6,13 +11,10 @@ use rcgen::{ Certificate, CertificateParams, DistinguishedName, ExtendedKeyUsagePurpose, IsCa, KeyUsagePurpose, SanType, PKCS_ECDSA_P256_SHA256, }; -use std::{ - fs::File, - io::{self, Write}, - path::{Path, PathBuf}, -}; use time::{Duration, OffsetDateTime}; +use crate::{error::BoxError, hpke::KeyPair}; + #[derive(Debug, Args)] #[clap( name = "keygen", diff --git a/src/cli/metric_collector.rs b/src/cli/metric_collector.rs index b60f39f6d..17fd72705 100644 --- a/src/cli/metric_collector.rs +++ b/src/cli/metric_collector.rs @@ -1,10 +1,12 @@ -use crate::telemetry::stats::Metrics; +use std::{io::stderr, thread}; + use metrics_tracing_context::TracingContextLayer; use metrics_util::{ debugging::{DebuggingRecorder, Snapshotter}, layers::Layer, }; -use std::{io::stderr, thread}; + +use crate::telemetry::stats::Metrics; /// Collects metrics using `DebuggingRecorder` and dumps them to `stderr` when dropped. pub struct CollectorHandle { diff --git a/src/cli/noise.rs b/src/cli/noise.rs index 6c49f7031..b92f1eae3 100644 --- a/src/cli/noise.rs +++ b/src/cli/noise.rs @@ -1,13 +1,14 @@ -use clap::Args; -use rand::rngs::StdRng; use std::{ collections::BTreeMap, fmt::{Debug, Display, Formatter}, }; -use crate::protocol::dp::InsecureDiscreteDp; +use clap::Args; +use rand::rngs::StdRng; use rand_core::SeedableRng; +use crate::protocol::dp::InsecureDiscreteDp; + #[derive(Debug, Args)] #[clap(about = "Apply differential privacy noise to the given input")] pub struct ApplyDpArgs { diff --git a/src/cli/playbook/input.rs b/src/cli/playbook/input.rs index 723b8828d..74bf03dcd 100644 --- a/src/cli/playbook/input.rs +++ b/src/cli/playbook/input.rs @@ -1,18 +1,16 @@ -use crate::ff::Field; - use std::{ any::type_name, fs::File, io, io::{stdin, BufRead, BufReader, Read}, + path::PathBuf, }; use crate::{ - ff::GaloisField, + ff::{Field, GaloisField}, ipa_test_input, test_fixture::{input::GenericReportTestInput, ipa::TestRawDataRecord}, }; -use std::path::PathBuf; pub trait InputItem { fn from_str(s: &str) -> Self; diff --git a/src/cli/playbook/ipa.rs b/src/cli/playbook/ipa.rs index 698c64e56..9047baeb3 100644 --- a/src/cli/playbook/ipa.rs +++ b/src/cli/playbook/ipa.rs @@ -1,4 +1,17 @@ #![cfg(all(feature = "web-app", feature = "cli"))] +use std::{ + cmp::min, + iter::zip, + time::{Duration, Instant}, +}; + +use futures_util::future::try_join_all; +use generic_array::GenericArray; +use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng}; +use rand_core::SeedableRng; +use tokio::time::sleep; +use typenum::Unsigned; + use crate::{ cli::IpaQueryResult, ff::{PrimeField, Serializable}, @@ -15,17 +28,6 @@ use crate::{ secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::{input::GenericReportTestInput, ipa::TestRawDataRecord, Reconstruct}, }; -use futures_util::future::try_join_all; -use generic_array::GenericArray; -use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng}; -use rand_core::SeedableRng; -use std::{ - cmp::min, - iter::zip, - time::{Duration, Instant}, -}; -use tokio::time::sleep; -use typenum::Unsigned; /// Semi-honest IPA protocol. /// Returns aggregated values per breakdown key represented as index in the returned vector diff --git a/src/cli/playbook/mod.rs b/src/cli/playbook/mod.rs index 8fb8629bb..cdc47be00 100644 --- a/src/cli/playbook/mod.rs +++ b/src/cli/playbook/mod.rs @@ -2,19 +2,21 @@ mod input; mod ipa; mod multiply; -pub use self::ipa::playbook_ipa; -use crate::{ - config::{ClientConfig, NetworkConfig, PeerConfig}, - net::{ClientIdentity, MpcHelperClient}, -}; -use comfy_table::{Cell, Color, Table}; use core::fmt::Debug; +use std::{fs, path::Path, time::Duration}; + +use comfy_table::{Cell, Color, Table}; use hyper::http::uri::Scheme; pub use input::InputSource; pub use multiply::secure_mul; -use std::{fs, path::Path, time::Duration}; use tokio::time::sleep; +pub use self::ipa::playbook_ipa; +use crate::{ + config::{ClientConfig, NetworkConfig, PeerConfig}, + net::{ClientIdentity, MpcHelperClient}, +}; + pub fn validate<'a, I, S>(expected: I, actual: I) where I: IntoIterator, diff --git a/src/cli/playbook/multiply.rs b/src/cli/playbook/multiply.rs index 2482f9fdb..caadd8d7b 100644 --- a/src/cli/playbook/multiply.rs +++ b/src/cli/playbook/multiply.rs @@ -1,5 +1,11 @@ #![cfg(feature = "web-app")] +use std::ops::Add; + +use futures::future::try_join_all; +use generic_array::{ArrayLength, GenericArray}; +use typenum::Unsigned; + use crate::{ ff::{Field, Serializable}, helpers::{query::QueryInput, BodyStream}, @@ -8,10 +14,6 @@ use crate::{ secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, IntoShares}, test_fixture::Reconstruct, }; -use futures::future::try_join_all; -use generic_array::{ArrayLength, GenericArray}; -use std::ops::Add; -use typenum::Unsigned; /// Secure multiplication. Each input must be a valid tuple of field values. /// `(a, b)` will produce `a` * `b`. diff --git a/src/cli/test_setup.rs b/src/cli/test_setup.rs index e50a9ac7c..83a456450 100644 --- a/src/cli/test_setup.rs +++ b/src/cli/test_setup.rs @@ -1,17 +1,19 @@ -use crate::{ - cli::{keygen, KeygenArgs}, - error::BoxError, -}; -use clap::Args; use std::{ fs::{DirBuilder, File}, iter::zip, path::PathBuf, }; -use crate::cli::{ - clientconf::{gen_client_config, HelperClientConf}, - paths::PathExt, +use clap::Args; + +use crate::{ + cli::{ + clientconf::{gen_client_config, HelperClientConf}, + keygen, + paths::PathExt, + KeygenArgs, + }, + error::BoxError, }; #[derive(Debug, Args)] diff --git a/src/cli/verbosity.rs b/src/cli/verbosity.rs index 476fff4a9..9fdff5c00 100644 --- a/src/cli/verbosity.rs +++ b/src/cli/verbosity.rs @@ -1,15 +1,17 @@ -use crate::{ - cli::{install_collector, metric_collector::CollectorHandle}, - error::set_global_panic_hook, -}; +use std::io::stderr; + use clap::Parser; use metrics_tracing_context::MetricsLayer; -use std::io::stderr; use tracing::{info, metadata::LevelFilter, Level}; use tracing_subscriber::{ fmt, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, }; +use crate::{ + cli::{install_collector, metric_collector::CollectorHandle}, + error::set_global_panic_hook, +}; + #[derive(Debug, Parser)] pub struct Verbosity { /// Silence all output diff --git a/src/config.rs b/src/config.rs index 252a33d13..24db8d149 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,17 +1,3 @@ -use crate::{ - error::BoxError, - helpers::HelperIdentity, - hpke::{ - Deserializable as _, IpaPrivateKey, IpaPublicKey, KeyPair, KeyRegistry, Serializable as _, - }, -}; - -use hyper::{client::Builder, http::uri::Scheme, Uri}; -use rustls_pemfile::Item; -use serde::{Deserialize, Deserializer, Serialize}; -use tokio::fs; -use tokio_rustls::rustls::Certificate; - use std::{ array, borrow::{Borrow, Cow}, @@ -22,6 +8,20 @@ use std::{ time::Duration, }; +use hyper::{client::Builder, http::uri::Scheme, Uri}; +use rustls_pemfile::Item; +use serde::{Deserialize, Deserializer, Serialize}; +use tokio::fs; +use tokio_rustls::rustls::Certificate; + +use crate::{ + error::BoxError, + helpers::HelperIdentity, + hpke::{ + Deserializable as _, IpaPrivateKey, IpaPublicKey, KeyPair, KeyRegistry, Serializable as _, + }, +}; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error(transparent)] @@ -426,13 +426,14 @@ impl Debug for Http2Configurator { #[cfg(all(test, unit_test))] mod tests { - use super::*; - use crate::{config::HpkeClientConfig, helpers::HelperIdentity, net::test::TestConfigBuilder}; use hpke::{kem::X25519HkdfSha256, Kem}; use hyper::Uri; use rand::rngs::StdRng; use rand_core::SeedableRng; + use super::*; + use crate::{config::HpkeClientConfig, helpers::HelperIdentity, net::test::TestConfigBuilder}; + const URI_1: &str = "http://localhost:3000"; const URI_2: &str = "http://localhost:3001"; const URI_3: &str = "http://localhost:3002"; diff --git a/src/error.rs b/src/error.rs index b82062c83..c68b325a8 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,7 +1,9 @@ -use crate::{report::InvalidReportError, task::JoinError}; use std::{backtrace::Backtrace, fmt::Debug}; + use thiserror::Error; +use crate::{report::InvalidReportError, task::JoinError}; + /// An error raised by the IPA protocol. /// /// This error type could be thought of as `ipa::protocol::Error`. There are other error types for diff --git a/src/exact.rs b/src/exact.rs index 6c9ba8c3d..d7ffb58be 100644 --- a/src/exact.rs +++ b/src/exact.rs @@ -1,10 +1,11 @@ -use futures::stream::Stream; -use pin_project::pin_project; use std::{ pin::Pin, task::{Context, Poll}, }; +use futures::stream::Stream; +use pin_project::pin_project; + /// An analogue of [`ExactSizeIterator`] for [`Stream`]. /// This behaves exactly as you might expect based on the documentation of /// [`ExactSizeIterator`]. @@ -83,10 +84,11 @@ impl ExactSizeStream for FixedLength {} #[cfg(all(test, unit_test))] mod test { - use crate::exact::{ExactSizeStream, FixedLength}; use futures::stream::iter; use futures_util::StreamExt; + use crate::exact::{ExactSizeStream, FixedLength}; + #[test] fn fixed_stream() { const COUNT: usize = 7; diff --git a/src/ff/field.rs b/src/ff/field.rs index de4a2678a..b7c8873d2 100644 --- a/src/ff/field.rs +++ b/src/ff/field.rs @@ -1,9 +1,11 @@ +use std::fmt::Debug; + +use typenum::{U1, U4}; + use crate::{ error, secret_sharing::{Block, SharedValue}, }; -use std::fmt::Debug; -use typenum::{U1, U4}; impl Block for u8 { type Size = U1; diff --git a/src/ff/galois_field.rs b/src/ff/galois_field.rs index ea36c74ed..6d2e44ffc 100644 --- a/src/ff/galois_field.rs +++ b/src/ff/galois_field.rs @@ -1,15 +1,17 @@ -use crate::{ - ff::{Field, Serializable}, - secret_sharing::{Block, SharedValue}, -}; -use bitvec::prelude::{bitarr, BitArr, Lsb0}; -use generic_array::GenericArray; use std::{ fmt::{Debug, Formatter}, ops::Index, }; + +use bitvec::prelude::{bitarr, BitArr, Lsb0}; +use generic_array::GenericArray; use typenum::{Unsigned, U1, U4, U5}; +use crate::{ + ff::{Field, Serializable}, + secret_sharing::{Block, SharedValue}, +}; + /// Trait for data types storing arbitrary number of bits. pub trait GaloisField: Field + Into + Index + Index diff --git a/src/ff/mod.rs b/src/ff/mod.rs index 9095070aa..453876175 100644 --- a/src/ff/mod.rs +++ b/src/ff/mod.rs @@ -6,15 +6,16 @@ mod field; mod galois_field; mod prime_field; +use std::ops::{Add, AddAssign, Mul, MulAssign, Neg, Sub, SubAssign}; + pub use field::{Field, FieldType}; pub use galois_field::{GaloisField, Gf2, Gf32Bit, Gf40Bit, Gf8Bit}; +use generic_array::{ArrayLength, GenericArray}; #[cfg(any(test, feature = "weak-field"))] pub use prime_field::Fp31; pub use prime_field::{Fp32BitPrime, PrimeField}; use crate::secret_sharing::SharedValue; -use generic_array::{ArrayLength, GenericArray}; -use std::ops::{Add, AddAssign, Mul, MulAssign, Neg, Sub, SubAssign}; #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum Error { diff --git a/src/ff/prime_field.rs b/src/ff/prime_field.rs index 5101124e5..ae35d59ab 100644 --- a/src/ff/prime_field.rs +++ b/src/ff/prime_field.rs @@ -1,9 +1,10 @@ +use generic_array::GenericArray; + use super::Field; use crate::{ ff::Serializable, secret_sharing::{Block, SharedValue}, }; -use generic_array::GenericArray; pub trait PrimeField: Field { type PrimeInteger: Into; @@ -181,11 +182,12 @@ macro_rules! field_impl { #[cfg(all(test, unit_test))] mod common_tests { - use super::*; - use crate::ff::Serializable; use generic_array::GenericArray; use proptest::proptest; + use super::*; + use crate::ff::Serializable; + #[test] fn zero() { let prime = u128::from($field::PRIME); diff --git a/src/helpers/buffers/mod.rs b/src/helpers/buffers/mod.rs index 6db63d59f..83a92c24e 100644 --- a/src/helpers/buffers/mod.rs +++ b/src/helpers/buffers/mod.rs @@ -9,9 +9,10 @@ pub use unordered_receiver::UnorderedReceiver; #[cfg(debug_assertions)] #[allow(unused)] // todo(alex): make test world print the state again mod waiting { - use crate::helpers::ChannelId; use std::collections::HashMap; + use crate::helpers::ChannelId; + pub(in crate::helpers) struct WaitingTasks<'a> { tasks: HashMap<&'a ChannelId, Vec>, } diff --git a/src/helpers/buffers/ordering_mpsc.rs b/src/helpers/buffers/ordering_mpsc.rs index 6e17d912f..1eb2bdd9b 100644 --- a/src/helpers/buffers/ordering_mpsc.rs +++ b/src/helpers/buffers/ordering_mpsc.rs @@ -1,13 +1,5 @@ #![allow(dead_code)] -use crate::{ - ff::Serializable, - helpers::{Error, Message}, - sync::{atomic::AtomicUsize, Arc}, -}; -use bitvec::{bitvec, vec::BitVec}; -use futures::{FutureExt, Stream}; -use generic_array::GenericArray; use std::{ fmt::{Debug, Formatter}, num::NonZeroUsize, @@ -15,12 +7,22 @@ use std::{ sync::atomic::Ordering::{AcqRel, Acquire}, task::{Context, Poll}, }; + +use bitvec::{bitvec, vec::BitVec}; +use futures::{FutureExt, Stream}; +use generic_array::GenericArray; use tokio::sync::{ mpsc::{self, Receiver, Sender}, Notify, }; use typenum::Unsigned; +use crate::{ + ff::Serializable, + helpers::{Error, Message}, + sync::{atomic::AtomicUsize, Arc}, +}; + pub struct OrderingMpscReceiver { rx: Receiver<(usize, M)>, buf: Vec, @@ -271,6 +273,13 @@ impl OrderingMpscEnd { #[cfg(test)] mod fixture { + use std::num::NonZeroUsize; + + use async_trait::async_trait; + use futures::future::join_all; + use tokio::sync::mpsc::{channel, Receiver}; + use typenum::Unsigned; + use crate::{ ff::{Fp32BitPrime, Serializable}, helpers::buffers::ordering_mpsc::{ @@ -278,11 +287,6 @@ mod fixture { }, rand::thread_rng, }; - use async_trait::async_trait; - use futures::future::join_all; - use std::num::NonZeroUsize; - use tokio::sync::mpsc::{channel, Receiver}; - use typenum::Unsigned; pub const FP32BIT_SIZE: usize = ::Size::USIZE; @@ -357,6 +361,11 @@ mod fixture { #[cfg(all(test, unit_test))] mod unit { + use std::{mem, num::NonZeroUsize}; + + use futures::{future::join, FutureExt, StreamExt}; + use generic_array::GenericArray; + use crate::{ ff::{Field, Fp31, Serializable}, helpers::buffers::ordering_mpsc::{ @@ -364,9 +373,6 @@ mod unit { ordering_mpsc, }, }; - use futures::{future::join, FutureExt, StreamExt}; - use generic_array::GenericArray; - use std::{mem, num::NonZeroUsize}; /// Test that a single value can be sent and received successfully. #[tokio::test] @@ -471,9 +477,10 @@ mod unit { #[cfg(all(test, feature = "shuttle"))] mod concurrency { - use crate::helpers::buffers::ordering_mpsc::fixture::{shuffle_indices, shuffled_send_recv}; use shuttle::{check_random, future::block_on}; + use crate::helpers::buffers::ordering_mpsc::fixture::{shuffle_indices, shuffled_send_recv}; + #[test] fn shuffle() { check_random( diff --git a/src/helpers/buffers/ordering_sender.rs b/src/helpers/buffers/ordering_sender.rs index 1284b996b..1c490a3e9 100644 --- a/src/helpers/buffers/ordering_sender.rs +++ b/src/helpers/buffers/ordering_sender.rs @@ -1,17 +1,5 @@ #![allow(dead_code)] // TODO remove -use crate::{ - helpers::Message, - sync::{ - atomic::{ - AtomicUsize, - Ordering::{AcqRel, Acquire}, - }, - Mutex, MutexGuard, - }, -}; -use futures::{task::Waker, Future, Stream}; -use generic_array::GenericArray; use std::{ borrow::Borrow, cmp::Ordering, @@ -22,8 +10,22 @@ use std::{ sync::Arc, task::{Context, Poll}, }; + +use futures::{task::Waker, Future, Stream}; +use generic_array::GenericArray; use typenum::Unsigned; +use crate::{ + helpers::Message, + sync::{ + atomic::{ + AtomicUsize, + Ordering::{AcqRel, Acquire}, + }, + Mutex, MutexGuard, + }, +}; + /// The operating state for an `OrderingSender`. struct State { /// A store of bytes to write into. @@ -394,13 +396,8 @@ impl + Unpin> Stream for OrderedStream { #[cfg(all(test, any(unit_test, feature = "shuttle")))] mod test { - use super::OrderingSender; - use crate::{ - ff::{Field, Fp31, Fp32BitPrime, Serializable}, - rand::thread_rng, - sync::Arc, - test_executor::run, - }; + use std::{iter::zip, num::NonZeroUsize}; + use futures::{ future::{join, join3, join_all}, stream::StreamExt, @@ -408,9 +405,16 @@ mod test { }; use generic_array::GenericArray; use rand::Rng; - use std::{iter::zip, num::NonZeroUsize}; use typenum::Unsigned; + use super::OrderingSender; + use crate::{ + ff::{Field, Fp31, Fp32BitPrime, Serializable}, + rand::thread_rng, + sync::Arc, + test_executor::run, + }; + fn sender() -> Arc { Arc::new(OrderingSender::new( NonZeroUsize::new(6).unwrap(), diff --git a/src/helpers/buffers/unordered_receiver.rs b/src/helpers/buffers/unordered_receiver.rs index d9689cd91..d578e6456 100644 --- a/src/helpers/buffers/unordered_receiver.rs +++ b/src/helpers/buffers/unordered_receiver.rs @@ -1,10 +1,3 @@ -use crate::{ - helpers::{Error, Message}, - protocol::RecordId, - sync::{Arc, Mutex}, -}; -use futures::{task::Waker, Future, Stream}; -use generic_array::GenericArray; use std::{ marker::PhantomData, mem::take, @@ -12,8 +5,17 @@ use std::{ pin::Pin, task::{Context, Poll}, }; + +use futures::{task::Waker, Future, Stream}; +use generic_array::GenericArray; use typenum::Unsigned; +use crate::{ + helpers::{Error, Message}, + protocol::RecordId, + sync::{Arc, Mutex}, +}; + /// A future for receiving item `i` from an `UnorderedReceiver`. pub struct Receiver where @@ -298,10 +300,8 @@ where #[cfg(all(test, any(unit_test, feature = "shuttle")))] mod test { - use crate::{ - ff::{Field, Fp31, Fp32BitPrime, Serializable}, - helpers::buffers::unordered_receiver::UnorderedReceiver, - }; + use std::num::NonZeroUsize; + use futures::{ future::{try_join, try_join_all}, stream::iter, @@ -311,11 +311,15 @@ mod test { use rand::Rng; #[cfg(feature = "shuttle")] use shuttle::future::spawn; - use std::num::NonZeroUsize; #[cfg(not(feature = "shuttle"))] use tokio::spawn; use typenum::Unsigned; + use crate::{ + ff::{Field, Fp31, Fp32BitPrime, Serializable}, + helpers::buffers::unordered_receiver::UnorderedReceiver, + }; + fn receiver(it: I) -> UnorderedReceiver, T> where I: IntoIterator + 'static, diff --git a/src/helpers/error.rs b/src/helpers/error.rs index 0912a6f6c..d73c38359 100644 --- a/src/helpers/error.rs +++ b/src/helpers/error.rs @@ -1,10 +1,11 @@ +use thiserror::Error; +use tokio::sync::mpsc::error::SendError; + use crate::{ error::BoxError, helpers::{ChannelId, HelperIdentity, Message, Role, TotalRecords}, protocol::{step::Gate, RecordId}, }; -use thiserror::Error; -use tokio::sync::mpsc::error::SendError; /// An error raised by the IPA supporting infrastructure. #[derive(Error, Debug)] diff --git a/src/helpers/gateway/mod.rs b/src/helpers/gateway/mod.rs index b191bdecb..142c7b738 100644 --- a/src/helpers/gateway/mod.rs +++ b/src/helpers/gateway/mod.rs @@ -2,7 +2,11 @@ mod receive; mod send; mod transport; +use std::{fmt::Debug, num::NonZeroUsize}; + pub use send::SendingEnd; +#[cfg(all(feature = "shuttle", test))] +use shuttle::future as tokio; use crate::{ helpers::{ @@ -15,9 +19,6 @@ use crate::{ }, protocol::QueryId, }; -#[cfg(all(feature = "shuttle", test))] -use shuttle::future as tokio; -use std::{fmt::Debug, num::NonZeroUsize}; /// Alias for the currently configured transport. /// @@ -145,6 +146,8 @@ impl GatewayConfig { #[cfg(all(test, unit_test))] mod tests { + use futures_util::future::{join, try_join}; + use super::*; use crate::{ ff::{Field, Fp31, Fp32BitPrime, Gf2}, @@ -152,7 +155,6 @@ mod tests { protocol::{context::Context, RecordId}, test_fixture::{Runner, TestWorld, TestWorldConfig}, }; - use futures_util::future::{join, try_join}; /// Verifies that [`Gateway`] send buffer capacity is adjusted to the message size. /// IPA protocol opens many channels to send values from different fields, while message size diff --git a/src/helpers/gateway/receive.rs b/src/helpers/gateway/receive.rs index fd92dec98..4f50e7b48 100644 --- a/src/helpers/gateway/receive.rs +++ b/src/helpers/gateway/receive.rs @@ -1,10 +1,12 @@ +use std::marker::PhantomData; + +use dashmap::DashMap; +use futures::Stream; + use crate::{ helpers::{buffers::UnorderedReceiver, ChannelId, Error, Message, Transport}, protocol::RecordId, }; -use dashmap::DashMap; -use futures::Stream; -use std::marker::PhantomData; /// Receiving end end of the gateway channel. pub struct ReceivingEnd { diff --git a/src/helpers/gateway/send.rs b/src/helpers/gateway/send.rs index 2f9c1e97d..c36875829 100644 --- a/src/helpers/gateway/send.rs +++ b/src/helpers/gateway/send.rs @@ -1,17 +1,18 @@ -use crate::sync::Arc; -use dashmap::DashMap; -use futures::Stream; use std::{ marker::PhantomData, num::NonZeroUsize, pin::Pin, task::{Context, Poll}, }; + +use dashmap::DashMap; +use futures::Stream; use typenum::Unsigned; use crate::{ helpers::{buffers::OrderingSender, ChannelId, Error, Message, Role, TotalRecords}, protocol::RecordId, + sync::Arc, telemetry::{ labels::{ROLE, STEP}, metrics::{BYTES_SENT, RECORDS_SENT}, diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index d6a07eb40..373070736 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -9,30 +9,29 @@ mod gateway; pub(crate) mod prss_protocol; mod transport; -pub use error::{Error, Result}; -pub use gateway::{GatewayConfig, ReceivingEnd, SendingEnd}; +use std::ops::{Index, IndexMut}; +/// to validate that transport can actually send streams of this type +#[cfg(test)] +pub use buffers::OrderingSender; +pub use error::{Error, Result}; // TODO: this type should only be available within infra. Right now several infra modules // are exposed at the root level. That makes it impossible to have a proper hierarchy here. pub use gateway::{Gateway, TransportError, TransportImpl}; - +pub use gateway::{GatewayConfig, ReceivingEnd, SendingEnd}; +use generic_array::GenericArray; pub use prss_protocol::negotiate as negotiate_prss; #[cfg(feature = "web-app")] pub use transport::WrappedAxumBodyStream; pub use transport::{ - callbacks::*, BodyStream, BytesStream, LengthDelimitedStream, LogErrors, NoResourceIdentifier, - QueryIdBinding, ReceiveRecords, RecordsStream, RouteId, RouteParams, StepBinding, - StreamCollection, StreamKey, Transport, WrappedBoxBodyStream, + callbacks::*, query, BodyStream, BytesStream, LengthDelimitedStream, LogErrors, + NoResourceIdentifier, QueryIdBinding, ReceiveRecords, RecordsStream, RouteId, RouteParams, + StepBinding, StreamCollection, StreamKey, Transport, WrappedBoxBodyStream, }; - #[cfg(feature = "in-memory-infra")] pub use transport::{InMemoryNetwork, InMemoryTransport}; - -pub use transport::query; - -/// to validate that transport can actually send streams of this type -#[cfg(test)] -pub use buffers::OrderingSender; +use typenum::{Unsigned, U8}; +use x25519_dalek::PublicKey; use crate::{ ff::Serializable, @@ -43,10 +42,6 @@ use crate::{ protocol::{step::Gate, RecordId}, secret_sharing::SharedValue, }; -use generic_array::GenericArray; -use std::ops::{Index, IndexMut}; -use typenum::{Unsigned, U8}; -use x25519_dalek::PublicKey; // TODO work with ArrayLength only pub type MessagePayloadArrayLen = U8; @@ -527,6 +522,7 @@ mod tests { } mod role_assignment_tests { + use super::*; use crate::{ ff::Fp31, protocol::{basics::SecureMul, context::Context, RecordId}, @@ -534,8 +530,6 @@ mod tests { test_fixture::{Reconstruct, Runner, TestWorld, TestWorldConfig}, }; - use super::*; - #[test] fn basic() { let identities = (1..=3) @@ -663,6 +657,10 @@ mod tests { #[cfg(all(test, feature = "shuttle"))] mod concurrency_tests { + use futures::future::try_join; + use rand_core::RngCore; + use shuttle_crate::rand::thread_rng; + use crate::{ ff::{Field, FieldType, Fp31, Fp32BitPrime}, helpers::{ @@ -676,9 +674,6 @@ mod concurrency_tests { seq_join::SeqJoin, test_fixture::{Reconstruct, Runner, TestApp, TestWorld, TestWorldConfig}, }; - use futures::future::try_join; - use rand_core::RngCore; - use shuttle_crate::rand::thread_rng; #[test] fn send_receive_sequential() { diff --git a/src/helpers/prss_protocol.rs b/src/helpers/prss_protocol.rs index 54d7c60a9..48a9b760e 100644 --- a/src/helpers/prss_protocol.rs +++ b/src/helpers/prss_protocol.rs @@ -1,3 +1,7 @@ +use futures_util::future::try_join4; +use rand_core::{CryptoRng, RngCore}; +use x25519_dalek::PublicKey; + use crate::{ helpers::{ChannelId, Direction, Error, Gateway, TotalRecords, Transport}, protocol::{ @@ -6,11 +10,6 @@ use crate::{ RecordId, }, }; -use futures_util::future::try_join4; - -use rand_core::{CryptoRng, RngCore}; - -use x25519_dalek::PublicKey; pub struct PrssExchangeStep; diff --git a/src/helpers/transport/callbacks.rs b/src/helpers/transport/callbacks.rs index 38d2ce5d8..ea32ee005 100644 --- a/src/helpers/transport/callbacks.rs +++ b/src/helpers/transport/callbacks.rs @@ -1,3 +1,5 @@ +use std::{future::Future, pin::Pin}; + use crate::{ helpers::query::{PrepareQuery, QueryConfig, QueryInput}, protocol::QueryId, @@ -6,7 +8,6 @@ use crate::{ QueryStatus, QueryStatusError, }, }; -use std::{future::Future, pin::Pin}; /// Macro for defining transport callbacks. /// diff --git a/src/helpers/transport/in_memory/mod.rs b/src/helpers/transport/in_memory/mod.rs index 37a58f47c..564caee0c 100644 --- a/src/helpers/transport/in_memory/mod.rs +++ b/src/helpers/transport/in_memory/mod.rs @@ -1,12 +1,12 @@ mod transport; +pub use transport::Setup; + use crate::{ helpers::{HelperIdentity, TransportCallbacks}, sync::{Arc, Weak}, }; -pub use transport::Setup; - pub type InMemoryTransport = Weak; /// Container for all active transports diff --git a/src/helpers/transport/in_memory/transport.rs b/src/helpers/transport/in_memory/transport.rs index cb6cb5427..97c9f6406 100644 --- a/src/helpers/transport/in_memory/transport.rs +++ b/src/helpers/transport/in_memory/transport.rs @@ -1,22 +1,3 @@ -use crate::{ - error::BoxError, - helpers::{ - query::{PrepareQuery, QueryConfig}, - HelperIdentity, NoResourceIdentifier, QueryIdBinding, ReceiveRecords, RouteId, RouteParams, - StepBinding, StreamCollection, Transport, TransportCallbacks, - }, - protocol::{step::Gate, QueryId}, -}; -use ::tokio::sync::{ - mpsc::{channel, Receiver, Sender}, - oneshot, -}; - -use async_trait::async_trait; -use futures::{Stream, StreamExt}; -use serde::de::DeserializeOwned; -#[cfg(all(feature = "shuttle", test))] -use shuttle::future as tokio; use std::{ borrow::Borrow, collections::{HashMap, HashSet}, @@ -27,9 +8,29 @@ use std::{ sync::{Arc, Weak}, task::{Context, Poll}, }; + +use ::tokio::sync::{ + mpsc::{channel, Receiver, Sender}, + oneshot, +}; +use async_trait::async_trait; +use futures::{Stream, StreamExt}; +use serde::de::DeserializeOwned; +#[cfg(all(feature = "shuttle", test))] +use shuttle::future as tokio; use tokio_stream::wrappers::ReceiverStream; use tracing::Instrument; +use crate::{ + error::BoxError, + helpers::{ + query::{PrepareQuery, QueryConfig}, + HelperIdentity, NoResourceIdentifier, QueryIdBinding, ReceiveRecords, RouteId, RouteParams, + StepBinding, StreamCollection, Transport, TransportCallbacks, + }, + protocol::{step::Gate, QueryId}, +}; + type Packet = (Addr, InMemoryStream, oneshot::Sender>); type ConnectionTx = Sender; type ConnectionRx = Receiver; @@ -368,6 +369,11 @@ impl Setup { #[cfg(all(test, unit_test))] mod tests { + use std::{io::ErrorKind, num::NonZeroUsize, panic::AssertUnwindSafe, sync::Mutex}; + + use futures_util::{stream::poll_immediate, FutureExt, StreamExt}; + use tokio::sync::{mpsc::channel, oneshot}; + use super::*; use crate::{ ff::{FieldType, Fp31}, @@ -376,9 +382,6 @@ mod tests { OrderingSender, }, }; - use futures_util::{stream::poll_immediate, FutureExt, StreamExt}; - use std::{io::ErrorKind, num::NonZeroUsize, panic::AssertUnwindSafe, sync::Mutex}; - use tokio::sync::{mpsc::channel, oneshot}; const STEP: &str = "in-memory-transport"; diff --git a/src/helpers/transport/mod.rs b/src/helpers/transport/mod.rs index 05cde5ef2..acbbb8e8e 100644 --- a/src/helpers/transport/mod.rs +++ b/src/helpers/transport/mod.rs @@ -1,10 +1,12 @@ +use std::borrow::Borrow; + +use async_trait::async_trait; +use futures::Stream; + use crate::{ helpers::HelperIdentity, protocol::{step::Gate, QueryId}, }; -use async_trait::async_trait; -use futures::Stream; -use std::borrow::Borrow; pub mod callbacks; #[cfg(feature = "in-memory-infra")] diff --git a/src/helpers/transport/query.rs b/src/helpers/transport/query.rs index c37b737ac..7a3de0b1d 100644 --- a/src/helpers/transport/query.rs +++ b/src/helpers/transport/query.rs @@ -1,3 +1,10 @@ +use std::{ + fmt::{Debug, Display, Formatter}, + num::NonZeroU32, +}; + +use serde::{Deserialize, Deserializer, Serialize}; + use crate::{ ff::FieldType, helpers::{ @@ -6,11 +13,6 @@ use crate::{ }, protocol::{step::Step, QueryId}, }; -use serde::{Deserialize, Deserializer, Serialize}; -use std::{ - fmt::{Debug, Display, Formatter}, - num::NonZeroU32, -}; #[derive(Copy, Clone, Debug, Ord, PartialOrd, Eq, PartialEq)] #[cfg_attr(feature = "enable-serde", derive(Serialize))] diff --git a/src/helpers/transport/receive.rs b/src/helpers/transport/receive.rs index 0c0862e57..e797e7dc9 100644 --- a/src/helpers/transport/receive.rs +++ b/src/helpers/transport/receive.rs @@ -1,15 +1,17 @@ -use crate::{ - error::BoxError, - helpers::transport::stream::{StreamCollection, StreamKey}, -}; -use futures::Stream; -use futures_util::StreamExt; use std::{ pin::Pin, task::{Context, Poll}, }; + +use futures::Stream; +use futures_util::StreamExt; use tracing::error; +use crate::{ + error::BoxError, + helpers::transport::stream::{StreamCollection, StreamKey}, +}; + /// Adapt a stream of `Result>, Error>` to a stream of `Vec`. /// /// If an error is encountered, the error is logged, and the stream is terminated. diff --git a/src/helpers/transport/stream/axum_body.rs b/src/helpers/transport/stream/axum_body.rs index 2fe36a416..f21daa11a 100644 --- a/src/helpers/transport/stream/axum_body.rs +++ b/src/helpers/transport/stream/axum_body.rs @@ -1,13 +1,14 @@ -use crate::error::BoxError; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; use axum::extract::{BodyStream, FromRequest, RequestParts}; use futures::{Stream, TryStreamExt}; use hyper::Body; use pin_project::pin_project; -use std::{ - pin::Pin, - task::{Context, Poll}, -}; + +use crate::error::BoxError; type AxumInner = futures::stream::MapErr crate::error::BoxError>; diff --git a/src/helpers/transport/stream/box_body.rs b/src/helpers/transport/stream/box_body.rs index ab8557178..e51d86f87 100644 --- a/src/helpers/transport/stream/box_body.rs +++ b/src/helpers/transport/stream/box_body.rs @@ -1,11 +1,12 @@ -use crate::helpers::transport::stream::BoxBytesStream; - -use futures::Stream; use std::{ pin::Pin, task::{Context, Poll}, }; +use futures::Stream; + +use crate::helpers::transport::stream::BoxBytesStream; + pub struct WrappedBoxBodyStream(BoxBytesStream); impl WrappedBoxBodyStream { diff --git a/src/helpers/transport/stream/collection.rs b/src/helpers/transport/stream/collection.rs index 7d5ca951f..33ce549b9 100644 --- a/src/helpers/transport/stream/collection.rs +++ b/src/helpers/transport/stream/collection.rs @@ -1,15 +1,17 @@ -use crate::{ - helpers::HelperIdentity, - protocol::{step::Gate, QueryId}, - sync::{Arc, Mutex}, -}; -use futures::Stream; use std::{ collections::{hash_map::Entry, HashMap}, fmt::{Debug, Formatter}, task::Waker, }; +use futures::Stream; + +use crate::{ + helpers::HelperIdentity, + protocol::{step::Gate, QueryId}, + sync::{Arc, Mutex}, +}; + /// Each stream is indexed by query id, the identity of helper where stream is originated from /// and step. pub type StreamKey = (QueryId, HelperIdentity, Gate); diff --git a/src/helpers/transport/stream/input.rs b/src/helpers/transport/stream/input.rs index 396889731..4f238930e 100644 --- a/src/helpers/transport/stream/input.rs +++ b/src/helpers/transport/stream/input.rs @@ -1,10 +1,3 @@ -use crate::{error::BoxError, ff::Serializable, helpers::BytesStream}; -use bytes::Bytes; -use futures::{ - stream::{iter, once, Fuse, FusedStream, Iter, Map, Once}, - Stream, StreamExt, -}; -use pin_project::pin_project; use std::{ cmp::max, collections::VecDeque, @@ -15,8 +8,17 @@ use std::{ pin::Pin, task::{Context, Poll}, }; + +use bytes::Bytes; +use futures::{ + stream::{iter, once, Fuse, FusedStream, Iter, Map, Once}, + Stream, StreamExt, +}; +use pin_project::pin_project; use typenum::{Unsigned, U2}; +use crate::{error::BoxError, ff::Serializable, helpers::BytesStream}; + #[derive(Debug)] pub struct BufDeque { buffered_size: usize, @@ -448,13 +450,14 @@ mod test { use super::*; mod unit_test { + use futures::{StreamExt, TryStreamExt}; + use generic_array::GenericArray; + use super::*; use crate::{ ff::{Fp31, Fp32BitPrime, Serializable}, secret_sharing::replicated::semi_honest::AdditiveShare, }; - use futures::{StreamExt, TryStreamExt}; - use generic_array::GenericArray; #[tokio::test] async fn records_stream_fp31() { @@ -578,9 +581,10 @@ mod test { } mod delimited { - use super::*; use futures::TryStreamExt; + use super::*; + #[tokio::test] async fn basic() { let input = vec![ @@ -669,14 +673,14 @@ mod test { } mod prop_test { - use crate::ff::Fp32BitPrime; - - use super::*; use futures::TryStreamExt; use generic_array::GenericArray; use proptest::prelude::*; use rand::{rngs::StdRng, SeedableRng}; + use super::*; + use crate::ff::Fp32BitPrime; + prop_compose! { fn arb_size_in_bytes(field_size: usize, max_multiplier: usize) (multiplier in 1..max_multiplier) @@ -727,12 +731,13 @@ mod test { mod delimited_prop_test { use std::{mem::size_of, ops::Range}; - use super::*; use bytes::BufMut; use futures::TryStreamExt; use proptest::prelude::*; use rand::{rngs::StdRng, SeedableRng}; + use super::*; + #[derive(Copy, Clone, Debug)] enum ItemSize { Small, diff --git a/src/helpers/transport/stream/mod.rs b/src/helpers/transport/stream/mod.rs index a4586a990..053b6033c 100644 --- a/src/helpers/transport/stream/mod.rs +++ b/src/helpers/transport/stream/mod.rs @@ -4,16 +4,17 @@ mod box_body; mod collection; mod input; +use std::pin::Pin; + #[cfg(feature = "web-app")] pub use axum_body::WrappedAxumBodyStream; pub use box_body::WrappedBoxBodyStream; +use bytes::Bytes; pub use collection::{StreamCollection, StreamKey}; +use futures::Stream; pub use input::{LengthDelimitedStream, RecordsStream}; use crate::error::BoxError; -use bytes::Bytes; -use futures::Stream; -use std::pin::Pin; pub trait BytesStream: Stream> + Send { /// Collects the entire stream into a vec; only intended for use in tests diff --git a/src/hpke/mod.rs b/src/hpke/mod.rs index 5ea16463a..bc63f4876 100644 --- a/src/hpke/mod.rs +++ b/src/hpke/mod.rs @@ -2,25 +2,27 @@ //! //! [`specification`]: https://github.com/patcg-individual-drafts/ipa/blob/main/details/encryption.md +use std::{fmt::Debug, io, ops::Add}; + use generic_array::ArrayLength; use hpke::{ aead::AeadTag, generic_array::typenum::Unsigned, single_shot_open_in_place_detached, single_shot_seal_in_place_detached, OpModeR, OpModeS, }; use rand_core::{CryptoRng, RngCore}; -use std::{fmt::Debug, io, ops::Add}; use typenum::U16; mod info; mod registry; +pub use info::Info; +pub use registry::{KeyPair, KeyRegistry, PublicKeyOnly, PublicKeyRegistry}; + use crate::{ ff::{GaloisField, Gf40Bit, Serializable as IpaSerializable}, report::KeyIdentifier, secret_sharing::replicated::semi_honest::AdditiveShare, }; -pub use info::Info; -pub use registry::{KeyPair, KeyRegistry, PublicKeyOnly, PublicKeyRegistry}; /// IPA ciphersuite type IpaKem = hpke::kem::X25519HkdfSha256; @@ -188,16 +190,16 @@ struct MatchKeyEncryption<'a> { #[cfg(all(test, unit_test))] mod tests { - use super::*; use generic_array::GenericArray; + use rand::rngs::StdRng; + use rand_core::{CryptoRng, RngCore, SeedableRng}; + use super::*; use crate::{ ff::Serializable as IpaSerializable, report::{Epoch, EventType}, secret_sharing::replicated::ReplicatedSecretSharing, }; - use rand::rngs::StdRng; - use rand_core::{CryptoRng, RngCore, SeedableRng}; struct EncryptionSuite { registry: KeyRegistry, @@ -353,10 +355,11 @@ mod tests { } mod proptests { - use super::*; use proptest::prelude::ProptestConfig; use rand::{distributions::Alphanumeric, Rng}; + use super::*; + proptest::proptest! { #![proptest_config(ProptestConfig::with_cases(50))] #[test] diff --git a/src/hpke/registry.rs b/src/hpke/registry.rs index 650db0737..5a14905e2 100644 --- a/src/hpke/registry.rs +++ b/src/hpke/registry.rs @@ -1,7 +1,9 @@ -use super::{IpaPrivateKey, IpaPublicKey, KeyIdentifier}; -use hpke::Serializable; use std::ops::Deref; +use hpke::Serializable; + +use super::{IpaPrivateKey, IpaPublicKey, KeyIdentifier}; + /// A pair of secret key and public key. Public keys used by UA to encrypt the data towards helpers /// secret keys used by helpers to open the ciphertexts. Each helper needs access to both pub struct KeyPair { @@ -121,12 +123,13 @@ impl PublicKeyRegistry for KeyRegistry { #[cfg(all(test, unit_test))] mod tests { - use super::*; - use crate::hpke::{IpaAead, IpaKdf, IpaKem}; use hpke::{kem::EncappedKey, HpkeError, OpModeR, OpModeS}; use rand::rngs::StdRng; use rand_core::{CryptoRng, RngCore, SeedableRng}; + use super::*; + use crate::hpke::{IpaAead, IpaKdf, IpaKem}; + const INFO_STR: &[u8] = b"This is an INFO string."; const AAD: &[u8] = b"This is AAD."; diff --git a/src/lib.rs b/src/lib.rs index 922f16c21..340601adc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,11 +57,10 @@ pub(crate) mod sync { #[cfg(all(feature = "shuttle", test))] pub(crate) mod rand { - pub use shuttle::rand::{thread_rng, Rng, RngCore}; - /// TODO: shuttle does not re-export `CryptoRng`. The only reason it works is because IPA uses /// the same version of `rand`. pub use rand::CryptoRng; + pub use shuttle::rand::{thread_rng, Rng, RngCore}; } #[cfg(not(all(feature = "shuttle", test)))] diff --git a/src/net/client/mod.rs b/src/net/client/mod.rs index 621ee1e2b..cfb3fb167 100644 --- a/src/net/client/mod.rs +++ b/src/net/client/mod.rs @@ -1,20 +1,3 @@ -use crate::{ - config::{ClientConfig, HyperClientConfigurator, NetworkConfig, PeerConfig}, - helpers::{ - query::{PrepareQuery, QueryConfig, QueryInput}, - HelperIdentity, - }, - net::{http_serde, server::HTTP_CLIENT_ID_HEADER, Error}, - protocol::{step::Gate, QueryId}, -}; -use axum::http::uri::{self, Parts, Scheme}; -use futures::{Stream, StreamExt}; -use hyper::{ - body, client::HttpConnector, header::HeaderName, http::HeaderValue, Body, Client, Request, - Response, StatusCode, Uri, -}; -use hyper_rustls::{ConfigBuilderExt, HttpsConnector, HttpsConnectorBuilder}; -use pin_project::pin_project; use std::{ collections::HashMap, future::Future, @@ -24,12 +7,31 @@ use std::{ pin::Pin, task::{ready, Context, Poll}, }; + +use axum::http::uri::{self, Parts, Scheme}; +use futures::{Stream, StreamExt}; +use hyper::{ + body, client::HttpConnector, header::HeaderName, http::HeaderValue, Body, Client, Request, + Response, StatusCode, Uri, +}; +use hyper_rustls::{ConfigBuilderExt, HttpsConnector, HttpsConnectorBuilder}; +use pin_project::pin_project; use tokio_rustls::{ rustls, rustls::{Certificate, PrivateKey, RootCertStore}, }; use tracing::error; +use crate::{ + config::{ClientConfig, HyperClientConfigurator, NetworkConfig, PeerConfig}, + helpers::{ + query::{PrepareQuery, QueryConfig, QueryInput}, + HelperIdentity, + }, + net::{http_serde, server::HTTP_CLIENT_ID_HEADER, Error}, + protocol::{step::Gate, QueryId}, +}; + #[derive(Clone, Default)] pub enum ClientIdentity { /// Claim the specified helper identity without any additional authentication. @@ -421,6 +423,15 @@ fn make_http_connector() -> HttpConnector { #[cfg(all(test, web_test))] pub(crate) mod tests { + use std::{ + fmt::Debug, + future::{ready, Future}, + iter::zip, + task::Poll, + }; + + use futures::stream::{once, poll_immediate}; + use super::*; use crate::{ ff::{FieldType, Fp31}, @@ -434,13 +445,6 @@ pub(crate) mod tests { secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, sync::Arc, }; - use futures::stream::{once, poll_immediate}; - use std::{ - fmt::Debug, - future::{ready, Future}, - iter::zip, - task::Poll, - }; // This is a kludgy way of working around `TransportCallbacks` not being `Clone`, so // that tests can run against both HTTP and HTTPS servers with one set. diff --git a/src/net/error.rs b/src/net/error.rs index 618ab9cc3..3c7ddf6d9 100644 --- a/src/net/error.rs +++ b/src/net/error.rs @@ -1,9 +1,10 @@ -use crate::{error::BoxError, net::client::ResponseFromEndpoint, protocol::QueryId}; use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; +use crate::{error::BoxError, net::client::ResponseFromEndpoint, protocol::QueryId}; + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("{0}")] diff --git a/src/net/http_serde.rs b/src/net/http_serde.rs index bdc40d0ef..96188c8f2 100644 --- a/src/net/http_serde.rs +++ b/src/net/http_serde.rs @@ -4,11 +4,13 @@ // to compose const strings at compile time is concat!() pub mod echo { - use crate::net::Error; + use std::collections::HashMap; + use async_trait::async_trait; use axum::extract::{FromRequest, Query, RequestParts}; use hyper::http::uri; - use std::collections::HashMap; + + use crate::net::Error; #[derive(Debug, Default, Clone, PartialEq, Eq)] #[cfg_attr(feature = "enable-serde", derive(serde::Serialize, serde::Deserialize))] @@ -78,17 +80,19 @@ pub mod echo { } pub mod query { + use std::{ + fmt::{Display, Formatter}, + num::NonZeroU32, + }; + + use async_trait::async_trait; + use axum::extract::{FromRequest, Query, RequestParts}; + use crate::{ ff::FieldType, helpers::query::{IpaQueryConfig, QueryConfig, QuerySize, QueryType}, net::Error, }; - use async_trait::async_trait; - use axum::extract::{FromRequest, Query, RequestParts}; - use std::{ - fmt::{Display, Formatter}, - num::NonZeroU32, - }; /// wrapper around [`QueryConfig`] to enable extraction from an `Axum` request. To be used with /// the `create` and `prepare` commands @@ -209,6 +213,10 @@ pub mod query { pub const BASE_AXUM_PATH: &str = "/query"; pub mod create { + use async_trait::async_trait; + use axum::extract::{FromRequest, RequestParts}; + use hyper::http::uri; + use crate::{ helpers::query::QueryConfig, net::{ @@ -217,9 +225,6 @@ pub mod query { }, protocol::QueryId, }; - use async_trait::async_trait; - use axum::extract::{FromRequest, RequestParts}; - use hyper::http::uri; #[derive(Debug, Clone)] pub struct Request { @@ -268,13 +273,6 @@ pub mod query { } pub mod prepare { - use crate::{ - helpers::{query::PrepareQuery, RoleAssignment}, - net::{ - http_serde::query::{QueryConfigQueryParams, BASE_AXUM_PATH}, - Error, - }, - }; use async_trait::async_trait; use axum::{ extract::{FromRequest, Path, RequestParts}, @@ -283,6 +281,14 @@ pub mod query { }; use hyper::header::CONTENT_TYPE; + use crate::{ + helpers::{query::PrepareQuery, RoleAssignment}, + net::{ + http_serde::query::{QueryConfigQueryParams, BASE_AXUM_PATH}, + Error, + }, + }; + #[derive(Debug, Clone)] pub struct Request { pub data: PrepareQuery, @@ -346,10 +352,6 @@ pub mod query { } pub mod input { - use crate::{ - helpers::query::QueryInput, - net::{http_serde::query::BASE_AXUM_PATH, Error}, - }; use async_trait::async_trait; use axum::{ extract::{FromRequest, Path, RequestParts}, @@ -357,6 +359,11 @@ pub mod query { }; use hyper::{header::CONTENT_TYPE, Body}; + use crate::{ + helpers::query::QueryInput, + net::{http_serde::query::BASE_AXUM_PATH, Error}, + }; + #[derive(Debug)] pub struct Request { pub query_input: QueryInput, @@ -410,17 +417,18 @@ pub mod query { } pub mod step { - use crate::{ - helpers::BodyStream, - net::{http_serde::query::BASE_AXUM_PATH, Error}, - protocol::{step::Gate, QueryId}, - }; use async_trait::async_trait; use axum::{ extract::{FromRequest, Path, RequestParts}, http::uri, }; + use crate::{ + helpers::BodyStream, + net::{http_serde::query::BASE_AXUM_PATH, Error}, + protocol::{step::Gate, QueryId}, + }; + // When this type is used on the client side, `B` is `hyper::Body`. When this type // is used on the server side, `B` can be any body type supported by axum. #[derive(Debug)] @@ -490,11 +498,12 @@ pub mod query { } pub mod status { - use crate::{net::Error, protocol::QueryId, query::QueryStatus}; use async_trait::async_trait; use axum::extract::{FromRequest, Path, RequestParts}; use serde::{Deserialize, Serialize}; + use crate::{net::Error, protocol::QueryId, query::QueryStatus}; + #[derive(Debug, Clone)] pub struct Request { pub query_id: QueryId, @@ -544,10 +553,11 @@ pub mod query { } pub mod results { - use crate::{net::Error, protocol::QueryId}; use async_trait::async_trait; use axum::extract::{FromRequest, Path, RequestParts}; + use crate::{net::Error, protocol::QueryId}; + #[derive(Debug, Clone)] pub struct Request { pub query_id: QueryId, diff --git a/src/net/server/handlers/echo.rs b/src/net/server/handlers/echo.rs index 470e3e6ae..c2cba5eb7 100644 --- a/src/net/server/handlers/echo.rs +++ b/src/net/server/handlers/echo.rs @@ -1,6 +1,7 @@ -use crate::net::{http_serde, server::Error}; use axum::{routing::get, Json, Router}; +use crate::net::{http_serde, server::Error}; + #[allow(clippy::unused_async)] // needs to be async for axum handler async fn handler(req: http_serde::echo::Request) -> Result, Error> { Ok(Json(req)) @@ -12,9 +13,10 @@ pub fn router() -> Router { #[cfg(all(test, unit_test))] mod tests { - use super::*; use std::collections::HashMap; + use super::*; + #[tokio::test] async fn happy_case() { let req = http_serde::echo::Request::new( diff --git a/src/net/server/handlers/mod.rs b/src/net/server/handlers/mod.rs index abe5961d2..14c9b4c49 100644 --- a/src/net/server/handlers/mod.rs +++ b/src/net/server/handlers/mod.rs @@ -1,11 +1,12 @@ mod echo; mod query; +use axum::Router; + use crate::{ net::{http_serde, HttpTransport}, sync::Arc, }; -use axum::Router; pub fn router(transport: Arc) -> Router { echo::router().nest( diff --git a/src/net/server/handlers/query/create.rs b/src/net/server/handlers/query/create.rs index b14563bd9..205be03a3 100644 --- a/src/net/server/handlers/query/create.rs +++ b/src/net/server/handlers/query/create.rs @@ -1,11 +1,12 @@ +use axum::{routing::post, Extension, Json, Router}; +use hyper::StatusCode; + use crate::{ helpers::Transport, net::{http_serde, Error, HttpTransport}, query::NewQueryError, sync::Arc, }; -use axum::{routing::post, Extension, Json, Router}; -use hyper::StatusCode; /// Takes details from the HTTP request and creates a `[TransportCommand]::CreateQuery` that is sent /// to the [`HttpTransport`]. @@ -31,6 +32,14 @@ pub fn router(transport: Arc) -> Router { #[cfg(all(test, unit_test))] mod tests { + use std::{future::ready, num::NonZeroU32}; + + use axum::http::Request; + use hyper::{ + http::uri::{Authority, Scheme}, + Body, StatusCode, + }; + use super::*; use crate::{ ff::FieldType, @@ -44,13 +53,6 @@ mod tests { }, protocol::QueryId, }; - use axum::http::Request; - - use hyper::{ - http::uri::{Authority, Scheme}, - Body, StatusCode, - }; - use std::{future::ready, num::NonZeroU32}; async fn create_test(expected_query_config: QueryConfig) { let cb = TransportCallbacks { diff --git a/src/net/server/handlers/query/input.rs b/src/net/server/handlers/query/input.rs index 001497dca..d1ea6621d 100644 --- a/src/net/server/handlers/query/input.rs +++ b/src/net/server/handlers/query/input.rs @@ -1,10 +1,11 @@ +use axum::{routing::post, Extension, Router}; +use hyper::StatusCode; + use crate::{ helpers::Transport, net::{http_serde, Error, HttpTransport}, sync::Arc, }; -use axum::{routing::post, Extension, Router}; -use hyper::StatusCode; async fn handler( transport: Extension>, @@ -25,6 +26,9 @@ pub fn router(transport: Arc) -> Router { #[cfg(all(test, unit_test))] mod tests { + use axum::http::Request; + use hyper::{Body, StatusCode}; + use super::*; use crate::{ helpers::{query::QueryInput, BytesStream, TransportCallbacks}, @@ -34,8 +38,6 @@ mod tests { }, protocol::QueryId, }; - use axum::http::Request; - use hyper::{Body, StatusCode}; #[tokio::test] async fn input_test() { diff --git a/src/net/server/handlers/query/mod.rs b/src/net/server/handlers/query/mod.rs index 882e9833e..94ee2f3aa 100644 --- a/src/net/server/handlers/query/mod.rs +++ b/src/net/server/handlers/query/mod.rs @@ -5,10 +5,8 @@ mod results; mod status; mod step; -use crate::{ - net::{server::ClientIdentity, HttpTransport}, - sync::Arc, -}; +use std::any::Any; + use axum::{ response::{IntoResponse, Response}, Router, @@ -18,9 +16,13 @@ use futures_util::{ FutureExt, }; use hyper::{http::request, Request, StatusCode}; -use std::any::Any; use tower::{layer::layer_fn, Service}; +use crate::{ + net::{server::ClientIdentity, HttpTransport}, + sync::Arc, +}; + /// Construct router for IPA query web service /// /// In principle, this web service could be backed by either an HTTP-interconnected helper network or @@ -115,11 +117,12 @@ impl MaybeExtensionExt for request::Builder { #[cfg(all(test, unit_test))] pub mod test_helpers { - use crate::net::test::TestServer; use futures_util::future::poll_immediate; use hyper::{service::Service, StatusCode}; use tower::ServiceExt; + use crate::net::test::TestServer; + /// types that implement `IntoFailingReq` are intended to induce some failure in the process of /// axum routing. Pair with `assert_req_fails_with` to detect specific [`StatusCode`] failures. pub trait IntoFailingReq { diff --git a/src/net/server/handlers/query/prepare.rs b/src/net/server/handlers/query/prepare.rs index 76d35896a..7e3ee711a 100644 --- a/src/net/server/handlers/query/prepare.rs +++ b/src/net/server/handlers/query/prepare.rs @@ -1,11 +1,12 @@ use std::sync::Arc; +use axum::{response::IntoResponse, routing::post, Extension, Router}; +use hyper::StatusCode; + use crate::{ net::{http_serde, server::ClientIdentity, HttpTransport}, query::PrepareQueryError, }; -use axum::{response::IntoResponse, routing::post, Extension, Router}; -use hyper::StatusCode; /// Called by whichever peer helper is the leader for an individual query, to initiatialize /// processing of that query. @@ -33,6 +34,9 @@ pub fn router(transport: Arc) -> Router { mod tests { use std::future::ready; + use axum::http::Request; + use hyper::{Body, StatusCode}; + use super::*; use crate::{ ff::FieldType, @@ -52,8 +56,6 @@ mod tests { }, protocol::QueryId, }; - use axum::http::Request; - use hyper::{Body, StatusCode}; #[tokio::test] async fn prepare_test() { diff --git a/src/net/server/handlers/query/results.rs b/src/net/server/handlers/query/results.rs index d74fa4629..9773fa9c9 100644 --- a/src/net/server/handlers/query/results.rs +++ b/src/net/server/handlers/query/results.rs @@ -1,11 +1,12 @@ use std::sync::Arc; +use axum::{routing::get, Extension, Router}; +use hyper::StatusCode; + use crate::{ helpers::Transport, net::{http_serde, server::Error, HttpTransport}, }; -use axum::{routing::get, Extension, Router}; -use hyper::StatusCode; /// Handles the completion of the query by blocking the sender until query is completed. async fn handler( @@ -30,6 +31,9 @@ pub fn router(transport: Arc) -> Router { mod tests { use std::future::ready; + use axum::http::Request; + use hyper::StatusCode; + use super::*; use crate::{ ff::Fp31, @@ -42,8 +46,6 @@ mod tests { query::ProtocolResult, secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, }; - use axum::http::Request; - use hyper::StatusCode; #[tokio::test] async fn results_test() { diff --git a/src/net/server/handlers/query/status.rs b/src/net/server/handlers/query/status.rs index 773b8cb67..7c067e0cb 100644 --- a/src/net/server/handlers/query/status.rs +++ b/src/net/server/handlers/query/status.rs @@ -1,11 +1,12 @@ use std::sync::Arc; +use axum::{routing::get, Extension, Json, Router}; +use hyper::StatusCode; + use crate::{ helpers::Transport, net::{http_serde::query::status, server::Error, HttpTransport}, }; -use axum::{routing::get, Extension, Json, Router}; -use hyper::StatusCode; async fn handler( transport: Extension>, @@ -28,6 +29,9 @@ pub fn router(transport: Arc) -> Router { mod tests { use std::future::ready; + use axum::http::Request; + use hyper::StatusCode; + use super::*; use crate::{ helpers::TransportCallbacks, @@ -39,8 +43,6 @@ mod tests { protocol::QueryId, query::QueryStatus, }; - use axum::http::Request; - use hyper::StatusCode; #[tokio::test] async fn status_test() { diff --git a/src/net/server/handlers/query/step.rs b/src/net/server/handlers/query/step.rs index 2520671e0..223dbbff6 100644 --- a/src/net/server/handlers/query/step.rs +++ b/src/net/server/handlers/query/step.rs @@ -1,3 +1,5 @@ +use axum::{routing::post, Extension, Router}; + use crate::{ helpers::{BodyStream, Transport}, net::{ @@ -7,7 +9,6 @@ use crate::{ }, sync::Arc, }; -use axum::{routing::post, Extension, Router}; #[allow(clippy::unused_async)] // axum doesn't like synchronous handler async fn handler( @@ -30,6 +31,10 @@ pub fn router(transport: Arc) -> Router { mod tests { use std::task::Poll; + use axum::http::Request; + use futures::{stream::poll_immediate, StreamExt}; + use hyper::{Body, StatusCode}; + use super::*; use crate::{ helpers::{HelperIdentity, MESSAGE_PAYLOAD_SIZE_BYTES}, @@ -45,9 +50,6 @@ mod tests { QueryId, }, }; - use axum::http::Request; - use futures::{stream::poll_immediate, StreamExt}; - use hyper::{Body, StatusCode}; const DATA_LEN: usize = 3; diff --git a/src/net/server/mod.rs b/src/net/server/mod.rs index 5d24e24f5..d61804850 100644 --- a/src/net/server/mod.rs +++ b/src/net/server/mod.rs @@ -1,13 +1,16 @@ mod handlers; -use crate::{ - config::{NetworkConfig, ServerConfig, TlsConfig}, - error::BoxError, - helpers::HelperIdentity, - net::{Error, HttpTransport}, - sync::Arc, - task::JoinHandle, - telemetry::metrics::{web::RequestProtocolVersion, REQUESTS_RECEIVED}, +use std::{ + borrow::Cow, + io, + net::{Ipv4Addr, SocketAddr, TcpListener}, + ops::Deref, + task::{Context, Poll}, +}; + +use ::tokio::{ + fs, + io::{AsyncRead, AsyncWrite}, }; use axum::{ response::{IntoResponse, Response}, @@ -28,13 +31,8 @@ use futures::{ use hyper::{header::HeaderName, server::conn::AddrStream, Request}; use metrics::increment_counter; use rustls_pemfile::Item; -use std::{ - borrow::Cow, - io, - net::{Ipv4Addr, SocketAddr, TcpListener}, - ops::Deref, - task::{Context, Poll}, -}; +#[cfg(all(feature = "shuttle", test))] +use shuttle::future as tokio; use tokio_rustls::{ rustls::{ server::AllowAnyAnonymousOrAuthenticatedClient, Certificate, PrivateKey, RootCertStore, @@ -46,14 +44,16 @@ use tower::{layer::layer_fn, Service}; use tower_http::trace::TraceLayer; use tracing::{error, Span}; -use ::tokio::{ - fs, - io::{AsyncRead, AsyncWrite}, +use crate::{ + config::{NetworkConfig, ServerConfig, TlsConfig}, + error::BoxError, + helpers::HelperIdentity, + net::{Error, HttpTransport}, + sync::Arc, + task::JoinHandle, + telemetry::metrics::{web::RequestProtocolVersion, REQUESTS_RECEIVED}, }; -#[cfg(all(feature = "shuttle", test))] -use shuttle::future as tokio; - pub trait TracingSpanMaker: Send + Sync + Clone + 'static { fn make_span(&self) -> Span; } @@ -482,15 +482,11 @@ impl, Response = Response>> Service> #[cfg(all(test, unit_test))] mod e2e_tests { - use super::*; - use crate::{ - net::{http_serde, test::TestServer}, - test_fixture::metrics::MetricsHandle, - }; + use std::{collections::HashMap, time::SystemTime}; + use hyper::{client::HttpConnector, http::uri, StatusCode, Version}; use hyper_rustls::HttpsConnector; use metrics_util::debugging::Snapshotter; - use std::{collections::HashMap, time::SystemTime}; use tokio_rustls::{ rustls, rustls::{ @@ -500,6 +496,12 @@ mod e2e_tests { }; use tracing::Level; + use super::*; + use crate::{ + net::{http_serde, test::TestServer}, + test_fixture::metrics::MetricsHandle, + }; + fn expected_req(host: String) -> http_serde::echo::Request { http_serde::echo::Request::new( HashMap::from([ diff --git a/src/net/test.rs b/src/net/test.rs index d47f7d830..aad797df6 100644 --- a/src/net/test.rs +++ b/src/net/test.rs @@ -9,6 +9,15 @@ #![allow(clippy::missing_panics_doc)] +use std::{ + array, + net::{SocketAddr, TcpListener}, +}; + +use once_cell::sync::Lazy; +use tokio::task::JoinHandle; +use tokio_rustls::rustls::Certificate; + use crate::{ config::{ ClientConfig, HpkeClientConfig, HpkeServerConfig, NetworkConfig, PeerConfig, ServerConfig, @@ -20,15 +29,6 @@ use crate::{ sync::Arc, test_fixture::metrics::MetricsHandle, }; -use once_cell::sync::Lazy; -use std::{ - array, - net::{SocketAddr, TcpListener}, -}; - -use tokio::task::JoinHandle; - -use tokio_rustls::rustls::Certificate; pub const DEFAULT_TEST_PORTS: [u16; 3] = [3000, 3001, 3002]; diff --git a/src/net/transport.rs b/src/net/transport.rs index 95e8867a1..a55ec9d25 100644 --- a/src/net/transport.rs +++ b/src/net/transport.rs @@ -1,3 +1,14 @@ +use std::{ + borrow::Borrow, + future::Future, + pin::Pin, + task::{Context, Poll}, +}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::{Stream, TryFutureExt}; + use crate::{ config::{NetworkConfig, ServerConfig}, error::BoxError, @@ -12,15 +23,6 @@ use crate::{ protocol::{step::Gate, QueryId}, sync::Arc, }; -use async_trait::async_trait; -use bytes::Bytes; -use futures::{Stream, TryFutureExt}; -use std::{ - borrow::Borrow, - future::Future, - pin::Pin, - task::{Context, Poll}, -}; type LogHttpErrors = LogErrors; @@ -185,6 +187,16 @@ impl Transport for Arc { #[cfg(all(test, web_test))] mod tests { + use std::{iter::zip, net::TcpListener, task::Poll}; + + use futures::stream::{poll_immediate, StreamExt}; + use futures_util::future::{join_all, try_join_all}; + use generic_array::GenericArray; + use once_cell::sync::Lazy; + use tokio::sync::mpsc::channel; + use tokio_stream::wrappers::ReceiverStream; + use typenum::Unsigned; + use super::*; use crate::{ config::{NetworkConfig, ServerConfig}, @@ -198,14 +210,6 @@ mod tests { test_fixture::Reconstruct, AppSetup, HelperApp, }; - use futures::stream::{poll_immediate, StreamExt}; - use futures_util::future::{join_all, try_join_all}; - use generic_array::GenericArray; - use once_cell::sync::Lazy; - use std::{iter::zip, net::TcpListener, task::Poll}; - use tokio::sync::mpsc::channel; - use tokio_stream::wrappers::ReceiverStream; - use typenum::Unsigned; static STEP: Lazy = Lazy::new(|| Gate::from("http-transport")); diff --git a/src/protocol/attribution/accumulate_credit.rs b/src/protocol/attribution/accumulate_credit.rs index 0ce696b90..82d5d4f94 100644 --- a/src/protocol/attribution/accumulate_credit.rs +++ b/src/protocol/attribution/accumulate_credit.rs @@ -1,3 +1,8 @@ +use std::num::NonZeroU32; + +use ipa_macros::step; +use strum::AsRefStr; + use super::{ do_the_binary_tree_thing, input::{AccumulateCreditInputRow, AccumulateCreditOutputRow}, @@ -8,9 +13,6 @@ use crate::{ protocol::{context::Context, BasicProtocols, RecordId}, secret_sharing::Linear as LinearSecretSharing, }; -use ipa_macros::step; -use std::num::NonZeroU32; -use strum::AsRefStr; /// /// When `PER_USER_CAP` is set to one, this function is called @@ -146,6 +148,8 @@ pub(crate) enum Step { #[cfg(all(test, unit_test))] mod tests { + use std::num::NonZeroU32; + use crate::{ accumulation_test_input, ff::Fp32BitPrime, @@ -160,7 +164,6 @@ mod tests { secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, test_fixture::{input::GenericReportTestInput, Reconstruct, Runner, TestWorld}, }; - use std::num::NonZeroU32; async fn accumulate_credit_test( input: Vec>, diff --git a/src/protocol/attribution/aggregate_credit.rs b/src/protocol/attribution/aggregate_credit.rs index 8dafcfa21..fc052256b 100644 --- a/src/protocol/attribution/aggregate_credit.rs +++ b/src/protocol/attribution/aggregate_credit.rs @@ -1,5 +1,9 @@ extern crate ipa_macros; +use futures::stream::{iter as stream_iter, StreamExt, TryStreamExt}; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::{Gf2, PrimeField, Serializable}, @@ -19,9 +23,6 @@ use crate::{ }, seq_join::seq_join, }; -use futures::stream::{iter as stream_iter, StreamExt, TryStreamExt}; -use ipa_macros::step; -use strum::AsRefStr; /// This is the number of breakdown keys above which it is more efficient to SORT by breakdown key. /// Below this number, it's more efficient to just do a ton of equality checks. diff --git a/src/protocol/attribution/apply_attribution_window.rs b/src/protocol/attribution/apply_attribution_window.rs index d75b69f7b..3f433e66a 100644 --- a/src/protocol/attribution/apply_attribution_window.rs +++ b/src/protocol/attribution/apply_attribution_window.rs @@ -1,3 +1,11 @@ +use std::{ + iter::{repeat, zip}, + num::NonZeroU32, +}; + +use ipa_macros::step; +use strum::AsRefStr; + use super::{ do_the_binary_tree_thing, input::{ApplyAttributionWindowInputRow, ApplyAttributionWindowOutputRow}, @@ -12,12 +20,6 @@ use crate::{ }, secret_sharing::Linear as LinearSecretSharing, }; -use ipa_macros::step; -use std::{ - iter::{repeat, zip}, - num::NonZeroU32, -}; -use strum::AsRefStr; /// This protocol applies the specified attribution window to trigger events. All trigger values of /// events that are outside the window will be replaced with 0, hence will not be attributed to @@ -191,6 +193,8 @@ pub(crate) enum Step { #[cfg(all(test, unit_test))] mod tests { + use std::{iter::zip, num::NonZeroU32}; + use crate::{ attribution_window_test_input, ff::{Field, Fp32BitPrime}, @@ -206,7 +210,6 @@ mod tests { secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, test_fixture::{input::GenericReportTestInput, Reconstruct, Runner, TestWorld}, }; - use std::{iter::zip, num::NonZeroU32}; #[tokio::test] pub async fn attribution_window() { diff --git a/src/protocol/attribution/credit_capping.rs b/src/protocol/attribution/credit_capping.rs index e0c0f528c..e9ca00170 100644 --- a/src/protocol/attribution/credit_capping.rs +++ b/src/protocol/attribution/credit_capping.rs @@ -1,3 +1,12 @@ +use std::iter::{repeat, zip}; + +use futures::{ + stream::{iter, once}, + StreamExt, TryStreamExt, +}; +use ipa_macros::step; +use strum::AsRefStr; + use super::{do_the_binary_tree_thing, input::CreditCappingInputRow, prefix_or_binary_tree_style}; use crate::{ error::Error, @@ -11,13 +20,6 @@ use crate::{ secret_sharing::Linear as LinearSecretSharing, seq_join::seq_join, }; -use futures::{ - stream::{iter, once}, - StreamExt, TryStreamExt, -}; -use ipa_macros::step; -use std::iter::{repeat, zip}; -use strum::AsRefStr; /// User-level credit capping protocol. /// diff --git a/src/protocol/attribution/input.rs b/src/protocol/attribution/input.rs index 019641e7d..c0f6808f6 100644 --- a/src/protocol/attribution/input.rs +++ b/src/protocol/attribution/input.rs @@ -1,3 +1,10 @@ +use std::marker::PhantomData; + +use async_trait::async_trait; +use futures::future::try_join4; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::Field, @@ -5,11 +12,6 @@ use crate::{ protocol::{basics::Reshare, context::Context, RecordId}, secret_sharing::Linear as LinearSecretSharing, }; -use async_trait::async_trait; -use futures::future::try_join4; -use ipa_macros::step; -use std::marker::PhantomData; -use strum::AsRefStr; // // `apply_attribution_window` protocol diff --git a/src/protocol/attribution/mod.rs b/src/protocol/attribution/mod.rs index 9846d9168..e10ebcf16 100644 --- a/src/protocol/attribution/mod.rs +++ b/src/protocol/attribution/mod.rs @@ -4,6 +4,15 @@ pub mod apply_attribution_window; pub mod credit_capping; pub mod input; +use std::iter::{once as iter_once, zip}; + +use futures::{ + future::try_join, + stream::{iter as stream_iter, TryStreamExt}, +}; +use ipa_macros::step; +use strum::AsRefStr; + use self::{ accumulate_credit::accumulate_credit, aggregate_credit::aggregate_credit, apply_attribution_window::apply_attribution_window, credit_capping::credit_capping, @@ -32,13 +41,6 @@ use crate::{ }, seq_join::assert_send, }; -use futures::{ - future::try_join, - stream::{iter as stream_iter, TryStreamExt}, -}; -use ipa_macros::step; -use std::iter::{once as iter_once, zip}; -use strum::AsRefStr; /// Performs a set of attribution protocols on the sorted IPA input. /// diff --git a/src/protocol/basics/check_zero.rs b/src/protocol/basics/check_zero.rs index aa8881a16..61a33f87f 100644 --- a/src/protocol/basics/check_zero.rs +++ b/src/protocol/basics/check_zero.rs @@ -1,3 +1,6 @@ +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::Field, @@ -9,8 +12,6 @@ use crate::{ }, secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, }; -use ipa_macros::step; -use strum::AsRefStr; #[step] pub(crate) enum Step { @@ -66,6 +67,8 @@ pub async fn check_zero( #[cfg(all(test, unit_test))] mod tests { + use futures_util::future::try_join3; + use crate::{ error::Error, ff::{Field, Fp31, PrimeField}, @@ -74,7 +77,6 @@ mod tests { secret_sharing::{IntoShares, SharedValue}, test_fixture::TestWorld, }; - use futures_util::future::try_join3; #[tokio::test] async fn basic() -> Result<(), Error> { diff --git a/src/protocol/basics/mul/malicious.rs b/src/protocol/basics/mul/malicious.rs index 4d2741851..3af7bc48b 100644 --- a/src/protocol/basics/mul/malicious.rs +++ b/src/protocol/basics/mul/malicious.rs @@ -1,3 +1,9 @@ +use std::fmt::Debug; + +use futures::future::try_join; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, protocol::{ @@ -11,10 +17,6 @@ use crate::{ ReplicatedSecretSharing, }, }; -use futures::future::try_join; -use ipa_macros::step; -use std::fmt::Debug; -use strum::AsRefStr; #[step] pub(crate) enum Step { diff --git a/src/protocol/basics/mul/mod.rs b/src/protocol/basics/mul/mod.rs index 63887ebfd..ed98e9d0b 100644 --- a/src/protocol/basics/mul/mod.rs +++ b/src/protocol/basics/mul/mod.rs @@ -1,3 +1,5 @@ +use async_trait::async_trait; + use crate::{ error::Error, ff::Field, @@ -10,7 +12,6 @@ use crate::{ semi_honest::AdditiveShare as Replicated, }, }; -use async_trait::async_trait; pub(crate) mod malicious; mod semi_honest; diff --git a/src/protocol/basics/mul/semi_honest.rs b/src/protocol/basics/mul/semi_honest.rs index 1eed79191..57f793a0a 100644 --- a/src/protocol/basics/mul/semi_honest.rs +++ b/src/protocol/basics/mul/semi_honest.rs @@ -83,6 +83,10 @@ where #[cfg(all(test, unit_test))] mod test { + use std::iter::{repeat, zip}; + + use rand::distributions::{Distribution, Standard}; + use crate::{ ff::{Field, Fp31}, protocol::{basics::SecureMul, context::Context, RecordId}, @@ -90,8 +94,6 @@ mod test { seq_join::SeqJoin, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use rand::distributions::{Distribution, Standard}; - use std::iter::{repeat, zip}; #[tokio::test] async fn basic() { diff --git a/src/protocol/basics/mul/sparse.rs b/src/protocol/basics/mul/sparse.rs index fef4ecbbf..97796931e 100644 --- a/src/protocol/basics/mul/sparse.rs +++ b/src/protocol/basics/mul/sparse.rs @@ -187,6 +187,11 @@ impl MultiplyWork for MultiplyZeroPositions { #[cfg(all(test, unit_test))] pub(in crate::protocol) mod test { + use std::{borrow::Borrow, iter::zip}; + + use futures::future::try_join; + use rand::distributions::{Distribution, Standard}; + use crate::{ ff::{Field, Fp31, Fp32BitPrime}, helpers::{ @@ -206,9 +211,6 @@ pub(in crate::protocol) mod test { }, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use futures::future::try_join; - use rand::distributions::{Distribution, Standard}; - use std::{borrow::Borrow, iter::zip}; #[derive(Clone, Copy)] pub struct SparseField { diff --git a/src/protocol/basics/reshare.rs b/src/protocol/basics/reshare.rs index 894980a03..2e9a868e6 100644 --- a/src/protocol/basics/reshare.rs +++ b/src/protocol/basics/reshare.rs @@ -1,3 +1,9 @@ +use std::iter::{repeat, zip}; + +use async_trait::async_trait; +use embed_doc_image::embed_doc_image; +use futures::future::try_join; + use crate::{ error::Error, ff::Field, @@ -20,10 +26,6 @@ use crate::{ BitDecomposed, }, }; -use async_trait::async_trait; -use embed_doc_image::embed_doc_image; -use futures::future::try_join; -use std::iter::{repeat, zip}; #[embed_doc_image("reshare", "images/sort/reshare.png")] /// Trait for reshare protocol to renew shares of a secret value for all 3 helpers. /// diff --git a/src/protocol/basics/reveal.rs b/src/protocol/basics/reveal.rs index 0d69b2cc9..28e54a904 100644 --- a/src/protocol/basics/reveal.rs +++ b/src/protocol/basics/reveal.rs @@ -1,5 +1,9 @@ use std::iter::{repeat, zip}; +use async_trait::async_trait; +use embed_doc_image::embed_doc_image; +use futures::future::try_join; + use crate::{ error::Error, ff::Field, @@ -17,9 +21,6 @@ use crate::{ SecretSharing, }, }; -use async_trait::async_trait; -use embed_doc_image::embed_doc_image; -use futures::future::try_join; /// Trait for reveal protocol to open a shared secret to all helpers inside the MPC ring. #[async_trait] @@ -153,31 +154,30 @@ where #[cfg(all(test, unit_test))] mod tests { - use crate::{ - protocol::context::{UpgradableContext, UpgradedContext, Validator}, - rand::{thread_rng, Rng}, - secret_sharing::replicated::malicious::ExtendableField, - test_fixture::Runner, - }; - use futures::future::{try_join, try_join3}; use std::iter::zip; + use futures::future::{try_join, try_join3}; + use crate::{ error::Error, ff::{Field, Fp31}, helpers::Direction, protocol::{ basics::Reveal, - context::{Context, UpgradedMaliciousContext}, + context::{ + Context, UpgradableContext, UpgradedContext, UpgradedMaliciousContext, Validator, + }, RecordId, }, + rand::{thread_rng, Rng}, secret_sharing::{ replicated::malicious::{ - AdditiveShare as MaliciousReplicated, ThisCodeIsAuthorizedToDowngradeFromMalicious, + AdditiveShare as MaliciousReplicated, ExtendableField, + ThisCodeIsAuthorizedToDowngradeFromMalicious, }, IntoShares, }, - test_fixture::{join3v, TestWorld}, + test_fixture::{join3v, Runner, TestWorld}, }; #[tokio::test] diff --git a/src/protocol/basics/share_known_value.rs b/src/protocol/basics/share_known_value.rs index 79857d9c5..bd811a3df 100644 --- a/src/protocol/basics/share_known_value.rs +++ b/src/protocol/basics/share_known_value.rs @@ -36,6 +36,8 @@ impl<'a, F: ExtendableField> ShareKnownValue, F> #[cfg(all(test, unit_test))] mod tests { + use rand::Rng; + use super::ShareKnownValue; use crate::{ ff::Fp31, @@ -45,7 +47,6 @@ mod tests { }, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use rand::Rng; #[tokio::test] pub async fn semi_honest_share_known_values() { diff --git a/src/protocol/basics/sum_of_product/malicious.rs b/src/protocol/basics/sum_of_product/malicious.rs index 4727cc3b0..866cb5012 100644 --- a/src/protocol/basics/sum_of_product/malicious.rs +++ b/src/protocol/basics/sum_of_product/malicious.rs @@ -1,3 +1,9 @@ +use std::fmt::Debug; + +use futures::future::try_join; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, helpers::Direction, @@ -12,10 +18,6 @@ use crate::{ ReplicatedSecretSharing, }, }; -use futures::future::try_join; -use ipa_macros::step; -use std::fmt::Debug; -use strum::AsRefStr; #[step] pub(crate) enum Step { diff --git a/src/protocol/basics/sum_of_product/mod.rs b/src/protocol/basics/sum_of_product/mod.rs index 4f9db605e..18937eff1 100644 --- a/src/protocol/basics/sum_of_product/mod.rs +++ b/src/protocol/basics/sum_of_product/mod.rs @@ -1,3 +1,5 @@ +use async_trait::async_trait; + use crate::{ error::Error, ff::Field, @@ -10,7 +12,6 @@ use crate::{ semi_honest::AdditiveShare as Replicated, }, }; -use async_trait::async_trait; pub(crate) mod malicious; mod semi_honest; diff --git a/src/protocol/boolean/add_constant.rs b/src/protocol/boolean/add_constant.rs index c99e42dd0..0708e4029 100644 --- a/src/protocol/boolean/add_constant.rs +++ b/src/protocol/boolean/add_constant.rs @@ -211,6 +211,9 @@ impl AsRef for Step { #[cfg(all(test, unit_test))] mod tests { + use bitvec::macros::internal::funty::Fundamental; + use rand::{distributions::Standard, prelude::Distribution}; + use crate::{ ff::{Field, Fp31, Fp32BitPrime, PrimeField}, protocol::{ @@ -221,8 +224,6 @@ mod tests { secret_sharing::{replicated::malicious::ExtendableField, SharedValue}, test_fixture::{into_bits, Reconstruct, Runner, TestWorld}, }; - use bitvec::macros::internal::funty::Fundamental; - use rand::{distributions::Standard, prelude::Distribution}; async fn add(world: &TestWorld, a: F, b: u128) -> Vec where diff --git a/src/protocol/boolean/bit_decomposition.rs b/src/protocol/boolean/bit_decomposition.rs index b3875922b..4343b0e8b 100644 --- a/src/protocol/boolean/bit_decomposition.rs +++ b/src/protocol/boolean/bit_decomposition.rs @@ -1,3 +1,6 @@ +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::PrimeField, @@ -12,8 +15,6 @@ use crate::{ }, secret_sharing::Linear as LinearSecretSharing, }; -use ipa_macros::step; -use strum::AsRefStr; /// This is an implementation of "3. Bit-Decomposition" from I. Damgård et al.. /// @@ -89,6 +90,8 @@ pub(crate) enum Step { #[cfg(all(test, unit_test))] mod tests { + use rand::{distributions::Standard, prelude::Distribution}; + use super::BitDecomposition; use crate::{ ff::{Field, Fp31, Fp32BitPrime, PrimeField}, @@ -100,7 +103,6 @@ mod tests { secret_sharing::replicated::malicious::ExtendableField, test_fixture::{bits_to_value, Reconstruct, Runner, TestWorld}, }; - use rand::{distributions::Standard, prelude::Distribution}; async fn bit_decomposition(world: &TestWorld, a: F) -> Vec where diff --git a/src/protocol/boolean/bitwise_equal.rs b/src/protocol/boolean/bitwise_equal.rs index 25bceb689..3657ade18 100644 --- a/src/protocol/boolean/bitwise_equal.rs +++ b/src/protocol/boolean/bitwise_equal.rs @@ -1,3 +1,8 @@ +use std::iter::zip; + +use ipa_macros::step; +use strum::AsRefStr; + use super::xor; use crate::{ error::Error, @@ -8,9 +13,6 @@ use crate::{ }, secret_sharing::Linear as LinearSecretSharing, }; -use ipa_macros::step; -use std::iter::zip; -use strum::AsRefStr; /// Compares `[a]` and `c`, and returns 1 iff `a == c` /// @@ -115,6 +117,7 @@ pub(crate) enum Step { #[cfg(all(test, unit_test))] mod tests { + use super::{bitwise_equal, bitwise_equal_constant}; use crate::{ ff::{Field, Fp31, Fp32BitPrime}, protocol::{context::Context, RecordId}, @@ -122,8 +125,6 @@ mod tests { test_fixture::{get_bits, Reconstruct, Runner, TestWorld}, }; - use super::{bitwise_equal, bitwise_equal_constant}; - #[tokio::test] pub async fn simple() { assert_eq!(1, run_bitwise_equal(45, 45, 9).await); diff --git a/src/protocol/boolean/bitwise_less_than_prime.rs b/src/protocol/boolean/bitwise_less_than_prime.rs index 9db9a09fc..87a707595 100644 --- a/src/protocol/boolean/bitwise_less_than_prime.rs +++ b/src/protocol/boolean/bitwise_less_than_prime.rs @@ -1,3 +1,9 @@ +use std::cmp::Ordering; + +use futures::future::try_join; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::PrimeField, @@ -9,10 +15,6 @@ use crate::{ }, secret_sharing::Linear as LinearSecretSharing, }; -use futures::future::try_join; -use ipa_macros::step; -use std::cmp::Ordering; -use strum::AsRefStr; /// This is an implementation of Bitwise Less-Than on bitwise-shared numbers. /// @@ -202,6 +204,8 @@ pub(crate) enum Step { #[cfg(all(test, unit_test))] mod tests { + use rand::{distributions::Standard, prelude::Distribution}; + use super::BitwiseLessThanPrime; use crate::{ ff::{Field, Fp31, Fp32BitPrime, PrimeField}, @@ -209,7 +213,6 @@ mod tests { secret_sharing::{replicated::malicious::ExtendableField, SharedValue}, test_fixture::{get_bits, Reconstruct, Runner, TestWorld}, }; - use rand::{distributions::Standard, prelude::Distribution}; #[tokio::test] pub async fn fp31() { diff --git a/src/protocol/boolean/comparison.rs b/src/protocol/boolean/comparison.rs index 464f547e5..971d197a6 100644 --- a/src/protocol/boolean/comparison.rs +++ b/src/protocol/boolean/comparison.rs @@ -1,3 +1,6 @@ +use ipa_macros::step; +use strum::AsRefStr; + use super::or::or; use crate::{ error::Error, @@ -10,8 +13,6 @@ use crate::{ }, secret_sharing::Linear as LinearSecretSharing, }; -use ipa_macros::step; -use strum::AsRefStr; // Compare an arithmetic-shared value `a` to a known value `c`. // @@ -296,6 +297,9 @@ pub(crate) enum Step { #[cfg(all(test, unit_test))] mod tests { + use proptest::proptest; + use rand::{distributions::Standard, prelude::Distribution, Rng}; + use super::{ bitwise_greater_than_constant, bitwise_less_than_constant, compute_r_bounds, greater_than_constant, @@ -311,8 +315,6 @@ mod tests { secret_sharing::{replicated::malicious::ExtendableField, SharedValue}, test_fixture::{into_bits, Reconstruct, Runner, TestWorld}, }; - use proptest::proptest; - use rand::{distributions::Standard, prelude::Distribution, Rng}; async fn bitwise_lt(world: &TestWorld, a: F, b: u128) -> F where diff --git a/src/protocol/boolean/generate_random_bits.rs b/src/protocol/boolean/generate_random_bits.rs index 017090cc6..f8e2d2e87 100644 --- a/src/protocol/boolean/generate_random_bits.rs +++ b/src/protocol/boolean/generate_random_bits.rs @@ -1,5 +1,7 @@ use std::marker::PhantomData; +use futures::stream::{iter as stream_iter, Stream, StreamExt}; + use crate::{ error::Error, ff::PrimeField, @@ -17,7 +19,6 @@ use crate::{ Linear as LinearSecretSharing, }, }; -use futures::stream::{iter as stream_iter, Stream, StreamExt}; #[derive(Debug)] struct RawRandomBits { diff --git a/src/protocol/boolean/mod.rs b/src/protocol/boolean/mod.rs index dd47da833..25f9f8f03 100644 --- a/src/protocol/boolean/mod.rs +++ b/src/protocol/boolean/mod.rs @@ -1,3 +1,5 @@ +use std::iter::repeat; + use crate::{ error::Error, ff::{Field, PrimeField}, @@ -9,7 +11,6 @@ use crate::{ }, secret_sharing::{Linear as LinearSecretSharing, SecretSharing}, }; -use std::iter::repeat; pub mod add_constant; pub mod bit_decomposition; diff --git a/src/protocol/boolean/or.rs b/src/protocol/boolean/or.rs index b1180de3b..e367264a8 100644 --- a/src/protocol/boolean/or.rs +++ b/src/protocol/boolean/or.rs @@ -22,6 +22,8 @@ pub async fn or + SecureMul>( #[cfg(all(test, unit_test))] mod tests { + use rand::distributions::{Distribution, Standard}; + use super::or; use crate::{ ff::{Field, Fp31}, @@ -29,7 +31,6 @@ mod tests { secret_sharing::{replicated::malicious::ExtendableField, SharedValue}, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use rand::distributions::{Distribution, Standard}; async fn run(world: &TestWorld, a: F, b: F) -> F where diff --git a/src/protocol/boolean/random_bits_generator.rs b/src/protocol/boolean/random_bits_generator.rs index 3f31d5811..068c0631f 100644 --- a/src/protocol/boolean/random_bits_generator.rs +++ b/src/protocol/boolean/random_bits_generator.rs @@ -1,3 +1,8 @@ +use std::{ + marker::PhantomData, + sync::atomic::{AtomicU32, Ordering}, +}; + use crate::{ error::Error, ff::PrimeField, @@ -10,10 +15,6 @@ use crate::{ }, secret_sharing::Linear as LinearSecretSharing, }; -use std::{ - marker::PhantomData, - sync::atomic::{AtomicU32, Ordering}, -}; /// A struct that generates random sharings of bits from the /// `SolvedBits` protocol. Any protocol who wish to use a random-bits can draw @@ -96,6 +97,10 @@ where #[cfg(all(test, unit_test))] mod tests { + use std::iter::zip; + + use futures::future::try_join_all; + use super::RandomBitsGenerator; use crate::{ ff::{Field, Fp31}, @@ -109,8 +114,6 @@ mod tests { }, test_fixture::{join3, join3v, Reconstruct, Runner, TestWorld}, }; - use futures::future::try_join_all; - use std::iter::zip; #[tokio::test] pub async fn semi_honest() { diff --git a/src/protocol/boolean/solved_bits.rs b/src/protocol/boolean/solved_bits.rs index 1ffb4a298..19b05bab1 100644 --- a/src/protocol/boolean/solved_bits.rs +++ b/src/protocol/boolean/solved_bits.rs @@ -1,3 +1,9 @@ +use std::marker::PhantomData; + +use async_trait::async_trait; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::{Field, PrimeField}, @@ -16,10 +22,6 @@ use crate::{ BitDecomposed, Linear as LinearSecretSharing, SecretSharing, }, }; -use async_trait::async_trait; -use ipa_macros::step; -use std::marker::PhantomData; -use strum::AsRefStr; #[derive(Debug)] pub struct RandomBitsShare @@ -146,6 +148,10 @@ pub(crate) enum Step { #[cfg(all(test, unit_test))] mod tests { + use std::iter::{repeat, zip}; + + use rand::{distributions::Standard, prelude::Distribution}; + use crate::{ ff::{Field, Fp31, Fp32BitPrime, PrimeField}, protocol::{ @@ -157,8 +163,6 @@ mod tests { seq_join::SeqJoin, test_fixture::{bits_to_value, Reconstruct, Runner, TestWorld}, }; - use rand::{distributions::Standard, prelude::Distribution}; - use std::iter::{repeat, zip}; /// Execute `solved_bits` `COUNT` times for `F`. The count should be chosen /// such that the probability of that many consecutive failures in `F` is diff --git a/src/protocol/boolean/xor.rs b/src/protocol/boolean/xor.rs index a69f77025..95e20059a 100644 --- a/src/protocol/boolean/xor.rs +++ b/src/protocol/boolean/xor.rs @@ -43,6 +43,8 @@ where #[cfg(all(test, unit_test))] mod tests { + use rand::distributions::{Distribution, Standard}; + use super::xor; use crate::{ ff::{Field, Fp31, Fp32BitPrime}, @@ -55,7 +57,6 @@ mod tests { secret_sharing::{replicated::malicious::ExtendableField, SharedValue}, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use rand::distributions::{Distribution, Standard}; async fn run(world: &TestWorld, a: F, b: F) -> F where diff --git a/src/protocol/context/malicious.rs b/src/protocol/context/malicious.rs index c60231f6d..3335397a7 100644 --- a/src/protocol/context/malicious.rs +++ b/src/protocol/context/malicious.rs @@ -1,3 +1,12 @@ +use std::{ + any::type_name, + fmt::{Debug, Formatter}, + num::NonZeroUsize, +}; + +use async_trait::async_trait; + +use super::{UpgradeContext, UpgradeToMalicious}; use crate::{ error::Error, helpers::{ChannelId, Gateway, Message, ReceivingEnd, Role, SendingEnd, TotalRecords}, @@ -24,14 +33,6 @@ use crate::{ seq_join::SeqJoin, sync::Arc, }; -use async_trait::async_trait; -use std::{ - any::type_name, - fmt::{Debug, Formatter}, - num::NonZeroUsize, -}; - -use super::{UpgradeContext, UpgradeToMalicious}; #[derive(Clone)] pub struct Context<'a> { diff --git a/src/protocol/context/mod.rs b/src/protocol/context/mod.rs index c9ac24b21..cf445ae95 100644 --- a/src/protocol/context/mod.rs +++ b/src/protocol/context/mod.rs @@ -4,6 +4,15 @@ pub mod semi_honest; pub mod upgrade; pub mod validator; +use std::{num::NonZeroUsize, sync::Arc}; + +use async_trait::async_trait; +pub use malicious::{Context as MaliciousContext, Upgraded as UpgradedMaliciousContext}; +use prss::{InstrumentedIndexedSharedRandomness, InstrumentedSequentialSharedRandomness}; +pub use semi_honest::{Context as SemiHonestContext, Upgraded as UpgradedSemiHonestContext}; +pub use upgrade::{UpgradeContext, UpgradeToMalicious}; +pub use validator::Validator; + use crate::{ error::Error, helpers::{ChannelId, Gateway, Message, ReceivingEnd, Role, SendingEnd, TotalRecords}, @@ -19,14 +28,6 @@ use crate::{ }, seq_join::SeqJoin, }; -use async_trait::async_trait; -use prss::{InstrumentedIndexedSharedRandomness, InstrumentedSequentialSharedRandomness}; -use std::{num::NonZeroUsize, sync::Arc}; - -pub use malicious::{Context as MaliciousContext, Upgraded as UpgradedMaliciousContext}; -pub use semi_honest::{Context as SemiHonestContext, Upgraded as UpgradedSemiHonestContext}; -pub use upgrade::{UpgradeContext, UpgradeToMalicious}; -pub use validator::Validator; /// Context used by each helper to perform secure computation. Provides access to shared randomness /// generator and communication channel. @@ -265,6 +266,16 @@ impl<'a> Inner<'a> { #[cfg(all(test, unit_test))] mod tests { + use std::iter::repeat; + + use futures_util::{future::join_all, try_join}; + use rand::{ + distributions::{Distribution, Standard}, + Rng, + }; + use typenum::Unsigned; + + use super::*; use crate::{ ff::{Field, Fp31, Serializable}, helpers::Direction, @@ -279,15 +290,6 @@ mod tests { }, test_fixture::{Reconstruct, Runner, TestWorld, TestWorldConfig}, }; - use futures_util::{future::join_all, try_join}; - use rand::{ - distributions::{Distribution, Standard}, - Rng, - }; - use std::iter::repeat; - use typenum::Unsigned; - - use super::*; trait AsReplicatedTestOnly { fn l(&self) -> F; diff --git a/src/protocol/context/prss.rs b/src/protocol/context/prss.rs index a76cf730b..fbec6255d 100644 --- a/src/protocol/context/prss.rs +++ b/src/protocol/context/prss.rs @@ -1,5 +1,7 @@ //! Metric-aware PRSS decorators +use rand_core::{Error, RngCore}; + use crate::{ helpers::Role, protocol::{ @@ -12,7 +14,6 @@ use crate::{ metrics::{INDEXED_PRSS_GENERATED, SEQUENTIAL_PRSS_GENERATED}, }, }; -use rand_core::{Error, RngCore}; /// Wrapper around `IndexedSharedRandomness` that instrument calls to `generate_values` pub struct InstrumentedIndexedSharedRandomness<'a> { diff --git a/src/protocol/context/semi_honest.rs b/src/protocol/context/semi_honest.rs index d306eb7fd..49d070910 100644 --- a/src/protocol/context/semi_honest.rs +++ b/src/protocol/context/semi_honest.rs @@ -1,5 +1,13 @@ +use std::{ + any::type_name, + fmt::{Debug, Formatter}, + marker::PhantomData, + num::NonZeroUsize, +}; + use async_trait::async_trait; +use super::{Context as SuperContext, UpgradeContext, UpgradeToMalicious}; use crate::{ error::Error, helpers::{Gateway, Message, ReceivingEnd, Role, SendingEnd, TotalRecords}, @@ -19,14 +27,6 @@ use crate::{ }, seq_join::SeqJoin, }; -use std::{ - any::type_name, - fmt::{Debug, Formatter}, - marker::PhantomData, - num::NonZeroUsize, -}; - -use super::{Context as SuperContext, UpgradeContext, UpgradeToMalicious}; #[derive(Clone)] pub struct Context<'a> { diff --git a/src/protocol/context/upgrade.rs b/src/protocol/context/upgrade.rs index ae874f3f4..02bb3aeea 100644 --- a/src/protocol/context/upgrade.rs +++ b/src/protocol/context/upgrade.rs @@ -1,3 +1,10 @@ +use std::marker::PhantomData; + +use async_trait::async_trait; +use futures::future::{try_join, try_join3}; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::Field, @@ -15,11 +22,6 @@ use crate::{ BitDecomposed, Linear as LinearSecretSharing, }, }; -use async_trait::async_trait; -use futures::future::{try_join, try_join3}; -use ipa_macros::step; -use std::marker::PhantomData; -use strum::AsRefStr; /// Special context type used for malicious upgrades. /// diff --git a/src/protocol/context/validator.rs b/src/protocol/context/validator.rs index 121dbd89a..e39185428 100644 --- a/src/protocol/context/validator.rs +++ b/src/protocol/context/validator.rs @@ -1,3 +1,14 @@ +use std::{ + any::type_name, + fmt::{Debug, Formatter}, + marker::PhantomData, +}; + +use async_trait::async_trait; +use futures::future::try_join; +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::Field, @@ -18,15 +29,6 @@ use crate::{ }, sync::{Arc, Mutex, Weak}, }; -use async_trait::async_trait; -use futures::future::try_join; -use ipa_macros::step; -use std::{ - any::type_name, - fmt::{Debug, Formatter}, - marker::PhantomData, -}; -use strum::AsRefStr; #[async_trait] pub trait Validator { @@ -311,6 +313,8 @@ impl Debug for Malicious<'_, F> { #[cfg(all(test, unit_test))] mod tests { + use std::iter::{repeat, zip}; + use crate::{ error::Error, ff::{Field, Fp31, Fp32BitPrime}, @@ -331,7 +335,6 @@ mod tests { seq_join::SeqJoin, test_fixture::{join3v, Reconstruct, Runner, TestWorld}, }; - use std::iter::{repeat, zip}; /// This is the simplest arithmetic circuit that allows us to test all of the pieces of this validator /// A - diff --git a/src/protocol/dp/distributions.rs b/src/protocol/dp/distributions.rs index 89d5b6c6b..3cff35214 100644 --- a/src/protocol/dp/distributions.rs +++ b/src/protocol/dp/distributions.rs @@ -1,10 +1,10 @@ +use std::{f64::consts::PI, fmt::Debug}; + use rand::{ distributions::{Distribution, Uniform}, Rng, }; -use std::{f64::consts::PI, fmt::Debug}; - /// Returns `true` iff `a` and `b` are close to each other. `a` and `b` are considered close if /// |a-b| < 10^(-precision). #[cfg(all(test, unit_test))] @@ -72,10 +72,12 @@ impl From for RoundedBoxMuller { #[cfg(all(test, unit_test))] mod tests { - use super::*; + use std::iter::repeat_with; + use rand::{distributions::Distribution, thread_rng}; use rand_core::RngCore; - use std::iter::repeat_with; + + use super::*; #[test] fn dp_normal_distribution_sample_standard() { diff --git a/src/protocol/dp/insecure.rs b/src/protocol/dp/insecure.rs index 12cebe9f7..c4bf7c5a0 100644 --- a/src/protocol/dp/insecure.rs +++ b/src/protocol/dp/insecure.rs @@ -1,9 +1,11 @@ #![allow(dead_code)] -use crate::protocol::dp::distributions::{BoxMuller, RoundedBoxMuller}; +use std::f64; + use rand::distributions::Distribution; use rand_core::{CryptoRng, RngCore}; -use std::f64; + +use crate::protocol::dp::distributions::{BoxMuller, RoundedBoxMuller}; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -101,12 +103,13 @@ impl DiscreteDp { #[cfg(all(test, unit_test))] mod test { - use super::*; - use crate::protocol::dp::distributions::is_close; use proptest::{prelude::ProptestConfig, proptest}; use rand::{rngs::StdRng, thread_rng, Rng}; use rand_core::SeedableRng; + use super::*; + use crate::protocol::dp::distributions::is_close; + #[test] fn dp_normal_distribution_generation_standard() { let delta = 1.25_f64 * ((1_f64 / std::f64::consts::E).sqrt()); diff --git a/src/protocol/ipa/mod.rs b/src/protocol/ipa/mod.rs index af64b5812..cc6a9868b 100644 --- a/src/protocol/ipa/mod.rs +++ b/src/protocol/ipa/mod.rs @@ -1,3 +1,15 @@ +use std::{iter::zip, marker::PhantomData, ops::Add}; + +use async_trait::async_trait; +use futures::{ + future::{try_join, try_join3}, + stream::iter as stream_iter, +}; +use generic_array::{ArrayLength, GenericArray}; +use ipa_macros::step; +use strum::AsRefStr; +use typenum::Unsigned; + use crate::{ error::Error, ff::{Field, GaloisField, Gf2, PrimeField, Serializable}, @@ -27,16 +39,6 @@ use crate::{ BitDecomposed, Linear as LinearSecretSharing, }, }; -use async_trait::async_trait; -use futures::{ - future::{try_join, try_join3}, - stream::iter as stream_iter, -}; -use generic_array::{ArrayLength, GenericArray}; -use ipa_macros::step; -use std::{iter::zip, marker::PhantomData, ops::Add}; -use strum::AsRefStr; -use typenum::Unsigned; #[step] pub(crate) enum Step { @@ -449,6 +451,8 @@ where #[cfg(all(test, any(unit_test, feature = "shuttle")))] pub mod tests { + use std::num::NonZeroU32; + use super::ipa; use crate::{ ff::{Field, Fp31, Fp32BitPrime}, @@ -464,7 +468,6 @@ pub mod tests { TestWorldConfig, }, }; - use std::num::NonZeroU32; #[test] fn semi_honest() { @@ -895,6 +898,14 @@ pub mod tests { #[cfg(all(test, unit_test))] mod serialization { + use generic_array::GenericArray; + use proptest::{ + proptest, + test_runner::{RngAlgorithm, TestRng}, + }; + use rand::distributions::{Distribution, Standard}; + use typenum::Unsigned; + use crate::{ ff::{Field, Fp31, PrimeField, Serializable}, ipa_test_input, @@ -905,13 +916,6 @@ pub mod tests { secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::input::GenericReportTestInput, }; - use generic_array::GenericArray; - use proptest::{ - proptest, - test_runner::{RngAlgorithm, TestRng}, - }; - use rand::distributions::{Distribution, Standard}; - use typenum::Unsigned; fn serde_internal( timestamp: u128, diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 530cb3196..0903ab939 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -9,10 +9,6 @@ pub mod prss; pub mod sort; pub mod step; -use crate::{ - error::Error, - ff::{Gf40Bit, Gf8Bit}, -}; use std::{ fmt::{Debug, Display, Formatter}, hash::Hash, @@ -21,6 +17,11 @@ use std::{ pub use basics::BasicProtocols; +use crate::{ + error::Error, + ff::{Gf40Bit, Gf8Bit}, +}; + pub type MatchKey = Gf40Bit; pub type BreakdownKey = Gf8Bit; diff --git a/src/protocol/modulus_conversion/convert_shares.rs b/src/protocol/modulus_conversion/convert_shares.rs index eacf5e513..e16fd2954 100644 --- a/src/protocol/modulus_conversion/convert_shares.rs +++ b/src/protocol/modulus_conversion/convert_shares.rs @@ -19,6 +19,17 @@ //! we know the secret-shared values are all either 0, or 1. As such, the XOR operation //! is equivalent to fn xor(a, b) { a + b - 2*a*b } +use std::{ + iter::zip, + marker::PhantomData, + ops::Range, + pin::Pin, + task::{Context as TaskContext, Poll}, +}; + +use futures::stream::{unfold, Stream, StreamExt}; +use pin_project::pin_project; + use crate::{ error::Error, exact::ExactSizeStream, @@ -37,15 +48,6 @@ use crate::{ }, seq_join::seq_join, }; -use futures::stream::{unfold, Stream, StreamExt}; -use pin_project::pin_project; -use std::{ - iter::zip, - marker::PhantomData, - ops::Range, - pin::Pin, - task::{Context as TaskContext, Poll}, -}; #[derive(PartialEq, Eq, Debug)] pub(crate) enum Step { @@ -318,6 +320,10 @@ where #[cfg(all(test, unit_test))] mod tests { + use std::future::ready; + + use futures::stream::{once, StreamExt, TryStreamExt}; + use crate::{ error::Error, ff::{Field, Fp31, Fp32BitPrime}, @@ -333,8 +339,6 @@ mod tests { }, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use futures::stream::{once, StreamExt, TryStreamExt}; - use std::future::ready; #[tokio::test] pub async fn one_bit() { diff --git a/src/protocol/prss/crypto.rs b/src/protocol/prss/crypto.rs index b99b8e159..e3eb415b7 100644 --- a/src/protocol/prss/crypto.rs +++ b/src/protocol/prss/crypto.rs @@ -1,9 +1,3 @@ -use crate::{ - ff::{Field, GaloisField}, - secret_sharing::replicated::{ - semi_honest::AdditiveShare as Replicated, ReplicatedSecretSharing, - }, -}; use aes::{ cipher::{generic_array::GenericArray, BlockEncrypt, KeyInit}, Aes256, @@ -13,6 +7,13 @@ use rand::{CryptoRng, RngCore}; use sha2::Sha256; use x25519_dalek::{EphemeralSecret, PublicKey}; +use crate::{ + ff::{Field, GaloisField}, + secret_sharing::replicated::{ + semi_honest::AdditiveShare as Replicated, ReplicatedSecretSharing, + }, +}; + pub trait SharedRandomness { /// Generate two random values, one that is known to the left helper /// and one that is known to the right helper. diff --git a/src/protocol/prss/mod.rs b/src/protocol/prss/mod.rs index f40102ebf..2102fc659 100644 --- a/src/protocol/prss/mod.rs +++ b/src/protocol/prss/mod.rs @@ -1,16 +1,16 @@ mod crypto; -use super::step::Gate; -use crate::{ - rand::{CryptoRng, RngCore}, - sync::{Arc, Mutex}, -}; use std::{collections::HashMap, fmt::Debug}; -use x25519_dalek::PublicKey; - #[cfg(debug_assertions)] use std::{collections::HashSet, fmt::Formatter}; pub use crypto::{Generator, GeneratorFactory, KeyExchange, SharedRandomness}; +use x25519_dalek::PublicKey; + +use super::step::Gate; +use crate::{ + rand::{CryptoRng, RngCore}, + sync::{Arc, Mutex}, +}; /// Keeps track of all indices used to generate shared randomness inside `IndexedSharedRandomness`. /// Any two indices provided to `IndexesSharedRandomness::generate_values` must be unique. @@ -256,6 +256,10 @@ impl EndpointSetup { #[cfg(all(test, unit_test))] pub mod test { + use std::mem::drop; + + use rand::prelude::SliceRandom; + use super::{Generator, KeyExchange, SequentialSharedRandomness}; use crate::{ ff::{Field, Fp31}, @@ -267,8 +271,6 @@ pub mod test { secret_sharing::SharedValue, test_fixture::make_participants, }; - use rand::prelude::SliceRandom; - use std::mem::drop; fn make() -> (Generator, Generator) { const CONTEXT: &[u8] = b"test generator"; diff --git a/src/protocol/sort/apply.rs b/src/protocol/sort/apply.rs index e46b7c8e0..571cd081f 100644 --- a/src/protocol/sort/apply.rs +++ b/src/protocol/sort/apply.rs @@ -48,9 +48,10 @@ pub fn apply_inv(permutation: &[u32], values: &mut [T]) { #[cfg(all(test, unit_test))] mod tests { + use rand::seq::SliceRandom; + use super::{apply, apply_inv}; use crate::rand::thread_rng; - use rand::seq::SliceRandom; #[test] fn apply_just_one_cycle() { diff --git a/src/protocol/sort/apply_sort/mod.rs b/src/protocol/sort/apply_sort/mod.rs index bd5601f37..10a783c09 100644 --- a/src/protocol/sort/apply_sort/mod.rs +++ b/src/protocol/sort/apply_sort/mod.rs @@ -43,6 +43,8 @@ where #[cfg(all(test, unit_test))] mod tests { + use futures::stream::iter as stream_iter; + use crate::{ accumulation_test_input, ff::{Fp32BitPrime, GaloisField}, @@ -59,7 +61,6 @@ mod tests { secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, SharedValue}, test_fixture::{input::GenericReportTestInput, Reconstruct, Runner, TestWorld}, }; - use futures::stream::iter as stream_iter; #[tokio::test] pub async fn semi_honest() { diff --git a/src/protocol/sort/apply_sort/shuffle.rs b/src/protocol/sort/apply_sort/shuffle.rs index 82fd4da2d..cd9aeb4bd 100644 --- a/src/protocol/sort/apply_sort/shuffle.rs +++ b/src/protocol/sort/apply_sort/shuffle.rs @@ -1,3 +1,5 @@ +use embed_doc_image::embed_doc_image; + use crate::{ error::Error, helpers::Direction, @@ -13,7 +15,6 @@ use crate::{ }, repeat64str, }; -use embed_doc_image::embed_doc_image; pub struct InnerVectorElementStep(usize); @@ -114,6 +115,8 @@ where mod tests { mod semi_honest { + use std::collections::HashSet; + use crate::{ accumulation_test_input, ff::{Fp31, Fp32BitPrime}, @@ -136,7 +139,6 @@ mod tests { TestWorld, }, }; - use std::collections::HashSet; #[tokio::test] async fn shuffle_attribution_input_row() { diff --git a/src/protocol/sort/bit_permutation.rs b/src/protocol/sort/bit_permutation.rs index 36f482207..3df58de89 100644 --- a/src/protocol/sort/bit_permutation.rs +++ b/src/protocol/sort/bit_permutation.rs @@ -1,11 +1,13 @@ +use std::iter::{repeat, zip}; + +use embed_doc_image::embed_doc_image; + use crate::{ error::Error, ff::Field, protocol::{context::Context, BasicProtocols, RecordId}, secret_sharing::Linear as LinearSecretSharing, }; -use embed_doc_image::embed_doc_image; -use std::iter::{repeat, zip}; #[embed_doc_image("bit_permutation", "images/sort/bit_permutations.png")] /// This is an implementation of `GenBitPerm` (Algorithm 3) described in: diff --git a/src/protocol/sort/compose.rs b/src/protocol/sort/compose.rs index 99f3ecc67..2a858af55 100644 --- a/src/protocol/sort/compose.rs +++ b/src/protocol/sort/compose.rs @@ -1,3 +1,5 @@ +use embed_doc_image::embed_doc_image; + use crate::{ error::Error, ff::Field, @@ -9,7 +11,6 @@ use crate::{ }, secret_sharing::SecretSharing, }; -use embed_doc_image::embed_doc_image; #[embed_doc_image("compose", "images/sort/compose.png")] /// This is an implementation of Compose (Algorithm 5) found in the paper: @@ -53,6 +54,8 @@ pub async fn compose + Reshare, C: Co #[cfg(all(test, unit_test))] mod tests { + use rand::seq::SliceRandom; + use crate::{ ff::{Field, Fp31}, protocol::{ @@ -65,7 +68,6 @@ mod tests { rand::thread_rng, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use rand::seq::SliceRandom; #[tokio::test] pub async fn semi_honest() { diff --git a/src/protocol/sort/generate_permutation.rs b/src/protocol/sort/generate_permutation.rs index 80358549a..11b02ea3b 100644 --- a/src/protocol/sort/generate_permutation.rs +++ b/src/protocol/sort/generate_permutation.rs @@ -1,3 +1,6 @@ +use async_trait::async_trait; +use futures::stream::Stream; + use crate::{ error::Error, ff::PrimeField, @@ -26,8 +29,6 @@ use crate::{ Linear as LinearSecretSharing, SecretSharing, }, }; -use async_trait::async_trait; -use futures::stream::Stream; #[derive(Debug)] /// This object contains the output of `shuffle_and_reveal_permutation` @@ -173,6 +174,11 @@ impl<'a, F: ExtendableField> DowngradeMalicious #[cfg(all(test, unit_test))] mod tests { + use std::iter::zip; + + use futures::stream::iter as stream_iter; + use rand::seq::SliceRandom; + use crate::{ ff::{Field, Fp31, GaloisField}, protocol::{ @@ -187,9 +193,6 @@ mod tests { secret_sharing::SharedValue, test_fixture::{generate_shares, join3, Reconstruct, Runner, TestWorld}, }; - use futures::stream::iter as stream_iter; - use rand::seq::SliceRandom; - use std::iter::zip; #[tokio::test] pub async fn semi_honest() { diff --git a/src/protocol/sort/generate_permutation_opt.rs b/src/protocol/sort/generate_permutation_opt.rs index c7d82b4b6..28a09e8b3 100644 --- a/src/protocol/sort/generate_permutation_opt.rs +++ b/src/protocol/sort/generate_permutation_opt.rs @@ -1,3 +1,8 @@ +use std::cmp::min; + +use embed_doc_image::embed_doc_image; +use futures::stream::{iter as stream_iter, Stream, StreamExt, TryStreamExt}; + use crate::{ error::Error, ff::PrimeField, @@ -28,9 +33,6 @@ use crate::{ Linear as LinearSecretSharing, }, }; -use embed_doc_image::embed_doc_image; -use futures::stream::{iter as stream_iter, Stream, StreamExt, TryStreamExt}; -use std::cmp::min; #[embed_doc_image("semi_honest_sort", "images/sort/semi-honest-sort.png")] #[embed_doc_image("malicious_sort", "images/sort/malicious-sort.png")] @@ -194,6 +196,10 @@ where #[cfg(all(test, unit_test))] mod tests { + use std::iter::zip; + + use futures::stream::iter as stream_iter; + use crate::{ ff::{Field, Fp31, Fp32BitPrime, GaloisField}, protocol::{ @@ -205,8 +211,6 @@ mod tests { secret_sharing::SharedValue, test_fixture::{join3, Reconstruct, Runner, TestWorld}, }; - use futures::stream::iter as stream_iter; - use std::iter::zip; #[tokio::test] pub async fn semi_honest() { diff --git a/src/protocol/sort/mod.rs b/src/protocol/sort/mod.rs index 1a3ca88e1..48fc98e3c 100644 --- a/src/protocol/sort/mod.rs +++ b/src/protocol/sort/mod.rs @@ -9,6 +9,11 @@ mod multi_bit_permutation; mod secureapplyinv; mod shuffle; +use std::fmt::Debug; + +use ipa_macros::step; +use strum::AsRefStr; + use crate::{ error::Error, ff::Field, @@ -20,9 +25,6 @@ use crate::{ repeat64str, secret_sharing::{BitDecomposed, Linear as LinearSecretSharing, SecretSharing}, }; -use ipa_macros::step; -use std::fmt::Debug; -use strum::AsRefStr; #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub enum SortStep { diff --git a/src/protocol/sort/multi_bit_permutation.rs b/src/protocol/sort/multi_bit_permutation.rs index 30edede8b..472ebaa1d 100644 --- a/src/protocol/sort/multi_bit_permutation.rs +++ b/src/protocol/sort/multi_bit_permutation.rs @@ -1,3 +1,5 @@ +use std::iter::repeat; + use crate::{ error::Error, ff::PrimeField, @@ -10,7 +12,6 @@ use crate::{ SecretSharing, }, }; -use std::iter::repeat; /// This is an implementation of `GenMultiBitSort` (Algorithm 11) described in: /// "An Efficient Secure Three-Party Sorting Protocol with an Honest Majority" diff --git a/src/protocol/sort/secureapplyinv.rs b/src/protocol/sort/secureapplyinv.rs index ba9189b30..8cfb8aca9 100644 --- a/src/protocol/sort/secureapplyinv.rs +++ b/src/protocol/sort/secureapplyinv.rs @@ -31,6 +31,10 @@ pub async fn secureapplyinv_multi + Send + S #[cfg(all(test, unit_test))] mod tests { mod semi_honest { + use std::iter::repeat_with; + + use rand::seq::SliceRandom; + use crate::{ ff::{Field, Fp31}, protocol::{ @@ -44,8 +48,6 @@ mod tests { secret_sharing::BitDecomposed, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use rand::seq::SliceRandom; - use std::iter::repeat_with; #[tokio::test] pub async fn multi() { diff --git a/src/protocol/sort/shuffle.rs b/src/protocol/sort/shuffle.rs index 9b01090c2..feff12ee1 100644 --- a/src/protocol/sort/shuffle.rs +++ b/src/protocol/sort/shuffle.rs @@ -1,6 +1,10 @@ use embed_doc_image::embed_doc_image; use rand::{seq::SliceRandom, Rng}; +use super::{ + apply::{apply, apply_inv}, + ShuffleStep::{self, Shuffle1, Shuffle2, Shuffle3}, +}; use crate::{ error::Error, ff::Field, @@ -9,11 +13,6 @@ use crate::{ secret_sharing::SecretSharing, }; -use super::{ - apply::{apply, apply_inv}, - ShuffleStep::{self, Shuffle1, Shuffle2, Shuffle3}, -}; - #[derive(Debug)] /// This is SHUFFLE(Algorithm 1) described in . /// This protocol shuffles the given inputs across 3 helpers making them indistinguishable to the helpers @@ -199,6 +198,8 @@ mod tests { } mod semi_honest { + use std::collections::HashSet; + use crate::{ ff::{Field, Fp31}, protocol::{ @@ -209,7 +210,6 @@ mod tests { }, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use std::collections::HashSet; #[tokio::test] async fn semi_honest() { @@ -293,6 +293,8 @@ mod tests { } mod malicious { + use std::collections::HashSet; + use crate::{ ff::{Field, Fp31}, protocol::{ @@ -303,7 +305,6 @@ mod tests { }, test_fixture::{Reconstruct, Runner, TestWorld}, }; - use std::collections::HashSet; #[tokio::test] async fn malicious() { diff --git a/src/protocol/step/compact.rs b/src/protocol/step/compact.rs index 5314a11bb..3bca8b8ef 100644 --- a/src/protocol/step/compact.rs +++ b/src/protocol/step/compact.rs @@ -1,7 +1,9 @@ -use super::{Step, StepNarrow}; -use ipa_macros::Gate; use std::fmt::{Debug, Display, Formatter}; +use ipa_macros::Gate; + +use super::{Step, StepNarrow}; + #[derive(Gate, Clone, Hash, PartialEq, Eq, Default)] #[cfg_attr( feature = "enable-serde", diff --git a/src/protocol/step/descriptive.rs b/src/protocol/step/descriptive.rs index 5d42ddcd8..4f41a4881 100644 --- a/src/protocol/step/descriptive.rs +++ b/src/protocol/step/descriptive.rs @@ -1,7 +1,8 @@ +use std::fmt::{Debug, Display, Formatter}; + use super::{Step, StepNarrow}; #[cfg(feature = "step-trace")] use crate::telemetry::{labels::STEP, metrics::STEP_NARROWED}; -use std::fmt::{Debug, Display, Formatter}; /// A descriptive representation of a unique step in protocol execution. /// diff --git a/src/query/completion.rs b/src/query/completion.rs index 6c9aa1393..caccbd1eb 100644 --- a/src/query/completion.rs +++ b/src/query/completion.rs @@ -1,14 +1,16 @@ -use crate::query::{ - runner::QueryResult, - state::{RemoveQuery, RunningQuery}, -}; -use futures::FutureExt; use std::{ future::Future, pin::Pin, task::{Context, Poll}, }; +use futures::FutureExt; + +use crate::query::{ + runner::QueryResult, + state::{RemoveQuery, RunningQuery}, +}; + /// Query completion polls the tokio task to get the results and cleans up the query state after. pub struct Handle<'a> { _query_state_guard: RemoveQuery<'a>, diff --git a/src/query/executor.rs b/src/query/executor.rs index 9d38fe810..d10f62989 100644 --- a/src/query/executor.rs +++ b/src/query/executor.rs @@ -1,3 +1,21 @@ +use std::{ + fmt::Debug, + future::{ready, Future}, + pin::Pin, + sync::Arc, +}; + +use ::tokio::sync::oneshot; +use futures::FutureExt; +use generic_array::GenericArray; +use rand::rngs::StdRng; +use rand_core::SeedableRng; +#[cfg(all(feature = "shuttle", test))] +use shuttle::future as tokio; +use typenum::Unsigned; + +#[cfg(any(test, feature = "cli", feature = "test-fixture"))] +use crate::query::runner::execute_test_multiply; use crate::{ ff::{FieldType, Fp32BitPrime, Serializable}, helpers::{ @@ -11,26 +29,11 @@ use crate::{ prss::Endpoint as PrssEndpoint, step::{Gate, StepNarrow}, }, - query::{runner::IpaQuery, state::RunningQuery}, -}; - -#[cfg(any(test, feature = "cli", feature = "test-fixture"))] -use crate::query::runner::execute_test_multiply; -use crate::query::runner::QueryResult; -use ::tokio::sync::oneshot; -use futures::FutureExt; -use generic_array::GenericArray; -use rand::rngs::StdRng; -use rand_core::SeedableRng; -#[cfg(all(feature = "shuttle", test))] -use shuttle::future as tokio; -use std::{ - fmt::Debug, - future::{ready, Future}, - pin::Pin, - sync::Arc, + query::{ + runner::{IpaQuery, QueryResult}, + state::RunningQuery, + }, }; -use typenum::Unsigned; pub trait Result: Send + Debug { fn into_bytes(self: Box) -> Vec; diff --git a/src/query/mod.rs b/src/query/mod.rs index 04e9a92f0..7577c8b13 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -4,13 +4,10 @@ mod processor; mod runner; mod state; +use completion::Handle as CompletionHandle; pub use executor::Result as ProtocolResult; - pub use processor::{ NewQueryError, PrepareQueryError, Processor as QueryProcessor, QueryCompletionError, QueryInputError, QueryStatusError, }; - pub use state::QueryStatus; - -use completion::Handle as CompletionHandle; diff --git a/src/query/processor.rs b/src/query/processor.rs index 3fafb06dd..017f1ed58 100644 --- a/src/query/processor.rs +++ b/src/query/processor.rs @@ -1,3 +1,11 @@ +use std::{ + collections::hash_map::Entry, + fmt::{Debug, Formatter}, + sync::Arc, +}; + +use futures::{future::try_join, stream}; + use crate::{ error::Error as ProtocolError, helpers::{ @@ -12,12 +20,6 @@ use crate::{ CompletionHandle, ProtocolResult, }, }; -use futures::{future::try_join, stream}; -use std::{ - collections::hash_map::Entry, - fmt::{Debug, Formatter}, - sync::Arc, -}; /// `Processor` accepts and tracks requests to initiate new queries on this helper party /// network. It makes sure queries are coordinated and each party starts processing it when @@ -310,6 +312,12 @@ impl Processor { #[cfg(all(test, unit_test))] mod tests { + use std::{array, future::Future, sync::Arc}; + + use futures::pin_mut; + use futures_util::future::poll_immediate; + use tokio::sync::Barrier; + use super::*; use crate::{ ff::FieldType, @@ -318,10 +326,6 @@ mod tests { HelperIdentity, InMemoryNetwork, PrepareQueryCallback, TransportCallbacks, }, }; - use futures::pin_mut; - use futures_util::future::poll_immediate; - use std::{array, future::Future, sync::Arc}; - use tokio::sync::Barrier; fn prepare_query_callback(cb: F) -> Box> where @@ -520,6 +524,10 @@ mod tests { } mod e2e { + use std::time::Duration; + + use tokio::time::sleep; + use super::*; use crate::{ error::BoxError, @@ -530,8 +538,6 @@ mod tests { secret_sharing::replicated::semi_honest, test_fixture::{input::GenericReportTestInput, Reconstruct, TestApp}, }; - use std::time::Duration; - use tokio::time::sleep; #[tokio::test] async fn complete_query_test_multiply() -> Result<(), BoxError> { diff --git a/src/query/runner/ipa.rs b/src/query/runner/ipa.rs index 9788923f2..94805bb7c 100644 --- a/src/query/runner/ipa.rs +++ b/src/query/runner/ipa.rs @@ -1,3 +1,10 @@ +use std::marker::PhantomData; + +use futures::{ + stream::{iter, repeat}, + Stream, StreamExt, TryStreamExt, +}; + use crate::{ error::Error, ff::{Gf2, PrimeField, Serializable}, @@ -21,11 +28,6 @@ use crate::{ }, sync::Arc, }; -use futures::{ - stream::{iter, repeat}, - Stream, StreamExt, TryStreamExt, -}; -use std::marker::PhantomData; pub struct IpaQuery { config: IpaQueryConfig, @@ -158,6 +160,11 @@ pub fn assert_stream_send<'a, T>( mod tests { use std::iter::zip; + use generic_array::GenericArray; + use rand::rngs::StdRng; + use rand_core::SeedableRng; + use typenum::Unsigned; + use super::*; use crate::{ ff::Fp31, @@ -166,10 +173,6 @@ mod tests { secret_sharing::IntoShares, test_fixture::{input::GenericReportTestInput, join3v, Reconstruct, TestWorld}, }; - use generic_array::GenericArray; - use rand::rngs::StdRng; - use rand_core::SeedableRng; - use typenum::Unsigned; #[tokio::test] async fn ipa() { diff --git a/src/query/runner/mod.rs b/src/query/runner/mod.rs index a2be242b9..ad712fb1d 100644 --- a/src/query/runner/mod.rs +++ b/src/query/runner/mod.rs @@ -2,10 +2,10 @@ mod ipa; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] mod test_multiply; -use crate::{error::Error, query::ProtocolResult}; - -pub(super) use self::ipa::IpaQuery; #[cfg(any(test, feature = "cli", feature = "test-fixture"))] pub(super) use test_multiply::execute_test_multiply; +pub(super) use self::ipa::IpaQuery; +use crate::{error::Error, query::ProtocolResult}; + pub(super) type QueryResult = Result, Error>; diff --git a/src/query/runner/test_multiply.rs b/src/query/runner/test_multiply.rs index 7267156b3..02f6ece81 100644 --- a/src/query/runner/test_multiply.rs +++ b/src/query/runner/test_multiply.rs @@ -1,3 +1,5 @@ +use futures::StreamExt; + use crate::{ error::Error, ff::{PrimeField, Serializable}, @@ -11,7 +13,6 @@ use crate::{ query::runner::QueryResult, secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, }; -use futures::StreamExt; pub async fn execute_test_multiply<'a, F>( prss: &'a PrssEndpoint, @@ -67,14 +68,15 @@ where #[cfg(all(test, unit_test))] mod tests { + use generic_array::GenericArray; + use typenum::Unsigned; + use super::*; use crate::{ ff::{Field, Fp31}, secret_sharing::IntoShares, test_fixture::{join3v, Reconstruct, TestWorld}, }; - use generic_array::GenericArray; - use typenum::Unsigned; #[tokio::test] async fn multiply() { diff --git a/src/query/state.rs b/src/query/state.rs index a97f59976..2cb9fdeb0 100644 --- a/src/query/state.rs +++ b/src/query/state.rs @@ -1,3 +1,14 @@ +use std::{ + collections::{hash_map::Entry, HashMap}, + fmt::{Debug, Formatter}, + future::Future, + task::Poll, +}; + +use ::tokio::sync::oneshot::{error::TryRecvError, Receiver}; +use futures::{ready, FutureExt}; +use serde::{Deserialize, Serialize}; + use crate::{ helpers::{query::QueryConfig, RoleAssignment}, protocol::QueryId, @@ -5,15 +16,6 @@ use crate::{ sync::Mutex, task::JoinHandle, }; -use ::tokio::sync::oneshot::{error::TryRecvError, Receiver}; -use futures::{ready, FutureExt}; -use serde::{Deserialize, Serialize}; -use std::{ - collections::{hash_map::Entry, HashMap}, - fmt::{Debug, Formatter}, - future::Future, - task::Poll, -}; /// The status of query processing #[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] diff --git a/src/report.rs b/src/report.rs index 424a117bd..a3066d98f 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,3 +1,15 @@ +use std::{ + fmt::{Display, Formatter}, + marker::PhantomData, + ops::Deref, +}; + +use bytes::{BufMut, Bytes}; +use generic_array::GenericArray; +use hpke::Serializable as _; +use rand_core::{CryptoRng, RngCore}; +use typenum::Unsigned; + use crate::{ ff::{GaloisField, Gf40Bit, Gf8Bit, PrimeField, Serializable}, hpke::{ @@ -6,16 +18,6 @@ use crate::{ }, secret_sharing::replicated::semi_honest::AdditiveShare as Replicated, }; -use bytes::{BufMut, Bytes}; -use generic_array::GenericArray; -use hpke::Serializable as _; -use rand_core::{CryptoRng, RngCore}; -use std::{ - fmt::{Display, Formatter}, - marker::PhantomData, - ops::Deref, -}; -use typenum::Unsigned; // TODO(679): This needs to come from configuration. static HELPER_ORIGIN: &str = "github.com/private-attribution"; @@ -386,13 +388,12 @@ where #[cfg(all(test, unit_test))] mod test { - use crate::ff::{Fp32BitPrime, Gf40Bit, Gf8Bit}; - - use super::*; - use rand::{distributions::Alphanumeric, rngs::StdRng, Rng}; use rand_core::SeedableRng; + use super::*; + use crate::ff::{Fp32BitPrime, Gf40Bit, Gf8Bit}; + #[test] fn enc_dec_roundtrip() { let mut rng = StdRng::from_seed([1_u8; 32]); diff --git a/src/secret_sharing/decomposed.rs b/src/secret_sharing/decomposed.rs index ee507b95b..0b8a441bc 100644 --- a/src/secret_sharing/decomposed.rs +++ b/src/secret_sharing/decomposed.rs @@ -1,6 +1,7 @@ -use crate::error::Error; use std::{fmt::Debug, ops::Deref}; +use crate::error::Error; + #[derive(Clone, Debug, PartialEq)] pub struct BitDecomposed { bits: Vec, diff --git a/src/secret_sharing/mod.rs b/src/secret_sharing/mod.rs index f1e382a4b..c4bf6551a 100644 --- a/src/secret_sharing/mod.rs +++ b/src/secret_sharing/mod.rs @@ -4,12 +4,11 @@ mod decomposed; mod into_shares; mod scheme; -pub use decomposed::BitDecomposed; -pub use into_shares::IntoShares; -pub use scheme::{Bitwise, Linear, SecretSharing}; +use std::fmt::Debug; -use crate::ff::{ArithmeticOps, Serializable}; +pub use decomposed::BitDecomposed; use generic_array::ArrayLength; +pub use into_shares::IntoShares; #[cfg(any(test, feature = "test-fixture", feature = "cli"))] use rand::{ distributions::{Distribution, Standard}, @@ -17,7 +16,9 @@ use rand::{ }; #[cfg(any(test, feature = "test-fixture", feature = "cli"))] use replicated::{semi_honest::AdditiveShare, ReplicatedSecretSharing}; -use std::fmt::Debug; +pub use scheme::{Bitwise, Linear, SecretSharing}; + +use crate::ff::{ArithmeticOps, Serializable}; // Trait for primitive integer types used to represent the underlying type for shared values pub trait Block: Sized + Copy + Debug { diff --git a/src/secret_sharing/replicated/malicious/additive_share.rs b/src/secret_sharing/replicated/malicious/additive_share.rs index 5a111d27e..043710dd3 100644 --- a/src/secret_sharing/replicated/malicious/additive_share.rs +++ b/src/secret_sharing/replicated/malicious/additive_share.rs @@ -1,24 +1,26 @@ -use crate::{ - ff::{Field, Gf2, Gf32Bit, PrimeField, Serializable}, - secret_sharing::{ - replicated::semi_honest::AdditiveShare as SemiHonestAdditiveShare, BitDecomposed, - Linear as LinearSecretSharing, SecretSharing, SharedValue, - }, - seq_join::seq_join, +use std::{ + fmt::{Debug, Formatter}, + num::NonZeroUsize, + ops::{Add, AddAssign, Mul, Neg, Sub, SubAssign}, }; + use async_trait::async_trait; use futures::{ future::{join, join_all}, stream::{iter as stream_iter, StreamExt}, }; use generic_array::{ArrayLength, GenericArray}; -use std::{ - fmt::{Debug, Formatter}, - num::NonZeroUsize, - ops::{Add, AddAssign, Mul, Neg, Sub, SubAssign}, -}; use typenum::Unsigned; +use crate::{ + ff::{Field, Gf2, Gf32Bit, PrimeField, Serializable}, + secret_sharing::{ + replicated::semi_honest::AdditiveShare as SemiHonestAdditiveShare, BitDecomposed, + Linear as LinearSecretSharing, SecretSharing, SharedValue, + }, + seq_join::seq_join, +}; + /// /// This code is an optimization to our malicious compiler that is drawn from: /// "Field Extension in Secret-Shared Form and Its Applications to Efficient Secure Computation" diff --git a/src/secret_sharing/replicated/semi_honest/additive_share.rs b/src/secret_sharing/replicated/semi_honest/additive_share.rs index c65384463..b310971b8 100644 --- a/src/secret_sharing/replicated/semi_honest/additive_share.rs +++ b/src/secret_sharing/replicated/semi_honest/additive_share.rs @@ -1,3 +1,11 @@ +use std::{ + fmt::{Debug, Formatter}, + ops::{Add, AddAssign, Mul, Neg, Sub, SubAssign}, +}; + +use generic_array::{ArrayLength, GenericArray}; +use typenum::Unsigned; + use crate::{ ff::Serializable, secret_sharing::{ @@ -5,12 +13,6 @@ use crate::{ SharedValue, }, }; -use generic_array::{ArrayLength, GenericArray}; -use std::{ - fmt::{Debug, Formatter}, - ops::{Add, AddAssign, Mul, Neg, Sub, SubAssign}, -}; -use typenum::Unsigned; #[derive(Clone, PartialEq, Eq)] pub struct AdditiveShare(V, V); diff --git a/src/secret_sharing/scheme.rs b/src/secret_sharing/scheme.rs index 80cc2f874..262891252 100644 --- a/src/secret_sharing/scheme.rs +++ b/src/secret_sharing/scheme.rs @@ -1,6 +1,7 @@ +use std::fmt::Debug; + use super::SharedValue; use crate::ff::{ArithmeticRefOps, GaloisField}; -use std::fmt::Debug; /// Secret sharing scheme i.e. Replicated secret sharing pub trait SecretSharing: Clone + Debug + Sized + Send + Sync { diff --git a/src/seq_join.rs b/src/seq_join.rs index a87f704ba..fbb582ad0 100644 --- a/src/seq_join.rs +++ b/src/seq_join.rs @@ -1,9 +1,3 @@ -use crate::exact::ExactSizeStream; -use futures::{ - stream::{iter, Iter as StreamIter, TryCollect}, - Future, Stream, StreamExt, TryStreamExt, -}; -use pin_project::pin_project; use std::{ collections::VecDeque, future::IntoFuture, @@ -12,6 +6,14 @@ use std::{ task::{Context, Poll}, }; +use futures::{ + stream::{iter, Iter as StreamIter, TryCollect}, + Future, Stream, StreamExt, TryStreamExt, +}; +use pin_project::pin_project; + +use crate::exact::ExactSizeStream; + /// This helper function might be necessary to convince the compiler that /// the return value from [`seq_try_join_all`] implements `Send`. /// Use this if you get higher-ranked lifetime errors that mention `std::marker::Send`. @@ -212,12 +214,6 @@ where #[cfg(all(test, unit_test))] mod test { - use crate::seq_join::{seq_join, seq_try_join_all}; - use futures::{ - future::{lazy, BoxFuture}, - stream::{iter, poll_fn, poll_immediate, repeat_with}, - Future, StreamExt, - }; use std::{ convert::Infallible, iter::once, @@ -227,6 +223,14 @@ mod test { task::{Context, Poll, Waker}, }; + use futures::{ + future::{lazy, BoxFuture}, + stream::{iter, poll_fn, poll_immediate, repeat_with}, + Future, StreamExt, + }; + + use crate::seq_join::{seq_join, seq_try_join_all}; + async fn immediate(count: u32) { let capacity = NonZeroUsize::new(3).unwrap(); let values = seq_join(capacity, iter((0..count).map(|i| async move { i }))) diff --git a/src/telemetry/stats.rs b/src/telemetry/stats.rs index 479bdbed5..638248f44 100644 --- a/src/telemetry/stats.rs +++ b/src/telemetry/stats.rs @@ -4,13 +4,13 @@ use std::{ }; use metrics::{KeyName, Label, SharedString}; - -use crate::{helpers::Role, protocol::step::Gate, telemetry::labels}; use metrics_util::{ debugging::{DebugValue, Snapshot}, CompositeKey, MetricKind, }; +use crate::{helpers::Role, protocol::step::Gate, telemetry::labels}; + /// Simple counter stats #[derive(Debug, Default)] pub struct CounterDetails { diff --git a/src/telemetry/step_stats.rs b/src/telemetry/step_stats.rs index 5fc4090e7..2f0e03ac0 100644 --- a/src/telemetry/step_stats.rs +++ b/src/telemetry/step_stats.rs @@ -1,6 +1,12 @@ //! //! Export metrics collected during protocol run in CSV format. Metrics are partitioned by step. +use std::{ + collections::{BTreeMap, HashMap}, + io, + io::{Error, Write}, +}; + use crate::telemetry::{ labels, metrics::{ @@ -8,11 +14,6 @@ use crate::telemetry::{ }, stats::Metrics, }; -use std::{ - collections::{BTreeMap, HashMap}, - io, - io::{Error, Write}, -}; pub trait CsvExporter { /// Writes the serialized version of this instance into the provided writer in CSV format. diff --git a/src/test_fixture/app.rs b/src/test_fixture/app.rs index 8caf679c2..d38e95da5 100644 --- a/src/test_fixture/app.rs +++ b/src/test_fixture/app.rs @@ -1,18 +1,21 @@ +use std::iter::zip; + +use generic_array::GenericArray; +use typenum::Unsigned; + use crate::{ app::Error, ff::Serializable, - helpers::query::{QueryConfig, QueryInput}, + helpers::{ + query::{QueryConfig, QueryInput}, + InMemoryNetwork, InMemoryTransport, + }, protocol::QueryId, query::QueryStatus, secret_sharing::IntoShares, test_fixture::try_join3_array, AppSetup, HelperApp, }; -use std::iter::zip; - -use crate::helpers::{InMemoryNetwork, InMemoryTransport}; -use generic_array::GenericArray; -use typenum::Unsigned; pub trait IntoBuf { fn into_buf(self) -> Vec; diff --git a/src/test_fixture/circuit.rs b/src/test_fixture/circuit.rs index 18b9882d6..28fd5ca82 100644 --- a/src/test_fixture/circuit.rs +++ b/src/test_fixture/circuit.rs @@ -1,3 +1,6 @@ +use futures_util::future::join_all; + +use super::join3v; use crate::{ ff::Field, helpers::TotalRecords, @@ -10,9 +13,6 @@ use crate::{ secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, IntoShares}, test_fixture::{narrow_contexts, Reconstruct, TestWorld}, }; -use futures_util::future::join_all; - -use super::join3v; /// Creates an arithmetic circuit with the given width and depth. /// diff --git a/src/test_fixture/event_gen.rs b/src/test_fixture/event_gen.rs index 3dc6d52eb..5ffab9020 100644 --- a/src/test_fixture/event_gen.rs +++ b/src/test_fixture/event_gen.rs @@ -97,12 +97,13 @@ impl Config { } } -use crate::{rand::Rng, test_fixture::ipa::TestRawDataRecord}; use std::{ collections::HashSet, num::{NonZeroU32, NonZeroU64}, }; +use crate::{rand::Rng, test_fixture::ipa::TestRawDataRecord}; + struct UserStats { user_id: UserId, generated: u32, @@ -265,9 +266,10 @@ impl Iterator for EventGenerator { #[cfg(all(test, unit_test))] mod tests { - use super::*; use rand::thread_rng; + use super::*; + #[test] fn iter() { let gen = EventGenerator::with_default_config(thread_rng()); @@ -294,14 +296,16 @@ mod tests { } mod proptests { - use super::*; + use std::collections::HashMap; + use proptest::{ prelude::{Just, Strategy}, prop_oneof, proptest, }; use rand::rngs::StdRng; use rand_core::SeedableRng; - use std::collections::HashMap; + + use super::*; fn report_filter_strategy() -> impl Strategy { prop_oneof![ diff --git a/src/test_fixture/input/sharing.rs b/src/test_fixture/input/sharing.rs index c1ffe6ba5..68e0bb0e6 100644 --- a/src/test_fixture/input/sharing.rs +++ b/src/test_fixture/input/sharing.rs @@ -1,3 +1,7 @@ +use std::iter::zip; + +use rand::{distributions::Standard, prelude::Distribution}; + use crate::{ ff::{Field, GaloisField, PrimeField, Serializable}, protocol::{ @@ -16,8 +20,6 @@ use crate::{ Reconstruct, }, }; -use rand::{distributions::Standard, prelude::Distribution}; -use std::iter::zip; impl IntoShares> for GenericReportTestInput where diff --git a/src/test_fixture/logging.rs b/src/test_fixture/logging.rs index 23af5a8cf..b8553eef0 100644 --- a/src/test_fixture/logging.rs +++ b/src/test_fixture/logging.rs @@ -1,5 +1,6 @@ -use metrics_tracing_context::MetricsLayer; use std::{str::FromStr, sync::Once}; + +use metrics_tracing_context::MetricsLayer; use tracing::Level; use tracing_subscriber::{ filter::Directive, fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, diff --git a/src/test_fixture/metrics.rs b/src/test_fixture/metrics.rs index 8f63f2be7..18fca9db3 100644 --- a/src/test_fixture/metrics.rs +++ b/src/test_fixture/metrics.rs @@ -1,8 +1,3 @@ -use crate::{ - rand::{thread_rng, Rng}, - telemetry::{metrics::register, stats::Metrics}, - test_fixture::logging, -}; use metrics::KeyName; use metrics_tracing_context::TracingContextLayer; use metrics_util::{ @@ -13,6 +8,12 @@ use once_cell::sync::OnceCell; use rand::distributions::Alphanumeric; use tracing::{Level, Span}; +use crate::{ + rand::{thread_rng, Rng}, + telemetry::{metrics::register, stats::Metrics}, + test_fixture::logging, +}; + // TODO: move to OnceCell from std once it is stabilized static ONCE: OnceCell = OnceCell::new(); diff --git a/src/test_fixture/mod.rs b/src/test_fixture/mod.rs index 6342183bb..acfb8f853 100644 --- a/src/test_fixture/mod.rs +++ b/src/test_fixture/mod.rs @@ -15,15 +15,8 @@ pub mod ipa; pub mod logging; pub mod metrics; -use crate::{ - ff::Field, - protocol::{ - context::Context, - prss::Endpoint as PrssEndpoint, - step::{Gate, Step, StepNarrow}, - }, - secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, IntoShares}, -}; +use std::fmt::Debug; + #[cfg(feature = "in-memory-infra")] pub use app::TestApp; pub use event_gen::{Config as EventGeneratorConfig, EventGenerator}; @@ -31,10 +24,19 @@ use futures::TryFuture; use rand::{distributions::Standard, prelude::Distribution, rngs::mock::StepRng}; use rand_core::{CryptoRng, RngCore}; pub use sharing::{get_bits, into_bits, Reconstruct}; -use std::fmt::Debug; #[cfg(feature = "in-memory-infra")] pub use world::{Runner, TestWorld, TestWorldConfig}; +use crate::{ + ff::Field, + protocol::{ + context::Context, + prss::Endpoint as PrssEndpoint, + step::{Gate, Step, StepNarrow}, + }, + secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, IntoShares}, +}; + /// Narrows a set of contexts all at once. /// Use by assigning like so: `let [c0, c1, c2] = narrow_contexts(&contexts, "test")` /// diff --git a/src/test_fixture/sharing.rs b/src/test_fixture/sharing.rs index 40adf940d..a9ac85cf3 100644 --- a/src/test_fixture/sharing.rs +++ b/src/test_fixture/sharing.rs @@ -1,3 +1,5 @@ +use std::{borrow::Borrow, iter::zip, ops::Deref}; + use crate::{ ff::{Field, PrimeField}, protocol::boolean::RandomBitsShare, @@ -10,7 +12,6 @@ use crate::{ BitDecomposed, SecretSharing, }, }; -use std::{borrow::Borrow, iter::zip, ops::Deref}; /// Deconstructs a field value into N values, one for each bit. pub fn into_bits(v: F) -> BitDecomposed { diff --git a/src/test_fixture/world.rs b/src/test_fixture/world.rs index 71f4ef2e8..e3502056d 100644 --- a/src/test_fixture/world.rs +++ b/src/test_fixture/world.rs @@ -1,3 +1,11 @@ +use std::{fmt::Debug, io::stdout, iter::zip}; + +use async_trait::async_trait; +use futures::{future::join_all, Future}; +use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng}; +use rand_core::{RngCore, SeedableRng}; +use tracing::{Instrument, Level, Span}; + use crate::{ helpers::{Gateway, GatewayConfig, InMemoryNetwork, Role, RoleAssignment}, protocol::{ @@ -22,12 +30,6 @@ use crate::{ logging, make_participants, metrics::MetricsHandle, sharing::ValidateMalicious, Reconstruct, }, }; -use async_trait::async_trait; -use futures::{future::join_all, Future}; -use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng}; -use rand_core::{RngCore, SeedableRng}; -use std::{fmt::Debug, io::stdout, iter::zip}; -use tracing::{Instrument, Level, Span}; /// Test environment for protocols to run tests that require communication between helpers. /// For now the messages sent through it never leave the test infra memory perimeter, so diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 4afdd2a36..7990bccb6 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,9 +1,3 @@ -use command_fds::CommandFdExt; -use ipa::{ - cli::IpaQueryResult, helpers::query::IpaQueryConfig, test_fixture::ipa::IpaSecurityModel, -}; -use rand::thread_rng; -use rand_core::RngCore; use std::{ array, error::Error, @@ -15,6 +9,13 @@ use std::{ path::Path, process::{Child, Command, ExitStatus, Stdio}, }; + +use command_fds::CommandFdExt; +use ipa::{ + cli::IpaQueryResult, helpers::query::IpaQueryConfig, test_fixture::ipa::IpaSecurityModel, +}; +use rand::thread_rng; +use rand_core::RngCore; use tempdir::TempDir; #[cfg(all(test, feature = "cli"))] diff --git a/tests/common/tempdir.rs b/tests/common/tempdir.rs index 521c0b05b..f1fd69465 100644 --- a/tests/common/tempdir.rs +++ b/tests/common/tempdir.rs @@ -1,4 +1,5 @@ use std::{mem, path::Path, thread}; + use tempfile::tempdir; /// Wrapper around [`TempDir`] that prevents the temp directory from being deleted diff --git a/tests/compact_gate.rs b/tests/compact_gate.rs index 034b5f05c..de99f7655 100644 --- a/tests/compact_gate.rs +++ b/tests/compact_gate.rs @@ -3,9 +3,10 @@ #[allow(dead_code)] mod common; +use std::num::NonZeroU32; + use common::test_ipa_with_config; use ipa::{helpers::query::IpaQueryConfig, test_fixture::ipa::IpaSecurityModel}; -use std::num::NonZeroU32; fn test_compact_gate>( mode: IpaSecurityModel, diff --git a/tests/helper_networks.rs b/tests/helper_networks.rs index 546173f64..d00340f75 100644 --- a/tests/helper_networks.rs +++ b/tests/helper_networks.rs @@ -1,11 +1,12 @@ mod common; +use std::{array, net::TcpListener, path::Path, process::Command}; + use common::{ spawn_helpers, tempdir::TempDir, test_ipa, test_multiply, test_network, CommandExt, UnwrapStatusExt, HELPER_BIN, }; use ipa::{cli::CliPaths, helpers::HelperIdentity, test_fixture::ipa::IpaSecurityModel}; -use std::{array, net::TcpListener, path::Path, process::Command}; #[test] #[cfg(all(test, web_test))]