Alg Opt with GLV

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "r1cs-core",
     "r1cs-std",
     "algebra-core/algebra-core-derive",
+    "scripts/glv_lattice_basis"
 ]
 
 [profile.release]

algebra-benches/Cargo.toml

@@ -32,6 +32,7 @@ paste = "0.1"
 
 [features]
 asm = [ "algebra/asm"]
+prefetch = [ "algebra/prefetch"]
 n_fold = []
 mnt4_298 = [ "algebra/mnt4_298"]
 mnt6_298 = [ "algebra/mnt6_298"]
@@ -42,6 +43,9 @@ bls12_381 = [ "algebra/bls12_381"]
 bls12_377 = [ "algebra/bls12_377"]
 cp6_782 = [ "algebra/cp6_782" ]
 bw6_761 = [ "algebra/bw6_761" ]
+timing = [ "algebra/timing"]
+timing_detailed = [ "algebra/timing_detailed" ]
+timing_thread_id = [ "algebra/timing_thread_id" ]
 
 [build-dependencies]
 rustc_version = "0.2"

algebra-benches/src/curves/bw6_761.rs

@@ -9,9 +9,11 @@ use algebra::{
         fq::Fq, fq3::Fq3, fr::Fr, Fq6, G1Affine, G1Projective as G1, G2Affine, G2Projective as G2,
         Parameters, BW6_761,
     },
+    curves::BatchGroupArithmeticSlice,
     BigInteger, Field, PairingEngine, PrimeField, ProjectiveCurve, SquareRootField, UniformRand,
 };
 
+batch_arith!();
 ec_bench!();
 f_bench!(1, Fq3, Fq3, fq3);
 f_bench!(2, Fq6, Fq6, fq6);

algebra-benches/src/macros/batch_arith.rs

@@ -0,0 +1,81 @@
+macro_rules! batch_arith {
+    () => {
+        #[bench]
+        fn bench_g1_batch_mul_affine(b: &mut ::test::Bencher) {
+            const SAMPLES: usize = 5000;
+
+            let mut rng = XorShiftRng::seed_from_u64(1231275789u64);
+
+            let mut g: Vec<G1Affine> = (0..SAMPLES)
+                .map(|_| G1::rand(&mut rng).into_affine())
+                .collect();
+
+            let s: Vec<FrRepr> = (0..SAMPLES)
+                .map(|_| Fr::rand(&mut rng).into_repr())
+                .collect();
+
+            let now = std::time::Instant::now();
+            println!("Start");
+            b.iter(|| {
+                g[..].batch_scalar_mul_in_place::<FrRepr>(&mut s.to_vec()[..], 4);
+                println!("G1 scalar mul batch affine {:?}", now.elapsed().as_micros());
+            });
+        }
+
+        #[bench]
+        fn bench_g1_batch_mul_projective(b: &mut ::test::Bencher) {
+            const SAMPLES: usize = 5000;
+
+            let mut rng = XorShiftRng::seed_from_u64(1231275789u64);
+
+            let mut g: Vec<G1> = (0..SAMPLES).map(|_| G1::rand(&mut rng)).collect();
+
+            let s: Vec<Fr> = (0..SAMPLES).map(|_| Fr::rand(&mut rng)).collect();
+
+            let now = std::time::Instant::now();
+            b.iter(|| {
+                g.iter_mut().zip(&s).for_each(|(p, sc)| p.mul_assign(*sc));
+                println!("G1 scalar mul proj {:?}", now.elapsed().as_micros());
+            });
+        }
+
+        #[bench]
+        fn bench_g2_batch_mul_affine(b: &mut ::test::Bencher) {
+            const SAMPLES: usize = 5000;
+
+            let mut rng = XorShiftRng::seed_from_u64(1231275789u64);
+
+            let mut g: Vec<G2Affine> = (0..SAMPLES)
+                .map(|_| G2::rand(&mut rng).into_affine())
+                .collect();
+
+            let s: Vec<FrRepr> = (0..SAMPLES)
+                .map(|_| Fr::rand(&mut rng).into_repr())
+                .collect();
+
+            let now = std::time::Instant::now();
+            println!("Start");
+            b.iter(|| {
+                g[..].batch_scalar_mul_in_place::<FrRepr>(&mut s.to_vec()[..], 4);
+                println!("G2 scalar mul batch affine {:?}", now.elapsed().as_micros());
+            });
+        }
+
+        #[bench]
+        fn bench_g2_batch_mul_projective(b: &mut ::test::Bencher) {
+            const SAMPLES: usize = 5000;
+
+            let mut rng = XorShiftRng::seed_from_u64(1231275789u64);
+
+            let mut g: Vec<G2> = (0..SAMPLES).map(|_| G2::rand(&mut rng)).collect();
+
+            let s: Vec<Fr> = (0..SAMPLES).map(|_| Fr::rand(&mut rng)).collect();
+
+            let now = std::time::Instant::now();
+            b.iter(|| {
+                g.iter_mut().zip(&s).for_each(|(p, sc)| p.mul_assign(*sc));
+                println!("G2 scalar mul proj {:?}", now.elapsed().as_micros());
+            });
+        }
+    };
+}

algebra-benches/src/macros/mod.rs

@@ -9,3 +9,6 @@ mod pairing;
 
 #[macro_use]
 mod utils;
+
+#[macro_use]
+mod batch_arith;

algebra-core/Cargo.toml

@@ -29,18 +29,27 @@ num-traits = { version = "0.2", default-features = false }
 rand = { version = "0.7", default-features = false }
 rayon = { version = "1", optional = true }
 unroll = { version = "=0.1.4" }
+itertools = { version = "0.9.0", default-features = false }
+voracious_radix_sort = { version = "1.0.0", optional = true }
+either = { version = "1.6.0", default-features = false }
+thread-id = { version = "3.3.0", optional = true }
+backtrace = { version = "0.3", optional = true }
 
 [build-dependencies]
-field-assembly = { path = "./field-assembly" }
+field-assembly = { path = "./field-assembly", optional = true }
 rustc_version = "0.2"
 
 [dev-dependencies]
 rand_xorshift = "0.2"
 
 [features]
-default = [ "std" ]
-std = []
-parallel = [ "std", "rayon" ]
+default = [ "std", "rand/default" ]
+std = [ "voracious_radix_sort" ]
+parallel = [ "std", "rayon", "rand/default" ]
 derive = [ "algebra-core-derive" ]
-llvm_asm = []
+llvm_asm = [ "field-assembly" ]
+prefetch = [ "std" ]
 
+timing = [ "std", "backtrace" ]
+timing_detailed = [ "std", "backtrace" ]
+timing_thread_id = [ "thread-id" ]

algebra-core/build.rs

@@ -1,26 +1,29 @@
-use std::env;
-use std::fs;
-use std::path::Path;
-
 extern crate rustc_version;
 use rustc_version::{version_meta, Channel};
 
-use field_assembly::generate_macro_string;
+#[cfg(feature = "llvm_asm")]
+use {
+    field_assembly::generate_macro_string,
+    std::{env, fs, path::Path},
+};
 
+#[cfg(feature = "llvm_asm")]
 const NUM_LIMBS: usize = 8;
 
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
 
     let is_nightly = version_meta().expect("nightly check failed").channel == Channel::Nightly;
 
-    let should_use_asm = cfg!(all(
+    let _should_use_asm = cfg!(all(
         feature = "llvm_asm",
         target_feature = "bmi2",
         target_feature = "adx",
         target_arch = "x86_64"
     )) && is_nightly;
-    if should_use_asm {
+
+    #[cfg(feature = "llvm_asm")]
+    if _should_use_asm {
         let out_dir = env::var_os("OUT_DIR").unwrap();
         let dest_path = Path::new(&out_dir).join("field_assembly.rs");
         fs::write(&dest_path, generate_macro_string(NUM_LIMBS)).unwrap();

algebra-core/field-assembly/Cargo.toml

@@ -8,3 +8,4 @@ edition = "2018"
 
 [dependencies]
 mince = { path = "../mince" }
+paste = "0.1"

algebra-core/field-assembly/src/lib.rs

@@ -13,29 +13,6 @@ use std::cell::RefCell;
 
 const MAX_REGS: usize = 6;
 
-pub fn generate_macro_string(num_limbs: usize) -> std::string::String {
-    if num_limbs > 3 * MAX_REGS {
-        panic!(
-            "Number of limbs must be <= {} and MAX_REGS >= 6",
-            3 * MAX_REGS
-        );
-    }
-    let mut macro_string = String::from(
-        "
-        macro_rules! llvm_asm_mul {
-        ($limbs:expr, $a:expr, $b:expr, $modulus:expr, $mod_prime:expr) => {
-            match $limbs {",
-    );
-    macro_string += &generate_matches(num_limbs, true);
-
-    macro_string += &"
-        macro_rules! llvm_asm_square {
-        ($limbs:expr, $a:expr, $modulus:expr, $mod_prime:expr) => {
-            match $limbs {";
-    macro_string += &generate_matches(num_limbs, false);
-    macro_string
-}
-
 #[assemble]
 fn generate_llvm_asm_mul_string(
     a: &str,
@@ -45,25 +22,25 @@ fn generate_llvm_asm_mul_string(
     mod_prime: &str,
     limbs: usize,
 ) -> String {
-    reg!(a0, a1, a, limbs);
-    reg!(b0, b1, b, limbs);
-    reg!(m, m1, modulus, limbs);
+    reg!(a_reg, a, limbs);
+    reg!(b_reg, b, limbs);
+    reg!(m_reg, modulus, limbs);
 
     xorq(RCX, RCX);
     for i in 0..limbs {
         if i == 0 {
-            mul_1!(a1[0], b1, zero, limbs);
+            mul_1!(a_reg[0], b_reg, zero, limbs);
         } else {
-            mul_add_1!(a1, b1, zero, i, limbs);
+            mul_add_1!(a_reg, b_reg, zero, i, limbs);
         }
-        mul_add_shift_1!(m1, mod_prime, zero, i, limbs);
+        mul_add_shift_1!(m_reg, mod_prime, zero, i, limbs);
     }
     for i in 0..limbs {
-        movq(R[i], a1[i]);
+        movq(R[i], a_reg[i]);
     }
 }
 
-fn generate_matches(num_limbs: usize, is_mul: bool) -> String {
+fn generate_match_arms(num_limbs: usize, is_mul: bool) -> String {
     let mut ctx = Context::new();
     for limbs in 2..(num_limbs + 1) {
         ctx.reset();
@@ -102,3 +79,26 @@ fn generate_matches(num_limbs: usize, is_mul: bool) -> String {
     ctx.end(num_limbs);
     ctx.get_string()
 }
+
+pub fn generate_macro_string(num_limbs: usize) -> std::string::String {
+    if num_limbs > 3 * MAX_REGS {
+        panic!(
+            "Number of limbs must be <= {} and MAX_REGS >= 6",
+            3 * MAX_REGS
+        );
+    }
+    let mut macro_string = String::from(
+        "
+        macro_rules! llvm_asm_mul {
+        ($limbs:expr, $a:expr, $b:expr, $modulus:expr, $mod_prime:expr) => {
+            match $limbs {",
+    );
+    macro_string += &generate_match_arms(num_limbs, true);
+
+    macro_string += &"
+        macro_rules! llvm_asm_square {
+        ($limbs:expr, $a:expr, $modulus:expr, $mod_prime:expr) => {
+            match $limbs {";
+    macro_string += &generate_match_arms(num_limbs, false);
+    macro_string
+}

algebra-core/field-assembly/src/utils.rs

@@ -7,14 +7,16 @@ pub const RSI: &'static str = "%rsi";
 pub const R: [&'static str; 8] = ["%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15"];
 
 macro_rules! reg {
-    ($a_0:ident, $a_1:ident, $a:ident, $range:expr) => {
-        let mut $a_0 = Vec::new();
-        let mut $a_1 = Vec::new();
-        for i in 0..$range {
-            $a_0.push(format!("{}({})", i * 8, $a));
-        }
-        for i in 0..$range {
-            $a_1.push(&*$a_0[i]);
+    ($a_reg:ident, $a:ident, $range:expr) => {
+        paste::item! {
+            let mut $a_reg = Vec::new();
+            let mut [<$a_reg _1>] = Vec::new();
+            for i in 0..$range {
+                [<$a_reg _1>].push(format!("{}({})", i * 8, $a));
+            }
+            for i in 0..$range {
+                $a_reg.push(&*[<$a_reg _1>][i]);
+            }
         }
     };
 }

algebra-core/src/biginteger/macros.rs

@@ -198,6 +198,48 @@ macro_rules! bigint_impl {
 
                 res
             }
+
+            #[inline]
+            fn mul_no_reduce(this: &[u64], other: &[u64]) -> Self {
+                assert!(this.len() == $num_limbs / 2);
+                assert!(other.len() == $num_limbs / 2);
+
+                let mut r = [0u64; $num_limbs];
+                for i in 0..$num_limbs / 2 {
+                    let mut carry = 0u64;
+                    for j in 0..$num_limbs / 2 {
+                        r[j + i] =
+                            arithmetic::mac_with_carry(r[j + i], this[i], other[j], &mut carry);
+                    }
+                    r[$num_limbs / 2 + i] = carry;
+                }
+                Self::new(r)
+            }
+
+            #[inline]
+            fn mul_no_reduce_lo(this: &[u64], other: &[u64]) -> Self {
+                assert!(this.len() == $num_limbs);
+                assert!(other.len() == $num_limbs);
+
+                let mut r = [0u64; $num_limbs];
+                for i in 0..$num_limbs {
+                    let mut carry = 0u64;
+                    for j in 0..($num_limbs - i) {
+                        r[j + i] =
+                            arithmetic::mac_with_carry(r[j + i], this[i], other[j], &mut carry);
+                    }
+                }
+                Self::new(r)
+            }
+
+            #[inline]
+            fn from_slice(slice: &[u64]) -> Self {
+                let mut repr = Self::default();
+                for (limb, &value) in repr.0.iter_mut().zip(slice) {
+                    *limb = value;
+                }
+                repr
+            }
         }
 
         impl ToBytes for $name {