Integrate aggregate signature + bit-vector QC validation trait + public hotshot-primitive repo + change signature scheme

Cargo.toml

@@ -133,10 +133,8 @@ hotshot-orchestrator = { version = "0.1.1", path = "orchestrator", default-featu
 hotshot-types = { path = "./types", version = "0.1.0", default-features = false }
 hotshot-utils = { path = "./utils" }
 itertools = "0.10"
-hotshot-primitives = { git = "ssh://[email protected]/EspressoSystems/hotshot-primitives.git", branch = 'update_jellyfish_0.4.0_pre.0' }
-jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", rev = "36dceb6", features = [
-        "std",
-] }
+hotshot-primitives = { git = "https://github.com/EspressoSystems/hotshot-primitives",  rev = "4b13928"} # rev after downgrade: rev = "b22c7f2" branch = 'update_jellyfish_0.4.0_pre.0'
+jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", rev = "c07a87f"} # downgrade rev = "36dceb63", branch = 'downgrade-curve25519'
 libp2p-swarm-derive = { version = "=0.32.0" }
 libp2p-networking = { path = "./libp2p-networking", version = "0.1.0", default-features = false }
 libp2p-identity = "0.1.2"
@@ -188,6 +186,9 @@ tokio = { version = "1", optional = true, features = [
         "tracing",
 ] }
 tracing = "0.1.37"
+ethereum-types = { version = "0.14.1", features = ["impl-serde"] }
+bitvec = { version = "1.0.1", default-features = false, features = ["alloc", "atomic", "serde"] }
+typenum = { version = "1.16.0" }
 
 [dev-dependencies]
 async-std = { version = "1.12.0", features = ["attributes"] }

centralized_server/Cargo.toml

@@ -55,4 +55,4 @@ tracing = "0.1.37"
 serde = { version = "1.0.163", features = ["derive"] }
 serde_json = "1.0.96"
 snafu = "0.7.4"
-toml = "0.7.4"
+toml = "0.7.4"

centralized_server/benchmark_client/Cargo.toml

@@ -36,4 +36,4 @@ futures = "0.3.28"
 hotshot-centralized-server = { path = ".." }
 hotshot-utils = { path = "../../utils" }
 hotshot-types = { path = "../../types" }
-tracing = "0.1.37"
+tracing = "0.1.37"

centralized_server/benchmark_client/src/main.rs

@@ -11,7 +11,7 @@ use hotshot_types::traits::signature_key::{
     SignatureKey,
 };
 use std::{net::ToSocketAddrs, time::Instant};
-use tracing::{error, info};
+use tracing::{error, warn, info};
 
 type ToServer = hotshot_centralized_server::ToServer<Ed25519Pub>;
 type FromServer = hotshot_centralized_server::FromServer<Ed25519Pub, ()>;

centralized_server/src/config.rs

@@ -183,6 +183,7 @@ impl<K, E> From<HotShotConfigFile> for HotShotConfig<K, E> {
             max_transactions: val.max_transactions,
             min_transactions: val.min_transactions,
             known_nodes: Vec::new(),
+            known_nodes_qc: Vec::new(),
             da_committee_size: val.total_nodes,
             next_view_timeout: val.next_view_timeout,
             timeout_ratio: val.timeout_ratio,

consensus/Cargo.toml

@@ -58,3 +58,7 @@ tokio = { version = "1", optional = true, features = [
 ] }
 tracing = "0.1.37"
 time = "0.3.21"
+bitvec = { version = "1.0.1", default-features = false, features = ["alloc", "atomic", "serde"] }
+hotshot-primitives = { git = "https://github.com/EspressoSystems/hotshot-primitives",  rev = "c99b58d"} # rev after downgrade: rev = "b22c7f2" branch = 'update_jellyfish_0.4.0_pre.0'
+jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", rev = "c07a87f"} # downgrade rev = "36dceb63", branch = 'downgrade-curve25519'
+blake3 = { version = "1.3.3", features = ["traits-preview"] }

consensus/src/da_member.rs

@@ -123,15 +123,13 @@ where
                                 if view_leader_key != sender {
                                     continue;
                                 }
-
                                 let block_commitment = p.data.deltas.commit();
                                 if !view_leader_key
-                                    .validate(&p.signature, block_commitment.as_ref())
+                                    .validate(p.ver_key, &p.signature, block_commitment.as_ref())
                                 {
                                     warn!(?p.signature, "Could not verify proposal.");
                                     continue;
                                 }
-
                                 let vote_token = self.exchange.make_vote_token(self.cur_view);
                                 match vote_token {
                                     Err(e) => {

consensus/src/leader.rs

@@ -185,7 +185,6 @@ where
             .add_point(task_start_time.elapsed().as_secs_f64());
 
         let proposal_build_start = Instant::now();
-
         if let Ok(new_state) = starting_state.append(&block, &self.cur_view) {
             let leaf = ValidatingLeaf {
                 view_number: self.cur_view,
@@ -198,14 +197,15 @@ where
                 timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(),
                 proposer_id: pk.to_bytes(),
             };
-            let signature = self
+            let (signature, ver_key) = self
                 .exchange
                 .sign_validating_or_commitment_proposal::<I>(&leaf.commit());
-            let data: ValidatingProposal<TYPES, ValidatingLeaf<TYPES>> = leaf.into();
+            let data: ValidatingProposal<TYPES, ValidatingLeaf<TYPES>> = leaf.clone().into();
             let message =
                 ValidatingMessage::<TYPES, I>(GeneralConsensusMessage::Proposal(Proposal {
                     data,
                     signature,
+                    ver_key,
                 }));
             consensus
                 .metrics

consensus/src/next_leader.rs

@@ -27,6 +27,7 @@ use std::{
     sync::Arc,
 };
 use tracing::{info, instrument, warn};
+use bitvec::prelude::*;
 
 /// The next view's validating leader
 #[derive(custom_debug::Debug, Clone)]
@@ -95,12 +96,16 @@ where
         let mut qcs = HashSet::<QuorumCertificate<TYPES, ValidatingLeaf<TYPES>>>::new();
         qcs.insert(self.generic_qc.clone());
 
+
         let mut accumlator = VoteAccumulator {
             total_vote_outcomes: HashMap::new(),
+            da_vote_outcomes: HashMap::new(),
             yes_vote_outcomes: HashMap::new(),
             no_vote_outcomes: HashMap::new(),
             success_threshold: self.exchange.success_threshold(),
             failure_threshold: self.exchange.failure_threshold(),
+            sig_lists: Vec::new(),
+            signers: bitvec![0; self.exchange.total_nodes()],
         };
 
         let lock = self.vote_collection_chan.lock().await;
@@ -121,6 +126,7 @@ where
                             match self.exchange.accumulate_vote(
                                 &vote.signature.0,
                                 &vote.signature.1,
+                                vote.signature.2,
                                 vote.leaf_commitment,
                                 vote.vote_data,
                                 vote.vote_token.clone(),

consensus/src/replica.rs

@@ -111,6 +111,7 @@ where
                 }
                 match msg {
                     ProcessedGeneralConsensusMessage::Proposal(p, sender) => {
+
                         if view_leader_key != sender {
                             continue;
                         }
@@ -177,7 +178,7 @@ where
                             continue;
                         };
 
-                        if !view_leader_key.validate(&p.signature, leaf.commit().as_ref()) {
+                        if !view_leader_key.validate(p.ver_key, &p.signature, leaf.commit().as_ref()) {
                             warn!(?p.signature, "Could not verify proposal.");
                             continue;
                         }

consensus/src/sequencing_leader.rs

@@ -21,7 +21,7 @@ use hotshot_types::traits::node_implementation::{
 };
 use hotshot_types::traits::state::State;
 use hotshot_types::{
-    certificate::{DACertificate, QuorumCertificate, YesNoSignature},
+    certificate::{DACertificate, QuorumCertificate, QCAssembledSignature},
     data::{DAProposal, QuorumProposal, SequencingLeaf},
     message::{
         CommitteeConsensusMessage, ConsensusMessageType, GeneralConsensusMessage, InternalTrigger,
@@ -42,6 +42,7 @@ use std::marker::PhantomData;
 use std::num::NonZeroU64;
 use std::{collections::HashSet, sync::Arc, time::Instant};
 use tracing::{error, info, instrument, warn};
+use bitvec::prelude::*;
 /// This view's DA committee leader
 #[derive(Debug, Clone)]
 pub struct DALeader<
@@ -101,16 +102,20 @@ where
         &self,
         cur_view: TYPES::Time,
         threshold: NonZeroU64,
+        total_nodes_num: usize,
         block_commitment: Commitment<<TYPES as NodeType>::BlockType>,
     ) -> Option<DACertificate<TYPES>> {
         let lock = self.vote_collection_chan.lock().await;
         let mut accumulator = VoteAccumulator {
             total_vote_outcomes: HashMap::new(),
+            da_vote_outcomes: HashMap::new(),
             yes_vote_outcomes: HashMap::new(),
             no_vote_outcomes: HashMap::new(),
             // TODO ED Revisit this once Yes/No votes are in place for DA
             success_threshold: threshold,
             failure_threshold: threshold,
+            sig_lists: Vec::new(),
+            signers: bitvec![0; total_nodes_num],
         };
 
         while let Ok(msg) = lock.recv().await {
@@ -147,6 +152,7 @@ where
                         match self.committee_exchange.accumulate_vote(
                             &vote.signature.0,
                             &vote.signature.1,
+                            vote.signature.2,
                             vote.block_commitment,
                             vote.vote_data,
                             vote.vote_token.clone(),
@@ -158,8 +164,11 @@ where
                             }
                             Either::Right(qc) => {
                                 match qc.clone().signatures {
-                                    YesNoSignature::Yes(map) => {
-                                        info!("Number of DA signatures in this QC: {}", map.len());
+                                    QCAssembledSignature::Yes(signature) => {
+                                        info!("Number of DA signatures in this QC: {}", signature.1.len());
+                                    }
+                                    QCAssembledSignature::DA(signature) => {
+                                        info!("Number of DA signatures in this QC: {}", signature.1.len());
                                     }
                                     _ => unimplemented!(),
                                 };
@@ -269,15 +278,17 @@ where
         let block_commitment = block.commit();
 
         let consensus = self.consensus.read().await;
-        let signature = self.committee_exchange.sign_da_proposal(&block.commit());
+        let (signature, ver_key) = self.committee_exchange.sign_da_proposal(&block.commit());
         let data: DAProposal<TYPES> = DAProposal {
             deltas: block.clone(),
             view_number: self.cur_view,
+            ver_key,
         };
         let message =
             SequencingMessage::<TYPES, I>(Right(CommitteeConsensusMessage::DAProposal(Proposal {
                 data,
                 signature,
+                ver_key,
             })));
         // Brodcast DA proposal
         if let Err(e) = self.api.send_da_broadcast(message.clone()).await {
@@ -295,6 +306,7 @@ where
             .wait_for_votes(
                 self.cur_view,
                 self.committee_exchange.success_threshold(),
+                self.committee_exchange.total_nodes(),
                 block_commitment,
             )
             .await
@@ -374,7 +386,7 @@ where
             timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(),
             proposer_id: self.api.public_key().to_bytes(),
         };
-        let signature = self
+        let (signature, ver_key) = self
             .quorum_exchange
             .sign_validating_or_commitment_proposal::<I>(&leaf.commit());
         // TODO: DA cert is sent as part of the proposal here, we should split this out so we don't have to wait for it.
@@ -385,12 +397,13 @@ where
             justify_qc: self.high_qc.clone(),
             dac: self.cert,
             proposer_id: leaf.proposer_id,
+            ver_key: ver_key,
         };
-
         let message =
             SequencingMessage::<TYPES, I>(Left(GeneralConsensusMessage::Proposal(Proposal {
                 data: proposal,
                 signature,
+                ver_key,
             })));
         if let Err(e) = self
             .api
@@ -463,13 +476,15 @@ where
     pub async fn run_view(self) -> QuorumCertificate<TYPES, SequencingLeaf<TYPES>> {
         let mut qcs = HashSet::<QuorumCertificate<TYPES, SequencingLeaf<TYPES>>>::new();
         qcs.insert(self.generic_qc.clone());
-
         let mut accumulator = VoteAccumulator {
             total_vote_outcomes: HashMap::new(),
+            da_vote_outcomes: HashMap::new(),
             yes_vote_outcomes: HashMap::new(),
             no_vote_outcomes: HashMap::new(),
             success_threshold: self.quorum_exchange.success_threshold(),
             failure_threshold: self.quorum_exchange.failure_threshold(),
+            sig_lists: Vec::new(),
+            signers: bitvec![0; self.quorum_exchange.total_nodes()],
         };
 
         let lock = self.vote_collection_chan.lock().await;
@@ -492,6 +507,7 @@ where
                                 match self.quorum_exchange.accumulate_vote(
                                     &vote.signature.0,
                                     &vote.signature.1,
+                                    vote.signature.2,
                                     vote.leaf_commitment,
                                     vote.vote_data,
                                     vote.vote_token.clone(),
@@ -503,9 +519,9 @@ where
                                     }
                                     Either::Right(qc) => {
                                         match qc.clone().signatures {
-                                            YesNoSignature::Yes(map) => info!(
+                                            QCAssembledSignature::Yes(signature) => info!(
                                                 "Number of qurorum signatures in this QC: {}",
-                                                map.len()
+                                                signature.1.len()
                                             ),
                                             _ => unimplemented!(),
                                         };

consensus/src/sequencing_replica.rs

@@ -253,7 +253,7 @@ where
                                         }
                                         // Validate the signature.
                                         else if !view_leader_key
-                                            .validate(&p.signature, leaf_commitment.as_ref())
+                                            .validate(p.ver_key, &p.signature, leaf_commitment.as_ref())
                                         {
                                             warn!(?p.signature, "Could not verify proposal.");
                                             message = self.quorum_exchange.create_no_message(

examples/infra/mod.rs

@@ -65,6 +65,13 @@ use surf_disco::Client;
 #[allow(deprecated)]
 use tracing::error;
 
+// Sishan NOTE: for QC aggregation
+use hotshot_types::traits::signature_key::ed25519::Ed25519Priv;
+use jf_primitives::signatures::bls_over_bn254::{KeyPair as QCKeyPair};
+use hotshot_primitives::quorum_certificate::StakeTableEntry;
+use rand::prelude::*;
+use rand_core::SeedableRng;
+
 // ORCHESTRATOR
 
 #[derive(Parser, Debug, Clone)]
@@ -106,6 +113,20 @@ pub fn load_config_from_file<TYPES: NodeType>(
         })
         .collect();
 
+    config.config.known_nodes_qc = (0..config.config.total_nodes.get())
+    .map(|node_id| {
+        let real_seed = Ed25519Priv::get_seed_from_seed_indexed(
+                config.seed,
+                node_id.try_into().unwrap(),
+            );
+        let entry = StakeTableEntry {
+                stake_key: QCKeyPair::generate(&mut ChaCha20Rng::from_seed(real_seed)).ver_key(),
+                stake_amount: U256::from(1u8),
+            };
+        entry
+    })
+    .collect();
+
     config
 }
 
@@ -229,10 +250,18 @@ pub trait Run<
 
         let (pk, sk) =
             TYPES::SignatureKey::generated_from_seed_indexed(config.seed, config.node_index);
-        let ek = jf_primitives::aead::KeyPair::generate(&mut rand_chacha::ChaChaRng::from_seed(
-            [0u8; 32],
-        ));
+        // Sishan Note: For QC Aggregation
+        let real_seed = Ed25519Priv::get_seed_from_seed_indexed(
+            config.seed,
+            node_id.try_into().unwrap(),
+        );
+        let key_pair = QCKeyPair::generate(&mut ChaCha20Rng::from_seed(real_seed));
+        let entry = StakeTableEntry {
+            stake_key: key_pair.ver_key(),
+            stake_amount: U256::from(1u8),
+        };
         let known_nodes = config.config.known_nodes.clone();
+        let known_nodes_qc = config.config.known_nodes_qc.clone();
 
         let network = self.get_network();
 
@@ -251,12 +280,14 @@ pub trait Run<
         });
 
         let exchanges = NODE::Exchanges::create(
+            known_nodes_qc.clone(),
             known_nodes.clone(),
             election_config.clone(),
             (network.clone(), ()),
             pk.clone(),
+            key_pair.clone(),
+            entry.clone(),
             sk.clone(),
-            ek.clone(),
         );
         let hotshot = HotShot::init(
             pk,

orchestrator/src/config.rs

@@ -182,6 +182,7 @@ impl<K, E> From<HotShotConfigFile> for HotShotConfig<K, E> {
             max_transactions: val.max_transactions,
             min_transactions: val.min_transactions,
             known_nodes: Vec::new(),
+            known_nodes_qc: Vec::new(),
             da_committee_size: val.total_nodes,
             next_view_timeout: val.next_view_timeout,
             timeout_ratio: val.timeout_ratio,