From 05284e2aa8ce80ce8d83c7b56bd87ff030c2c9b5 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Fri, 29 Sep 2023 16:05:49 -0700 Subject: [PATCH 01/13] Save before switching branch --- crates/hotshot/examples/infra/modDA.rs | 16 +++--- crates/hotshot/src/demo.rs | 12 ++--- crates/hotshot/src/lib.rs | 6 +-- crates/hotshot/src/tasks/mod.rs | 10 ++-- crates/hotshot/src/traits/election/vrf.rs | 2 +- .../src/traits/networking/memory_network.rs | 2 +- .../src/traits/storage/memory_storage.rs | 2 +- crates/task-impls/src/consensus.rs | 10 ++-- crates/task-impls/src/da.rs | 14 ++--- crates/task-impls/src/events.rs | 4 +- crates/task-impls/src/transactions.rs | 14 ++--- crates/testing/src/node_types.rs | 2 +- crates/types/src/block_impl.rs | 11 +++- crates/types/src/certificate.rs | 10 ++-- crates/types/src/consensus.rs | 2 +- crates/types/src/data.rs | 53 ++++++++++--------- crates/types/src/traits/block_contents.rs | 33 +++++++++--- crates/types/src/traits/election.rs | 22 ++++---- .../types/src/traits/node_implementation.rs | 28 +++++----- crates/types/src/traits/state.rs | 18 +++---- crates/types/src/traits/storage.rs | 2 +- crates/types/src/vote.rs | 8 +-- testing-macros/src/lib.rs | 4 +- 23 files changed, 157 insertions(+), 128 deletions(-) diff --git a/crates/hotshot/examples/infra/modDA.rs b/crates/hotshot/examples/infra/modDA.rs index 82cc12bf68..cb6cf5e289 100644 --- a/crates/hotshot/examples/infra/modDA.rs +++ b/crates/hotshot/examples/infra/modDA.rs @@ -161,7 +161,7 @@ pub trait RunDA< >, > where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, Self: Sync, SystemContext: HotShotType, @@ -180,7 +180,7 @@ pub trait RunDA< /// get the anchored view /// Note: sequencing leaf does not have state, so does not return state async fn initialize_state_and_hotshot(&self) -> SystemContextHandle { - let genesis_block = TYPES::BlockType::genesis(); + let genesis_block = TYPES::BlockPayload::genesis(); let initializer = hotshot::HotShotInitializer::>::from_genesis( genesis_block, @@ -408,7 +408,7 @@ pub struct WebServerDARun< #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -452,7 +452,7 @@ impl< > for WebServerDARun where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, Self: Sync, { @@ -552,7 +552,7 @@ pub struct Libp2pDARun, MEMBERSHIP #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -596,7 +596,7 @@ impl< > for Libp2pDARun where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, Self: Sync, { @@ -766,7 +766,7 @@ where /// Main entry point for validators pub async fn main_entry_point< - TYPES: NodeType, + TYPES: NodeType, MEMBERSHIP: Membership + Debug, DANETWORK: CommunicationChannel, MEMBERSHIP> + Debug, QUORUMNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, @@ -802,7 +802,7 @@ pub async fn main_entry_point< args: ValidatorArgs, ) where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, { setup_logging(); diff --git a/crates/hotshot/src/demo.rs b/crates/hotshot/src/demo.rs index 81cac76f4b..33ac15dd28 100644 --- a/crates/hotshot/src/demo.rs +++ b/crates/hotshot/src/demo.rs @@ -66,11 +66,11 @@ impl Default for SDemoState { impl State for SDemoState { type Error = BlockPayloadError; - type BlockType = VIDBlockPayload; + type BlockPayload = VIDBlockPayload; type Time = ViewNumber; - fn validate_block(&self, _block: &Self::BlockType, view_number: &Self::Time) -> bool { + fn validate_block(&self, _block: &Self::BlockPayload, view_number: &Self::Time) -> bool { if view_number == &ViewNumber::genesis() { &self.view_number == view_number } else { @@ -80,7 +80,7 @@ impl State for SDemoState { fn append( &self, - block: &Self::BlockType, + block: &Self::BlockPayload, view_number: &Self::Time, ) -> Result { if !self.validate_block(block, view_number) { @@ -102,7 +102,7 @@ impl TestableState for SDemoState { _state: Option<&Self>, _rng: &mut dyn rand::RngCore, padding: u64, - ) -> ::Transaction { + ) -> ::Transaction { /// clippy appeasement for `RANDOM_TX_BASE_SIZE` const RANDOM_TX_BASE_SIZE: usize = 8; VIDTransaction(vec![0; RANDOM_TX_BASE_SIZE + (padding as usize)]) @@ -126,7 +126,7 @@ pub struct DemoTypes; impl NodeType for DemoTypes { type Time = ViewNumber; - type BlockType = VIDBlockPayload; + type BlockPayload = VIDBlockPayload; type SignatureKey = BLSPubKey; type VoteTokenType = StaticVoteToken; type Transaction = VIDTransaction; @@ -187,7 +187,7 @@ pub fn random_quorum_certificate( - deltas: Either>, + deltas: Either>, rng: &mut dyn rand::RngCore, ) -> SequencingLeaf { let justify_qc = random_quorum_certificate(rng); diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index b7872ce59e..2281a1af08 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -634,7 +634,7 @@ pub trait HotShotType> { #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -656,7 +656,7 @@ where Message, Proposal = DAProposal, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, Membership = MEMBERSHIP, > + 'static, ViewSyncEx: ConsensusExchange< @@ -1052,7 +1052,7 @@ impl> HotShotInitializer Result> { + pub fn from_genesis(genesis_block: TYPES::BlockPayload) -> Result> { let state = TYPES::StateType::default() .append(&genesis_block, &TYPES::Time::new(0)) .map_err(|err| HotShotError::Misc { diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 66f9fb20da..1a0fc8ad4e 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -250,7 +250,7 @@ where /// # Panics /// Is unable to panic. This section here is just to satisfy clippy pub async fn add_consensus_task< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -274,7 +274,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { let consensus = handle.hotshot.get_consensus(); @@ -364,7 +364,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { // build the da task @@ -416,7 +416,7 @@ where /// # Panics /// Is unable to panic. This section here is just to satisfy clippy pub async fn add_transaction_task< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -433,7 +433,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { // build the transactions task diff --git a/crates/hotshot/src/traits/election/vrf.rs b/crates/hotshot/src/traits/election/vrf.rs index d1151e0ab2..f99dabd7e8 100644 --- a/crates/hotshot/src/traits/election/vrf.rs +++ b/crates/hotshot/src/traits/election/vrf.rs @@ -884,7 +884,7 @@ impl Clone for VRFStakeTable; // type VoteTokenType = VRFVoteToken< // BLSVerKey, diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 51f7bbbdf7..087333d77a 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -629,7 +629,7 @@ mod tests { // // TODO (da) can this be SequencingConsensus? // type ConsensusType = ValidatingConsensus; // type Time = ViewNumber; - // type BlockType = VDemoBlock; + // type BlockPayload = VDemoBlock; // type SignatureKey = Ed25519Pub; // type VoteTokenType = StaticVoteToken; // type Transaction = VDemoTransaction; diff --git a/crates/hotshot/src/traits/storage/memory_storage.rs b/crates/hotshot/src/traits/storage/memory_storage.rs index dd6e0217d5..73d86fc1d9 100644 --- a/crates/hotshot/src/traits/storage/memory_storage.rs +++ b/crates/hotshot/src/traits/storage/memory_storage.rs @@ -148,7 +148,7 @@ mod test { impl NodeType for DummyTypes { type Time = ViewNumber; - type BlockType = DummyBlock; + type BlockPayload = DummyBlock; type SignatureKey = BLSPubKey; type VoteTokenType = StaticVoteToken; type Transaction = ::Transaction; diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index 236b5ae6b8..1fb8d383e8 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -71,7 +71,7 @@ pub struct SequencingConsensusTaskState< TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// The global task registry @@ -84,7 +84,7 @@ pub struct SequencingConsensusTaskState< pub cur_view: TYPES::Time, /// Current block submitted to DA - pub block: TYPES::BlockType, + pub block: TYPES::BlockPayload, /// the quorum exchange pub quorum_exchange: Arc>, @@ -279,7 +279,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "Consensus genesis leaf", level = "error")] @@ -1217,7 +1217,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { } @@ -1266,7 +1266,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { if let SequencingHotShotEvent::Shutdown = event { diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index 1100f4e078..fbe156aec6 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -55,7 +55,7 @@ pub struct DATaskState< TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// The state's api @@ -91,7 +91,7 @@ pub struct DAVoteCollectionTaskState< TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// the committee exchange @@ -103,7 +103,7 @@ pub struct DAVoteCollectionTaskState< TYPES, TYPES::Time, TYPES::VoteTokenType, - Commitment, + Commitment, >>::VoteAccumulator, DACertificate, >, @@ -122,7 +122,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { } @@ -140,7 +140,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { match event { @@ -252,7 +252,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// main task event handler @@ -703,7 +703,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { } diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs index f9fc7f8ba3..fbb7e227b0 100644 --- a/crates/task-impls/src/events.rs +++ b/crates/task-impls/src/events.rs @@ -61,9 +61,9 @@ pub enum SequencingHotShotEvent> { /// Send transactions to the network TransactionSend(TYPES::Transaction, TYPES::SignatureKey), /// Event to send DA block data from DA leader to next quorum leader (which should always be the same node); internal event only - SendDABlockData(TYPES::BlockType), + SendDABlockData(TYPES::BlockPayload), /// Event when the transactions task has a block formed - BlockReady(TYPES::BlockType, TYPES::Time), + BlockReady(TYPES::BlockPayload, TYPES::Time), /// Event when consensus decided on a leaf LeafDecided(Vec), /// Send VID shares to VID storage nodes; emitted by the DA leader diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index c682f5f38d..e868c50ba5 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -56,7 +56,7 @@ pub struct TransactionTaskState< TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// The state's api @@ -87,10 +87,10 @@ pub struct TransactionTaskState< } // We have two `TransactionTaskState` implementations with different bounds. The implementation -// here requires `TYPES: NodeType`, +// here requires `TYPES: NodeType`, // whereas it's just `TYPES: NodeType` in the second implementation. impl< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -103,7 +103,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// main task event handler @@ -289,7 +289,7 @@ where } // We have two `TransactionTaskState` implementations with different bounds. The implementation -// above requires `TYPES: NodeType`, +// above requires `TYPES: NodeType`, // whereas here it's just `TYPES: NodeType`. impl< TYPES: NodeType, @@ -305,7 +305,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { #[instrument(skip_all, fields(id = self.id, view = *self.cur_view), name = "Transaction Handling Task", level = "error")] @@ -403,7 +403,7 @@ where TYPES, Message, Certificate = DACertificate, - Commitment = Commitment, + Commitment = Commitment, >, { } diff --git a/crates/testing/src/node_types.rs b/crates/testing/src/node_types.rs index 4ffd74e2bf..3cfb013851 100644 --- a/crates/testing/src/node_types.rs +++ b/crates/testing/src/node_types.rs @@ -42,7 +42,7 @@ use serde::{Deserialize, Serialize}; pub struct SequencingTestTypes; impl NodeType for SequencingTestTypes { type Time = ViewNumber; - type BlockType = VIDBlockPayload; + type BlockPayload = VIDBlockPayload; type SignatureKey = BLSPubKey; type VoteTokenType = StaticVoteToken; type Transaction = VIDTransaction; diff --git a/crates/types/src/block_impl.rs b/crates/types/src/block_impl.rs index 2360cb3a24..2e9f75cd09 100644 --- a/crates/types/src/block_impl.rs +++ b/crates/types/src/block_impl.rs @@ -5,7 +5,7 @@ use std::{ }; use crate::{ - data::{test_srs, VidScheme, VidSchemeTrait}, + data::{test_srs, VidScheme, VidSchemeTrait, SequencingLeaf}, traits::{block_contents::Transaction, state::TestableBlock, BlockPayload}, }; use commit::{Commitment, Committable}; @@ -125,3 +125,12 @@ impl BlockPayload for VIDBlockPayload { .collect() } } + +/// A [`BlockHeader`] that commits to [`VIDBlockPayload`]. +#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] +pub struct VIDBlockHeader { + /// Previous leaf. + pub previous_leaf: SequencingLeaf, + /// VID commitment. + pub commitment: ::Commit, +} \ No newline at end of file diff --git a/crates/types/src/certificate.rs b/crates/types/src/certificate.rs index 09c3235744..2abf94b34b 100644 --- a/crates/types/src/certificate.rs +++ b/crates/types/src/certificate.rs @@ -40,7 +40,7 @@ pub struct DACertificate { pub view_number: TYPES::Time, /// committment to the block - pub block_commitment: Commitment, + pub block_commitment: Commitment, /// Assembled signature for certificate aggregation pub signatures: AssembledSignature, @@ -237,11 +237,11 @@ impl Committable } impl - SignedCertificate> + SignedCertificate> for DACertificate { type Vote = DAVote; - type VoteAccumulator = DAVoteAccumulator, Self::Vote>; + type VoteAccumulator = DAVoteAccumulator, Self::Vote>; fn from_signatures_and_commitment( signatures: AssembledSignature, @@ -262,11 +262,11 @@ impl self.signatures.clone() } - fn leaf_commitment(&self) -> Commitment { + fn leaf_commitment(&self) -> Commitment { self.block_commitment } - fn set_leaf_commitment(&mut self, _commitment: Commitment) { + fn set_leaf_commitment(&mut self, _commitment: Commitment) { // This function is only useful for QC. Will be removed after we have separated cert traits. } diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index e10900aa7e..959c31604e 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -47,7 +47,7 @@ pub struct Consensus> { /// Saved blocks /// /// Contains the full block for every leaf in `saved_leaves` if that block is available. - pub saved_blocks: BlockStore, + pub saved_blocks: BlockStore, /// The `locked_qc` view number pub locked_view: TYPES::Time, diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 44a1937161..66c5ebfe31 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -108,7 +108,7 @@ pub fn genesis_proposer_id() -> EncodedPublicKey { } /// The `Transaction` type associated with a `State`, as a syntactic shortcut -pub type Transaction = <::BlockType as BlockPayload>::Transaction; +pub type Transaction = <::BlockPayload as BlockPayload>::Transaction; /// `Commitment` to the `Transaction` type associated with a `State`, as a syntactic shortcut pub type TxnCommitment = Commitment>; @@ -122,7 +122,7 @@ where LEAF: Committable, { /// current view's block commitment - pub block_commitment: Commitment, + pub block_commitment: Commitment, /// CurView from leader when proposing leaf pub view_number: TYPES::Time, @@ -139,13 +139,13 @@ where pub parent_commitment: Commitment, /// BlockPayload leaf wants to apply - pub deltas: TYPES::BlockType, + pub deltas: TYPES::BlockPayload, /// What the state should be after applying `self.deltas` pub state_commitment: Commitment, /// Transactions that were marked for rejection while collecting deltas - pub rejected: Vec<::Transaction>, + pub rejected: Vec<::Transaction>, /// the propser id pub proposer_id: EncodedPublicKey, @@ -155,7 +155,7 @@ where #[derive(custom_debug::Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] pub struct DAProposal { /// BlockPayload leaf wants to apply - pub deltas: TYPES::BlockType, + pub deltas: TYPES::BlockPayload, /// View this proposal applies to pub view_number: TYPES::Time, } @@ -172,7 +172,7 @@ pub struct VidDisperse { /// The view number for which this VID data is intended pub view_number: TYPES::Time, /// Block commitment - pub commitment: Commitment, + pub commitment: Commitment, /// VID shares dispersed among storage nodes pub shares: Vec<::Share>, /// VID common data sent to all storage nodes @@ -203,7 +203,7 @@ pub fn test_srs( #[serde(bound(deserialize = ""))] pub struct QuorumProposal> { /// The commitment to append. - pub block_commitment: Commitment, + pub block_commitment: Commitment, /// CurView from leader when proposing leaf pub view_number: TYPES::Time, @@ -281,7 +281,7 @@ pub trait ProposalType: /// A state change encoded in a leaf. /// -/// [`DeltasType`] represents a [block](NodeType::BlockType), but it may not contain the block in +/// [`DeltasType`] represents a [block](NodeType::BlockPayload), but it may not contain the block in /// full. It is guaranteed to contain, at least, a cryptographic commitment to the block, and it /// provides an interface for resolving the commitment to a full block if the full block is /// available. @@ -495,7 +495,7 @@ pub type LeafNode = ::NodeType; /// The [`StateType`] in a [`LeafType`]. pub type LeafState = as NodeType>::StateType; /// The [`BlockPayload`] in a [`LeafType`]. -pub type LeafBlock = as NodeType>::BlockType; +pub type LeafBlock = as NodeType>::BlockPayload; /// The [`Transaction`] in a [`LeafType`]. pub type LeafTransaction = as BlockPayload>::Transaction; /// The [`ConsensusTime`] used by a [`LeafType`]. @@ -511,7 +511,7 @@ pub trait TestableLeaf { &self, rng: &mut dyn rand::RngCore, padding: u64, - ) -> <::BlockType as BlockPayload>::Transaction; + ) -> <::BlockPayload as BlockPayload>::Transaction; } /// This is the consensus-internal analogous concept to a block, and it contains the block proper, @@ -535,13 +535,13 @@ pub struct ValidatingLeaf { pub parent_commitment: Commitment, /// BlockPayload leaf wants to apply - pub deltas: TYPES::BlockType, + pub deltas: TYPES::BlockPayload, /// What the state should be AFTER applying `self.deltas` pub state: TYPES::StateType, /// Transactions that were marked for rejection while collecting deltas - pub rejected: Vec<::Transaction>, + pub rejected: Vec<::Transaction>, /// the timestamp the leaf was constructed at, in nanoseconds. Only exposed for dashboard stats #[derivative(PartialEq = "ignore")] @@ -574,11 +574,12 @@ pub struct SequencingLeaf { pub parent_commitment: Commitment, /// The block or block commitment to be applied - pub deltas: Either>, + pub deltas: Either>, /// Transactions that were marked for rejection while collecting deltas - pub rejected: Vec<::Transaction>, + pub rejected: Vec<::Transaction>, + // TODO (Keyao) Remove. /// the timestamp the leaf was constructed at, in nanoseconds. Only exposed for dashboard stats pub timestamp: i128, @@ -636,13 +637,13 @@ impl Display for ValidatingLeaf { impl LeafType for ValidatingLeaf { type NodeType = TYPES; - type DeltasType = TYPES::BlockType; + type DeltasType = TYPES::BlockPayload; type MaybeState = TYPES::StateType; fn new( view_number: ::Time, justify_qc: QuorumCertificate>, - deltas: ::BlockType, + deltas: ::BlockPayload, state: ::StateType, ) -> Self { Self { @@ -682,7 +683,7 @@ impl LeafType for ValidatingLeaf { self.deltas.clone() } - fn get_deltas_commitment(&self) -> Commitment<::BlockType> { + fn get_deltas_commitment(&self) -> Commitment<::BlockPayload> { self.deltas.block_commitment() } @@ -694,7 +695,7 @@ impl LeafType for ValidatingLeaf { self.state.clone() } - fn get_rejected(&self) -> Vec<::Transaction> { + fn get_rejected(&self) -> Vec<::Transaction> { self.rejected.clone() } @@ -724,7 +725,7 @@ impl LeafType for ValidatingLeaf { impl TestableLeaf for ValidatingLeaf where TYPES::StateType: TestableState, - TYPES::BlockType: TestableBlock, + TYPES::BlockPayload: TestableBlock, { type NodeType = TYPES; @@ -732,7 +733,7 @@ where &self, rng: &mut dyn rand::RngCore, padding: u64, - ) -> <::BlockType as BlockPayload>::Transaction { + ) -> <::BlockPayload as BlockPayload>::Transaction { ::create_random_transaction( Some(&self.state), rng, @@ -753,13 +754,13 @@ impl Display for SequencingLeaf { impl LeafType for SequencingLeaf { type NodeType = TYPES; - type DeltasType = Either>; + type DeltasType = Either>; type MaybeState = (); fn new( view_number: ::Time, justify_qc: QuorumCertificate>, - deltas: ::BlockType, + deltas: ::BlockPayload, _state: ::StateType, ) -> Self { Self { @@ -798,7 +799,7 @@ impl LeafType for SequencingLeaf { self.deltas.clone() } - fn get_deltas_commitment(&self) -> Commitment<::BlockType> { + fn get_deltas_commitment(&self) -> Commitment<::BlockPayload> { self.deltas.block_commitment() } @@ -809,7 +810,7 @@ impl LeafType for SequencingLeaf { // The Sequencing Leaf doesn't have a state. fn get_state(&self) -> Self::MaybeState {} - fn get_rejected(&self) -> Vec<::Transaction> { + fn get_rejected(&self) -> Vec<::Transaction> { self.rejected.clone() } @@ -838,7 +839,7 @@ impl LeafType for SequencingLeaf { impl TestableLeaf for SequencingLeaf where TYPES::StateType: TestableState, - TYPES::BlockType: TestableBlock, + TYPES::BlockPayload: TestableBlock, { type NodeType = TYPES; @@ -846,7 +847,7 @@ where &self, rng: &mut dyn rand::RngCore, padding: u64, - ) -> <::BlockType as BlockPayload>::Transaction { + ) -> <::BlockPayload as BlockPayload>::Transaction { TYPES::StateType::create_random_transaction(None, rng, padding) } } diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index 2fc3383a14..ab851745d4 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -1,7 +1,7 @@ //! Abstraction over the contents of a block //! -//! This module provides the [`BlockPayload`] and [`BlockHeader`] traits, which describe the -//! behaviors that a block is expected to have. +//! This module provides the [`Transaction`], [`BlockPayload`], and [`BlockHeader`] traits, which +//! describe the behaviors that a block is expected to have. use commit::{Commitment, Committable}; use serde::{de::DeserializeOwned, Serialize}; @@ -13,6 +13,14 @@ use std::{ hash::Hash, }; +// TODO (Keyao) Determine whether we can refactor BlockPayload and Transaction from traits to structs. +// +/// Abstraction over any type of transaction. Used by [`BlockPayload`]. +pub trait Transaction: + Clone + Serialize + DeserializeOwned + Debug + PartialEq + Eq + Sync + Send + Committable + Hash +{ +} + // TODO (Keyao) Determine whether we can refactor BlockPayload and Transaction from traits to structs. // /// Abstraction over the full contents of a block @@ -47,12 +55,23 @@ pub trait BlockPayload: fn contained_transactions(&self) -> HashSet>; } -// TODO (Keyao) Determine whether we can refactor BlockPayload and Transaction from traits to structs. -// -/// Abstraction over any type of transaction. Used by [`BlockPayload`]. -pub trait Transaction: - Clone + Serialize + DeserializeOwned + Debug + PartialEq + Eq + Sync + Send + Committable + Hash +pub trait BlockHeader: + Serialize + + Clone + + Debug + + Display + + Hash + + PartialEq + + Eq + + Send + + Sync + + Committable + + DeserializeOwned { + type Payload: BlockPayload; + + /// Get the payload commitment. + fn commitment(&self) -> Commitment; } /// Dummy implementation of `BlockPayload` for unit tests diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 6b3ef7e790..7d9c0d0c2d 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -534,7 +534,7 @@ pub trait CommitteeExchangeType: ConsensusExchange { /// Sign a DA proposal. - fn sign_da_proposal(&self, block_commitment: &Commitment) + fn sign_da_proposal(&self, block_commitment: &Commitment) -> EncodedSignature; /// Sign a vote on DA proposal. @@ -543,13 +543,13 @@ pub trait CommitteeExchangeType: /// of information necessary for checking that this node voted on that block. fn sign_da_vote( &self, - block_commitment: Commitment, + block_commitment: Commitment, ) -> (EncodedPublicKey, EncodedSignature); /// Create a message with a vote on DA proposal. fn create_da_message( &self, - block_commitment: Commitment, + block_commitment: Commitment, current_view: TYPES::Time, vote_token: TYPES::VoteTokenType, ) -> DAVote; @@ -559,7 +559,7 @@ pub trait CommitteeExchangeType: /// Create a message with a vote on VID disperse data. fn create_vid_message( &self, - block_commitment: Commitment, + block_commitment: Commitment, current_view: TYPES::Time, vote_token: TYPES::VoteTokenType, ) -> DAVote; @@ -567,7 +567,7 @@ pub trait CommitteeExchangeType: /// Sign a vote on VID proposal. fn sign_vid_vote( &self, - block_commitment: Commitment, + block_commitment: Commitment, ) -> (EncodedPublicKey, EncodedSignature); } @@ -605,7 +605,7 @@ impl< /// Sign a DA proposal. fn sign_da_proposal( &self, - block_commitment: &Commitment, + block_commitment: &Commitment, ) -> EncodedSignature { let signature = TYPES::SignatureKey::sign(&self.private_key, block_commitment.as_ref()); signature @@ -616,7 +616,7 @@ impl< /// of information necessary for checking that this node voted on that block. fn sign_da_vote( &self, - block_commitment: Commitment, + block_commitment: Commitment, ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, @@ -627,7 +627,7 @@ impl< /// Create a message with a vote on DA proposal. fn create_da_message( &self, - block_commitment: Commitment, + block_commitment: Commitment, current_view: TYPES::Time, vote_token: TYPES::VoteTokenType, ) -> DAVote { @@ -643,7 +643,7 @@ impl< fn create_vid_message( &self, - block_commitment: Commitment, + block_commitment: Commitment, current_view: ::Time, vote_token: ::VoteTokenType, ) -> DAVote { @@ -659,7 +659,7 @@ impl< fn sign_vid_vote( &self, - block_commitment: Commitment<::BlockType>, + block_commitment: Commitment<::BlockPayload>, ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, @@ -681,7 +681,7 @@ impl< type Certificate = DACertificate; type Membership = MEMBERSHIP; type Networking = NETWORK; - type Commitment = Commitment; + type Commitment = Commitment; fn create( entries: Vec<::StakeTableEntry>, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index 49d9184bc4..a48f3152d3 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -359,7 +359,7 @@ pub trait TestableNodeImplementation: NodeImplementation state: Option<&TYPES::StateType>, rng: &mut dyn rand::RngCore, padding: u64, - ) -> ::Transaction; + ) -> ::Transaction; /// Creates random transaction if possible /// otherwise panics @@ -368,13 +368,13 @@ pub trait TestableNodeImplementation: NodeImplementation leaf: &Self::Leaf, rng: &mut dyn rand::RngCore, padding: u64, - ) -> ::Transaction; + ) -> ::Transaction; /// generate a genesis block - fn block_genesis() -> TYPES::BlockType; + fn block_genesis() -> TYPES::BlockPayload; /// the number of transactions in a block - fn txn_count(block: &TYPES::BlockType) -> u64; + fn txn_count(block: &TYPES::BlockPayload) -> u64; /// Create ephemeral storage /// Will be deleted/lost immediately after storage is dropped @@ -413,7 +413,7 @@ where QuorumNetwork, >, TYPES::StateType: TestableState, - TYPES::BlockType: TestableBlock, + TYPES::BlockPayload: TestableBlock, I::Storage: TestableStorage, I::Leaf: TestableLeaf, { @@ -428,7 +428,7 @@ where state: Option<&TYPES::StateType>, rng: &mut dyn rand::RngCore, padding: u64, - ) -> ::Transaction { + ) -> ::Transaction { ::create_random_transaction(state, rng, padding) } @@ -436,16 +436,16 @@ where leaf: &Self::Leaf, rng: &mut dyn rand::RngCore, padding: u64, - ) -> ::Transaction { + ) -> ::Transaction { ::create_random_transaction(leaf, rng, padding) } - fn block_genesis() -> TYPES::BlockType { - ::genesis() + fn block_genesis() -> TYPES::BlockPayload { + ::genesis() } - fn txn_count(block: &TYPES::BlockType) -> u64 { - ::txn_count(block) + fn txn_count(block: &TYPES::BlockPayload) -> u64 { + ::txn_count(block) } fn construct_tmp_storage() -> Result { @@ -548,8 +548,8 @@ pub trait NodeType: type Time: ConsensusTime; /// The block type that this hotshot setup is using. /// - /// This should be the same block that `StateType::BlockType` is using. - type BlockType: BlockPayload; + /// This should be the same block that `StateType::BlockPayload` is using. + type BlockPayload: BlockPayload; /// The signature key that this hotshot setup is using. type SignatureKey: SignatureKey; /// The vote token that this hotshot setup is using. @@ -562,5 +562,5 @@ pub trait NodeType: type ElectionConfigType: ElectionConfig; /// The state type that this hotshot setup is using. - type StateType: State; + type StateType: State; } diff --git a/crates/types/src/traits/state.rs b/crates/types/src/traits/state.rs index bd8fe7d2ed..c143efdbc8 100644 --- a/crates/types/src/traits/state.rs +++ b/crates/types/src/traits/state.rs @@ -19,7 +19,7 @@ use std::{ /// /// This trait represents the behaviors that the 'global' ledger state must have: /// * A defined error type ([`Error`](State::Error)) -/// * The type of block that modifies this type of state ([`BlockPayload`](State::BlockType)) +/// * The type of block that modifies this type of state ([`BlockPayload`](State::BlockPayload)) /// * The ability to validate that a block is actually a valid extension of this state /// ([`validate_block`](State::validate_block)) /// * The ability to produce a new state, with the modifications from the block applied @@ -40,12 +40,12 @@ pub trait State: /// The error type for this particular type of ledger state type Error: Error + Debug + Send + Sync; /// The type of block this state is associated with - type BlockType: BlockPayload; + type BlockPayload: BlockPayload; /// Time compatibility needed for reward collection type Time: ConsensusTime; /// Returns true if and only if the provided block is valid and can extend this state - fn validate_block(&self, block: &Self::BlockType, view_number: &Self::Time) -> bool; + fn validate_block(&self, block: &Self::BlockPayload, view_number: &Self::Time) -> bool; /// Appends the given block to this state, returning an new state /// @@ -54,7 +54,7 @@ pub trait State: /// Should produce and error if appending this block would lead to an invalid state fn append( &self, - block: &Self::BlockType, + block: &Self::BlockPayload, view_number: &Self::Time, ) -> Result; @@ -95,7 +95,7 @@ pub trait ConsensusTime: /// extra functions required on state to be usable by hotshot-testing pub trait TestableState: State where - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, { /// Creates random transaction if possible /// otherwise panics @@ -104,7 +104,7 @@ where state: Option<&Self>, rng: &mut dyn rand::RngCore, padding: u64, - ) -> ::Transaction; + ) -> ::Transaction; } /// extra functions required on block to be usable by hotshot-testing @@ -157,16 +157,16 @@ pub mod dummy { impl State for DummyState { type Error = DummyError; - type BlockType = DummyBlock; + type BlockPayload = DummyBlock; type Time = ViewNumber; - fn validate_block(&self, _block: &Self::BlockType, _view_number: &Self::Time) -> bool { + fn validate_block(&self, _block: &Self::BlockPayload, _view_number: &Self::Time) -> bool { false } fn append( &self, - _block: &Self::BlockType, + _block: &Self::BlockPayload, _view_number: &Self::Time, ) -> Result { Ok(Self { diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index 122698486c..996ea53e02 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -161,7 +161,7 @@ where state: LEAF::MaybeState, height: u64, parent_commitment: Commitment, - rejected: Vec<::Transaction>, + rejected: Vec<::Transaction>, proposer_id: EncodedPublicKey, ) -> Self { Self { diff --git a/crates/types/src/vote.rs b/crates/types/src/vote.rs index 9e232b755a..daf244d7aa 100644 --- a/crates/types/src/vote.rs +++ b/crates/types/src/vote.rs @@ -50,13 +50,13 @@ pub struct DAVote { /// The signature share associated with this vote pub signature: (EncodedPublicKey, EncodedSignature), /// The block commitment being voted on. - pub block_commitment: Commitment, + pub block_commitment: Commitment, /// The view this vote was cast for pub current_view: TYPES::Time, /// The vote token generated by this replica pub vote_token: TYPES::VoteTokenType, /// The vote data this vote is signed over - pub vote_data: VoteData>, + pub vote_data: VoteData>, } /// A positive or negative vote on validating or commitment proposal. @@ -196,7 +196,7 @@ pub enum QuorumVote { Timeout(TimeoutVote), } -impl VoteType> for DAVote { +impl VoteType> for DAVote { fn get_view(&self) -> TYPES::Time { self.current_view } @@ -206,7 +206,7 @@ impl VoteType> for DAVote EncodedSignature { self.signature.1.clone() } - fn get_data(&self) -> VoteData> { + fn get_data(&self) -> VoteData> { self.vote_data.clone() } fn get_vote_token(&self) -> ::VoteTokenType { diff --git a/testing-macros/src/lib.rs b/testing-macros/src/lib.rs index 3f720691e2..bbe332f1a2 100644 --- a/testing-macros/src/lib.rs +++ b/testing-macros/src/lib.rs @@ -308,9 +308,9 @@ impl TestData { impl hotshot_types::traits::node_implementation::NodeType for TestTypes { type ConsensusType = #consensus_type; type Time = #time_type; - type BlockType = <#demo_state as hotshot_types::traits::State>::BlockType; + type BlockPayload = <#demo_state as hotshot_types::traits::State>::BlockPayload; type SignatureKey = #signature_key_type; - type Transaction = <<#demo_state as hotshot_types::traits::State>::BlockType as hotshot_types::traits::BlockPayload>::Transaction; + type Transaction = <<#demo_state as hotshot_types::traits::State>::BlockPayload as hotshot_types::traits::BlockPayload>::Transaction; type StateType = #demo_state; type VoteTokenType = hotshot::traits::election::static_committee::StaticVoteToken; type ElectionConfigType = hotshot::traits::election::static_committee::StaticElectionConfig; From 90c5800d948bccd3b6b7e1c1731d8c4c3f4fdc02 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Mon, 23 Oct 2023 17:50:09 -0700 Subject: [PATCH 02/13] Fix build --- crates/hotshot/src/demo.rs | 30 ++----------- crates/hotshot/src/lib.rs | 8 +++- crates/hotshot/src/tasks/mod.rs | 8 ++-- .../src/traits/storage/memory_storage.rs | 1 + crates/task-impls/src/consensus.rs | 43 +++++++++---------- crates/task-impls/src/da.rs | 7 ++- crates/task-impls/src/events.rs | 4 +- crates/testing/src/node_types.rs | 3 +- crates/testing/src/task_helpers.rs | 13 +++--- crates/testing/tests/consensus_task.rs | 4 +- crates/testing/tests/da_task.rs | 5 ++- crates/testing/tests/memory_network.rs | 5 ++- crates/testing/tests/network_task.rs | 5 ++- crates/types/src/block_impl.rs | 28 ++++++++---- crates/types/src/data.rs | 7 +-- crates/types/src/traits/block_contents.rs | 21 +++++++-- crates/types/src/traits/election.rs | 6 ++- .../types/src/traits/node_implementation.rs | 4 +- testing-macros/src/lib.rs | 2 +- 19 files changed, 112 insertions(+), 92 deletions(-) diff --git a/crates/hotshot/src/demo.rs b/crates/hotshot/src/demo.rs index 33ac15dd28..0d1b3993ad 100644 --- a/crates/hotshot/src/demo.rs +++ b/crates/hotshot/src/demo.rs @@ -8,15 +8,11 @@ use crate::traits::election::static_committee::{StaticElectionConfig, StaticVoteToken}; use commit::{Commitment, Committable}; use derivative::Derivative; -use either::Either; use hotshot_signature_key::bn254::BLSPubKey; use hotshot_types::{ - block_impl::{BlockPayloadError, VIDBlockPayload, VIDTransaction}, + block_impl::{BlockPayloadError, VIDBlockHeader, VIDBlockPayload, VIDTransaction}, certificate::{AssembledSignature, QuorumCertificate}, - data::{ - fake_commitment, genesis_proposer_id, random_commitment, LeafType, SequencingLeaf, - ViewNumber, - }, + data::{fake_commitment, random_commitment, LeafType, ViewNumber}, traits::{ election::Membership, node_implementation::NodeType, @@ -126,6 +122,7 @@ pub struct DemoTypes; impl NodeType for DemoTypes { type Time = ViewNumber; + type BlockHeader = VIDBlockHeader; type BlockPayload = VIDBlockPayload; type SignatureKey = BLSPubKey; type VoteTokenType = StaticVoteToken; @@ -184,24 +181,3 @@ pub fn random_quorum_certificate( - deltas: Either>, - rng: &mut dyn rand::RngCore, -) -> SequencingLeaf { - let justify_qc = random_quorum_certificate(rng); - // let state = TYPES::StateType::default() - // .append(&deltas, &TYPES::Time::new(42)) - // .unwrap_or_default(); - SequencingLeaf { - view_number: justify_qc.view_number, - height: rng.next_u64(), - justify_qc, - parent_commitment: random_commitment(rng), - deltas, - rejected: Vec::new(), - timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), - proposer_id: genesis_proposer_id(), - } -} diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 2281a1af08..444538c3ad 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -55,7 +55,7 @@ use hotshot_task::{ use hotshot_task_impls::{events::SequencingHotShotEvent, network::NetworkTaskKind}; use hotshot_types::{ - block_impl::{VIDBlockPayload, VIDTransaction}, + block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction}, certificate::{DACertificate, ViewSyncCertificate}, consensus::{BlockStore, Consensus, ConsensusMetrics, View, ViewInner, ViewQueue}, data::{DAProposal, DeltasType, LeafType, QuorumProposal, SequencingLeaf}, @@ -634,7 +634,11 @@ pub trait HotShotType> { #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType< + BlockHeader = VIDBlockHeader, + BlockPayload = VIDBlockPayload, + Transaction = VIDTransaction, + >, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 1a0fc8ad4e..77e230ec55 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -5,7 +5,7 @@ use crate::{ QuorumCertificate, SequencingQuorumEx, }; use async_compatibility_layer::art::async_sleep; -use commit::{Commitment, CommitmentBounds}; +use commit::{Commitment, CommitmentBounds, Committable}; use futures::FutureExt; use hotshot_task::{ boxed_sync, @@ -27,7 +27,7 @@ use hotshot_task_impls::{ view_sync::{ViewSyncTaskState, ViewSyncTaskStateTypes}, }; use hotshot_types::{ - block_impl::{VIDBlockPayload, VIDTransaction}, + block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction}, certificate::ViewSyncCertificate, data::{ProposalType, QuorumProposal, SequencingLeaf}, event::Event, @@ -250,7 +250,7 @@ where /// # Panics /// Is unable to panic. This section here is just to satisfy clippy pub async fn add_consensus_task< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -288,7 +288,7 @@ where consensus, timeout: handle.hotshot.inner.config.next_view_timeout, cur_view: TYPES::Time::new(0), - block: VIDBlockPayload::genesis(), + block_commitment: VIDBlockPayload::genesis().commit(), quorum_exchange: c_api.inner.exchanges.quorum_exchange().clone().into(), api: c_api.clone(), committee_exchange: c_api.inner.exchanges.committee_exchange().clone().into(), diff --git a/crates/hotshot/src/traits/storage/memory_storage.rs b/crates/hotshot/src/traits/storage/memory_storage.rs index 73d86fc1d9..c57363bca7 100644 --- a/crates/hotshot/src/traits/storage/memory_storage.rs +++ b/crates/hotshot/src/traits/storage/memory_storage.rs @@ -148,6 +148,7 @@ mod test { impl NodeType for DummyTypes { type Time = ViewNumber; + type BlockHeader = DummyBlock; type BlockPayload = DummyBlock; type SignatureKey = BLSPubKey; type VoteTokenType = StaticVoteToken; diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index 1fb8d383e8..581348e8b0 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -15,7 +15,7 @@ use hotshot_task::{ task::{FilterEvent, HandleEvent, HotShotTaskCompleted, HotShotTaskTypes, TS}, task_impls::{HSTWithEvent, TaskBuilder}, }; -use hotshot_types::vote::QuorumVoteAccumulator; +use hotshot_types::{block_impl::VIDBlockPayload, vote::QuorumVoteAccumulator}; use hotshot_types::{ certificate::{DACertificate, QuorumCertificate}, consensus::{Consensus, View}, @@ -23,6 +23,7 @@ use hotshot_types::{ event::{Event, EventType}, message::{GeneralConsensusMessage, Message, Proposal, SequencingMessage}, traits::{ + block_contents::BlockHeader, consensus_api::SequencingConsensusApi, election::{ConsensusExchange, QuorumExchangeType, SignedCertificate}, network::{CommunicationChannel, ConsensusIntentEvent}, @@ -83,8 +84,8 @@ pub struct SequencingConsensusTaskState< /// View number this view is executing in. pub cur_view: TYPES::Time, - /// Current block submitted to DA - pub block: TYPES::BlockPayload, + /// The commitment to the current block submitted to DA + pub block_commitment: Commitment, /// the quorum exchange pub quorum_exchange: Arc>, @@ -259,7 +260,7 @@ where } impl< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -348,10 +349,10 @@ where let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.height, + height: proposal.block_header.block_number, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_commitment), + deltas: Right(self.block_commitment), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -414,10 +415,10 @@ where let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.height, + height: proposal.block_header.block_number, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_commitment), + deltas: Right(proposal.block_header.commitment), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -426,7 +427,7 @@ where // Validate the DAC. if self .committee_exchange - .is_valid_cert(cert, proposal.block_commitment) + .is_valid_cert(cert, proposal.block_header.commitment) { self.quorum_exchange.create_yes_message( proposal.justify_qc.commit(), @@ -605,10 +606,10 @@ where let parent_commitment = parent.commit(); let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.data.height, + height: proposal.data.block_header.block_number, justify_qc: justify_qc.clone(), parent_commitment, - deltas: Right(proposal.data.block_commitment), + deltas: Right(proposal.data.block_header.commitment), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -1076,9 +1077,9 @@ where *view ); } - SequencingHotShotEvent::SendDABlockData(block) => { + SequencingHotShotEvent::SendBlockCommitment(block_commitment) => { // ED TODO Should make sure this is actually the most recent block - self.block = block; + self.block_commitment = block_commitment; } _ => {} } @@ -1150,16 +1151,12 @@ where // TODO do some sort of sanity check on the view number that it matches decided } - let block_commitment = self.block.commit(); - let leaf = SequencingLeaf { view_number: view, height: parent_leaf.height + 1, justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), - // Use the block commitment rather than the block, so that the replica can construct - // the same leaf with the commitment. - deltas: Right(block_commitment), + deltas: Right(self.block_commitment), rejected: vec![], timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.api.public_key().to_bytes(), @@ -1170,9 +1167,11 @@ where .sign_validating_or_commitment_proposal::(&leaf.commit()); // TODO: DA cert is sent as part of the proposal here, we should split this out so we don't have to wait for it. let proposal = QuorumProposal { - block_commitment, + block_header: VIDBlockHeader { + block_number: parent_leaf.height + 1, + commitment: self.block_commitment, + }, view_number: leaf.view_number, - height: leaf.height, justify_qc: consensus.high_qc.clone(), // TODO ED Update this to be the actual TC if there is one timeout_certificate: None, @@ -1240,7 +1239,7 @@ pub type ConsensusTaskTypes = HSTWithEvent< /// Event handle for consensus pub async fn sequencing_consensus_handle< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -1289,7 +1288,7 @@ pub fn consensus_event_filter>( | SequencingHotShotEvent::DACRecv(_) | SequencingHotShotEvent::VidCertRecv(_) | SequencingHotShotEvent::ViewChange(_) - | SequencingHotShotEvent::SendDABlockData(_) + | SequencingHotShotEvent::SendBlockCommitment(_) | SequencingHotShotEvent::Timeout(_) | SequencingHotShotEvent::Shutdown, ) diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index fbe156aec6..83d27874ba 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -628,7 +628,8 @@ where .inject_consensus_info(ConsensusIntentEvent::CancelPollForTransactions(*view)) .await; - let signature = self.committee_exchange.sign_da_proposal(&block.commit()); + let block_commitment = block.commit(); + let signature = self.committee_exchange.sign_da_proposal(&block_commitment); let data: DAProposal = DAProposal { deltas: block.clone(), // Upon entering a new view we want to send a DA Proposal for the next view -> Is it always the case that this is cur_view + 1? @@ -644,7 +645,9 @@ where // TODO ED We should send an event to do this, but just getting it to work for now self.event_stream - .publish(SequencingHotShotEvent::SendDABlockData(block.clone())) + .publish(SequencingHotShotEvent::SendBlockCommitment( + block_commitment, + )) .await; self.event_stream diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs index fbb7e227b0..001bd60c12 100644 --- a/crates/task-impls/src/events.rs +++ b/crates/task-impls/src/events.rs @@ -60,8 +60,8 @@ pub enum SequencingHotShotEvent> { TransactionsRecv(Vec), /// Send transactions to the network TransactionSend(TYPES::Transaction, TYPES::SignatureKey), - /// Event to send DA block data from DA leader to next quorum leader (which should always be the same node); internal event only - SendDABlockData(TYPES::BlockPayload), + /// Event to send block commitment from DA leader to the quorum; internal event only + SendBlockCommitment(Commitment), /// Event when the transactions task has a block formed BlockReady(TYPES::BlockPayload, TYPES::Time), /// Event when consensus decided on a leaf diff --git a/crates/testing/src/node_types.rs b/crates/testing/src/node_types.rs index 3cfb013851..2970819d42 100644 --- a/crates/testing/src/node_types.rs +++ b/crates/testing/src/node_types.rs @@ -14,7 +14,7 @@ use hotshot::{ types::bn254::BLSPubKey, }; use hotshot_types::{ - block_impl::{VIDBlockPayload, VIDTransaction}, + block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction}, certificate::ViewSyncCertificate, data::{QuorumProposal, SequencingLeaf, ViewNumber}, message::{Message, SequencingMessage}, @@ -42,6 +42,7 @@ use serde::{Deserialize, Serialize}; pub struct SequencingTestTypes; impl NodeType for SequencingTestTypes { type Time = ViewNumber; + type BlockHeader = VIDBlockHeader; type BlockPayload = VIDBlockPayload; type SignatureKey = BLSPubKey; type VoteTokenType = StaticVoteToken; diff --git a/crates/testing/src/task_helpers.rs b/crates/testing/src/task_helpers.rs index f4e7fe49a8..8d41a8e11f 100644 --- a/crates/testing/src/task_helpers.rs +++ b/crates/testing/src/task_helpers.rs @@ -13,7 +13,7 @@ use hotshot::{ use hotshot_task::event_stream::ChannelStream; use hotshot_task_impls::events::SequencingHotShotEvent; use hotshot_types::{ - block_impl::{VIDBlockPayload, NUM_CHUNKS, NUM_STORAGE_NODES}, + block_impl::{VIDBlockHeader, VIDBlockPayload, NUM_CHUNKS, NUM_STORAGE_NODES}, data::{QuorumProposal, SequencingLeaf, VidScheme, ViewNumber}, message::{Message, Proposal}, traits::{ @@ -118,23 +118,24 @@ async fn build_quorum_proposal_and_signature( // every event input is seen on the event stream in the output. let block = ::genesis(); + let block_commitment = block.commit(); let leaf = SequencingLeaf { view_number: ViewNumber::new(view), height: parent_leaf.height + 1, justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), - // Use the block commitment rather than the block, so that the replica can construct - // the same leaf with the commitment. - deltas: Right(block.commit()), + deltas: Right(block_commitment), rejected: vec![], timestamp: 0, proposer_id: api.public_key().to_bytes(), }; let signature = ::sign(private_key, leaf.commit().as_ref()); let proposal = QuorumProposal::> { - block_commitment: block.commit(), + block_header: VIDBlockHeader { + block_number: 1, + commitment: block_commitment, + }, view_number: ViewNumber::new(view), - height: 1, justify_qc: QuorumCertificate::genesis(), timeout_certificate: None, proposer_id: leaf.proposer_id, diff --git a/crates/testing/tests/consensus_task.rs b/crates/testing/tests/consensus_task.rs index ec1bd7b05c..8c4bd866e9 100644 --- a/crates/testing/tests/consensus_task.rs +++ b/crates/testing/tests/consensus_task.rs @@ -62,10 +62,10 @@ async fn build_vote( let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.height, + height: proposal.block_header.block_number, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_commitment), + deltas: Right(proposal.block_header.commitment), rejected: Vec::new(), timestamp: 0, proposer_id: quorum_exchange.get_leader(view).to_bytes(), diff --git a/crates/testing/tests/da_task.rs b/crates/testing/tests/da_task.rs index e31c34e6aa..4f43f6beb4 100644 --- a/crates/testing/tests/da_task.rs +++ b/crates/testing/tests/da_task.rs @@ -95,7 +95,10 @@ async fn test_da_task() { SequencingHotShotEvent::BlockReady(block.clone(), ViewNumber::new(2)), 1, ); - output.insert(SequencingHotShotEvent::SendDABlockData(block.clone()), 1); + output.insert( + SequencingHotShotEvent::SendBlockCommitment(block.commit()), + 1, + ); output.insert( SequencingHotShotEvent::DAProposalSend(message.clone(), pub_key), 1, diff --git a/crates/testing/tests/memory_network.rs b/crates/testing/tests/memory_network.rs index 27f6ae6531..31f25f3dbb 100644 --- a/crates/testing/tests/memory_network.rs +++ b/crates/testing/tests/memory_network.rs @@ -13,7 +13,7 @@ use hotshot::traits::implementations::{ use hotshot::traits::NodeImplementation; use hotshot::types::bn254::{BLSPrivKey, BLSPubKey}; use hotshot::types::SignatureKey; -use hotshot_types::block_impl::{VIDBlockPayload, VIDTransaction}; +use hotshot_types::block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction}; use hotshot_types::certificate::ViewSyncCertificate; use hotshot_types::data::{DAProposal, QuorumProposal, SequencingLeaf}; use hotshot_types::message::{Message, SequencingMessage}; @@ -52,7 +52,8 @@ pub struct Test; impl NodeType for Test { type Time = ViewNumber; - type BlockType = VIDBlockPayload; + type BlockHeader = VIDBlockHeader; + type BlockPayload = VIDBlockPayload; type SignatureKey = BLSPubKey; type VoteTokenType = StaticVoteToken; type Transaction = VIDTransaction; diff --git a/crates/testing/tests/network_task.rs b/crates/testing/tests/network_task.rs index c4165a8c31..fe50497132 100644 --- a/crates/testing/tests/network_task.rs +++ b/crates/testing/tests/network_task.rs @@ -124,7 +124,10 @@ async fn test_network_task() { SequencingHotShotEvent::QuorumProposalSend(quorum_proposal.clone(), pub_key), 1, ); - output.insert(SequencingHotShotEvent::SendDABlockData(block), 1); + output.insert( + SequencingHotShotEvent::SendBlockCommitment(block.commit()), + 1, + ); output.insert( SequencingHotShotEvent::DAProposalRecv(da_proposal, pub_key), 1, diff --git a/crates/types/src/block_impl.rs b/crates/types/src/block_impl.rs index 2e9f75cd09..6501732dd4 100644 --- a/crates/types/src/block_impl.rs +++ b/crates/types/src/block_impl.rs @@ -5,8 +5,12 @@ use std::{ }; use crate::{ - data::{test_srs, VidScheme, VidSchemeTrait, SequencingLeaf}, - traits::{block_contents::Transaction, state::TestableBlock, BlockPayload}, + data::{test_srs, VidScheme, VidSchemeTrait}, + traits::{ + block_contents::{BlockHeader, Transaction}, + state::TestableBlock, + BlockPayload, + }, }; use commit::{Commitment, Committable}; use serde::{Deserialize, Serialize}; @@ -127,10 +131,18 @@ impl BlockPayload for VIDBlockPayload { } /// A [`BlockHeader`] that commits to [`VIDBlockPayload`]. -#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] -pub struct VIDBlockHeader { - /// Previous leaf. - pub previous_leaf: SequencingLeaf, +#[derive(PartialEq, Eq, Hash, Clone, Debug, Deserialize, Serialize)] +pub struct VIDBlockHeader { + /// Block number. + pub block_number: u64, /// VID commitment. - pub commitment: ::Commit, -} \ No newline at end of file + pub commitment: Commitment, +} + +impl BlockHeader for VIDBlockHeader { + type Payload = VIDBlockPayload; + + fn commitment(&self) -> Commitment { + self.commitment + } +} diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 66c5ebfe31..50550d685e 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -202,15 +202,12 @@ pub fn test_srs( #[derive(custom_debug::Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] #[serde(bound(deserialize = ""))] pub struct QuorumProposal> { - /// The commitment to append. - pub block_commitment: Commitment, + /// The block header to append + pub block_header: TYPES::BlockHeader, /// CurView from leader when proposing leaf pub view_number: TYPES::Time, - /// Height from leader when proposing leaf - pub height: u64, - /// Per spec, justification pub justify_qc: QuorumCertificate>, diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index ab851745d4..9d8fd8196f 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -50,24 +50,27 @@ pub trait BlockPayload: /// The type of the transitions we are applying type Transaction: Transaction; + // type Header: BlockHeader; + /// returns hashes of all the transactions in this block /// TODO make this ordered with a vec fn contained_transactions(&self) -> HashSet>; } +/// Header of a block, which commits to a [`BlockPayload`]. pub trait BlockHeader: Serialize + Clone + Debug - + Display + // + Display + Hash + PartialEq + Eq + Send + Sync - + Committable + DeserializeOwned { + /// Block payload associated with the commitment. type Payload: BlockPayload; /// Get the payload commitment. @@ -78,7 +81,9 @@ pub trait BlockHeader: pub mod dummy { use std::fmt::Display; - use super::{BlockPayload, Commitment, Committable, Debug, Hash, HashSet, Serialize}; + use super::{ + BlockHeader, BlockPayload, Commitment, Committable, Debug, Hash, HashSet, Serialize, + }; use rand::Rng; use serde::Deserialize; @@ -139,6 +144,16 @@ pub mod dummy { } } + impl BlockHeader for DummyBlock { + type Payload = Self; + + fn commitment(&self) -> commit::Commitment { + commit::RawCommitmentBuilder::new("Dummy BlockPayload Comm") + .u64_field("Nonce", self.nonce) + .finalize() + } + } + impl BlockPayload for DummyBlock { type Error = DummyError; diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 7d9c0d0c2d..ceef4f1f8c 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -534,8 +534,10 @@ pub trait CommitteeExchangeType: ConsensusExchange { /// Sign a DA proposal. - fn sign_da_proposal(&self, block_commitment: &Commitment) - -> EncodedSignature; + fn sign_da_proposal( + &self, + block_commitment: &Commitment, + ) -> EncodedSignature; /// Sign a vote on DA proposal. /// diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index a48f3152d3..df322948d2 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -4,7 +4,7 @@ //! describing the overall behavior of a node, as a composition of implementations of the node trait. use super::{ - block_contents::Transaction, + block_contents::{BlockHeader, Transaction}, election::{ CommitteeExchangeType, ConsensusExchange, ElectionConfig, QuorumExchangeType, ViewSyncExchangeType, VoteToken, @@ -546,6 +546,8 @@ pub trait NodeType: /// /// This should be the same `Time` that `StateType::Time` is using. type Time: ConsensusTime; + /// The block header type that this hotshot setup is using. + type BlockHeader: BlockHeader; /// The block type that this hotshot setup is using. /// /// This should be the same block that `StateType::BlockPayload` is using. diff --git a/testing-macros/src/lib.rs b/testing-macros/src/lib.rs index bbe332f1a2..a9494e03a0 100644 --- a/testing-macros/src/lib.rs +++ b/testing-macros/src/lib.rs @@ -308,7 +308,7 @@ impl TestData { impl hotshot_types::traits::node_implementation::NodeType for TestTypes { type ConsensusType = #consensus_type; type Time = #time_type; - type BlockPayload = <#demo_state as hotshot_types::traits::State>::BlockPayload; + type BlockHeader = <#demo_state as hotshot_types::traits::State>::BlockHeader; type SignatureKey = #signature_key_type; type Transaction = <<#demo_state as hotshot_types::traits::State>::BlockPayload as hotshot_types::traits::BlockPayload>::Transaction; type StateType = #demo_state; From 829c2528ba58a2223e887d4ab2173a3d1f8ffb6e Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Tue, 24 Oct 2023 09:21:03 -0700 Subject: [PATCH 03/13] Fix build after merge --- crates/hotshot/src/lib.rs | 2 +- crates/hotshot/src/tasks/mod.rs | 2 +- crates/task-impls/src/consensus.rs | 28 +++++++++++----------------- crates/task-impls/src/vid.rs | 14 +++++++------- crates/types/src/certificate.rs | 8 ++++---- crates/types/src/traits/election.rs | 8 ++++---- crates/types/src/vote.rs | 8 ++++---- 7 files changed, 32 insertions(+), 38 deletions(-) diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 7490527571..a970c4f72b 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -670,7 +670,7 @@ where Message, Proposal = VidDisperse, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, Membership = MEMBERSHIP, > + 'static, SequencingTimeoutEx: ConsensusExchange< diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index ffa5eb132e..0420e793ab 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -385,7 +385,7 @@ where TYPES, Message, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, >, { // build the vid task diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index 022a26dce4..d670c7f097 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -16,14 +16,13 @@ use hotshot_task::{ task_impls::{HSTWithEvent, TaskBuilder}, }; use hotshot_types::{ - block_impl::VIDBlockPayload + block_impl::{VIDBlockHeader, VIDBlockPayload}, certificate::{DACertificate, QuorumCertificate, TimeoutCertificate, VIDCertificate}, consensus::{Consensus, View}, data::{LeafType, ProposalType, QuorumProposal, SequencingLeaf}, event::{Event, EventType}, message::{GeneralConsensusMessage, Message, Proposal, SequencingMessage}, traits::{ - block_contents::BlockHeader, consensus_api::SequencingConsensusApi, election::{ConsensusExchange, QuorumExchangeType, SignedCertificate, TimeoutExchangeType}, network::{CommunicationChannel, ConsensusIntentEvent}, @@ -461,7 +460,7 @@ where height: proposal.block_header.block_number, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(self.block_commitment), + deltas: Right(proposal.block_header.commitment), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -543,7 +542,7 @@ where .is_valid_cert(cert) { // Validate the block commitment for non-genesis DAC. - if !cert.is_genesis() && cert.leaf_commitment() != proposal.block_commitment { + if !cert.is_genesis() && cert.leaf_commitment() != proposal.block_header.commitment { error!("Block commitment does not equal parent commitment"); return false; } @@ -745,10 +744,10 @@ where ); let leaf = SequencingLeaf { view_number: view, - height: proposal.data.height, + height: proposal.data.block_header.block_number, justify_qc: justify_qc.clone(), parent_commitment: justify_qc.leaf_commitment(), - deltas: Right(proposal.data.block_commitment), + deltas: Right(proposal.data.block_header.commitment), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -770,10 +769,10 @@ where let parent_commitment = parent.commit(); let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.data.height, + height: proposal.data.block_header.block_number, justify_qc: justify_qc.clone(), parent_commitment, - deltas: Right(proposal.data.block_commitment), + deltas: Right(proposal.data.block_header.commitment), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -1399,10 +1398,7 @@ where // TODO do some sort of sanity check on the view number that it matches decided } - // let block_commitment = Some(self.block.commit()); - if let Some(block) = &self.block { - let block_commitment = block.commit(); - + if let Some(block_commitment) = &self.block_commitment { let leaf = SequencingLeaf { view_number: view, height: parent_leaf.height + 1, @@ -1410,7 +1406,7 @@ where parent_commitment: parent_leaf.commit(), // Use the block commitment rather than the block, so that the replica can construct // the same leaf with the commitment. - deltas: Right(block_commitment), + deltas: Right(*block_commitment), rejected: vec![], timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.api.public_key().to_bytes(), @@ -1423,9 +1419,8 @@ where let proposal = QuorumProposal { block_header: VIDBlockHeader { block_number: leaf.height, - commitment: block_commitment, + commitment: *block_commitment, }, - block_commitment, view_number: leaf.view_number, justify_qc: consensus.high_qc.clone(), timeout_certificate: timeout_certificate.or_else(|| None), @@ -1441,14 +1436,13 @@ where "Sending proposal for view {:?} \n {:?}", leaf.view_number, "" ); ->>>>>>> develop self.event_stream .publish(SequencingHotShotEvent::QuorumProposalSend( message, self.quorum_exchange.public_key().clone(), )) .await; - self.block = None; + self.block_commitment = None; return true; } debug!("Self block was None"); diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs index bde44d7781..cf3589192d 100644 --- a/crates/task-impls/src/vid.rs +++ b/crates/task-impls/src/vid.rs @@ -53,7 +53,7 @@ pub struct VIDTaskState< TYPES, Message, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// The state's api @@ -89,7 +89,7 @@ pub struct VIDVoteCollectionTaskState< TYPES, Message, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// the vid exchange @@ -101,7 +101,7 @@ pub struct VIDVoteCollectionTaskState< TYPES, TYPES::Time, TYPES::VoteTokenType, - Commitment, + Commitment, >>::VoteAccumulator, VIDCertificate, >, @@ -120,7 +120,7 @@ where TYPES, Message, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, >, { } @@ -140,7 +140,7 @@ where TYPES, Message, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, >, { match event { @@ -199,7 +199,7 @@ where TYPES, Message, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, >, { /// main task event handler @@ -415,7 +415,7 @@ where TYPES, Message, Certificate = VIDCertificate, - Commitment = Commitment, + Commitment = Commitment, >, { } diff --git a/crates/types/src/certificate.rs b/crates/types/src/certificate.rs index 9f9c36b697..1c415ce5f8 100644 --- a/crates/types/src/certificate.rs +++ b/crates/types/src/certificate.rs @@ -51,7 +51,7 @@ pub struct VIDCertificate { pub view_number: TYPES::Time, /// committment to the block - pub block_commitment: Commitment, + pub block_commitment: Commitment, /// Assembled signature for certificate aggregation pub signatures: AssembledSignature, @@ -298,11 +298,11 @@ impl } impl - SignedCertificate> + SignedCertificate> for VIDCertificate { type Vote = VIDVote; - type VoteAccumulator = VIDVoteAccumulator, Self::Vote>; + type VoteAccumulator = VIDVoteAccumulator, Self::Vote>; fn create_certificate(signatures: AssembledSignature, vote: Self::Vote) -> Self { VIDCertificate { @@ -320,7 +320,7 @@ impl self.signatures.clone() } - fn leaf_commitment(&self) -> Commitment { + fn leaf_commitment(&self) -> Commitment { self.block_commitment } diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 5c41378edf..4718e2d033 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -595,7 +595,7 @@ impl< type Certificate = DACertificate; type Membership = MEMBERSHIP; type Networking = NETWORK; - type Commitment = Commitment; + type Commitment = Commitment; fn create( entries: Vec<::StakeTableEntry>, @@ -722,7 +722,7 @@ impl< /// Sign a VID proposal. fn sign_vid_proposal( &self, - block_commitment: &Commitment, + block_commitment: &Commitment, ) -> EncodedSignature { let signature = TYPES::SignatureKey::sign(&self.private_key, block_commitment.as_ref()); signature @@ -810,7 +810,7 @@ pub trait QuorumExchangeType, /// Sign a block commitment. fn sign_block_commitment( &self, - block_commitment: Commitment, + block_commitment: Commitment, ) -> EncodedSignature; /// Sign a positive vote on validating or commitment proposal. @@ -914,7 +914,7 @@ impl< fn sign_block_commitment( &self, - block_commitment: Commitment<::BlockType>, + block_commitment: Commitment<::BlockPayload>, ) -> EncodedSignature { TYPES::SignatureKey::sign(&self.private_key, block_commitment.as_ref()) } diff --git a/crates/types/src/vote.rs b/crates/types/src/vote.rs index 6014ce7ea4..2b721819bb 100644 --- a/crates/types/src/vote.rs +++ b/crates/types/src/vote.rs @@ -66,13 +66,13 @@ pub struct VIDVote { /// The signature share associated with this vote pub signature: (EncodedPublicKey, EncodedSignature), /// The block commitment being voted on. - pub block_commitment: Commitment, + pub block_commitment: Commitment, /// The view this vote was cast for pub current_view: TYPES::Time, /// The vote token generated by this replica pub vote_token: TYPES::VoteTokenType, /// The vote data this vote is signed over - pub vote_data: VoteData>, + pub vote_data: VoteData>, } /// A positive or negative vote on validating or commitment proposal. @@ -255,7 +255,7 @@ impl DAVote { } } -impl VoteType> for VIDVote { +impl VoteType> for VIDVote { fn get_view(&self) -> TYPES::Time { self.current_view } @@ -265,7 +265,7 @@ impl VoteType> for VIDVote< fn get_signature(&self) -> EncodedSignature { self.signature.1.clone() } - fn get_data(&self) -> VoteData> { + fn get_data(&self) -> VoteData> { self.vote_data.clone() } fn get_vote_token(&self) -> ::VoteTokenType { From 057f514d1825c5a6e716d87d22047e4a4f169904 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Tue, 24 Oct 2023 09:26:25 -0700 Subject: [PATCH 04/13] Fix clippy --- crates/hotshot/examples/infra/modDA.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/hotshot/examples/infra/modDA.rs b/crates/hotshot/examples/infra/modDA.rs index 2a273f22e4..fdec0f8a43 100644 --- a/crates/hotshot/examples/infra/modDA.rs +++ b/crates/hotshot/examples/infra/modDA.rs @@ -21,7 +21,7 @@ use hotshot_orchestrator::{ config::{NetworkConfig, WebServerConfig}, }; use hotshot_task::task::FilterEvent; -use hotshot_types::traits::election::VIDExchange; +use hotshot_types::{block_impl::VIDBlockHeader, traits::election::VIDExchange}; use hotshot_types::{ block_impl::{VIDBlockPayload, VIDTransaction}, certificate::ViewSyncCertificate, @@ -408,7 +408,11 @@ pub struct WebServerDARun< #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType< + Transaction = VIDTransaction, + BlockPayload = VIDBlockPayload, + BlockHeader = VIDBlockHeader, + >, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -576,7 +580,11 @@ pub struct Libp2pDARun, MEMBERSHIP #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType< + Transaction = VIDTransaction, + BlockPayload = VIDBlockPayload, + BlockHeader = VIDBlockHeader, + >, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -805,7 +813,11 @@ where /// Main entry point for validators pub async fn main_entry_point< - TYPES: NodeType, + TYPES: NodeType< + Transaction = VIDTransaction, + BlockPayload = VIDBlockPayload, + BlockHeader = VIDBlockHeader, + >, MEMBERSHIP: Membership + Debug, DANETWORK: CommunicationChannel, MEMBERSHIP> + Debug, QUORUMNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, From bbf2656508cf33f49b19992d62bf2bcfd8947c97 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Tue, 24 Oct 2023 11:13:28 -0700 Subject: [PATCH 05/13] Remove a commented-out line --- crates/types/src/traits/block_contents.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index 9d8fd8196f..59fd565140 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -59,16 +59,7 @@ pub trait BlockPayload: /// Header of a block, which commits to a [`BlockPayload`]. pub trait BlockHeader: - Serialize - + Clone - + Debug - // + Display - + Hash - + PartialEq - + Eq - + Send - + Sync - + DeserializeOwned + Serialize + Clone + Debug + Hash + PartialEq + Eq + Send + Sync + DeserializeOwned { /// Block payload associated with the commitment. type Payload: BlockPayload; From 286e52b3c79cacfbb602079c1caa2a9632097e4a Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Thu, 26 Oct 2023 11:28:50 -0700 Subject: [PATCH 06/13] Address comments --- crates/hotshot/src/demo.rs | 15 +- crates/hotshot/src/lib.rs | 10 +- crates/hotshot/src/tasks/mod.rs | 6 +- .../atomic_storage/dual_key_value_store.rs | 6 +- .../src/traits/storage/memory_storage.rs | 1 - crates/task-impls/src/consensus.rs | 83 +++++----- crates/task-impls/src/da.rs | 22 +-- crates/task-impls/src/events.rs | 4 +- crates/task-impls/src/transactions.rs | 8 +- crates/task-impls/src/vid.rs | 12 +- crates/testing/src/overall_safety_task.rs | 8 +- crates/testing/src/task_helpers.rs | 21 ++- crates/testing/tests/consensus_task.rs | 3 +- crates/testing/tests/da_task.rs | 8 +- crates/testing/tests/network_task.rs | 10 +- crates/testing/tests/vid_task.rs | 6 +- crates/types/src/block_impl.rs | 34 +++- crates/types/src/certificate.rs | 16 +- crates/types/src/consensus.rs | 4 +- crates/types/src/data.rs | 156 +++++++++++------- crates/types/src/traits/block_contents.rs | 25 ++- crates/types/src/traits/election.rs | 60 +++---- crates/types/src/traits/state.rs | 35 +++- crates/types/src/utils.rs | 8 +- crates/types/src/vote.rs | 8 +- 25 files changed, 332 insertions(+), 237 deletions(-) diff --git a/crates/hotshot/src/demo.rs b/crates/hotshot/src/demo.rs index 0d1b3993ad..2a86f960b6 100644 --- a/crates/hotshot/src/demo.rs +++ b/crates/hotshot/src/demo.rs @@ -62,11 +62,13 @@ impl Default for SDemoState { impl State for SDemoState { type Error = BlockPayloadError; + type BlockHeader = VIDBlockHeader; + type BlockPayload = VIDBlockPayload; type Time = ViewNumber; - fn validate_block(&self, _block: &Self::BlockPayload, view_number: &Self::Time) -> bool { + fn validate_block(&self, _block_header: &Self::BlockHeader, view_number: &Self::Time) -> bool { if view_number == &ViewNumber::genesis() { &self.view_number == view_number } else { @@ -74,12 +76,18 @@ impl State for SDemoState { } } + fn initialize() -> Self { + let mut state = Self::default(); + state.block_height += 1; + state + } + fn append( &self, - block: &Self::BlockPayload, + block_header: &Self::BlockHeader, view_number: &Self::Time, ) -> Result { - if !self.validate_block(block, view_number) { + if !self.validate_block(block_header, view_number) { return Err(BlockPayloadError::InvalidBlock); } @@ -174,7 +182,6 @@ pub fn random_quorum_certificate QuorumCertificate> { QuorumCertificate { - // block_commitment: random_commitment(rng), leaf_commitment: random_commitment(rng), view_number: TYPES::Time::new(rng.gen()), signatures: AssembledSignature::Genesis(), diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index a970c4f72b..67d94b02e7 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -1087,17 +1087,13 @@ impl> HotShotInitializer Result> { - let state = TYPES::StateType::default() - .append(&genesis_block, &TYPES::Time::new(0)) - .map_err(|err| HotShotError::Misc { - context: err.to_string(), - })?; + pub fn from_genesis(genesis_payload: TYPES::BlockPayload) -> Result> { + let state = TYPES::StateType::initialize(); let time = TYPES::Time::genesis(); let justify_qc = QuorumCertificate::>::genesis(); Ok(Self { - inner: LEAF::new(time, justify_qc, genesis_block, state), + inner: LEAF::new(time, justify_qc, genesis_payload, state), }) } diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 0420e793ab..37fc59cd52 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -28,7 +28,7 @@ use hotshot_task_impls::{ view_sync::{ViewSyncTaskState, ViewSyncTaskStateTypes}, }; use hotshot_types::{ - block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction}, + block_impl::{VIDBlockPayload, VIDTransaction}, certificate::{TimeoutCertificate, VIDCertificate, ViewSyncCertificate}, data::{ProposalType, QuorumProposal, SequencingLeaf}, event::Event, @@ -252,7 +252,7 @@ where /// # Panics /// Is unable to panic. This section here is just to satisfy clippy pub async fn add_consensus_task< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, @@ -297,7 +297,7 @@ where consensus, timeout: handle.hotshot.inner.config.next_view_timeout, cur_view: TYPES::Time::new(0), - block_commitment: Some(VIDBlockPayload::genesis().commit()), + payload_commitment: Some(VIDBlockPayload::genesis().commit()), quorum_exchange: c_api.inner.exchanges.quorum_exchange().clone().into(), timeout_exchange: c_api.inner.exchanges.timeout_exchange().clone().into(), api: c_api.clone(), diff --git a/crates/hotshot/src/traits/storage/atomic_storage/dual_key_value_store.rs b/crates/hotshot/src/traits/storage/atomic_storage/dual_key_value_store.rs index 84cf8c76a2..c890099a1d 100644 --- a/crates/hotshot/src/traits/storage/atomic_storage/dual_key_value_store.rs +++ b/crates/hotshot/src/traits/storage/atomic_storage/dual_key_value_store.rs @@ -184,11 +184,11 @@ impl DualKeyValue for QuorumCertificate { type Key1 = Commitment; type Key2 = ViewNumber; - const KEY_1_NAME: &'static str = "block_commitment"; + const KEY_1_NAME: &'static str = "payload_commitment"; const KEY_2_NAME: &'static str = "view_number"; fn key_1(&self) -> Self::Key1 { - self.block_commitment + self.payload_commitment } fn key_2(&self) -> Self::Key2 { self.view_number @@ -203,7 +203,7 @@ where type Key2 = Commitment; const KEY_1_NAME: &'static str = "leaf_commitment"; - const KEY_2_NAME: &'static str = "block_commitment"; + const KEY_2_NAME: &'static str = "payload_commitment"; fn key_1(&self) -> Self::Key1 { self.commit() diff --git a/crates/hotshot/src/traits/storage/memory_storage.rs b/crates/hotshot/src/traits/storage/memory_storage.rs index c57363bca7..45c02bf6fe 100644 --- a/crates/hotshot/src/traits/storage/memory_storage.rs +++ b/crates/hotshot/src/traits/storage/memory_storage.rs @@ -167,7 +167,6 @@ mod test { let dummy_leaf_commit = fake_commitment::>(); StoredView::from_qc_block_and_state( QuorumCertificate { - // block_commitment: dummy_block_commit, is_genesis: view_number == ::Time::genesis(), leaf_commitment: dummy_leaf_commit, signatures: AssembledSignature::Genesis(), diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index d670c7f097..08e75c6323 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -16,13 +16,14 @@ use hotshot_task::{ task_impls::{HSTWithEvent, TaskBuilder}, }; use hotshot_types::{ - block_impl::{VIDBlockHeader, VIDBlockPayload}, + block_impl::VIDBlockPayload, certificate::{DACertificate, QuorumCertificate, TimeoutCertificate, VIDCertificate}, consensus::{Consensus, View}, data::{LeafType, ProposalType, QuorumProposal, SequencingLeaf}, event::{Event, EventType}, message::{GeneralConsensusMessage, Message, Proposal, SequencingMessage}, traits::{ + block_contents::BlockHeader, consensus_api::SequencingConsensusApi, election::{ConsensusExchange, QuorumExchangeType, SignedCertificate, TimeoutExchangeType}, network::{CommunicationChannel, ConsensusIntentEvent}, @@ -94,8 +95,8 @@ pub struct SequencingConsensusTaskState< /// View number this view is executing in. pub cur_view: TYPES::Time, - /// The commitment to the current block submitted to DA - pub block_commitment: Option>, + /// The commitment to the current block payload submitted to DA + pub payload_commitment: Option>, /// the quorum exchange pub quorum_exchange: Arc>, @@ -360,13 +361,14 @@ where } impl< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, ConsensusMessage = SequencingMessage, >, A: SequencingConsensusApi, I> + 'static, + H: BlockHeader, > SequencingConsensusTaskState where SequencingQuorumEx: ConsensusExchange< @@ -457,10 +459,9 @@ where let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.block_header.block_number, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_header.commitment), + deltas: Right(proposal.block_header.clone()), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -527,10 +528,9 @@ where let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.block_header.block_number, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_header.commitment), + deltas: Right(proposal.block_header.clone()), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -541,9 +541,9 @@ where .committee_exchange .is_valid_cert(cert) { - // Validate the block commitment for non-genesis DAC. - if !cert.is_genesis() && cert.leaf_commitment() != proposal.block_header.commitment { - error!("Block commitment does not equal parent commitment"); + // Validate the block payload commitment for non-genesis DAC. + if !cert.is_genesis() && cert.leaf_commitment() != proposal.block_header.payload_commitment() { + error!("Block payload commitment does not equal parent commitment"); return false; } self.quorum_exchange.create_yes_message( @@ -744,10 +744,9 @@ where ); let leaf = SequencingLeaf { view_number: view, - height: proposal.data.block_header.block_number, justify_qc: justify_qc.clone(), parent_commitment: justify_qc.leaf_commitment(), - deltas: Right(proposal.data.block_header.commitment), + deltas: Right(proposal.data.block_header), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -769,29 +768,17 @@ where let parent_commitment = parent.commit(); let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.data.block_header.block_number, justify_qc: justify_qc.clone(), parent_commitment, - deltas: Right(proposal.data.block_header.commitment), + deltas: Right(proposal.data.block_header), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), }; let leaf_commitment = leaf.commit(); - // Validate the `height` - // TODO Remove height from proposal validation; view number is sufficient - // https://github.com/EspressoSystems/HotShot/issues/1796 - if leaf.height != parent.height + 1 { - error!( - "Incorrect height in proposal (expected {}, got {})", - parent.height + 1, - leaf.height - ); - return; - } // Validate the signature. This should also catch if the leaf_commitment does not equal our calculated parent commitment - else if !view_leader_key.validate(&proposal.signature, leaf_commitment.as_ref()) { + if !view_leader_key.validate(&proposal.signature, leaf_commitment.as_ref()) { error!(?proposal.signature, "Could not verify proposal."); return; } @@ -869,7 +856,7 @@ where consensus .metrics .last_synced_block_height - .set(usize::try_from(leaf.height).unwrap_or(0)); + .set(usize::try_from(leaf.get_height()).unwrap_or(0)); // If the full block is available for this leaf, include it in the leaf // chain that we send to the client. @@ -884,7 +871,7 @@ where leaf_views.push(leaf.clone()); match &leaf.deltas { - Left(block) => { + Left((_,block)) => { let txns = block.contained_transactions(); for txn in txns { included_txns.insert(txn); @@ -1323,8 +1310,8 @@ where let consensus = self.consensus.read().await; consensus.metrics.number_of_timeouts.add(1); } - SequencingHotShotEvent::SendBlockCommitment(block_commitment) => { - self.block_commitment = Some(block_commitment); + SequencingHotShotEvent::SendPayloadCommitment(payload_commitment) => { + self.payload_commitment = Some(payload_commitment); } _ => {} } @@ -1380,6 +1367,16 @@ where } let parent_leaf = leaf.clone(); + let parent_header = match parent_leaf.deltas { + Left((_, ref payload)) => { + if parent_leaf.view_number != TYPES::Time::new(0) { + error!("Non-genesis parent leaf should contain the block header rather than payload."); + return false; + } + TYPES::BlockHeader::genesis(payload.clone()) + } + Right(ref header) => header.clone(), + }; let original_parent_hash = parent_leaf.commit(); @@ -1398,15 +1395,17 @@ where // TODO do some sort of sanity check on the view number that it matches decided } - if let Some(block_commitment) = &self.block_commitment { + if let Some(payload_commitment) = &self.payload_commitment { let leaf = SequencingLeaf { view_number: view, - height: parent_leaf.height + 1, justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), - // Use the block commitment rather than the block, so that the replica can construct - // the same leaf with the commitment. - deltas: Right(*block_commitment), + // Use the payload commitment rather than the payload, so that the replica can + // construct the same leaf with the commitment. + deltas: Right(TYPES::BlockHeader::new( + *payload_commitment, + parent_header.clone(), + )), rejected: vec![], timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.api.public_key().to_bytes(), @@ -1417,10 +1416,7 @@ where .sign_validating_or_commitment_proposal::(&leaf.commit()); // TODO: DA cert is sent as part of the proposal here, we should split this out so we don't have to wait for it. let proposal = QuorumProposal { - block_header: VIDBlockHeader { - block_number: leaf.height, - commitment: *block_commitment, - }, + block_header: TYPES::BlockHeader::new(*payload_commitment, parent_header.clone()), view_number: leaf.view_number, justify_qc: consensus.high_qc.clone(), timeout_certificate: timeout_certificate.or_else(|| None), @@ -1442,7 +1438,7 @@ where self.quorum_exchange.public_key().clone(), )) .await; - self.block_commitment = None; + self.payload_commitment = None; return true; } debug!("Self block was None"); @@ -1501,13 +1497,14 @@ pub type ConsensusTaskTypes = HSTWithEvent< /// Event handle for consensus pub async fn sequencing_consensus_handle< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, ConsensusMessage = SequencingMessage, >, A: SequencingConsensusApi, I> + 'static, + H: BlockHeader, >( event: SequencingHotShotEvent, mut state: SequencingConsensusTaskState, @@ -1557,7 +1554,7 @@ pub fn consensus_event_filter>( | SequencingHotShotEvent::DACRecv(_) | SequencingHotShotEvent::VidCertRecv(_) | SequencingHotShotEvent::ViewChange(_) - | SequencingHotShotEvent::SendBlockCommitment(_) + | SequencingHotShotEvent::SendPayloadCommitment(_) | SequencingHotShotEvent::Timeout(_) | SequencingHotShotEvent::TimeoutVoteRecv(_) | SequencingHotShotEvent::Shutdown, diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index dc4b8280f7..d23aadfa5f 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -156,7 +156,7 @@ where match state.committee_exchange.accumulate_vote( accumulator, &vote, - &vote.block_commitment, + &vote.payload_commitment, ) { Left(new_accumulator) => { state.accumulator = either::Left(new_accumulator); @@ -242,7 +242,7 @@ where "Got a DA block with {} transactions!", proposal.data.deltas.contained_transactions().len() ); - let block_commitment = proposal.data.deltas.commit(); + let payload_commitment = proposal.data.deltas.commit(); // ED Is this the right leader? let view_leader_key = self.committee_exchange.get_leader(view); @@ -251,7 +251,7 @@ where return None; } - if !view_leader_key.validate(&proposal.signature, block_commitment.as_ref()) { + if !view_leader_key.validate(&proposal.signature, payload_commitment.as_ref()) { error!("Could not verify proposal."); return None; } @@ -267,7 +267,7 @@ where Ok(Some(vote_token)) => { // Generate and send vote let vote = self.committee_exchange.create_da_message( - block_commitment, + payload_commitment, view, vote_token, ); @@ -286,7 +286,7 @@ where // contains strictly more information. consensus.state_map.entry(view).or_insert(View { view_inner: ViewInner::DA { - block: block_commitment, + block: payload_commitment, }, }); @@ -334,7 +334,7 @@ where let accumulator = self.committee_exchange.accumulate_vote( new_accumulator, &vote, - &vote.clone().block_commitment, + &vote.clone().payload_commitment, ); if view > collection_view { @@ -428,8 +428,10 @@ where .inject_consensus_info(ConsensusIntentEvent::CancelPollForTransactions(*view)) .await; - let block_commitment = block.commit(); - let signature = self.committee_exchange.sign_da_proposal(&block_commitment); + let payload_commitment = block.commit(); + let signature = self + .committee_exchange + .sign_da_proposal(&payload_commitment); let data: DAProposal = DAProposal { deltas: block.clone(), // Upon entering a new view we want to send a DA Proposal for the next view -> Is it always the case that this is cur_view + 1? @@ -440,8 +442,8 @@ where let message = Proposal { data, signature }; self.event_stream - .publish(SequencingHotShotEvent::SendBlockCommitment( - block_commitment, + .publish(SequencingHotShotEvent::SendPayloadCommitment( + payload_commitment, )) .await; self.event_stream diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs index f4666d8748..a2c2db415d 100644 --- a/crates/task-impls/src/events.rs +++ b/crates/task-impls/src/events.rs @@ -65,8 +65,8 @@ pub enum SequencingHotShotEvent> { TransactionsRecv(Vec), /// Send transactions to the network TransactionSend(TYPES::Transaction, TYPES::SignatureKey), - /// Event to send block commitment from DA leader to the quorum; internal event only - SendBlockCommitment(Commitment), + /// Event to send block payload commitment from DA leader to the quorum; internal event only + SendPayloadCommitment(Commitment), /// Event when the transactions task has a block formed BlockReady(TYPES::BlockPayload, TYPES::Time), /// Event when consensus decided on a leaf diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index cd390d4702..bc8d7798ed 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -144,7 +144,7 @@ where let mut included_txn_count = 0; for leaf in leaf_chain { match &leaf.deltas { - Left(block) => { + Left((_, block)) => { let txns = block.contained_transactions(); for txn in txns { included_txns.insert(txn); @@ -263,7 +263,7 @@ where let vid_disperse = vid.disperse(&txns_flatten).unwrap(); let block = VIDBlockPayload { transactions: txns, - commitment: vid_disperse.commit, + payload_commitment: vid_disperse.commit, }; // TODO never clone a block @@ -279,12 +279,12 @@ where Proposal { data: VidDisperse { view_number: view + 1, - commitment: block.commit(), + payload_commitment: block.commit(), shares: vid_disperse.shares, common: vid_disperse.common, }, // TODO (Keyao) This is also signed in DA task. - signature: self.quorum_exchange.sign_block_commitment(block.commit()), + signature: self.quorum_exchange.sign_payload_commitment(block.commit()), }, self.quorum_exchange.public_key().clone(), )) diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs index cf3589192d..5e59c6f7b4 100644 --- a/crates/task-impls/src/vid.rs +++ b/crates/task-impls/src/vid.rs @@ -154,7 +154,7 @@ where match state .vid_exchange - .accumulate_vote(accumulator, &vote, &vote.block_commitment) + .accumulate_vote(accumulator, &vote, &vote.payload_commitment) { Left(new_accumulator) => { state.accumulator = either::Left(new_accumulator); @@ -254,7 +254,7 @@ where let accumulator = self.vid_exchange.accumulate_vote( new_accumulator, &vote, - &vote.clone().block_commitment, + &vote.clone().payload_commitment, ); if view > collection_view { @@ -311,7 +311,7 @@ where } debug!("VID disperse data is fresh."); - let block_commitment = disperse.data.commitment; + let payload_commitment = disperse.data.payload_commitment; // ED Is this the right leader? let view_leader_key = self.vid_exchange.get_leader(view); @@ -320,7 +320,7 @@ where return None; } - if !view_leader_key.validate(&disperse.signature, block_commitment.as_ref()) { + if !view_leader_key.validate(&disperse.signature, payload_commitment.as_ref()) { error!("Could not verify VID proposal sig."); return None; } @@ -336,7 +336,7 @@ where Ok(Some(vote_token)) => { // Generate and send vote let vote = self.vid_exchange.create_vid_message( - block_commitment, + payload_commitment, view, vote_token, ); @@ -355,7 +355,7 @@ where // contains strictly more information. consensus.state_map.entry(view).or_insert(View { view_inner: ViewInner::DA { - block: block_commitment, + block: payload_commitment, }, }); diff --git a/crates/testing/src/overall_safety_task.rs b/crates/testing/src/overall_safety_task.rs index b2a9a88451..c10d7391ca 100644 --- a/crates/testing/src/overall_safety_task.rs +++ b/crates/testing/src/overall_safety_task.rs @@ -21,7 +21,7 @@ use hotshot_task::{ }; use hotshot_types::{ certificate::QuorumCertificate, - data::{DeltasType, LeafBlock, LeafType}, + data::{DeltasType, LeafBlockPayload, LeafType}, error::RoundTimedoutState, event::{Event, EventType}, traits::node_implementation::NodeType, @@ -109,7 +109,7 @@ pub struct RoundResult> { pub leaf_map: HashMap, /// block -> # entries decided on that block - pub block_map: HashMap>, usize>, + pub block_map: HashMap>, usize>, /// state -> # entries decided on that state pub state_map: HashMap<::MaybeState, usize>, @@ -215,7 +215,7 @@ impl> RoundResult v.insert(1); } } - match self.block_map.entry(block.clone().block_commitment()) { + match self.block_map.entry(block.clone().payload_commitment()) { std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() += 1; } @@ -291,7 +291,7 @@ impl> RoundResult // if neither, continue through let state_key = key.get_state(); - let block_key = key.get_deltas().block_commitment(); + let block_key = key.get_deltas().payload_commitment(); if *self.block_map.get(&block_key).unwrap() == threshold && *self.state_map.get(&state_key).unwrap() == threshold diff --git a/crates/testing/src/task_helpers.rs b/crates/testing/src/task_helpers.rs index 8d34440500..9a20f6fe1e 100644 --- a/crates/testing/src/task_helpers.rs +++ b/crates/testing/src/task_helpers.rs @@ -3,7 +3,7 @@ use crate::{ test_builder::TestMetadata, }; use commit::Committable; -use either::Right; +use either::{Either::Left, Right}; use hotshot::{ certificate::QuorumCertificate, traits::{NodeImplementation, TestableNodeImplementation}, @@ -18,6 +18,7 @@ use hotshot_types::{ data::{QuorumProposal, SequencingLeaf, VidScheme, ViewNumber}, message::{Message, Proposal}, traits::{ + block_contents::BlockHeader, consensus_api::ConsensusSharedApi, election::{ConsensusExchange, Membership, SignedCertificate}, node_implementation::{CommitteeEx, ExchangesType, NodeType, QuorumEx}, @@ -115,26 +116,30 @@ async fn build_quorum_proposal_and_signature( panic!("Failed to find high QC parent."); }; let parent_leaf = leaf.clone(); + let parent_header = match parent_leaf.deltas { + Left((block_number, ref payload)) => VIDBlockHeader { + block_number, + payload_commitment: payload.commit(), + }, + Right(ref header) => header.clone(), + }; // every event input is seen on the event stream in the output. let block = ::genesis(); - let block_commitment = block.commit(); + let payload_commitment = block.commit(); + let block_header = VIDBlockHeader::new(payload_commitment, parent_header); let leaf = SequencingLeaf { view_number: ViewNumber::new(view), - height: parent_leaf.height + 1, justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), - deltas: Right(block_commitment), + deltas: Right(block_header.clone()), rejected: vec![], timestamp: 0, proposer_id: api.public_key().to_bytes(), }; let signature = ::sign(private_key, leaf.commit().as_ref()); let proposal = QuorumProposal::> { - block_header: VIDBlockHeader { - block_number: 1, - commitment: block_commitment, - }, + block_header, view_number: ViewNumber::new(view), justify_qc: QuorumCertificate::genesis(), timeout_certificate: None, diff --git a/crates/testing/tests/consensus_task.rs b/crates/testing/tests/consensus_task.rs index f4beb9d563..726b4ed0e9 100644 --- a/crates/testing/tests/consensus_task.rs +++ b/crates/testing/tests/consensus_task.rs @@ -63,10 +63,9 @@ async fn build_vote( let leaf: SequencingLeaf<_> = SequencingLeaf { view_number: view, - height: proposal.block_header.block_number, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_header.commitment), + deltas: Right(proposal.block_header), rejected: Vec::new(), timestamp: 0, proposer_id: quorum_exchange.get_leader(view).to_bytes(), diff --git a/crates/testing/tests/da_task.rs b/crates/testing/tests/da_task.rs index f9380b6991..8732a93fbb 100644 --- a/crates/testing/tests/da_task.rs +++ b/crates/testing/tests/da_task.rs @@ -42,10 +42,10 @@ async fn test_da_task() { let vid = vid_init(); let txn = vec![0u8]; let vid_disperse = vid.disperse(&txn).unwrap(); - let block_commitment = vid_disperse.commit; + let payload_commitment = vid_disperse.commit; let block = VIDBlockPayload { transactions: vec![VIDTransaction(txn)], - commitment: block_commitment, + payload_commitment, }; let signature = committee_exchange.sign_da_proposal(&block.commit()); @@ -58,7 +58,7 @@ async fn test_da_task() { signature, }; - // TODO for now reuse the same block commitment and signature as DA committee + // TODO for now reuse the same block payload commitment and signature as DA committee // https://github.com/EspressoSystems/jellyfish/issues/369 // Every event input is seen on the event stream in the output. @@ -85,7 +85,7 @@ async fn test_da_task() { 1, ); output.insert( - SequencingHotShotEvent::SendBlockCommitment(block.commit()), + SequencingHotShotEvent::SendPayloadCommitment(block.commit()), 1, ); output.insert( diff --git a/crates/testing/tests/network_task.rs b/crates/testing/tests/network_task.rs index fe50497132..03c0b13d63 100644 --- a/crates/testing/tests/network_task.rs +++ b/crates/testing/tests/network_task.rs @@ -45,10 +45,10 @@ async fn test_network_task() { let vid = vid_init(); let txn = vec![0u8]; let vid_disperse = vid.disperse(&txn).unwrap(); - let block_commitment = vid_disperse.commit; + let payload_commitment = vid_disperse.commit; let block = VIDBlockPayload { transactions: vec![VIDTransaction(txn)], - commitment: block_commitment, + payload_commitment, }; let signature = committee_exchange.sign_da_proposal(&block.commit()); let da_proposal = Proposal { @@ -59,12 +59,12 @@ async fn test_network_task() { signature, }; let quorum_proposal = build_quorum_proposal(&handle, priv_key, 2).await; - // TODO for now reuse the same block commitment and signature as DA committee + // TODO for now reuse the same block payload commitment and signature as DA committee // https://github.com/EspressoSystems/jellyfish/issues/369 let da_vid_disperse = Proposal { data: VidDisperse { view_number: da_proposal.data.view_number, - commitment: block.commit(), + payload_commitment: block.commit(), shares: vid_disperse.shares, common: vid_disperse.common, }, @@ -125,7 +125,7 @@ async fn test_network_task() { 1, ); output.insert( - SequencingHotShotEvent::SendBlockCommitment(block.commit()), + SequencingHotShotEvent::SendPayloadCommitment(block.commit()), 1, ); output.insert( diff --git a/crates/testing/tests/vid_task.rs b/crates/testing/tests/vid_task.rs index cf19740e1d..d4edf7b6ea 100644 --- a/crates/testing/tests/vid_task.rs +++ b/crates/testing/tests/vid_task.rs @@ -41,10 +41,10 @@ async fn test_vid_task() { let vid = vid_init(); let txn = vec![0u8]; let vid_disperse = vid.disperse(&txn).unwrap(); - let block_commitment = vid_disperse.commit; + let payload_commitment = vid_disperse.commit; let block = VIDBlockPayload { transactions: vec![VIDTransaction(txn)], - commitment: block_commitment, + payload_commitment, }; let signature = vid_exchange.sign_vid_proposal(&block.commit()); @@ -59,7 +59,7 @@ async fn test_vid_task() { let vid_proposal = Proposal { data: VidDisperse { view_number: message.data.view_number, - commitment: block.commit(), + payload_commitment: block.commit(), shares: vid_disperse.shares, common: vid_disperse.common, }, diff --git a/crates/types/src/block_impl.rs b/crates/types/src/block_impl.rs index 0de9365e79..90aa210b6f 100644 --- a/crates/types/src/block_impl.rs +++ b/crates/types/src/block_impl.rs @@ -65,8 +65,8 @@ pub enum BlockPayloadError { pub struct VIDBlockPayload { /// List of transactions. pub transactions: Vec, - /// VID commitment. - pub commitment: ::Commit, + /// VID commitment to the block payload. + pub payload_commitment: ::Commit, } impl VIDBlockPayload { @@ -86,14 +86,14 @@ impl VIDBlockPayload { let vid_disperse = vid.disperse(&txn).unwrap(); VIDBlockPayload { transactions: vec![VIDTransaction(txn)], - commitment: vid_disperse.commit, + payload_commitment: vid_disperse.commit, } } } impl Committable for VIDBlockPayload { fn commit(&self) -> Commitment { - as CanonicalDeserialize>::deserialize(&*self.commitment) + as CanonicalDeserialize>::deserialize(&*self.payload_commitment) .expect("conversion from VidScheme::Commit to Commitment should succeed") } @@ -136,14 +136,32 @@ impl BlockPayload for VIDBlockPayload { pub struct VIDBlockHeader { /// Block number. pub block_number: u64, - /// VID commitment. - pub commitment: Commitment, + /// VID commitment to the payload. + pub payload_commitment: Commitment, } impl BlockHeader for VIDBlockHeader { type Payload = VIDBlockPayload; - fn commitment(&self) -> Commitment { - self.commitment + fn new(payload_commitment: Commitment, parent_header: Self) -> Self { + Self { + block_number: parent_header.block_number + 1, + payload_commitment, + } + } + + fn genesis(payload: Self::Payload) -> Self { + Self { + block_number: 0, + payload_commitment: payload.commit(), + } + } + + fn block_number(&self) -> u64 { + self.block_number + } + + fn payload_commitment(&self) -> Commitment { + self.payload_commitment } } diff --git a/crates/types/src/certificate.rs b/crates/types/src/certificate.rs index 1c415ce5f8..adb37fa56e 100644 --- a/crates/types/src/certificate.rs +++ b/crates/types/src/certificate.rs @@ -35,8 +35,8 @@ pub struct DACertificate { /// This value is covered by the threshold signature. pub view_number: TYPES::Time, - /// committment to the block - pub block_commitment: Commitment, + /// committment to the block payload + pub payload_commitment: Commitment, /// Assembled signature for certificate aggregation pub signatures: AssembledSignature, @@ -50,8 +50,8 @@ pub struct VIDCertificate { /// The view number this VID certificate was generated during pub view_number: TYPES::Time, - /// committment to the block - pub block_commitment: Commitment, + /// Committment to the block payload + pub payload_commitment: Commitment, /// Assembled signature for certificate aggregation pub signatures: AssembledSignature, @@ -270,7 +270,7 @@ impl DACertificate { view_number: vote.get_view(), signatures, - block_commitment: vote.block_commitment, + payload_commitment: vote.payload_commitment, } } @@ -283,7 +283,7 @@ impl } fn leaf_commitment(&self) -> Commitment { - self.block_commitment + self.payload_commitment } fn is_genesis(&self) -> bool { @@ -308,7 +308,7 @@ impl VIDCertificate { view_number: vote.get_view(), signatures, - block_commitment: vote.block_commitment, + payload_commitment: vote.payload_commitment, } } @@ -321,7 +321,7 @@ impl } fn leaf_commitment(&self) -> Commitment { - self.block_commitment + self.payload_commitment } fn is_genesis(&self) -> bool { diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index 516b8879d6..fad59f0508 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -321,7 +321,7 @@ impl> Consensus { // perform gc self.state_map .range(old_anchor_view..new_anchor_view) - .filter_map(|(_view_number, view)| view.get_block_commitment()) + .filter_map(|(_view_number, view)| view.get_payload_commitment()) .for_each(|block| { self.saved_blocks.remove(block); }); @@ -351,7 +351,7 @@ impl> Consensus { } } -/// Mapping from block commitments to full blocks. +/// Mapping from block payload commitments to full blocks. /// /// Entries in this mapping are reference-counted, so multiple consensus objects can refer to the /// same block, and the block will only be deleted after _all_ such objects are garbage collected. diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 4a4f604ae6..5f1cef024d 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -9,6 +9,7 @@ use crate::{ ViewSyncCertificate, }, traits::{ + block_contents::BlockHeader, node_implementation::NodeType, signature_key::{EncodedPublicKey, SignatureKey}, state::{ConsensusTime, TestableBlock, TestableState}, @@ -21,7 +22,7 @@ use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Read, Serializatio use bincode::Options; use commit::{Commitment, Committable}; use derivative::Derivative; -use either::Either; +use either::{Either, Left, Right}; use espresso_systems_common::hotshot::tag; use hotshot_constants::GENESIS_PROPOSER_ID; use hotshot_utils::bincode::bincode_opts; @@ -125,8 +126,8 @@ pub struct ValidatingProposal> where LEAF: Committable, { - /// current view's block commitment - pub block_commitment: Commitment, + /// Current view's block payload commitment + pub payload_commitment: Commitment, /// CurView from leader when proposing leaf pub view_number: TYPES::Time, @@ -175,8 +176,8 @@ pub use jf_primitives::vid::VidScheme as VidSchemeTrait; pub struct VidDisperse { /// The view number for which this VID data is intended pub view_number: TYPES::Time, - /// Block commitment - pub commitment: Commitment, + /// Block payload commitment + pub payload_commitment: Commitment, /// VID shares dispersed among storage nodes pub shares: Vec<::Share>, /// VID common data sent to all storage nodes @@ -293,7 +294,7 @@ pub trait DeltasType: type Error: std::error::Error; /// Get a cryptographic commitment to the block represented by this delta. - fn block_commitment(&self) -> Commitment; + fn payload_commitment(&self) -> Commitment; /// Get the full block if it is available, otherwise return this object unchanged. /// @@ -310,13 +311,13 @@ pub trait DeltasType: /// /// # Errors /// - /// Fails if `block` does not match `self.block_commitment()`, or if the block is not able to be + /// Fails if `block` does not match `self.payload_commitment()`, or if the block is not able to be /// stored for some implementation-defined reason. fn fill(&mut self, block: BlockPayload) -> Result<(), Self::Error>; } /// Error which occurs when [`DeltasType::fill`] is called with a block that does not match the -/// deltas' internal block commitment. +/// deltas' internal block payload commitment. #[derive(Clone, Copy, Debug, Snafu)] #[snafu(display("the block {:?} has commitment {} (expected {})", block, block.commit(), commitment))] pub struct InconsistentDeltasError { @@ -341,7 +342,7 @@ where { type Error = InconsistentDeltasError; - fn block_commitment(&self) -> Commitment { + fn payload_commitment(&self) -> Commitment { self.commit() } @@ -378,7 +379,7 @@ where { type Error = InconsistentDeltasError; - fn block_commitment(&self) -> Commitment { + fn payload_commitment(&self) -> Commitment { match self { Either::Left(block) => block.commit(), Either::Right(comm) => *comm, @@ -410,6 +411,45 @@ where } } +impl DeltasType for Either<(u64, PAYLOAD), HEADER> +where + HEADER: BlockHeader, + PAYLOAD: BlockPayload, +{ + type Error = InconsistentDeltasError; + + fn payload_commitment(&self) -> Commitment { + match self { + Either::Left((_, block)) => block.commit(), + Either::Right(header) => header.payload_commitment(), + } + } + + fn try_resolve(self) -> Result { + match self { + Either::Left((_, block)) => Ok(block), + Either::Right(_) => Err(self), + } + } + + fn fill(&mut self, block: PAYLOAD) -> Result<(), Self::Error> { + match self { + Either::Left((_, curr)) => curr.fill(block), + Either::Right(header) => { + ensure!( + header.payload_commitment() == block.commit(), + InconsistentDeltasSnafu { + block, + commitment: header.payload_commitment() + } + ); + *self = Either::Left((header.block_number(), block)); + Ok(()) + } + } + } +} + /// An item which is appended to a blockchain. pub trait LeafType: Debug @@ -427,7 +467,7 @@ pub trait LeafType: /// Type of nodes participating in the network. type NodeType: NodeType; /// Type of block contained by this leaf. - type DeltasType: DeltasType>; + type DeltasType: DeltasType>; /// Either state or empty type MaybeState: Clone + Debug @@ -443,7 +483,7 @@ pub trait LeafType: fn new( view_number: LeafTime, justify_qc: QuorumCertificate>, - deltas: LeafBlock, + deltas: LeafBlockPayload, state: LeafState, ) -> Self; /// Time when this leaf was created. @@ -452,8 +492,6 @@ pub trait LeafType: /// /// Equivalently, this is the number of leaves before this one in the chain. fn get_height(&self) -> u64; - /// Change the height of this leaf. - fn set_height(&mut self, height: u64); /// The QC linking this leaf to its parent in the chain. fn get_justify_qc(&self) -> QuorumCertificate>; /// Commitment to this leaf's parent. @@ -469,7 +507,7 @@ pub trait LeafType: /// /// Fails if `block` does not match `self.get_deltas_commitment()`, or if the block is not able /// to be stored for some implementation-defined reason. - fn fill_deltas(&mut self, block: LeafBlock) -> Result<(), LeafDeltasError>; + fn fill_deltas(&mut self, block: LeafBlockPayload) -> Result<(), LeafDeltasError>; /// The blockchain state after appending this leaf. fn get_state(&self) -> Self::MaybeState; /// Transactions rejected or invalidated by the application of this leaf. @@ -481,24 +519,26 @@ pub trait LeafType: /// Create a leaf from information stored about a view. fn from_stored_view(stored_view: StoredView) -> Self; - /// A commitment to the block contained in this leaf. - fn get_deltas_commitment(&self) -> Commitment> { - self.get_deltas().block_commitment() + /// A commitment to the block payload contained in this leaf. + fn get_deltas_commitment(&self) -> Commitment> { + self.get_deltas().payload_commitment() } } /// The [`DeltasType`] in a [`LeafType`]. pub type LeafDeltas = ::DeltasType; /// Errors reported by the [`DeltasType`] in a [`LeafType`]. -pub type LeafDeltasError = as DeltasType>>::Error; +pub type LeafDeltasError = as DeltasType>>::Error; /// The [`NodeType`] in a [`LeafType`]. pub type LeafNode = ::NodeType; /// The [`StateType`] in a [`LeafType`]. pub type LeafState = as NodeType>::StateType; +/// The [`BlockHeader`] in a [`LeafType`]. +pub type LeafBlockHeader = as NodeType>::BlockHeader; /// The [`BlockPayload`] in a [`LeafType`]. -pub type LeafBlock = as NodeType>::BlockPayload; +pub type LeafBlockPayload = as NodeType>::BlockPayload; /// The [`Transaction`] in a [`LeafType`]. -pub type LeafTransaction = as BlockPayload>::Transaction; +pub type LeafTransaction = as BlockPayload>::Transaction; /// The [`ConsensusTime`] used by a [`LeafType`]. pub type LeafTime = as NodeType>::Time; @@ -564,9 +604,6 @@ pub struct SequencingLeaf { /// CurView from leader when proposing leaf pub view_number: TYPES::Time, - /// Number of leaves before this one in the chain - pub height: u64, - /// Per spec, justification pub justify_qc: QuorumCertificate>, @@ -574,8 +611,8 @@ pub struct SequencingLeaf { /// So we can ask if it extends pub parent_commitment: Commitment, - /// The block or block commitment to be applied - pub deltas: Either>, + /// Either the block number and payload, or the block header. + pub deltas: Either<(u64, TYPES::BlockPayload), TYPES::BlockHeader>, /// Transactions that were marked for rejection while collecting deltas pub rejected: Vec<::Transaction>, @@ -591,15 +628,15 @@ pub struct SequencingLeaf { impl PartialEq for SequencingLeaf { fn eq(&self, other: &Self) -> bool { let delta_left = match &self.deltas { - Either::Left(deltas) => deltas.commit(), - Either::Right(deltas) => *deltas, + Either::Left(deltas) => (deltas.0, deltas.1.commit()), + Either::Right(deltas) => (deltas.block_number(), deltas.payload_commitment()), }; let delta_right = match &other.deltas { - Either::Left(deltas) => deltas.commit(), - Either::Right(deltas) => *deltas, + Either::Left(deltas) => (deltas.0, deltas.1.commit()), + Either::Right(deltas) => (deltas.block_number(), deltas.payload_commitment()), }; self.view_number == other.view_number - && self.height == other.height + // && self.height == other.height && self.justify_qc == other.justify_qc && self.parent_commitment == other.parent_commitment && delta_left == delta_right @@ -610,15 +647,17 @@ impl PartialEq for SequencingLeaf { impl Hash for SequencingLeaf { fn hash(&self, state: &mut H) { self.view_number.hash(state); - self.height.hash(state); + // self.height.hash(state); self.justify_qc.hash(state); self.parent_commitment.hash(state); match &self.deltas { Either::Left(deltas) => { - deltas.commit().hash(state); + deltas.0.hash(state); + deltas.1.commit().hash(state); } - Either::Right(commitment) => { - commitment.hash(state); + Either::Right(header) => { + header.block_number().hash(state); + header.payload_commitment().hash(state); } } // self.deltas.hash(state.commit()); @@ -668,10 +707,6 @@ impl LeafType for ValidatingLeaf { self.height } - fn set_height(&mut self, height: u64) { - self.height = height; - } - fn get_justify_qc(&self) -> QuorumCertificate> { self.justify_qc.clone() } @@ -685,10 +720,10 @@ impl LeafType for ValidatingLeaf { } fn get_deltas_commitment(&self) -> Commitment<::BlockPayload> { - self.deltas.block_commitment() + self.deltas.payload_commitment() } - fn fill_deltas(&mut self, block: LeafBlock) -> Result<(), LeafDeltasError> { + fn fill_deltas(&mut self, block: LeafBlockPayload) -> Result<(), LeafDeltasError> { self.deltas.fill(block) } @@ -748,14 +783,16 @@ impl Display for SequencingLeaf { write!( f, "view: {:?}, height: {:?}, justify: {}", - self.view_number, self.height, self.justify_qc + self.view_number, + self.get_height(), + self.justify_qc ) } } impl LeafType for SequencingLeaf { type NodeType = TYPES; - type DeltasType = Either>; + type DeltasType = Either<(u64, TYPES::BlockPayload), TYPES::BlockHeader>; type MaybeState = (); fn new( @@ -766,10 +803,9 @@ impl LeafType for SequencingLeaf { ) -> Self { Self { view_number, - height: 0, justify_qc, parent_commitment: fake_commitment(), - deltas: Either::Left(deltas), + deltas: Either::Left((0, deltas)), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: genesis_proposer_id(), @@ -781,11 +817,10 @@ impl LeafType for SequencingLeaf { } fn get_height(&self) -> u64 { - self.height - } - - fn set_height(&mut self, height: u64) { - self.height = height; + match &self.deltas { + Left((height, _)) => *height, + Right(header) => header.block_number(), + } } fn get_justify_qc(&self) -> QuorumCertificate> { @@ -801,10 +836,10 @@ impl LeafType for SequencingLeaf { } fn get_deltas_commitment(&self) -> Commitment<::BlockPayload> { - self.deltas.block_commitment() + self.deltas.payload_commitment() } - fn fill_deltas(&mut self, block: LeafBlock) -> Result<(), LeafDeltasError> { + fn fill_deltas(&mut self, block: LeafBlockPayload) -> Result<(), LeafDeltasError> { self.deltas.fill(block) } @@ -826,7 +861,6 @@ impl LeafType for SequencingLeaf { fn from_stored_view(stored_view: StoredView) -> Self { Self { view_number: stored_view.view_number, - height: 0, justify_qc: stored_view.justify_qc, parent_commitment: stored_view.parent, deltas: stored_view.deltas, @@ -935,7 +969,7 @@ impl Committable for ValidatingLeaf { .u64_field("view number", *self.view_number) .u64_field("height", self.height) .field("parent Leaf commitment", self.parent_commitment) - .field("block commitment", self.deltas.commit()) + .field("block payload commitment", self.deltas.commit()) .field("state commitment", self.state.commit()) .constant_str("justify_qc view number") .u64(*self.justify_qc.view_number) @@ -955,20 +989,20 @@ impl Committable for ValidatingLeaf { impl Committable for SequencingLeaf { fn commit(&self) -> commit::Commitment { - // Commit the block commitment, rather than the block, so that the replicas can reconstruct + // Commit the block payload, rather than the block, so that the replicas can reconstruct // the leaf. - let block_commitment = match &self.deltas { - Either::Left(block) => block.commit(), - Either::Right(commitment) => *commitment, + let (height, payload_commitment) = match &self.deltas { + Either::Left((height, payload)) => (*height, payload.commit()), + Either::Right(header) => (header.block_number(), header.payload_commitment()), }; let signatures_bytes = serialize_signature(&self.justify_qc.signatures); commit::RawCommitmentBuilder::new("leaf commitment") .u64_field("view number", *self.view_number) - .u64_field("height", self.height) + .u64_field("height", height) .field("parent Leaf commitment", self.parent_commitment) - .field("block commitment", block_commitment) + .field("block payload commitment", payload_commitment) .constant_str("justify_qc view number") .u64(*self.justify_qc.view_number) .field( @@ -994,7 +1028,7 @@ impl From> state_commitment: leaf.state.commit(), rejected: leaf.rejected, proposer_id: leaf.proposer_id, - block_commitment: leaf.deltas.commit(), + payload_commitment: leaf.deltas.commit(), } } } diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index 59fd565140..6922b5c011 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -64,8 +64,17 @@ pub trait BlockHeader: /// Block payload associated with the commitment. type Payload: BlockPayload; + /// Build a header with the payload commitment and parent header. + fn new(payload_commitment: Commitment, parent_header: Self) -> Self; + + /// Build a genesis header with the genesis payload. + fn genesis(payload: Self::Payload) -> Self; + + /// Get the block number. + fn block_number(&self) -> u64; + /// Get the payload commitment. - fn commitment(&self) -> Commitment; + fn payload_commitment(&self) -> Commitment; } /// Dummy implementation of `BlockPayload` for unit tests @@ -138,7 +147,19 @@ pub mod dummy { impl BlockHeader for DummyBlock { type Payload = Self; - fn commitment(&self) -> commit::Commitment { + fn new(_payload_commitment: Commitment, _parent_header: Self) -> Self { + Self { nonce: 0 } + } + + fn genesis(_payload: Self::Payload) -> Self { + Self { nonce: 0 } + } + + fn block_number(&self) -> u64 { + 0 + } + + fn payload_commitment(&self) -> commit::Commitment { commit::RawCommitmentBuilder::new("Dummy BlockPayload Comm") .u64_field("Nonce", self.nonce) .finalize() diff --git a/crates/types/src/traits/election.rs b/crates/types/src/traits/election.rs index 4718e2d033..0254dfbb53 100644 --- a/crates/types/src/traits/election.rs +++ b/crates/types/src/traits/election.rs @@ -491,22 +491,22 @@ pub trait CommitteeExchangeType: /// Sign a DA proposal. fn sign_da_proposal( &self, - block_commitment: &Commitment, + payload_commitment: &Commitment, ) -> EncodedSignature; /// Sign a vote on DA proposal. /// - /// The block commitment and the type of the vote (DA) are signed, which is the minimum amount + /// The block payload commitment and the type of the vote (DA) are signed, which is the minimum amount /// of information necessary for checking that this node voted on that block. fn sign_da_vote( &self, - block_commitment: Commitment, + payload_commitment: Commitment, ) -> (EncodedPublicKey, EncodedSignature); /// Create a message with a vote on DA proposal. fn create_da_message( &self, - block_commitment: Commitment, + payload_commitment: Commitment, current_view: TYPES::Time, vote_token: TYPES::VoteTokenType, ) -> DAVote; @@ -546,39 +546,39 @@ impl< /// Sign a DA proposal. fn sign_da_proposal( &self, - block_commitment: &Commitment, + payload_commitment: &Commitment, ) -> EncodedSignature { - let signature = TYPES::SignatureKey::sign(&self.private_key, block_commitment.as_ref()); + let signature = TYPES::SignatureKey::sign(&self.private_key, payload_commitment.as_ref()); signature } /// Sign a vote on DA proposal. /// - /// The block commitment and the type of the vote (DA) are signed, which is the minimum amount + /// The block payload commitment and the type of the vote (DA) are signed, which is the minimum amount /// of information necessary for checking that this node voted on that block. fn sign_da_vote( &self, - block_commitment: Commitment, + payload_commitment: Commitment, ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::DA(block_commitment).commit().as_ref(), + VoteData::DA(payload_commitment).commit().as_ref(), ); (self.public_key.to_bytes(), signature) } /// Create a message with a vote on DA proposal. fn create_da_message( &self, - block_commitment: Commitment, + payload_commitment: Commitment, current_view: TYPES::Time, vote_token: TYPES::VoteTokenType, ) -> DAVote { - let signature = self.sign_da_vote(block_commitment); + let signature = self.sign_da_vote(payload_commitment); DAVote { signature, - block_commitment, + payload_commitment, current_view, vote_token, - vote_data: VoteData::DA(block_commitment), + vote_data: VoteData::DA(payload_commitment), } } } @@ -643,7 +643,7 @@ pub trait VIDExchangeType: ConsensusExchange, + payload_commitment: Commitment, current_view: TYPES::Time, vote_token: TYPES::VoteTokenType, ) -> VIDVote; @@ -651,13 +651,13 @@ pub trait VIDExchangeType: ConsensusExchange, + payload_commitment: Commitment, ) -> (EncodedPublicKey, EncodedSignature); /// Sign a VID proposal. fn sign_vid_proposal( &self, - block_commitment: &Commitment, + payload_commitment: &Commitment, ) -> EncodedSignature; } @@ -694,27 +694,27 @@ impl< { fn create_vid_message( &self, - block_commitment: Commitment, + payload_commitment: Commitment, current_view: ::Time, vote_token: ::VoteTokenType, ) -> VIDVote { - let signature = self.sign_vid_vote(block_commitment); + let signature = self.sign_vid_vote(payload_commitment); VIDVote { signature, - block_commitment, + payload_commitment, current_view, vote_token, - vote_data: VoteData::DA(block_commitment), + vote_data: VoteData::DA(payload_commitment), } } fn sign_vid_vote( &self, - block_commitment: Commitment<::BlockPayload>, + payload_commitment: Commitment<::BlockPayload>, ) -> (EncodedPublicKey, EncodedSignature) { let signature = TYPES::SignatureKey::sign( &self.private_key, - VoteData::DA(block_commitment).commit().as_ref(), + VoteData::DA(payload_commitment).commit().as_ref(), ); (self.public_key.to_bytes(), signature) } @@ -722,9 +722,9 @@ impl< /// Sign a VID proposal. fn sign_vid_proposal( &self, - block_commitment: &Commitment, + payload_commitment: &Commitment, ) -> EncodedSignature { - let signature = TYPES::SignatureKey::sign(&self.private_key, block_commitment.as_ref()); + let signature = TYPES::SignatureKey::sign(&self.private_key, payload_commitment.as_ref()); signature } } @@ -807,10 +807,10 @@ pub trait QuorumExchangeType, leaf_commitment: &Commitment, ) -> EncodedSignature; - /// Sign a block commitment. - fn sign_block_commitment( + /// Sign a block payload commitment. + fn sign_payload_commitment( &self, - block_commitment: Commitment, + payload_commitment: Commitment, ) -> EncodedSignature; /// Sign a positive vote on validating or commitment proposal. @@ -912,11 +912,11 @@ impl< signature } - fn sign_block_commitment( + fn sign_payload_commitment( &self, - block_commitment: Commitment<::BlockPayload>, + payload_commitment: Commitment<::BlockPayload>, ) -> EncodedSignature { - TYPES::SignatureKey::sign(&self.private_key, block_commitment.as_ref()) + TYPES::SignatureKey::sign(&self.private_key, payload_commitment.as_ref()) } /// Sign a positive vote on validating or commitment proposal. diff --git a/crates/types/src/traits/state.rs b/crates/types/src/traits/state.rs index 8c7efadcfc..3d6b8f6d29 100644 --- a/crates/types/src/traits/state.rs +++ b/crates/types/src/traits/state.rs @@ -15,6 +15,8 @@ use std::{ ops::{Deref, Sub}, }; +use super::block_contents::BlockHeader; + /// Abstraction over the state that blocks modify /// /// This trait represents the behaviors that the 'global' ledger state must have: @@ -39,22 +41,27 @@ pub trait State: { /// The error type for this particular type of ledger state type Error: Error + Debug + Send + Sync; - /// The type of block this state is associated with + /// The type of block header this state is associated with + type BlockHeader: BlockHeader; + /// The type of block payload this state is associated with type BlockPayload: BlockPayload; /// Time compatibility needed for reward collection type Time: ConsensusTime; - /// Returns true if and only if the provided block is valid and can extend this state - fn validate_block(&self, block: &Self::BlockPayload, view_number: &Self::Time) -> bool; + /// Returns true if and only if the provided block header is valid and can extend this state + fn validate_block(&self, block_header: &Self::BlockHeader, view_number: &Self::Time) -> bool; + + /// Initialize the state. + fn initialize() -> Self; - /// Appends the given block to this state, returning an new state + /// Appends the given block header to this state, returning an new state /// /// # Errors /// - /// Should produce and error if appending this block would lead to an invalid state + /// Should produce and error if appending this block header would lead to an invalid state fn append( &self, - block: &Self::BlockPayload, + block_header: &Self::BlockHeader, view_number: &Self::Time, ) -> Result; @@ -158,17 +165,27 @@ pub mod dummy { impl State for DummyState { type Error = DummyError; - + type BlockHeader = DummyBlock; type BlockPayload = DummyBlock; type Time = ViewNumber; - fn validate_block(&self, _block: &Self::BlockPayload, _view_number: &Self::Time) -> bool { + fn validate_block( + &self, + _block_header: &Self::BlockHeader, + _view_number: &Self::Time, + ) -> bool { false } + fn initialize() -> Self { + let mut state = Self::default(); + state.nonce += 1; + state + } + fn append( &self, - _block: &Self::BlockPayload, + _block_header: &Self::BlockHeader, _view_number: &Self::Time, ) -> Result { Ok(Self { diff --git a/crates/types/src/utils.rs b/crates/types/src/utils.rs index 38c4b5a852..cb06abf8d0 100644 --- a/crates/types/src/utils.rs +++ b/crates/types/src/utils.rs @@ -1,7 +1,7 @@ //! Utility functions, type aliases, helper structs and enum definitions. use crate::{ - data::{LeafBlock, LeafType}, + data::{LeafBlockPayload, LeafType}, traits::node_implementation::NodeType, }; use commit::Commitment; @@ -17,7 +17,7 @@ pub enum ViewInner> { /// leaders repeatedly request availability for blocks that they never propose. DA { /// Available block. - block: Commitment>, + block: Commitment>, }, /// Undecided view Leaf { @@ -39,9 +39,9 @@ impl> ViewInner { } } - /// return the underlying block hash if it exists + /// return the underlying block paylod commitment if it exists #[must_use] - pub fn get_block_commitment(&self) -> Option>> { + pub fn get_payload_commitment(&self) -> Option>> { if let Self::DA { block } = self { Some(*block) } else { diff --git a/crates/types/src/vote.rs b/crates/types/src/vote.rs index 2b721819bb..d63f8cc0fb 100644 --- a/crates/types/src/vote.rs +++ b/crates/types/src/vote.rs @@ -49,8 +49,8 @@ pub trait VoteType: pub struct DAVote { /// The signature share associated with this vote pub signature: (EncodedPublicKey, EncodedSignature), - /// The block commitment being voted on. - pub block_commitment: Commitment, + /// The block payload commitment being voted on. + pub payload_commitment: Commitment, /// The view this vote was cast for pub current_view: TYPES::Time, /// The vote token generated by this replica @@ -65,8 +65,8 @@ pub struct DAVote { pub struct VIDVote { /// The signature share associated with this vote pub signature: (EncodedPublicKey, EncodedSignature), - /// The block commitment being voted on. - pub block_commitment: Commitment, + /// The block payload commitment being voted on. + pub payload_commitment: Commitment, /// The view this vote was cast for pub current_view: TYPES::Time, /// The vote token generated by this replica From 51a7bf8c5e5f495b72ffcad2ddc859651004980b Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Thu, 26 Oct 2023 11:33:48 -0700 Subject: [PATCH 07/13] Fix after merge --- crates/hotshot/examples/infra/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/hotshot/examples/infra/mod.rs b/crates/hotshot/examples/infra/mod.rs index ba774d52e8..66b311f2be 100644 --- a/crates/hotshot/examples/infra/mod.rs +++ b/crates/hotshot/examples/infra/mod.rs @@ -233,7 +233,7 @@ pub trait RunDA< >, > where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, Self: Sync, SystemContext: HotShotType, @@ -252,7 +252,7 @@ pub trait RunDA< /// get the anchored view /// Note: sequencing leaf does not have state, so does not return state async fn initialize_state_and_hotshot(&self) -> SystemContextHandle { - let genesis_block = TYPES::BlockType::genesis(); + let genesis_block = TYPES::BlockPayload::genesis(); let initializer = hotshot::HotShotInitializer::>::from_genesis( genesis_block, @@ -460,7 +460,7 @@ pub struct WebServerDARun< #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -511,7 +511,7 @@ impl< > for WebServerDARun where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, Self: Sync, { @@ -628,7 +628,7 @@ pub struct Libp2pDARun, MEMBERSHIP #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -679,7 +679,7 @@ impl< > for Libp2pDARun where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, Self: Sync, { @@ -857,7 +857,7 @@ where /// Main entry point for validators pub async fn main_entry_point< - TYPES: NodeType, + TYPES: NodeType, MEMBERSHIP: Membership + Debug, DANETWORK: CommunicationChannel, MEMBERSHIP> + Debug, QUORUMNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, @@ -895,7 +895,7 @@ pub async fn main_entry_point< args: ValidatorArgs, ) where ::StateType: TestableState, - ::BlockType: TestableBlock, + ::BlockPayload: TestableBlock, SequencingLeaf: TestableLeaf, { setup_logging(); From fe81d42d2c398f188bd56b833b47f880ca7bda33 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Thu, 26 Oct 2023 14:39:31 -0700 Subject: [PATCH 08/13] Fix clippy --- crates/task-impls/src/consensus.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index 08e75c6323..1d2ca253ab 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -1318,6 +1318,7 @@ where } /// Sends a proposal if possible from the high qc we have + #[allow(clippy::too_many_lines)] pub async fn publish_proposal_if_able( &mut self, _qc: QuorumCertificate>, From 1ced006eecac898b856689510ec137276794e60c Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Thu, 26 Oct 2023 14:58:53 -0700 Subject: [PATCH 09/13] More fixes after merge --- crates/hotshot/examples/infra/mod.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/hotshot/examples/infra/mod.rs b/crates/hotshot/examples/infra/mod.rs index 66b311f2be..1dcc401698 100644 --- a/crates/hotshot/examples/infra/mod.rs +++ b/crates/hotshot/examples/infra/mod.rs @@ -20,7 +20,7 @@ use hotshot_orchestrator::{ config::{NetworkConfig, NetworkConfigFile, WebServerConfig}, }; use hotshot_task::task::FilterEvent; -use hotshot_types::traits::election::VIDExchange; +use hotshot_types::{block_impl::VIDBlockHeader, traits::election::VIDExchange}; use hotshot_types::{ block_impl::{VIDBlockPayload, VIDTransaction}, certificate::ViewSyncCertificate, @@ -460,7 +460,11 @@ pub struct WebServerDARun< #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType< + Transaction = VIDTransaction, + BlockPayload = VIDBlockPayload, + BlockHeader = VIDBlockHeader, + >, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -628,7 +632,11 @@ pub struct Libp2pDARun, MEMBERSHIP #[async_trait] impl< - TYPES: NodeType, + TYPES: NodeType< + Transaction = VIDTransaction, + BlockPayload = VIDBlockPayload, + BlockHeader = VIDBlockHeader, + >, MEMBERSHIP: Membership + Debug, NODE: NodeImplementation< TYPES, @@ -857,7 +865,11 @@ where /// Main entry point for validators pub async fn main_entry_point< - TYPES: NodeType, + TYPES: NodeType< + Transaction = VIDTransaction, + BlockPayload = VIDBlockPayload, + BlockHeader = VIDBlockHeader, + >, MEMBERSHIP: Membership + Debug, DANETWORK: CommunicationChannel, MEMBERSHIP> + Debug, QUORUMNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, From 2dfcf5d40a1ec37247fec71a13dfaa3ee8c478df Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Fri, 27 Oct 2023 12:17:46 -0700 Subject: [PATCH 10/13] Remove header bound, refactor leaf WIP --- crates/task-impls/src/consensus.rs | 47 +++++++----------- crates/task-impls/src/transactions.rs | 16 ++----- crates/testing/src/task_helpers.rs | 11 ++--- crates/testing/tests/consensus_task.rs | 3 +- crates/types/src/data.rs | 66 +++++++++----------------- crates/types/src/traits/storage.rs | 21 +++++--- 6 files changed, 63 insertions(+), 101 deletions(-) diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index 1d2ca253ab..0d64aa6eec 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -361,16 +361,16 @@ where } impl< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, ConsensusMessage = SequencingMessage, >, A: SequencingConsensusApi, I> + 'static, - H: BlockHeader, > SequencingConsensusTaskState where + TYPES::BlockHeader: BlockHeader, SequencingQuorumEx: ConsensusExchange< TYPES, Message, @@ -461,7 +461,8 @@ where view_number: view, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_header.clone()), + block_header: proposal.block_header.clone(), + transaction_commitments: HashSet::new(), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -530,7 +531,8 @@ where view_number: view, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_header.clone()), + block_header: proposal.block_header.clone(), + transaction_commitments: HashSet::new(), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -746,7 +748,8 @@ where view_number: view, justify_qc: justify_qc.clone(), parent_commitment: justify_qc.leaf_commitment(), - deltas: Right(proposal.data.block_header), + block_header: proposal.data.block_header, + transaction_commitments: HashSet::new(), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -770,7 +773,8 @@ where view_number: view, justify_qc: justify_qc.clone(), parent_commitment, - deltas: Right(proposal.data.block_header), + block_header: proposal.data.block_header, + transaction_commitments: HashSet::new(), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -870,15 +874,9 @@ where } leaf_views.push(leaf.clone()); - match &leaf.deltas { - Left((_,block)) => { - let txns = block.contained_transactions(); - for txn in txns { + for txn in leaf.transaction_commitments { included_txns.insert(txn); } - } - Right(_) => {} - } } true }, @@ -1368,16 +1366,7 @@ where } let parent_leaf = leaf.clone(); - let parent_header = match parent_leaf.deltas { - Left((_, ref payload)) => { - if parent_leaf.view_number != TYPES::Time::new(0) { - error!("Non-genesis parent leaf should contain the block header rather than payload."); - return false; - } - TYPES::BlockHeader::genesis(payload.clone()) - } - Right(ref header) => header.clone(), - }; + let parent_header = parent_leaf.block_header; let original_parent_hash = parent_leaf.commit(); @@ -1401,12 +1390,8 @@ where view_number: view, justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), - // Use the payload commitment rather than the payload, so that the replica can - // construct the same leaf with the commitment. - deltas: Right(TYPES::BlockHeader::new( - *payload_commitment, - parent_header.clone(), - )), + block_header: TYPES::BlockHeader::new(*payload_commitment, parent_header.clone()), + transaction_commitments: HashSet::new(), rejected: vec![], timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.api.public_key().to_bytes(), @@ -1498,14 +1483,13 @@ pub type ConsensusTaskTypes = HSTWithEvent< /// Event handle for consensus pub async fn sequencing_consensus_handle< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = SequencingLeaf, ConsensusMessage = SequencingMessage, >, A: SequencingConsensusApi, I> + 'static, - H: BlockHeader, >( event: SequencingHotShotEvent, mut state: SequencingConsensusTaskState, @@ -1514,6 +1498,7 @@ pub async fn sequencing_consensus_handle< SequencingConsensusTaskState, ) where + TYPES::BlockHeader: BlockHeader, SequencingQuorumEx: ConsensusExchange< TYPES, Message, diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index bc8d7798ed..72171de679 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -143,14 +143,8 @@ where let mut included_txn_size = 0; let mut included_txn_count = 0; for leaf in leaf_chain { - match &leaf.deltas { - Left((_, block)) => { - let txns = block.contained_transactions(); - for txn in txns { - included_txns.insert(txn); - } - } - Right(_) => {} + for txn in leaf.transaction_commitments { + included_txns.insert(txn); } } let consensus = self.consensus.read().await; @@ -329,9 +323,9 @@ where // TODO (Keyao) Investigate the use of transaction hash // // let parent_leaf = self.parent_leaf().await?; - // let previous_used_txns = match parent_leaf.deltas { - // Either::Left(block) => block.contained_transactions(), - // Either::Right(_commitment) => HashSet::new(), + // let previous_used_txns = match parent_leaf.tarnsaction_commitments { + // Some(txns) => txns, + // None => HashSet::new(), // }; let receiver = self.transactions.subscribe().await; diff --git a/crates/testing/src/task_helpers.rs b/crates/testing/src/task_helpers.rs index 9a20f6fe1e..f929eb4fee 100644 --- a/crates/testing/src/task_helpers.rs +++ b/crates/testing/src/task_helpers.rs @@ -116,13 +116,7 @@ async fn build_quorum_proposal_and_signature( panic!("Failed to find high QC parent."); }; let parent_leaf = leaf.clone(); - let parent_header = match parent_leaf.deltas { - Left((block_number, ref payload)) => VIDBlockHeader { - block_number, - payload_commitment: payload.commit(), - }, - Right(ref header) => header.clone(), - }; + let parent_header = parent_leaf.block_header; // every event input is seen on the event stream in the output. let block = ::genesis(); @@ -132,7 +126,8 @@ async fn build_quorum_proposal_and_signature( view_number: ViewNumber::new(view), justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), - deltas: Right(block_header.clone()), + block_header: block_header.clone(), + transaction_commitments: HashSet::new(), rejected: vec![], timestamp: 0, proposer_id: api.public_key().to_bytes(), diff --git a/crates/testing/tests/consensus_task.rs b/crates/testing/tests/consensus_task.rs index 726b4ed0e9..38163b6de5 100644 --- a/crates/testing/tests/consensus_task.rs +++ b/crates/testing/tests/consensus_task.rs @@ -65,7 +65,8 @@ async fn build_vote( view_number: view, justify_qc: proposal.justify_qc.clone(), parent_commitment, - deltas: Right(proposal.block_header), + block_header: proposal.block_header, + transaction_commitments: HashSet::new(), rejected: Vec::new(), timestamp: 0, proposer_id: quorum_exchange.get_leader(view).to_bytes(), diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 5f1cef024d..9f18e0745c 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -31,6 +31,7 @@ use rand::Rng; use serde::{Deserialize, Serialize}; use snafu::{ensure, Snafu}; use std::{ + collections::HashSet, fmt::{Debug, Display}, hash::Hash, }; @@ -611,10 +612,16 @@ pub struct SequencingLeaf { /// So we can ask if it extends pub parent_commitment: Commitment, - /// Either the block number and payload, or the block header. - pub deltas: Either<(u64, TYPES::BlockPayload), TYPES::BlockHeader>, + /// Block header. + pub block_header: TYPES::BlockHeader, - /// Transactions that were marked for rejection while collecting deltas + /// Set of commitments to the contained transactions. + /// + /// It may be empty for nodes not in the DA committee. + pub transaction_commitments: + HashSet::Transaction>>, + + /// Transactions that were marked for rejection while collecting the block. pub rejected: Vec<::Transaction>, // TODO (Keyao) Remove. @@ -627,19 +634,11 @@ pub struct SequencingLeaf { impl PartialEq for SequencingLeaf { fn eq(&self, other: &Self) -> bool { - let delta_left = match &self.deltas { - Either::Left(deltas) => (deltas.0, deltas.1.commit()), - Either::Right(deltas) => (deltas.block_number(), deltas.payload_commitment()), - }; - let delta_right = match &other.deltas { - Either::Left(deltas) => (deltas.0, deltas.1.commit()), - Either::Right(deltas) => (deltas.block_number(), deltas.payload_commitment()), - }; self.view_number == other.view_number - // && self.height == other.height && self.justify_qc == other.justify_qc && self.parent_commitment == other.parent_commitment - && delta_left == delta_right + && self.block_header == other.block_header + && self.transaction_commitments == other.transaction_commitments && self.rejected == other.rejected } } @@ -647,20 +646,10 @@ impl PartialEq for SequencingLeaf { impl Hash for SequencingLeaf { fn hash(&self, state: &mut H) { self.view_number.hash(state); - // self.height.hash(state); self.justify_qc.hash(state); self.parent_commitment.hash(state); - match &self.deltas { - Either::Left(deltas) => { - deltas.0.hash(state); - deltas.1.commit().hash(state); - } - Either::Right(header) => { - header.block_number().hash(state); - header.payload_commitment().hash(state); - } - } - // self.deltas.hash(state.commit()); + self.block_header.hasher(state); + self.transaction_commitments.hash(state); self.rejected.hash(state); } } @@ -749,7 +738,8 @@ impl LeafType for ValidatingLeaf { height: 0, justify_qc: stored_view.justify_qc, parent_commitment: stored_view.parent, - deltas: stored_view.deltas, + block_header: stored_view.block_header, + transaction_commitments: stored_view.transaction_commitments, state: stored_view.state, rejected: stored_view.rejected, timestamp: stored_view.timestamp, @@ -817,10 +807,7 @@ impl LeafType for SequencingLeaf { } fn get_height(&self) -> u64 { - match &self.deltas { - Left((height, _)) => *height, - Right(header) => header.block_number(), - } + self.block_header.block_number() } fn get_justify_qc(&self) -> QuorumCertificate> { @@ -831,12 +818,12 @@ impl LeafType for SequencingLeaf { self.parent_commitment } - fn get_deltas(&self) -> Self::DeltasType { - self.deltas.clone() + fn get_block_header(&self) -> ::BlockHeader { + self.block_header.clone() } - fn get_deltas_commitment(&self) -> Commitment<::BlockPayload> { - self.deltas.payload_commitment() + fn get_block_commitment(&self) -> Commitment<::BlockPayload> { + self.block_header.payload_commitment() } fn fill_deltas(&mut self, block: LeafBlockPayload) -> Result<(), LeafDeltasError> { @@ -989,20 +976,13 @@ impl Committable for ValidatingLeaf { impl Committable for SequencingLeaf { fn commit(&self) -> commit::Commitment { - // Commit the block payload, rather than the block, so that the replicas can reconstruct - // the leaf. - let (height, payload_commitment) = match &self.deltas { - Either::Left((height, payload)) => (*height, payload.commit()), - Either::Right(header) => (header.block_number(), header.payload_commitment()), - }; - let signatures_bytes = serialize_signature(&self.justify_qc.signatures); + // Skip the transaction commitments, so that the repliacs can reconstruct the leaf. commit::RawCommitmentBuilder::new("leaf commitment") .u64_field("view number", *self.view_number) - .u64_field("height", height) .field("parent Leaf commitment", self.parent_commitment) - .field("block payload commitment", payload_commitment) + .field("block header", self.block_header) .constant_str("justify_qc view number") .u64(*self.justify_qc.view_number) .field( diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index 996ea53e02..7da4c0b59f 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -10,7 +10,7 @@ use async_trait::async_trait; use commit::Commitment; use derivative::Derivative; use snafu::Snafu; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; /// Errors that can occur in the storage layer. #[derive(Clone, Debug, Snafu)] #[snafu(visibility(pub))] @@ -127,16 +127,19 @@ where pub struct StoredView> { /// The view number of this view pub view_number: TYPES::Time, - /// The index of `parent` in the chain - pub height: u64, /// The parent of this view pub parent: Commitment, /// The justify QC of this view. See the hotstuff paper for more information on this. pub justify_qc: QuorumCertificate>, /// The state of this view pub state: LEAF::MaybeState, - /// The deltas of this view - pub deltas: LEAF::DeltasType, + /// Block header. + pub block_header: TYPES::BlockHeader, + /// Set of commitments to the contained transactions. + /// + /// It may be empty for nodes not in the DA committee. + pub transaction_commitments: + HashSet::Transaction>>, /// transactions rejected in this view pub rejected: Vec, /// the timestamp this view was recv-ed in nanonseconds @@ -157,7 +160,10 @@ where /// Note that this will set the `parent` to `LeafHash::default()`, so this will not have a parent. pub fn from_qc_block_and_state( qc: QuorumCertificate>, - deltas: LEAF::DeltasType, + block_header: TYPES::BlockHeader, + transaction_commitments: HashSet< + Commitment<::Transaction>, + >, state: LEAF::MaybeState, height: u64, parent_commitment: Commitment, @@ -165,12 +171,13 @@ where proposer_id: EncodedPublicKey, ) -> Self { Self { - deltas, view_number: qc.view_number(), height, parent: parent_commitment, justify_qc: qc, state, + block_header, + transaction_commitments, rejected, timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id, From 9dbd45ed7de207a1ef0baef2cca032547be642d6 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Fri, 27 Oct 2023 12:25:42 -0700 Subject: [PATCH 11/13] Fmt after merge --- crates/hotshot/src/tasks/mod.rs | 12 ++---------- crates/task-impls/src/consensus.rs | 8 ++------ crates/task-impls/src/da.rs | 4 +--- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 9bbc8c8488..4c0917f565 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -244,11 +244,7 @@ where /// Is unable to panic. This section here is just to satisfy clippy pub async fn add_consensus_task< TYPES: NodeType, - I: NodeImplementation< - TYPES, - Leaf = Leaf, - ConsensusMessage = SequencingMessage, - >, + I: NodeImplementation, ConsensusMessage = SequencingMessage>, >( task_runner: TaskRunner, event_stream: ChannelStream>, @@ -486,11 +482,7 @@ where /// Is unable to panic. This section here is just to satisfy clippy pub async fn add_transaction_task< TYPES: NodeType, - I: NodeImplementation< - TYPES, - Leaf = Leaf, - ConsensusMessage = SequencingMessage, - >, + I: NodeImplementation, ConsensusMessage = SequencingMessage>, >( task_runner: TaskRunner, event_stream: ChannelStream>, diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index d21d8ea5c7..b609d6717b 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -1297,7 +1297,7 @@ where let consensus = self.consensus.read().await; consensus.metrics.number_of_timeouts.add(1); } -HotShotEvent::SendPayloadCommitment(payload_commitment) => { + HotShotEvent::SendPayloadCommitment(payload_commitment) => { self.payload_commitment = Some(payload_commitment); } _ => {} @@ -1473,11 +1473,7 @@ pub type ConsensusTaskTypes = HSTWithEvent< /// Event handle for consensus pub async fn sequencing_consensus_handle< TYPES: NodeType, - I: NodeImplementation< - TYPES, - Leaf = Leaf, - ConsensusMessage = SequencingMessage, - >, + I: NodeImplementation, ConsensusMessage = SequencingMessage>, A: ConsensusApi, I> + 'static, >( event: HotShotEvent, diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index 46235a321b..5271c4da90 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -438,9 +438,7 @@ where let message = Proposal { data, signature }; self.event_stream - .publish(HotShotEvent::SendPayloadCommitment( - payload_commitment, - )) + .publish(HotShotEvent::SendPayloadCommitment(payload_commitment)) .await; self.event_stream .publish(HotShotEvent::DAProposalSend( From ee6b138a0888077618f0ac018d0323d5771cf5ea Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Fri, 27 Oct 2023 17:53:29 -0700 Subject: [PATCH 12/13] Refactor leaf --- crates/hotshot/src/demo.rs | 2 +- crates/hotshot/src/lib.rs | 15 +- crates/hotshot/src/tasks/mod.rs | 2 +- .../src/traits/storage/memory_storage.rs | 2 +- crates/task-impls/src/consensus.rs | 113 ++++---- crates/task-impls/src/da.rs | 11 +- crates/task-impls/src/transactions.rs | 2 - crates/task-impls/src/vid.rs | 2 +- crates/testing/src/overall_safety_task.rs | 8 +- crates/testing/src/task_helpers.rs | 7 +- crates/testing/tests/consensus_task.rs | 2 +- crates/testing/tests/da_task.rs | 4 +- crates/testing/tests/network_task.rs | 4 +- crates/testing/tests/vid_task.rs | 2 +- crates/types/src/block_impl.rs | 4 +- crates/types/src/consensus.rs | 86 +++--- crates/types/src/data.rs | 248 ++++-------------- crates/types/src/traits/block_contents.rs | 8 +- crates/types/src/traits/storage.rs | 2 - 19 files changed, 204 insertions(+), 320 deletions(-) diff --git a/crates/hotshot/src/demo.rs b/crates/hotshot/src/demo.rs index f70c060701..000deb351b 100644 --- a/crates/hotshot/src/demo.rs +++ b/crates/hotshot/src/demo.rs @@ -12,7 +12,7 @@ use hotshot_signature_key::bn254::BLSPubKey; use hotshot_types::{ block_impl::{BlockPayloadError, VIDBlockHeader, VIDBlockPayload, VIDTransaction}, certificate::{AssembledSignature, QuorumCertificate}, - data::{fake_commitment, random_commitment, Leaf, LeafType, ViewNumber}, + data::{fake_commitment, random_commitment, LeafType, ViewNumber}, traits::{ election::Membership, node_implementation::NodeType, diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index ab753eaa08..7ebc80dafb 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -62,8 +62,8 @@ use hotshot_types::{ use hotshot_types::{ block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction}, certificate::{DACertificate, ViewSyncCertificate}, - consensus::{BlockStore, Consensus, ConsensusMetricsValue, View, ViewInner, ViewQueue}, - data::{DAProposal, DeltasType, Leaf, LeafType, QuorumProposal}, + consensus::{Consensus, ConsensusMetricsValue, TransactionStore, View, ViewInner, ViewQueue}, + data::{DAProposal, Leaf, LeafType, QuorumProposal}, error::StorageSnafu, message::{ ConsensusMessageType, DataMessage, InternalTrigger, Message, MessageKind, @@ -204,10 +204,13 @@ impl> SystemContext { ); let mut saved_leaves = HashMap::new(); - let mut saved_blocks = BlockStore::default(); + let mut saved_transaction_commitments = TransactionStore::default(); saved_leaves.insert(anchored_leaf.commit(), anchored_leaf.clone()); - if let Ok(block) = anchored_leaf.get_deltas().try_resolve() { - saved_blocks.insert(block); + if !anchored_leaf.get_transanction_commitments().is_empty() { + saved_transaction_commitments.insert( + anchored_leaf.get_payload_commitment(), + anchored_leaf.get_transanction_commitments(), + ); } let start_view = anchored_leaf.get_view_number(); @@ -217,7 +220,7 @@ impl> SystemContext { cur_view: start_view, last_decided_view: anchored_leaf.get_view_number(), saved_leaves, - saved_blocks, + saved_transaction_commitments, // TODO this is incorrect // https://github.com/EspressoSystems/HotShot/issues/560 locked_view: anchored_leaf.get_view_number(), diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 4c0917f565..bbabfb1ca8 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -243,7 +243,7 @@ where /// # Panics /// Is unable to panic. This section here is just to satisfy clippy pub async fn add_consensus_task< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation, ConsensusMessage = SequencingMessage>, >( task_runner: TaskRunner, diff --git a/crates/hotshot/src/traits/storage/memory_storage.rs b/crates/hotshot/src/traits/storage/memory_storage.rs index 45c02bf6fe..5511e4d4cd 100644 --- a/crates/hotshot/src/traits/storage/memory_storage.rs +++ b/crates/hotshot/src/traits/storage/memory_storage.rs @@ -173,8 +173,8 @@ mod test { view_number, }, DummyBlock::random(rng), + DummyBlock::random(rng).transaction_commitments(), DummyState::random(rng), - rng.next_u64(), dummy_leaf_commit, Vec::new(), genesis_proposer_id(), diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index b609d6717b..7d588c7a8d 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -6,7 +6,7 @@ use async_std::task::JoinHandle; use bitvec::prelude::*; use commit::{Commitment, Committable}; use core::time::Duration; -use either::{Either, Left, Right}; +use either::Either; use futures::FutureExt; use hotshot_constants::LOOK_AHEAD; use hotshot_task::{ @@ -16,7 +16,7 @@ use hotshot_task::{ task_impls::{HSTWithEvent, TaskBuilder}, }; use hotshot_types::{ - block_impl::VIDBlockPayload, + block_impl::{VIDBlockPayload, VIDTransaction}, certificate::{DACertificate, QuorumCertificate, TimeoutCertificate, VIDCertificate}, consensus::{Consensus, View}, data::{Leaf, LeafType, ProposalType, QuorumProposal}, @@ -30,7 +30,6 @@ use hotshot_types::{ node_implementation::{CommitteeEx, NodeImplementation, NodeType, QuorumEx, TimeoutEx}, signature_key::SignatureKey, state::ConsensusTime, - BlockPayload, }, utils::{Terminator, ViewInner}, vote::{QuorumVote, QuorumVoteAccumulator, TimeoutVoteAccumulator, VoteType}, @@ -55,7 +54,7 @@ pub struct ConsensusTaskError {} /// The state for the consensus task. Contains all of the information for the implementation /// of consensus pub struct ConsensusTaskState< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation, ConsensusMessage = SequencingMessage>, A: ConsensusApi, I> + 'static, > where @@ -355,7 +354,7 @@ where } impl< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = Leaf, @@ -823,62 +822,62 @@ where if parent_view + 1 == view { current_chain_length += 1; if let Err(e) = consensus.visit_leaf_ancestors( - parent_view, - Terminator::Exclusive(old_anchor_view), - true, - |leaf| { - if !new_decide_reached { - if last_view_number_visited == leaf.view_number + 1 { - last_view_number_visited = leaf.view_number; - current_chain_length += 1; - if current_chain_length == 2 { - new_locked_view = leaf.view_number; - new_commit_reached = true; - // The next leaf in the chain, if there is one, is decided, so this - // leaf's justify_qc would become the QC for the decided chain. - new_decide_qc = Some(leaf.justify_qc.clone()); - } else if current_chain_length == 3 { - new_anchor_view = leaf.view_number; - new_decide_reached = true; - } - } else { - // nothing more to do here... we don't have a new chain extension - return false; + parent_view, + Terminator::Exclusive(old_anchor_view), + true, + |leaf| { + if !new_decide_reached { + if last_view_number_visited == leaf.view_number + 1 { + last_view_number_visited = leaf.view_number; + current_chain_length += 1; + if current_chain_length == 2 { + new_locked_view = leaf.view_number; + new_commit_reached = true; + // The next leaf in the chain, if there is one, is decided, so this + // leaf's justify_qc would become the QC for the decided chain. + new_decide_qc = Some(leaf.justify_qc.clone()); + } else if current_chain_length == 3 { + new_anchor_view = leaf.view_number; + new_decide_reached = true; } + } else { + // nothing more to do here... we don't have a new chain extension + return false; } - // starting from the first iteration with a three chain, e.g. right after the else if case nested in the if case above - if new_decide_reached { - let mut leaf = leaf.clone(); - consensus + } + // starting from the first iteration with a three chain, e.g. right after the else if case nested in the if case above + if new_decide_reached { + let mut leaf = leaf.clone(); + consensus .metrics .last_synced_block_height .set(usize::try_from(leaf.get_height()).unwrap_or(0)); - // If the full block is available for this leaf, include it in the leaf - // chain that we send to the client. - if let Some(block) = - consensus.saved_blocks.get(leaf.get_deltas_commitment()) - { - if let Err(err) = leaf.fill_deltas(block.clone()) { - error!("unable to fill leaf {} with block {}, block will not be available: {}", - leaf.commit(), block.commit(), err); - } - } - - leaf_views.push(leaf.clone()); - for txn in leaf.transaction_commitments { - included_txns.insert(txn); - } + // If the full block is available for this leaf, include it in the leaf + // chain that we send to the client. + if let Some(comm) = consensus + .saved_transaction_commitments + .get(leaf.get_payload_commitment()) + { + leaf.fill_transaction_commitments(comm.clone()); + } + + leaf_views.push(leaf.clone()); + for txn in leaf.transaction_commitments { + included_txns.insert(txn); + } } - true - }, - ) { - error!("publishing view error"); - self.output_event_stream.publish(Event { + true + }, + ) { + error!("publishing view error"); + self.output_event_stream + .publish(Event { view_number: view, event: EventType::Error { error: e.into() }, - }).await; - } + }) + .await; + } } let included_txns_set: HashSet<_> = if new_decide_reached { @@ -1355,7 +1354,7 @@ where } let parent_leaf = leaf.clone(); - let parent_header = parent_leaf.block_header; + let parent_header = parent_leaf.block_header.clone(); let original_parent_hash = parent_leaf.commit(); @@ -1379,7 +1378,7 @@ where view_number: view, justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), - block_header: TYPES::BlockHeader::new(*payload_commitment, parent_header.clone()), + block_header: TYPES::BlockHeader::new(*payload_commitment, &parent_header), transaction_commitments: HashSet::new(), rejected: vec![], timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), @@ -1391,7 +1390,7 @@ where .sign_validating_or_commitment_proposal::(&leaf.commit()); // TODO: DA cert is sent as part of the proposal here, we should split this out so we don't have to wait for it. let proposal = QuorumProposal { - block_header: TYPES::BlockHeader::new(*payload_commitment, parent_header.clone()), + block_header: TYPES::BlockHeader::new(*payload_commitment, &parent_header), view_number: leaf.view_number, justify_qc: consensus.high_qc.clone(), timeout_certificate: timeout_certificate.or_else(|| None), @@ -1422,7 +1421,7 @@ where } impl< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation< TYPES, Leaf = Leaf, @@ -1472,7 +1471,7 @@ pub type ConsensusTaskTypes = HSTWithEvent< /// Event handle for consensus pub async fn sequencing_consensus_handle< - TYPES: NodeType, + TYPES: NodeType, I: NodeImplementation, ConsensusMessage = SequencingMessage>, A: ConsensusApi, I> + 'static, >( diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index 5271c4da90..f2f8237c2f 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -236,9 +236,9 @@ where debug!( "Got a DA block with {} transactions!", - proposal.data.deltas.contained_transactions().len() + proposal.data.block_payload.transaction_commitments().len() ); - let payload_commitment = proposal.data.deltas.commit(); + let payload_commitment = proposal.data.block_payload.commit(); // ED Is this the right leader? let view_leader_key = self.committee_exchange.get_leader(view); @@ -287,7 +287,10 @@ where }); // Record the block we have promised to make available. - consensus.saved_blocks.insert(proposal.data.deltas); + consensus.saved_transaction_commitments.insert( + proposal.data.block_payload.commit(), + proposal.data.block_payload.transaction_commitments(), + ); } } } @@ -429,7 +432,7 @@ where .committee_exchange .sign_da_proposal(&payload_commitment); let data: DAProposal = DAProposal { - deltas: block.clone(), + block_payload: block.clone(), // Upon entering a new view we want to send a DA Proposal for the next view -> Is it always the case that this is cur_view + 1? view_number: view, }; diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index cce883f9bd..4b5b356d4a 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -6,7 +6,6 @@ use async_compatibility_layer::{ use async_lock::RwLock; use bincode::config::Options; use commit::{Commitment, Committable}; -use either::{Left, Right}; use hotshot_task::{ event_stream::{ChannelStream, EventStream}, global_registry::GlobalRegistry, @@ -23,7 +22,6 @@ use hotshot_types::{ consensus_api::ConsensusApi, election::{ConsensusExchange, Membership, QuorumExchangeType}, node_implementation::{NodeImplementation, NodeType, QuorumEx}, - BlockPayload, }, }; use hotshot_utils::bincode::bincode_opts; diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs index 5a602dbc88..63883ca897 100644 --- a/crates/task-impls/src/vid.rs +++ b/crates/task-impls/src/vid.rs @@ -357,7 +357,7 @@ where // Record the block we have promised to make available. // TODO https://github.com/EspressoSystems/HotShot/issues/1692 - // consensus.saved_blocks.insert(proposal.data.deltas); + // consensus.saved_transaction_commitments.insert(proposal.data.block_payload); } } } diff --git a/crates/testing/src/overall_safety_task.rs b/crates/testing/src/overall_safety_task.rs index 535b5c1832..7bc4e83bdc 100644 --- a/crates/testing/src/overall_safety_task.rs +++ b/crates/testing/src/overall_safety_task.rs @@ -21,7 +21,7 @@ use hotshot_task::{ }; use hotshot_types::{ certificate::QuorumCertificate, - data::{DeltasType, LeafBlockPayload, LeafType}, + data::{LeafBlockPayload, LeafType}, error::RoundTimedoutState, event::{Event, EventType}, traits::node_implementation::NodeType, @@ -205,7 +205,7 @@ impl> RoundResult } } - let (state, block) = (leaf.get_state(), leaf.get_deltas()); + let (state, payload_commitment) = (leaf.get_state(), leaf.get_payload_commitment()); match self.state_map.entry(state.clone()) { std::collections::hash_map::Entry::Occupied(mut o) => { @@ -215,7 +215,7 @@ impl> RoundResult v.insert(1); } } - match self.block_map.entry(block.clone().payload_commitment()) { + match self.block_map.entry(payload_commitment) { std::collections::hash_map::Entry::Occupied(mut o) => { *o.get_mut() += 1; } @@ -291,7 +291,7 @@ impl> RoundResult // if neither, continue through let state_key = key.get_state(); - let block_key = key.get_deltas().payload_commitment(); + let block_key = key.get_payload_commitment(); if *self.block_map.get(&block_key).unwrap() == threshold && *self.state_map.get(&state_key).unwrap() == threshold diff --git a/crates/testing/src/task_helpers.rs b/crates/testing/src/task_helpers.rs index 2077f2ab34..32a3d5f2cd 100644 --- a/crates/testing/src/task_helpers.rs +++ b/crates/testing/src/task_helpers.rs @@ -1,9 +1,10 @@ +use std::collections::HashSet; + use crate::{ node_types::{MemoryImpl, TestTypes}, test_builder::TestMetadata, }; use commit::Committable; -use either::{Either::Left, Right}; use hotshot::{ certificate::QuorumCertificate, traits::{NodeImplementation, TestableNodeImplementation}, @@ -111,12 +112,12 @@ async fn build_quorum_proposal_and_signature( panic!("Failed to find high QC parent."); }; let parent_leaf = leaf.clone(); - let parent_header = parent_leaf.block_header; + let parent_header = parent_leaf.block_header.clone(); // every event input is seen on the event stream in the output. let block = ::genesis(); let payload_commitment = block.commit(); - let block_header = VIDBlockHeader::new(payload_commitment, parent_header); + let block_header = VIDBlockHeader::new(payload_commitment, &parent_header); let leaf = Leaf { view_number: ViewNumber::new(view), justify_qc: consensus.high_qc.clone(), diff --git a/crates/testing/tests/consensus_task.rs b/crates/testing/tests/consensus_task.rs index 9526b89df2..7ddc824c19 100644 --- a/crates/testing/tests/consensus_task.rs +++ b/crates/testing/tests/consensus_task.rs @@ -1,6 +1,5 @@ use commit::Commitment; use commit::Committable; -use either::Right; use hotshot::{ tasks::add_consensus_task, types::{SignatureKey, SystemContextHandle}, @@ -23,6 +22,7 @@ use hotshot_types::{ }; use std::collections::HashMap; +use std::collections::HashSet; async fn build_vote( handle: &SystemContextHandle, diff --git a/crates/testing/tests/da_task.rs b/crates/testing/tests/da_task.rs index fc91642c7c..3994c14b19 100644 --- a/crates/testing/tests/da_task.rs +++ b/crates/testing/tests/da_task.rs @@ -49,7 +49,7 @@ async fn test_da_task() { let signature = committee_exchange.sign_da_proposal(&block.commit()); let proposal = DAProposal { - deltas: block.clone(), + block_payload: block.clone(), view_number: ViewNumber::new(2), }; let message = Proposal { @@ -77,7 +77,7 @@ async fn test_da_task() { HotShotEvent::BlockReady(block.clone(), ViewNumber::new(2)), 1, ); - output.insert(HotShotEvent::SendDABlockData(block.clone()), 1); + output.insert(HotShotEvent::SendPayloadCommitment(block.commit()), 1); output.insert(HotShotEvent::DAProposalSend(message.clone(), pub_key), 1); let vote_token = committee_exchange .make_vote_token(ViewNumber::new(2)) diff --git a/crates/testing/tests/network_task.rs b/crates/testing/tests/network_task.rs index d88ce95ba2..e9409bc69b 100644 --- a/crates/testing/tests/network_task.rs +++ b/crates/testing/tests/network_task.rs @@ -52,7 +52,7 @@ async fn test_network_task() { let signature = committee_exchange.sign_da_proposal(&block.commit()); let da_proposal = Proposal { data: DAProposal { - deltas: block.clone(), + block_payload: block.clone(), view_number: ViewNumber::new(2), }, signature, @@ -117,7 +117,7 @@ async fn test_network_task() { HotShotEvent::QuorumProposalSend(quorum_proposal.clone(), pub_key), 1, ); - output.insert(HotShotEvent::SendDABlockData(block), 1); + output.insert(HotShotEvent::SendPayloadCommitment(block.commit()), 1); output.insert(HotShotEvent::DAProposalRecv(da_proposal, pub_key), 1); output.insert( HotShotEvent::QuorumProposalRecv(quorum_proposal, pub_key), diff --git a/crates/testing/tests/vid_task.rs b/crates/testing/tests/vid_task.rs index d97f158c4f..3070f05cf8 100644 --- a/crates/testing/tests/vid_task.rs +++ b/crates/testing/tests/vid_task.rs @@ -48,7 +48,7 @@ async fn test_vid_task() { let signature = vid_exchange.sign_vid_proposal(&block.commit()); let proposal: DAProposal = DAProposal { - deltas: block.clone(), + block_payload: block.clone(), view_number: ViewNumber::new(2), }; let message = Proposal { diff --git a/crates/types/src/block_impl.rs b/crates/types/src/block_impl.rs index 90aa210b6f..def1698078 100644 --- a/crates/types/src/block_impl.rs +++ b/crates/types/src/block_impl.rs @@ -123,7 +123,7 @@ impl BlockPayload for VIDBlockPayload { type Transaction = VIDTransaction; - fn contained_transactions(&self) -> HashSet> { + fn transaction_commitments(&self) -> HashSet> { self.transactions .iter() .map(commit::Committable::commit) @@ -143,7 +143,7 @@ pub struct VIDBlockHeader { impl BlockHeader for VIDBlockHeader { type Payload = VIDBlockPayload; - fn new(payload_commitment: Commitment, parent_header: Self) -> Self { + fn new(payload_commitment: Commitment, parent_header: &Self) -> Self { Self { block_number: parent_header.block_number + 1, payload_commitment, diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index fad59f0508..e61685ad69 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -13,13 +13,14 @@ use crate::{ traits::{ metrics::{Counter, Gauge, Histogram, Label, Metrics}, node_implementation::NodeType, + BlockPayload, }, utils::Terminator, }; -use commit::{Commitment, Committable}; +use commit::Commitment; use derivative::Derivative; use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap}, + collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, sync::{Arc, Mutex}, }; use tracing::error; @@ -47,10 +48,11 @@ pub struct Consensus> { /// - includes the MOST RECENT decided leaf pub saved_leaves: CommitmentMap, - /// Saved blocks + /// Saved transaction commitments /// - /// Contains the full block for every leaf in `saved_leaves` if that block is available. - pub saved_blocks: BlockStore, + /// Contains the transaction commitments of the block for every leaf in `saved_leaves` if that + /// commitment set is available. + pub saved_transaction_commitments: TransactionStore, /// The `locked_qc` view number pub locked_view: TYPES::Time, @@ -297,7 +299,7 @@ impl> Consensus { } /// garbage collects based on state change - /// right now, this removes from both the `saved_blocks` + /// right now, this removes from both the `saved_transaction_commitments` /// and `state_map` fields of `Consensus` /// # Panics /// On inconsistent stored entries @@ -323,14 +325,15 @@ impl> Consensus { .range(old_anchor_view..new_anchor_view) .filter_map(|(_view_number, view)| view.get_payload_commitment()) .for_each(|block| { - self.saved_blocks.remove(block); + self.saved_transaction_commitments.remove(block); }); self.state_map .range(old_anchor_view..new_anchor_view) .filter_map(|(_view_number, view)| view.get_leaf_commitment()) .for_each(|leaf| { if let Some(removed) = self.saved_leaves.remove(&leaf) { - self.saved_blocks.remove(removed.get_deltas_commitment()); + self.saved_transaction_commitments + .remove(removed.get_payload_commitment()); } }); self.state_map = self.state_map.split_off(&new_anchor_view); @@ -351,7 +354,10 @@ impl> Consensus { } } -/// Mapping from block payload commitments to full blocks. +/// Alias for the set of transaction commitments. +type TransactionCommitments = HashSet::Transaction>>; + +/// Mapping from block payload commitments to the set of transaction commitents. /// /// Entries in this mapping are reference-counted, so multiple consensus objects can refer to the /// same block, and the block will only be deleted after _all_ such objects are garbage collected. @@ -359,46 +365,60 @@ impl> Consensus { /// before all but one branch are ultimately garbage collected. #[derive(Clone, Debug, Derivative)] #[derivative(Default(bound = ""))] -pub struct BlockStore(HashMap, (BLOCK, u64)>); +pub struct TransactionStore( + HashMap, (TransactionCommitments, u64)>, +); -impl BlockStore { - /// Save `block` for later retrieval. +impl TransactionStore { + /// Save payload commitment for later retrieval. /// /// After calling this function, and before the corresponding call to [`remove`](Self::remove), - /// `self.get(block.commit())` will return `Some(block)`. + /// `self.get(payload_commitment)` will return `Some(transaction_commitments)`. /// - /// This function will increment a reference count on the saved block, so that multiple calls to - /// [`insert`](Self::insert) for the same block result in multiple owning references to the - /// block. [`remove`](Self::remove) must be called once for each reference before the block will - /// be deallocated. - pub fn insert(&mut self, block: BLOCK) { + /// This function will increment a reference count on the saved payload commitment, so that + /// multiple calls to [`insert`](Self::insert) for the same payload commitment result in + /// multiple owning references to the payload commitment. [`remove`](Self::remove) must be + /// called once for each reference before the payload commitment will be deallocated. + pub fn insert( + &mut self, + payload_commitment: Commitment, + transaction_commitments: TransactionCommitments, + ) { self.0 - .entry(block.commit()) + .entry(payload_commitment) .and_modify(|(_, refcount)| *refcount += 1) - .or_insert((block, 1)); + .or_insert((transaction_commitments, 1)); } - /// Get a saved block, if available. + /// Get a saved set of transaction commitments, if available. /// - /// If a block has been saved with [`insert`](Self::insert), this function will retrieve it. It - /// may return [`None`] if a block with the given commitment has not been saved or if the block - /// has been dropped with [`remove`](Self::remove). + /// If a set of transaction commitments has been saved with [`insert`](Self::insert), this + /// function will retrieve it. It may return [`None`] if a block with the given commitment has + /// not been saved or if the block has been dropped with [`remove`](Self::remove). #[must_use] - pub fn get(&self, block: Commitment) -> Option<&BLOCK> { - self.0.get(&block).map(|(block, _)| block) + pub fn get( + &self, + payload_commitment: Commitment, + ) -> Option<&HashSet::Transaction>>> { + self.0 + .get(&payload_commitment) + .map(|(txn_comm, _)| txn_comm) } - /// Drop a reference to a saved block. + /// Drop a reference to a saved set of transaction commitments. /// - /// If the block exists and this call drops the last reference to it, the block will be - /// returned. Otherwise, the return value is [`None`]. - pub fn remove(&mut self, block: Commitment) -> Option { - if let Entry::Occupied(mut e) = self.0.entry(block) { + /// If the set exists and this call drops the last reference to it, the set will be returned, + /// Otherwise, the return value is [`None`]. + pub fn remove( + &mut self, + payload_commitment: Commitment, + ) -> Option::Transaction>>> { + if let Entry::Occupied(mut e) = self.0.entry(payload_commitment) { let (_, refcount) = e.get_mut(); *refcount -= 1; if *refcount == 0 { - let (block, _) = e.remove(); - return Some(block); + let (txn_comm, _) = e.remove(); + return Some(txn_comm); } } None diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 0495ae1805..418fbd64b9 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -22,14 +22,12 @@ use ark_serialize::{CanonicalDeserialize, CanonicalSerialize, Read, Serializatio use bincode::Options; use commit::{Commitment, Committable}; use derivative::Derivative; -use either::{Either, Left, Right}; use espresso_systems_common::hotshot::tag; use hotshot_constants::GENESIS_PROPOSER_ID; use hotshot_utils::bincode::bincode_opts; use jf_primitives::pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}; use rand::Rng; use serde::{Deserialize, Serialize}; -use snafu::{ensure, Snafu}; use std::{ collections::HashSet, fmt::{Debug, Display}, @@ -161,7 +159,7 @@ where #[derive(custom_debug::Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] pub struct DAProposal { /// BlockPayload leaf wants to apply - pub deltas: TYPES::BlockPayload, + pub block_payload: TYPES::BlockPayload, /// View this proposal applies to pub view_number: TYPES::Time, } @@ -317,140 +315,6 @@ pub trait DeltasType: fn fill(&mut self, block: BlockPayload) -> Result<(), Self::Error>; } -/// Error which occurs when [`DeltasType::fill`] is called with a block that does not match the -/// deltas' internal block payload commitment. -#[derive(Clone, Copy, Debug, Snafu)] -#[snafu(display("the block {:?} has commitment {} (expected {})", block, block.commit(), commitment))] -pub struct InconsistentDeltasError { - /// The block with the wrong commitment. - block: BLOCK, - /// The expected commitment. - commitment: Commitment, -} - -impl DeltasType for BLOCK -where - BLOCK: Committable - + Clone - + Debug - + for<'a> Deserialize<'a> - + PartialEq - + Eq - + std::hash::Hash - + Send - + Serialize - + Sync, -{ - type Error = InconsistentDeltasError; - - fn payload_commitment(&self) -> Commitment { - self.commit() - } - - fn try_resolve(self) -> Result { - Ok(self) - } - - fn fill(&mut self, block: BLOCK) -> Result<(), Self::Error> { - ensure!( - block.commit() == self.commit(), - InconsistentDeltasSnafu { - block, - commitment: self.commit() - } - ); - // If the commitments are equal the blocks are equal, and we already have the block, so we - // don't have to do anything. - Ok(()) - } -} - -impl DeltasType for Either> -where - BLOCK: Committable - + Clone - + Debug - + for<'a> Deserialize<'a> - + PartialEq - + Eq - + std::hash::Hash - + Send - + Serialize - + Sync, -{ - type Error = InconsistentDeltasError; - - fn payload_commitment(&self) -> Commitment { - match self { - Either::Left(block) => block.commit(), - Either::Right(comm) => *comm, - } - } - - fn try_resolve(self) -> Result { - match self { - Either::Left(block) => Ok(block), - Either::Right(_) => Err(self), - } - } - - fn fill(&mut self, block: BLOCK) -> Result<(), Self::Error> { - match self { - Either::Left(curr) => curr.fill(block), - Either::Right(comm) => { - ensure!( - *comm == block.commit(), - InconsistentDeltasSnafu { - block, - commitment: *comm - } - ); - *self = Either::Left(block); - Ok(()) - } - } - } -} - -impl DeltasType for Either<(u64, PAYLOAD), HEADER> -where - HEADER: BlockHeader, - PAYLOAD: BlockPayload, -{ - type Error = InconsistentDeltasError; - - fn payload_commitment(&self) -> Commitment { - match self { - Either::Left((_, block)) => block.commit(), - Either::Right(header) => header.payload_commitment(), - } - } - - fn try_resolve(self) -> Result { - match self { - Either::Left((_, block)) => Ok(block), - Either::Right(_) => Err(self), - } - } - - fn fill(&mut self, block: PAYLOAD) -> Result<(), Self::Error> { - match self { - Either::Left((_, curr)) => curr.fill(block), - Either::Right(header) => { - ensure!( - header.payload_commitment() == block.commit(), - InconsistentDeltasSnafu { - block, - commitment: header.payload_commitment() - } - ); - *self = Either::Left((header.block_number(), block)); - Ok(()) - } - } - } -} - /// An item which is appended to a blockchain. pub trait LeafType: Debug @@ -467,8 +331,8 @@ pub trait LeafType: { /// Type of nodes participating in the network. type NodeType: NodeType; - /// Type of block contained by this leaf. - type DeltasType: DeltasType>; + // /// Type of block contained by this leaf. + // type DeltasType: DeltasType>; /// Either state or empty type MaybeState: Clone + Debug @@ -497,18 +361,21 @@ pub trait LeafType: fn get_justify_qc(&self) -> QuorumCertificate>; /// Commitment to this leaf's parent. fn get_parent_commitment(&self) -> Commitment; - /// The block contained in this leaf. - fn get_deltas(&self) -> Self::DeltasType; - /// Fill this leaf with the entire corresponding block. - /// - /// After this function succeeds, `self.get_deltas().try_resolve()` is guaranteed to return - /// `Ok(block)`. - /// - /// # Errors - /// - /// Fails if `block` does not match `self.get_deltas_commitment()`, or if the block is not able - /// to be stored for some implementation-defined reason. - fn fill_deltas(&mut self, block: LeafBlockPayload) -> Result<(), LeafDeltasError>; + /// The block header contained in this leaf. + fn get_block_header(&self) -> ::BlockHeader; + /// A commitment to the block payload contained in this leaf. + fn get_payload_commitment(&self) -> Commitment> { + self.get_block_header().payload_commitment() + } + /// Fill the transaciton commitments of this leaf with the corresponding block payload. + fn fill_transaction_commitments( + &mut self, + transaction_commitments: HashSet::Transaction>>, + ); + /// Optional set of commitments to the transactions. + fn get_transanction_commitments( + &self, + ) -> HashSet::Transaction>>; /// The blockchain state after appending this leaf. fn get_state(&self) -> Self::MaybeState; /// Transactions rejected or invalidated by the application of this leaf. @@ -519,17 +386,8 @@ pub trait LeafType: fn get_proposer_id(&self) -> EncodedPublicKey; /// Create a leaf from information stored about a view. fn from_stored_view(stored_view: StoredView) -> Self; - - /// A commitment to the block payload contained in this leaf. - fn get_deltas_commitment(&self) -> Commitment> { - self.get_deltas().payload_commitment() - } } -/// The [`DeltasType`] in a [`LeafType`]. -pub type LeafDeltas = ::DeltasType; -/// Errors reported by the [`DeltasType`] in a [`LeafType`]. -pub type LeafDeltasError = as DeltasType>>::Error; /// The [`NodeType`] in a [`LeafType`]. pub type LeafNode = ::NodeType; /// The [`StateType`] in a [`LeafType`]. @@ -648,8 +506,10 @@ impl Hash for Leaf { self.view_number.hash(state); self.justify_qc.hash(state); self.parent_commitment.hash(state); - self.block_header.hasher(state); - self.transaction_commitments.hash(state); + self.block_header.hash(state); + for com in &self.transaction_commitments { + com.hash(state); + } self.rejected.hash(state); } } @@ -666,7 +526,7 @@ impl Display for ValidatingLeaf { impl LeafType for ValidatingLeaf { type NodeType = TYPES; - type DeltasType = TYPES::BlockPayload; + // type DeltasType = TYPES::BlockPayload; type MaybeState = TYPES::StateType; fn new( @@ -704,16 +564,21 @@ impl LeafType for ValidatingLeaf { self.parent_commitment } - fn get_deltas(&self) -> Self::DeltasType { - self.deltas.clone() + fn get_block_header(&self) -> ::BlockHeader { + unimplemented!("Unimplemented for validating consensus which will be removed."); } - fn get_deltas_commitment(&self) -> Commitment<::BlockPayload> { - self.deltas.payload_commitment() + fn fill_transaction_commitments( + &mut self, + _transaction_commitments: HashSet::Transaction>>, + ) { + unimplemented!("Unimplemented for validating consensus which will be removed."); } - fn fill_deltas(&mut self, block: LeafBlockPayload) -> Result<(), LeafDeltasError> { - self.deltas.fill(block) + fn get_transanction_commitments( + &self, + ) -> HashSet::Transaction>> { + unimplemented!("Unimplemented for validating consensus which will be removed."); } fn get_state(&self) -> Self::MaybeState { @@ -732,19 +597,8 @@ impl LeafType for ValidatingLeaf { self.proposer_id.clone() } - fn from_stored_view(stored_view: StoredView) -> Self { - Self { - view_number: stored_view.view_number, - height: 0, - justify_qc: stored_view.justify_qc, - parent_commitment: stored_view.parent, - block_header: stored_view.block_header, - transaction_commitments: stored_view.transaction_commitments, - state: stored_view.state, - rejected: stored_view.rejected, - timestamp: stored_view.timestamp, - proposer_id: stored_view.proposer_id, - } + fn from_stored_view(_stored_view: StoredView) -> Self { + unimplemented!("Unimplemented for validating consensus which will be removed."); } } @@ -782,20 +636,21 @@ impl Display for Leaf { impl LeafType for Leaf { type NodeType = TYPES; - type DeltasType = Either<(u64, TYPES::BlockPayload), TYPES::BlockHeader>; + // type DeltasType = Either<(u64, TYPES::BlockPayload), TYPES::BlockHeader>; type MaybeState = (); fn new( view_number: ::Time, justify_qc: QuorumCertificate>, - deltas: ::BlockPayload, + payload: ::BlockPayload, _state: ::StateType, ) -> Self { Self { view_number, justify_qc, parent_commitment: fake_commitment(), - deltas: Either::Left((0, deltas)), + block_header: TYPES::BlockHeader::genesis(payload.clone()), + transaction_commitments: payload.transaction_commitments(), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: genesis_proposer_id(), @@ -822,12 +677,17 @@ impl LeafType for Leaf { self.block_header.clone() } - fn get_block_commitment(&self) -> Commitment<::BlockPayload> { - self.block_header.payload_commitment() + fn fill_transaction_commitments( + &mut self, + transaction_commitments: HashSet::Transaction>>, + ) { + self.transaction_commitments = transaction_commitments; } - fn fill_deltas(&mut self, block: LeafBlockPayload) -> Result<(), LeafDeltasError> { - self.deltas.fill(block) + fn get_transanction_commitments( + &self, + ) -> HashSet::Transaction>> { + self.transaction_commitments.clone() } // The Sequencing Leaf doesn't have a state. @@ -850,7 +710,8 @@ impl LeafType for Leaf { view_number: stored_view.view_number, justify_qc: stored_view.justify_qc, parent_commitment: stored_view.parent, - deltas: stored_view.deltas, + block_header: stored_view.block_header, + transaction_commitments: stored_view.transaction_commitments, rejected: stored_view.rejected, timestamp: stored_view.timestamp, proposer_id: stored_view.proposer_id, @@ -981,8 +842,9 @@ impl Committable for Leaf { // Skip the transaction commitments, so that the repliacs can reconstruct the leaf. commit::RawCommitmentBuilder::new("leaf commitment") .u64_field("view number", *self.view_number) + .u64_field("block number", self.get_height()) .field("parent Leaf commitment", self.parent_commitment) - .field("block header", self.block_header) + .field("block payload commitment", self.get_payload_commitment()) .constant_str("justify_qc view number") .u64(*self.justify_qc.view_number) .field( @@ -1021,11 +883,11 @@ where fn from(leaf: LEAF) -> Self { StoredView { view_number: leaf.get_view_number(), - height: leaf.get_height(), parent: leaf.get_parent_commitment(), justify_qc: leaf.get_justify_qc(), state: leaf.get_state(), - deltas: leaf.get_deltas(), + block_header: leaf.get_block_header(), + transaction_commitments: leaf.get_transanction_commitments(), rejected: leaf.get_rejected(), timestamp: leaf.get_timestamp(), proposer_id: leaf.get_proposer_id(), diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index 6922b5c011..a61a2613d9 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -54,7 +54,7 @@ pub trait BlockPayload: /// returns hashes of all the transactions in this block /// TODO make this ordered with a vec - fn contained_transactions(&self) -> HashSet>; + fn transaction_commitments(&self) -> HashSet>; } /// Header of a block, which commits to a [`BlockPayload`]. @@ -65,7 +65,7 @@ pub trait BlockHeader: type Payload: BlockPayload; /// Build a header with the payload commitment and parent header. - fn new(payload_commitment: Commitment, parent_header: Self) -> Self; + fn new(payload_commitment: Commitment, parent_header: &Self) -> Self; /// Build a genesis header with the genesis payload. fn genesis(payload: Self::Payload) -> Self; @@ -147,7 +147,7 @@ pub mod dummy { impl BlockHeader for DummyBlock { type Payload = Self; - fn new(_payload_commitment: Commitment, _parent_header: Self) -> Self { + fn new(_payload_commitment: Commitment, _parent_header: &Self) -> Self { Self { nonce: 0 } } @@ -171,7 +171,7 @@ pub mod dummy { type Transaction = DummyTransaction; - fn contained_transactions(&self) -> HashSet> { + fn transaction_commitments(&self) -> HashSet> { HashSet::new() } } diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index 7da4c0b59f..80df3b9841 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -165,14 +165,12 @@ where Commitment<::Transaction>, >, state: LEAF::MaybeState, - height: u64, parent_commitment: Commitment, rejected: Vec<::Transaction>, proposer_id: EncodedPublicKey, ) -> Self { Self { view_number: qc.view_number(), - height, parent: parent_commitment, justify_qc: qc, state, From 172c38f8da40c3c318c50690c1c88abeee3b3e1c Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Mon, 30 Oct 2023 16:02:58 -0700 Subject: [PATCH 13/13] Replace txn set with payload in leaf, return a reference for get_block_header. --- crates/hotshot/src/lib.rs | 13 +- crates/hotshot/src/traits/election/vrf.rs | 1024 ----------------- .../src/traits/storage/memory_storage.rs | 2 +- crates/task-impls/src/consensus.rs | 32 +- crates/task-impls/src/da.rs | 9 +- crates/task-impls/src/transactions.rs | 7 +- crates/task-impls/src/vid.rs | 2 +- crates/testing/src/task_helpers.rs | 4 +- crates/testing/tests/consensus_task.rs | 3 +- crates/types/src/consensus.rs | 56 +- crates/types/src/data.rs | 91 +- crates/types/src/traits/storage.rs | 18 +- 12 files changed, 116 insertions(+), 1145 deletions(-) delete mode 100644 crates/hotshot/src/traits/election/vrf.rs diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 7ebc80dafb..3d5c770ea9 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -62,7 +62,7 @@ use hotshot_types::{ use hotshot_types::{ block_impl::{VIDBlockHeader, VIDBlockPayload, VIDTransaction}, certificate::{DACertificate, ViewSyncCertificate}, - consensus::{Consensus, ConsensusMetricsValue, TransactionStore, View, ViewInner, ViewQueue}, + consensus::{BlockPayloadStore, Consensus, ConsensusMetricsValue, View, ViewInner, ViewQueue}, data::{DAProposal, Leaf, LeafType, QuorumProposal}, error::StorageSnafu, message::{ @@ -204,13 +204,10 @@ impl> SystemContext { ); let mut saved_leaves = HashMap::new(); - let mut saved_transaction_commitments = TransactionStore::default(); + let mut saved_block_payloads = BlockPayloadStore::default(); saved_leaves.insert(anchored_leaf.commit(), anchored_leaf.clone()); - if !anchored_leaf.get_transanction_commitments().is_empty() { - saved_transaction_commitments.insert( - anchored_leaf.get_payload_commitment(), - anchored_leaf.get_transanction_commitments(), - ); + if let Some(payload) = anchored_leaf.get_block_payload() { + saved_block_payloads.insert(payload); } let start_view = anchored_leaf.get_view_number(); @@ -220,7 +217,7 @@ impl> SystemContext { cur_view: start_view, last_decided_view: anchored_leaf.get_view_number(), saved_leaves, - saved_transaction_commitments, + saved_block_payloads, // TODO this is incorrect // https://github.com/EspressoSystems/HotShot/issues/560 locked_view: anchored_leaf.get_view_number(), diff --git a/crates/hotshot/src/traits/election/vrf.rs b/crates/hotshot/src/traits/election/vrf.rs deleted file mode 100644 index f99dabd7e8..0000000000 --- a/crates/hotshot/src/traits/election/vrf.rs +++ /dev/null @@ -1,1024 +0,0 @@ -use hotshot_types::traits::signature_key::EncodedPublicKey; - -#[allow(deprecated)] -use serde::{Deserialize, Serialize}; -use std::{collections::BTreeMap, fmt::Debug, marker::PhantomData, num::NonZeroU64}; - -// TODO wrong palce for this -/// the sortition committee size parameter -pub const SORTITION_PARAMETER: u64 = 100; - -// TODO compatibility this function's impl into a trait -// TODO do we necessariy want the units of stake to be a u64? or generics -/// The stake table for VRFs -#[derive(Serialize, Deserialize, Debug)] -pub struct VRFStakeTable { - /// the mapping of id -> stake - mapping: BTreeMap, - /// total stake present - total_stake: NonZeroU64, - /// PhantomData - _pd: PhantomData<(VRF, VRFHASHER, VRFPARAMS)>, -} - -impl Clone for VRFStakeTable { - fn clone(&self) -> Self { - Self { - mapping: self.mapping.clone(), - total_stake: self.total_stake, - _pd: PhantomData, - } - } -} - -// impl VRFStakeTable { -// /// get total stake -// #[must_use] -// pub fn get_all_stake(&self) -> NonZeroU64 { -// self.total_stake -// } -// } - -// impl VRFStakeTable -// where -// VRF: Vrf, -// VRFPARAMS: Bls12Parameters, -// ::G1Parameters: TEHashToGroup, -// VRF::PublicKey: Clone, -// { -// /// get total stake -// /// # Panics -// /// If converting non-zero stake into `NonZeroU64` fails -// pub fn get_stake(&self, pk: &JfPubKey) -> Option -// where -// SIGSCHEME: SignatureScheme< -// VerificationKey = VRF::PublicKey, -// PublicParameter = (), -// MessageUnit = u8, -// >, -// SIGSCHEME::VerificationKey: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// SIGSCHEME::SigningKey: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// SIGSCHEME::Signature: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// { -// let encoded = pk.to_bytes(); -// let stake = self.mapping.get(&encoded).map(|val| val.get()); -// stake.and_then(NonZeroU64::new) -// } -// } - -// /// the vrf implementation -// #[derive(Derivative)] -// #[derivative(Debug, Eq, PartialEq)] -// pub struct VrfImpl, SIGSCHEME, VRF, VRFHASHER, VRFPARAMS> -// where -// VRF: Vrf + Sync + Send, -// TYPES: NodeType, -// { -// /// the stake table -// #[derivative(Debug = "ignore", PartialEq = "ignore")] -// stake_table: VRFStakeTable, -// /// the proof params -// #[derivative(Debug = "ignore", PartialEq = "ignore")] -// proof_parameters: VRF::PublicParameter, -// /// the rng -// #[derivative(PartialEq = "ignore")] -// prng: std::sync::Arc>, -// /// the committee parameter -// sortition_parameter: NonZeroU64, -// /// the chain commitment seed -// chain_seed: [u8; 32], -// /// pdf cache -// #[derivative(PartialEq = "ignore")] -// _sortition_cache: std::sync::Arc>>>, - -// /// phantom data -// _pd: PhantomData<(TYPES, LEAF, SIGSCHEME, VRF, VRFHASHER, VRFPARAMS)>, -// } - -// impl, SIGSCHEME, VRF, VRFHASHER, VRFPARAMS> Clone -// for VrfImpl -// where -// VRF: Vrf + Sync + Send, -// TYPES: NodeType, -// { -// fn clone(&self) -> Self { -// Self { -// stake_table: self.stake_table.clone(), -// proof_parameters: (), -// prng: self.prng.clone(), -// sortition_parameter: self.sortition_parameter, -// chain_seed: self.chain_seed, -// _sortition_cache: Arc::default(), -// _pd: PhantomData, -// } -// } -// } - -// /// TODO doc me -// #[derive(Serialize, Deserialize, Clone)] -// pub struct VRFVoteToken { -// /// The public key assocaited with this token -// pub pub_key: PUBKEY, -// /// The list of signatures -// pub proof: PROOF, -// /// The number of signatures that are valid -// /// TODO (ct) this should be the sorition outbput -// pub count: NonZeroU64, -// } - -// impl Hash for VRFVoteToken -// where -// PUBKEY: serde::Serialize, -// PROOF: serde::Serialize, -// { -// fn hash(&self, state: &mut H) { -// bincode_opts().serialize(&self.pub_key).unwrap().hash(state); -// bincode_opts().serialize(&self.proof).unwrap().hash(state); -// self.count.hash(state); -// } -// } - -// impl Debug for VRFVoteToken { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// f.debug_struct("VRFVoteToken") -// .field("pub_key", &std::any::type_name::()) -// .field("proof", &std::any::type_name::()) -// .field("count", &self.count) -// .finish() -// } -// } - -// impl PartialEq for VRFVoteToken -// where -// PUBKEY: serde::Serialize, -// PROOF: serde::Serialize, -// { -// fn eq(&self, other: &Self) -> bool { -// self.count == other.count -// && bincode_opts().serialize(&self.pub_key).unwrap() -// == bincode_opts().serialize(&other.pub_key).unwrap() -// && bincode_opts().serialize(&self.proof).unwrap() -// == bincode_opts().serialize(&other.proof).unwrap() -// } -// } - -// impl VoteToken for VRFVoteToken -// where -// PUBKEY: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static, -// PROOF: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static, -// { -// fn vote_count(&self) -> NonZeroU64 { -// self.count -// } -// } - -// impl Committable for VRFVoteToken -// where -// PUBKEY: serde::Serialize, -// PROOF: serde::Serialize, -// { -// fn commit(&self) -> Commitment { -// RawCommitmentBuilder::new(std::any::type_name::()) -// .u64(self.count.get()) -// .var_size_bytes(bincode_opts().serialize(&self.pub_key).unwrap().as_slice()) -// .var_size_bytes(bincode_opts().serialize(&self.proof).unwrap().as_slice()) -// .finalize() -// } - -// fn tag() -> String { -// tag::VRF_VOTE_TOKEN.to_string() -// } -// } - -// // KEY is VRFPubKey -// impl> -// Membership for VrfImpl -// where -// SIGSCHEME: SignatureScheme + Sync + Send + 'static, -// SIGSCHEME::VerificationKey: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// SIGSCHEME::SigningKey: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// SIGSCHEME::Signature: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// VRF: Vrf< -// PublicParameter = (), -// Input = Vec, -// Output = Vec, -// PublicKey = SIGSCHEME::VerificationKey, -// SecretKey = SIGSCHEME::SigningKey, -// > + Sync -// + Send -// + 'static, -// VRF::Proof: Clone + Sync + Send + Serialize + for<'a> Deserialize<'a>, -// VRF::PublicParameter: Sync + Send, -// VRFHASHER: digest::Digest + Clone + Sync + Send + 'static, -// VRFPARAMS: Sync + Send + Bls12Parameters, -// ::G1Parameters: TEHashToGroup, -// TYPES: NodeType< -// VoteTokenType = VRFVoteToken, -// ElectionConfigType = VRFStakeTableConfig, -// SignatureKey = JfPubKey, -// >, -// { -// // pubkey -> unit of stake -// type StakeTable = VRFStakeTable; - -// // FIXED STAKE -// // just return the state -// fn get_stake_table( -// &self, -// _view_number: TYPES::Time, -// _state: &TYPES::StateType, -// ) -> Self::StakeTable { -// self.stake_table.clone() -// } - -// fn get_leader(&self, view_number: TYPES::Time) -> JfPubKey { -// // TODO fst2 (ct) this is round robin, we should make this dependent on -// // the VRF + some source of randomness - -// // TODO for now do by stake table of how much stake each -// // participant has -// let mapping = &self.stake_table.mapping; -// let index = ((*view_number) as usize) % mapping.len(); -// let encoded = mapping.keys().nth(index).unwrap(); -// SignatureKey::from_bytes(encoded).unwrap() -// } - -// // what this is doing: -// // - -// fn make_vote_token( -// // TODO see if we can make this take &mut self -// // because we're using a mutable prng -// &self, -// view_number: TYPES::Time, -// private_key: &(SIGSCHEME::SigningKey, SIGSCHEME::VerificationKey), -// ) -> Result, ElectionError> { -// let pub_key = JfPubKey::::from_native(private_key.1.clone()); -// let Some(replicas_stake) = self.stake_table.get_stake(&pub_key) else { return Ok(None) }; - -// let view_seed = generate_view_seed::(view_number, &self.chain_seed); - -// let proof = Self::internal_get_vrf_proof( -// &private_key.0, -// &self.proof_parameters, -// &mut self.prng.lock().unwrap(), -// &view_seed, -// )?; - -// let selected_stake = Self::internal_get_sortition_for_proof( -// &self.proof_parameters, -// &proof, -// self.stake_table.get_all_stake(), -// replicas_stake, -// self.sortition_parameter, -// ); - -// match selected_stake { -// Some(count) => { -// // TODO (ct) this can fail, return Result::Err -// let proof = VRF::prove( -// &self.proof_parameters, -// &private_key.0, -// &view_seed, -// &mut *self.prng.lock().unwrap(), -// ) -// .unwrap(); - -// Ok(Some(VRFVoteToken { -// pub_key: private_key.1.clone(), -// proof, -// count, -// })) -// } -// None => Ok(None), -// } -// } - -// fn validate_vote_token( -// &self, -// view_number: TYPES::Time, -// pub_key: JfPubKey, -// token: Checked, -// ) -> Result, ElectionError> { -// match token { -// Checked::Unchecked(token) => { -// let stake: Option = self.stake_table.get_stake(&pub_key); -// let view_seed = -// generate_view_seed::(view_number, &self.chain_seed); -// if let Some(stake) = stake { -// Self::internal_check_sortition( -// &pub_key.pk, -// &self.proof_parameters, -// &token.proof, -// self.stake_table.get_all_stake(), -// stake, -// self.sortition_parameter, -// token.count, -// &view_seed, -// ) -// .map(|c| match c { -// Checked::Inval(_) => Checked::Inval(token), -// Checked::Valid(_) => Checked::Valid(token), -// Checked::Unchecked(_) => Checked::Unchecked(token), -// }) -// } else { -// // TODO better error -// Err(ElectionError::StubError) -// } -// } -// already_checked => Ok(already_checked), -// } -// } - -// fn create_election(keys: Vec>, config: TYPES::ElectionConfigType) -> Self { -// // This all needs to be refactored. For one thing, having the stake table - even an initial -// // stake table - hardcoded like this is flat-out broken. This is, obviously, an artifact -// let genesis_seed = [0u8; 32]; -// VrfImpl::with_initial_stake(keys, &config, genesis_seed) -// } - -// fn default_election_config(num_nodes: u64) -> TYPES::ElectionConfigType { -// let mut stake = Vec::new(); -// let units_of_stake_per_node = NonZeroU64::new(100).unwrap(); -// for _ in 0..num_nodes { -// stake.push(units_of_stake_per_node); -// } -// VRFStakeTableConfig { -// sortition_parameter: NonZeroU64::new(SORTITION_PARAMETER).unwrap(), -// distribution: stake, -// } -// } - -// fn success_threshold(&self) -> NonZeroU64 { -// NonZeroU64::new(((u64::from(self.sortition_parameter) * 2) / 3) + 1).unwrap() -// } - -// fn failure_threshold(&self) -> NonZeroU64 { -// NonZeroU64::new(((u64::from(self.sortition_parameter)) / 3) + 1).unwrap() -// } -// /// TODO if we ever come back to using this, we'll need to change this -// /// this stub is incorrect as it stands right now -// fn get_committee( -// &self, -// _view_number: ::Time, -// ) -> std::collections::BTreeSet<::SignatureKey> { -// self.stake_table -// .mapping -// .keys() -// .clone() -// .filter_map(::SignatureKey::from_bytes) -// .collect() -// } -// } - -// /// checks that the expected aomunt of stake matches the VRF output -// /// TODO this can be optimized most likely -// fn check_bin_idx( -// expected_amount_of_stake: u64, -// replicas_stake: u64, -// total_stake: u64, -// sortition_parameter: u64, -// unnormalized_seed: &[u8; 32], -// cache: &mut HashMap>, -// ) -> Option { -// let bin_idx = find_bin_idx( -// replicas_stake, -// total_stake, -// sortition_parameter, -// unnormalized_seed, -// cache, -// ); -// bin_idx.map(|idx| idx == NonZeroU64::new(expected_amount_of_stake).unwrap()) -// } - -// /// generates the seed from algorand paper -// /// baseed on `view_number` and a constant as of now, but in the future will be other things -// /// this is a stop-gap -// fn generate_view_seed( -// view_number: TYPES::Time, -// vrf_seed: &[u8; 32], -// ) -> [u8; 32] { -// let mut hasher = HASHER::new(); -// hasher.update(vrf_seed); -// hasher.update(view_number.deref().to_le_bytes()); -// let mut output = [0u8; 32]; -// output.copy_from_slice(hasher.finalize().as_ref()); -// output -// } - -// /// represents a binomial query made by sortition -// /// `B(stake_attempt; replicas_stake; sortition_parameter / total_stake)` -// #[derive(Hash, Eq, PartialEq, Clone, Debug)] -// pub struct BinomialQuery { -// /// the number of heads -// stake_attempt: u64, -// /// the total number of coin flips -// replicas_stake: u64, -// /// the total amount of stake -// total_stake: u64, -// /// the sortition parameter -// sortition_parameter: u64, -// } - -// impl BinomialQuery { -// /// get the committee parameter -// /// for this query -// #[must_use] -// pub fn get_p(&self) -> Ratio { -// let sortition_parameter_big: BigUint = BigUint::from(self.sortition_parameter); -// let total_stake_big: BigUint = BigUint::from(self.total_stake); -// Ratio::new(sortition_parameter_big, total_stake_big) -// } -// } - -// #[instrument] -// fn calculate_threshold_from_cache( -// previous_calculation: Option<(BinomialQuery, Ratio)>, -// query: BinomialQuery, -// ) -> Option> { -// if let Some((previous_query, previous_result)) = previous_calculation { -// let expected_previous_query = BinomialQuery { -// stake_attempt: query.stake_attempt - 1, -// ..query -// }; -// if previous_query == expected_previous_query { -// let permutation = Ratio::new( -// BigUint::from(query.replicas_stake - query.stake_attempt + 1), -// BigUint::from(query.stake_attempt), -// ); -// let p = query.get_p(); -// assert!(p.numer() < p.denom()); -// let reciprocal = Ratio::recip(&(Ratio::from_integer(BigUint::from(1_u32)) - p.clone())); -// let result = previous_result * p * reciprocal * permutation; -// assert!(result.numer() < result.denom()); - -// return Some(result); -// } -// } -// calculate_threshold(query) -// } - -// // Calculates B(j; w; p) where B means bernoulli distribution. -// // That is: run w trials, with p probability of success for each trial, and return the probability -// // of j successes. -// // p = tau / W, where tau is the sortition parameter (controlling committee size) -// // this is the only usage of W and tau -// // -// // Translation: -// // stake_attempt: our guess at what the stake might be. This is j -// // replicas_stake: the units of stake owned by the replica. This is w -// // total_stake: the units of stake owned in total. This is W -// // sorition_parameter: the parameter controlling the committee size. This is tau -// // -// // TODO (ct) better error handling -// // returns none if one of our calculations fails -// // -// // TODO keep data around from last iteration so less calculation is needed -// // TODO test this "correct/simple" implementation against any optimized version -// #[instrument] -// // fn calculate_threshold(stake_attempt: u32, replicas_stake: u64, total_stake: u64, sortition_parameter: u64) -> Option> { -// fn calculate_threshold(query: BinomialQuery) -> Option> { -// let stake_attempt = query.stake_attempt; -// tracing::info!("Running calculate threshold"); -// // TODO (ct) better error handling -// if stake_attempt > query.replicas_stake { -// error!("j is larger than amount of stake we are allowed"); -// return None; -// } - -// let sortition_parameter_big: BigUint = BigUint::from(query.sortition_parameter); -// let total_stake_big: BigUint = BigUint::from(query.total_stake); -// let one_big = BigUint::from(1_u32); - -// // this is the p parameter for the bernoulli distribution -// let p = Ratio::new(sortition_parameter_big, total_stake_big); - -// assert!(p.numer() <= p.denom()); - -// info!("p is {p:?}"); - -// // number of tails in bernoulli -// let failed_num = query.replicas_stake - stake_attempt; - -// // TODO cancel things out (avoid calculating factorial) -// // TODO can just do division -// let num_permutations = Ratio::new( -// factorial(query.replicas_stake), -// factorial(stake_attempt) * factorial(failed_num), -// ); - -// info!("num permutations is {num_permutations:?}, failed_num is {failed_num:?}"); - -// let one = Ratio::from_integer(one_big); - -// // TODO can keep results from last try -// let result = num_permutations -// * (p.pow(i32::try_from(stake_attempt).ok()?) -// * (one - p).pow(i32::try_from(failed_num).ok()?)); - -// assert!(result.numer() < result.denom()); - -// info!("result is is {result:?}"); - -// Some(result) -// } - -// /// compute i! as a biguint -// fn factorial(mut i: u64) -> BigUint { -// if i == 0 { -// return BigUint::from(1u32); -// } - -// let mut result = BigUint::from(1u32); -// while i > 0 { -// result *= i; -// i -= 1; -// } -// result -// } - -// /// find the amount of stake we rolled. -// /// NOTE: in the future this requires a view numb -// /// Returns None if zero stake was rolled -// #[instrument] -// fn find_bin_idx( -// replicas_stake: u64, -// total_stake: u64, -// sortition_parameter: u64, -// unnormalized_seed: &[u8; 32], -// cache: &mut HashMap>, -// ) -> Option { -// let unnormalized_seed = BigUint::from_bytes_le(unnormalized_seed); -// let normalized_seed = Ratio::new(unnormalized_seed, BigUint::from(2_u32).pow(256)); -// assert!(normalized_seed.numer() < normalized_seed.denom()); -// let mut j: u64 = 0; - -// // [j, j+1) -// // [cdf(j),cdf(j+1)) - -// // left_threshold corresponds to the sum of all bernoulli distributions -// // from i in 0 to j: B(i; replicas_stake; p). Where p is calculated later and corresponds to -// // algorands paper -// let mut left_threshold = Ratio::from_integer(BigUint::from(0u32)); - -// loop { -// // check cache - -// // if cache miss, feed in with previous val from cache -// // that *probably* exists - -// assert!(left_threshold.numer() < left_threshold.denom()); -// let query = BinomialQuery { -// stake_attempt: j + 1, -// replicas_stake, -// total_stake, -// sortition_parameter, -// }; - -// let bin_val = { -// // we already computed this value -// if let Some(result) = cache.get(&query) { -// result.clone() -// } else { -// // we haven't computed this value, but maybe -// // we already computed the previous value - -// let mut maybe_old_query = query.clone(); -// maybe_old_query.stake_attempt -= 1; -// let old_result = cache -// .get(&maybe_old_query) -// .map(|x| (maybe_old_query, x.clone())); -// let result = calculate_threshold_from_cache(old_result, query.clone())?; -// cache.insert(query, result.clone()); -// result -// } -// }; - -// // corresponds to right range from apper -// let right_threshold = left_threshold + bin_val.clone(); - -// // debugging info. Unnecessary -// { -// let right_threshold_float = ToPrimitive::to_f64(&right_threshold.clone()); -// let bin_val_float = ToPrimitive::to_f64(&bin_val.clone()); -// let normalized_seed_float = ToPrimitive::to_f64(&normalized_seed.clone()); -// info!("rightthreshold: {right_threshold_float:?}, bin: {bin_val_float:?}, seed: {normalized_seed_float:?}"); -// } - -// // from i in 0 to j + 1: B(i; replicas_stake; p) -// if normalized_seed < right_threshold { -// match j { -// 0 => return None, -// _ => return Some(NonZeroU64::new(j).unwrap()), -// } -// } -// left_threshold = right_threshold; -// j += 1; -// } -// } - -// impl, SIGSCHEME, VRF, VRFHASHER, VRFPARAMS> -// VrfImpl -// where -// SIGSCHEME: SignatureScheme + Sync + Send, -// SIGSCHEME::VerificationKey: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// SIGSCHEME::SigningKey: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// SIGSCHEME::Signature: Clone + Serialize + for<'a> Deserialize<'a> + Sync + Send, -// VRF: Vrf< -// PublicParameter = (), -// Input = [u8; 32], -// Output = [u8; 32], -// PublicKey = SIGSCHEME::VerificationKey, -// SecretKey = SIGSCHEME::SigningKey, -// > + Sync -// + Send, -// VRF::Proof: Clone + Sync + Send + Serialize + for<'a> Deserialize<'a>, -// VRF::PublicParameter: Sync + Send, -// VRFHASHER: digest::Digest + Clone + Sync + Send, -// VRFPARAMS: Sync + Send + Bls12Parameters, -// ::G1Parameters: TEHashToGroup, -// TYPES: NodeType, -// { -// /// create stake table with this initial stake -// /// # Panics -// /// TODO -// #[must_use] -// pub fn with_initial_stake( -// known_nodes: Vec>, -// config: &VRFStakeTableConfig, -// genesis_seed: [u8; 32], -// ) -> Self { -// assert_eq!(known_nodes.iter().len(), config.distribution.len()); -// let key_with_stake = known_nodes -// .into_iter() -// .map(|x| x.to_bytes()) -// .zip(config.distribution.clone()) -// .collect(); -// VrfImpl { -// stake_table: { -// let st = VRFStakeTable { -// mapping: key_with_stake, -// total_stake: NonZeroU64::new(config.distribution.iter().map(|x| x.get()).sum()) -// .unwrap(), -// _pd: PhantomData, -// }; -// st -// }, -// proof_parameters: (), -// chain_seed: genesis_seed, -// prng: Arc::new(Mutex::new(ChaChaRng::from_seed(Default::default()))), -// _pd: PhantomData, -// sortition_parameter: config.sortition_parameter, -// _sortition_cache: Arc::default(), -// } -// } - -// /// stateless delegate for VRF proof generation -// /// # Errors -// /// - -// fn internal_get_vrf_proof( -// private_key: &SIGSCHEME::SigningKey, -// proof_param: &VRF::PublicParameter, -// to_refactor: &mut rand_chacha::ChaChaRng, -// vrf_in_seed: &VRF::Input, -// ) -> Result { -// VRF::prove(proof_param, private_key, vrf_in_seed, to_refactor) -// .map_err(|_| ElectionError::StubError) -// } - -// /// stateless delegate for VRF sortition generation -// fn internal_get_sortition_for_proof( -// proof_param: &VRF::PublicParameter, -// proof: &VRF::Proof, -// total_stake: NonZeroU64, -// voter_stake: NonZeroU64, -// sortition_parameter: NonZeroU64, -// ) -> Option { -// // TODO (ct) this can fail, return result::err -// let hash = VRF::evaluate(proof_param, proof).unwrap(); -// let mut cache: HashMap> = HashMap::new(); - -// find_bin_idx( -// u64::from(voter_stake), -// u64::from(total_stake), -// sortition_parameter.into(), -// &hash, -// &mut cache, -// ) -// } - -// /// stateless delegate for VRF sortition confirmation -// /// # Errors -// /// if the proof is malformed -// #[allow(clippy::too_many_arguments)] -// fn internal_check_sortition( -// public_key: &SIGSCHEME::VerificationKey, -// proof_param: &VRF::PublicParameter, -// proof: &VRF::Proof, -// total_stake: NonZeroU64, -// voter_stake: NonZeroU64, -// sortition_parameter: NonZeroU64, -// sortition_claim: NonZeroU64, -// vrf_in_seed: &VRF::Input, -// ) -> Result, hotshot_types::traits::election::ElectionError> { -// if let Ok(true) = VRF::verify(proof_param, proof, public_key, vrf_in_seed) { -// let seed = VRF::evaluate(proof_param, proof).map_err(|_| ElectionError::StubError)?; -// if let Some(res) = check_bin_idx( -// u64::from(sortition_claim), -// u64::from(voter_stake), -// u64::from(total_stake), -// u64::from(sortition_parameter), -// &seed, -// &mut HashMap::new(), -// ) { -// if res { -// Ok(Checked::Valid(())) -// } else { -// Ok(Checked::Inval(())) -// } -// } else { -// Ok(Checked::Unchecked(())) -// } -// } else { -// Ok(Checked::Inval(())) -// } -// } - -// /// Stateless method to produce VRF proof and sortition for a given view number -// /// # Errors -// /// -// pub fn get_sortition_proof( -// private_key: &SIGSCHEME::SigningKey, -// proof_param: &VRF::PublicParameter, -// chain_seed: &VRF::Input, -// view_number: TYPES::Time, -// total_stake: NonZeroU64, -// voter_stake: NonZeroU64, -// sortition_parameter: NonZeroU64, -// ) -> Result<(VRF::Proof, Option), hotshot_types::traits::election::ElectionError> -// { -// let mut rng = ChaChaRng::from_seed(Default::default()); // maybe use something else that isn't deterministic? -// let view_seed = generate_view_seed::(view_number, chain_seed); -// let proof = Self::internal_get_vrf_proof(private_key, proof_param, &mut rng, &view_seed)?; -// let sortition = Self::internal_get_sortition_for_proof( -// proof_param, -// &proof, -// total_stake, -// voter_stake, -// sortition_parameter, -// ); -// Ok((proof, sortition)) -// } - -// /// Stateless method to verify VRF proof and sortition for a given view number -// /// # Errors -// /// -// #[allow(clippy::too_many_arguments)] -// pub fn check_sortition_proof( -// public_key: &JfPubKey, -// proof_param: &VRF::PublicParameter, -// proof: &VRF::Proof, -// total_stake: NonZeroU64, -// voter_stake: NonZeroU64, -// sortition_parameter: NonZeroU64, -// sortition_claim: NonZeroU64, -// chain_seed: &VRF::Input, -// view_number: TYPES::Time, -// ) -> Result { -// let view_seed = generate_view_seed::(view_number, chain_seed); -// Self::internal_check_sortition( -// &public_key.pk, -// proof_param, -// proof, -// total_stake, -// voter_stake, -// sortition_parameter, -// sortition_claim, -// &view_seed, -// ) -// .map(|c| matches!(c, Checked::Valid(_))) -// } -// } - -// impl> TestableElection -// for VrfImpl -// where -// TYPES: NodeType< -// VoteTokenType = VRFVoteToken< -// BLSVerKey, -// BLSSignature, -// >, -// ElectionConfigType = VRFStakeTableConfig, -// SignatureKey = JfPubKey, -// >, -// { -// fn generate_test_vote_token() -> TYPES::VoteTokenType { -// VRFVoteToken { -// count: NonZeroU64::new(1234).unwrap(), -// proof: BLSSignature::default(), -// pub_key: BLSVerKey::default(), -// } -// } -// } - -// /// configuration specifying the stake table -// #[derive(Clone, Serialize, Deserialize, core::fmt::Debug)] -// pub struct VRFStakeTableConfig { -// /// the committee size parameter -// pub sortition_parameter: NonZeroU64, -// /// the ordered distribution of stake across nodes -// pub distribution: Vec, -// } - -// impl Default for VRFStakeTableConfig { -// fn default() -> Self { -// VRFStakeTableConfig { -// sortition_parameter: NonZeroU64::new(SORTITION_PARAMETER).unwrap(), -// distribution: Vec::new(), -// } -// } -// } - -// impl ElectionConfig for VRFStakeTableConfig {} - -// Tests have been commented out, so `mod tests` isn't used. -// #[cfg(test)] -// mod tests { -// use super::*; -// use ark_bls12_381::Parameters as Param381; -// use ark_std::test_rng; - -// use blake3::Hasher; -// use hotshot_types::{ -// data::ViewNumber, -// traits::{ -// block_contents::dummy::{DummyBlock, DummyTransaction}, -// consensus_type::validating_consensus::ValidatingConsensus, -// state::dummy::DummyState, -// }, -// }; -// use jf_primitives::{ -// signatures::{ -// bls::{BLSSignature, BLSVerKey}, -// BLSSignatureScheme, -// }, -// vrf::blsvrf::BLSVRFScheme, -// }; -// use std::{num::NonZeroUsize, time::Duration}; - -// #[derive( -// Copy, -// Clone, -// Debug, -// Default, -// Hash, -// PartialEq, -// Eq, -// PartialOrd, -// Ord, -// serde::Serialize, -// serde::Deserialize, -// )] -// struct TestTypes; -// impl NodeType for TestTypes { -// // TODO (da) can this be SequencingConsensus? -// type ConsensusType = ValidatingConsensus; -// type Time = ViewNumber; -// type BlockPayload = DummyBlock; -// type SignatureKey = JfPubKey; -// type VoteTokenType = VRFVoteToken< -// BLSVerKey, -// BLSSignature, -// >; -// type Transaction = DummyTransaction; -// type ElectionConfigType = VRFStakeTableConfig; -// type StateType = DummyState; -// } - -// fn gen_vrf_impl>( -// num_nodes: usize, -// ) -> ( -// VrfImpl< -// TestTypes, -// LEAF, -// BLSSignatureScheme, -// BLSVRFScheme, -// Hasher, -// Param381, -// >, -// Vec<( -// jf_primitives::signatures::bls::BLSSignKey, -// jf_primitives::signatures::bls::BLSVerKey, -// )>, -// ) { -// let mut known_nodes = Vec::new(); -// let mut keys = Vec::new(); -// let rng = &mut test_rng(); -// let mut stake_distribution = Vec::new(); -// let stake_per_node = NonZeroU64::new(100).unwrap(); -// let genesis_seed = [0u8; 32]; -// for _i in 0..num_nodes { -// let (sk, pk) = BLSSignatureScheme::::key_gen(&(), rng).unwrap(); -// keys.push((sk.clone(), pk.clone())); -// known_nodes.push(JfPubKey::from_native(pk.clone())); -// stake_distribution.push(stake_per_node); -// } -// let stake_table = VrfImpl::with_initial_stake( -// known_nodes, -// &VRFStakeTableConfig { -// sortition_parameter: std::num::NonZeroU64::new(SORTITION_PARAMETER).unwrap(), -// distribution: stake_distribution, -// }, -// genesis_seed, -// ); -// (stake_table, keys) -// } - -// pub fn check_if_valid(token: &Checked) -> bool { -// match token { -// Checked::Valid(_) => true, -// Checked::Inval(_) | Checked::Unchecked(_) => false, -// } -// } - -// // #[test] -// // pub fn test_sortition() { -// // setup_logging(); -// // let (vrf_impl, keys) = gen_vrf_impl::>(10); -// // let views = 100; - -// // for view in 0..views { -// // for (node_idx, (sk, pk)) in keys.iter().enumerate() { -// // let token_result = vrf_impl -// // .make_vote_token(ViewNumber::new(view), &(sk.clone(), pk.clone())) -// // .unwrap(); -// // match token_result { -// // Some(token) => { -// // let count = token.count; -// // let result = vrf_impl -// // .validate_vote_token( -// // ViewNumber::new(view), -// // JfPubKey::from_native(pk.clone()), -// // Checked::Unchecked(token), -// // ) -// // .unwrap(); -// // let result_is_valid = check_if_valid(&result); -// // error!("view {view:?}, node_idx {node_idx:?}, stake {count:?} "); -// // assert!(result_is_valid); -// // } -// // _ => continue, -// // } -// // } -// // } -// // } - -// #[test] -// pub fn test_factorial() { -// assert_eq!(factorial(0), BigUint::from(1u32)); -// assert_eq!(factorial(1), BigUint::from(1u32)); -// assert_eq!(factorial(2), BigUint::from(2u32)); -// assert_eq!(factorial(3), BigUint::from(6u32)); -// assert_eq!(factorial(4), BigUint::from(24u32)); -// assert_eq!(factorial(5), BigUint::from(120u32)); -// } - -// // TODO add failure case - -// #[test] -// fn network_config_is_serializable() { -// // validate that `RunResults` can be serialized -// // Note that there is currently an issue with `VRFPubKey` where it can't be serialized with toml -// // so instead we only test with serde_json -// let key = -// as TestableSignatureKey>::generate_test_key(1); -// let pub_key = JfPubKey::::from_private(&key); -// let mut config = hotshot_centralized_server::NetworkConfig { -// config: hotshot_types::HotShotConfig { -// election_config: Some(super::VRFStakeTableConfig { -// distribution: vec![NonZeroU64::new(1).unwrap()], -// sortition_parameter: NonZeroU64::new(1).unwrap(), -// }), -// known_nodes: vec![pub_key], -// execution_type: hotshot_types::ExecutionType::Incremental, -// total_nodes: NonZeroUsize::new(1).unwrap(), -// min_transactions: 1, -// max_transactions: NonZeroUsize::new(1).unwrap(), -// next_view_timeout: 1, -// timeout_ratio: (1, 1), -// round_start_delay: 1, -// start_delay: 1, -// num_bootstrap: 1, -// propose_min_round_time: Duration::from_secs(1), -// propose_max_round_time: Duration::from_secs(1), -// }, -// ..Default::default() -// }; -// serde_json::to_string(&config).unwrap(); -// assert!(toml::to_string(&config).is_err()); - -// // validate that this is indeed a `pub_key` issue -// config.config.known_nodes.clear(); -// serde_json::to_string(&config).unwrap(); -// toml::to_string(&config).unwrap(); -// } -// } diff --git a/crates/hotshot/src/traits/storage/memory_storage.rs b/crates/hotshot/src/traits/storage/memory_storage.rs index 5511e4d4cd..12e4d0ddde 100644 --- a/crates/hotshot/src/traits/storage/memory_storage.rs +++ b/crates/hotshot/src/traits/storage/memory_storage.rs @@ -173,7 +173,7 @@ mod test { view_number, }, DummyBlock::random(rng), - DummyBlock::random(rng).transaction_commitments(), + Some(DummyBlock::random(rng)), DummyState::random(rng), dummy_leaf_commit, Vec::new(), diff --git a/crates/task-impls/src/consensus.rs b/crates/task-impls/src/consensus.rs index 7d588c7a8d..2b1355f29d 100644 --- a/crates/task-impls/src/consensus.rs +++ b/crates/task-impls/src/consensus.rs @@ -30,6 +30,7 @@ use hotshot_types::{ node_implementation::{CommitteeEx, NodeImplementation, NodeType, QuorumEx, TimeoutEx}, signature_key::SignatureKey, state::ConsensusTime, + BlockPayload, }, utils::{Terminator, ViewInner}, vote::{QuorumVote, QuorumVoteAccumulator, TimeoutVoteAccumulator, VoteType}, @@ -455,7 +456,7 @@ where justify_qc: proposal.justify_qc.clone(), parent_commitment, block_header: proposal.block_header.clone(), - transaction_commitments: HashSet::new(), + block_payload: None, rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -525,7 +526,7 @@ where justify_qc: proposal.justify_qc.clone(), parent_commitment, block_header: proposal.block_header.clone(), - transaction_commitments: HashSet::new(), + block_payload: None, rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.quorum_exchange.get_leader(view).to_bytes(), @@ -740,7 +741,7 @@ where justify_qc: justify_qc.clone(), parent_commitment: justify_qc.leaf_commitment(), block_header: proposal.data.block_header, - transaction_commitments: HashSet::new(), + block_payload: None, rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -765,7 +766,7 @@ where justify_qc: justify_qc.clone(), parent_commitment, block_header: proposal.data.block_header, - transaction_commitments: HashSet::new(), + block_payload: None, rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: sender.to_bytes(), @@ -853,18 +854,25 @@ where .last_synced_block_height .set(usize::try_from(leaf.get_height()).unwrap_or(0)); - // If the full block is available for this leaf, include it in the leaf - // chain that we send to the client. - if let Some(comm) = consensus - .saved_transaction_commitments + // If the block payload is available for this leaf, include it in + // the leaf chain that we send to the client. + if let Some(payload) = consensus + .saved_block_payloads .get(leaf.get_payload_commitment()) { - leaf.fill_transaction_commitments(comm.clone()); + if let Err(e) = leaf.fill_block_payload(payload.clone()) { + error!( + "Saved block payload and commitment don't match: {:?}", + e + ); + } } leaf_views.push(leaf.clone()); - for txn in leaf.transaction_commitments { - included_txns.insert(txn); + if let Some(payload) = leaf.block_payload { + for txn in payload.transaction_commitments() { + included_txns.insert(txn); + } } } true @@ -1379,7 +1387,7 @@ where justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), block_header: TYPES::BlockHeader::new(*payload_commitment, &parent_header), - transaction_commitments: HashSet::new(), + block_payload: None, rejected: vec![], timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: self.api.public_key().to_bytes(), diff --git a/crates/task-impls/src/da.rs b/crates/task-impls/src/da.rs index f2f8237c2f..6d62b11a77 100644 --- a/crates/task-impls/src/da.rs +++ b/crates/task-impls/src/da.rs @@ -286,11 +286,10 @@ where }, }); - // Record the block we have promised to make available. - consensus.saved_transaction_commitments.insert( - proposal.data.block_payload.commit(), - proposal.data.block_payload.transaction_commitments(), - ); + // Record the block payload we have promised to make available. + consensus + .saved_block_payloads + .insert(proposal.data.block_payload); } } } diff --git a/crates/task-impls/src/transactions.rs b/crates/task-impls/src/transactions.rs index 4b5b356d4a..509a8b45fa 100644 --- a/crates/task-impls/src/transactions.rs +++ b/crates/task-impls/src/transactions.rs @@ -22,6 +22,7 @@ use hotshot_types::{ consensus_api::ConsensusApi, election::{ConsensusExchange, Membership, QuorumExchangeType}, node_implementation::{NodeImplementation, NodeType, QuorumEx}, + BlockPayload, }, }; use hotshot_utils::bincode::bincode_opts; @@ -137,8 +138,10 @@ where let mut included_txn_size = 0; let mut included_txn_count = 0; for leaf in leaf_chain { - for txn in leaf.transaction_commitments { - included_txns.insert(txn); + if let Some(payload) = leaf.block_payload { + for txn in payload.transaction_commitments() { + included_txns.insert(txn); + } } } let consensus = self.consensus.read().await; diff --git a/crates/task-impls/src/vid.rs b/crates/task-impls/src/vid.rs index 63883ca897..dda95a09af 100644 --- a/crates/task-impls/src/vid.rs +++ b/crates/task-impls/src/vid.rs @@ -357,7 +357,7 @@ where // Record the block we have promised to make available. // TODO https://github.com/EspressoSystems/HotShot/issues/1692 - // consensus.saved_transaction_commitments.insert(proposal.data.block_payload); + // consensus.saved_block_payloads.insert(proposal.data.block_payload); } } } diff --git a/crates/testing/src/task_helpers.rs b/crates/testing/src/task_helpers.rs index 32a3d5f2cd..3aa8b87ffb 100644 --- a/crates/testing/src/task_helpers.rs +++ b/crates/testing/src/task_helpers.rs @@ -1,5 +1,3 @@ -use std::collections::HashSet; - use crate::{ node_types::{MemoryImpl, TestTypes}, test_builder::TestMetadata, @@ -123,7 +121,7 @@ async fn build_quorum_proposal_and_signature( justify_qc: consensus.high_qc.clone(), parent_commitment: parent_leaf.commit(), block_header: block_header.clone(), - transaction_commitments: HashSet::new(), + block_payload: None, rejected: vec![], timestamp: 0, proposer_id: api.public_key().to_bytes(), diff --git a/crates/testing/tests/consensus_task.rs b/crates/testing/tests/consensus_task.rs index 7ddc824c19..12feb914a1 100644 --- a/crates/testing/tests/consensus_task.rs +++ b/crates/testing/tests/consensus_task.rs @@ -22,7 +22,6 @@ use hotshot_types::{ }; use std::collections::HashMap; -use std::collections::HashSet; async fn build_vote( handle: &SystemContextHandle, @@ -65,7 +64,7 @@ async fn build_vote( justify_qc: proposal.justify_qc.clone(), parent_commitment, block_header: proposal.block_header, - transaction_commitments: HashSet::new(), + block_payload: None, rejected: Vec::new(), timestamp: 0, proposer_id: quorum_exchange.get_leader(view).to_bytes(), diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index e61685ad69..99a8b00c51 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -20,7 +20,7 @@ use crate::{ use commit::Commitment; use derivative::Derivative; use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, + collections::{hash_map::Entry, BTreeMap, HashMap}, sync::{Arc, Mutex}, }; use tracing::error; @@ -48,11 +48,10 @@ pub struct Consensus> { /// - includes the MOST RECENT decided leaf pub saved_leaves: CommitmentMap, - /// Saved transaction commitments + /// Saved block payloads /// - /// Contains the transaction commitments of the block for every leaf in `saved_leaves` if that - /// commitment set is available. - pub saved_transaction_commitments: TransactionStore, + /// Contains the block payload for every leaf in `saved_leaves` if that payload is available. + pub saved_block_payloads: BlockPayloadStore, /// The `locked_qc` view number pub locked_view: TYPES::Time, @@ -299,7 +298,7 @@ impl> Consensus { } /// garbage collects based on state change - /// right now, this removes from both the `saved_transaction_commitments` + /// right now, this removes from both the `saved_block_payloads` /// and `state_map` fields of `Consensus` /// # Panics /// On inconsistent stored entries @@ -325,14 +324,14 @@ impl> Consensus { .range(old_anchor_view..new_anchor_view) .filter_map(|(_view_number, view)| view.get_payload_commitment()) .for_each(|block| { - self.saved_transaction_commitments.remove(block); + self.saved_block_payloads.remove(block); }); self.state_map .range(old_anchor_view..new_anchor_view) .filter_map(|(_view_number, view)| view.get_leaf_commitment()) .for_each(|leaf| { if let Some(removed) = self.saved_leaves.remove(&leaf) { - self.saved_transaction_commitments + self.saved_block_payloads .remove(removed.get_payload_commitment()); } }); @@ -354,10 +353,7 @@ impl> Consensus { } } -/// Alias for the set of transaction commitments. -type TransactionCommitments = HashSet::Transaction>>; - -/// Mapping from block payload commitments to the set of transaction commitents. +/// Mapping from block payload commitments to the payloads. /// /// Entries in this mapping are reference-counted, so multiple consensus objects can refer to the /// same block, and the block will only be deleted after _all_ such objects are garbage collected. @@ -365,29 +361,23 @@ type TransactionCommitments = HashSet( - HashMap, (TransactionCommitments, u64)>, -); +pub struct BlockPayloadStore(HashMap, (PAYLOAD, u64)>); -impl TransactionStore { +impl BlockPayloadStore { /// Save payload commitment for later retrieval. /// /// After calling this function, and before the corresponding call to [`remove`](Self::remove), - /// `self.get(payload_commitment)` will return `Some(transaction_commitments)`. + /// `self.get(payload_commitment)` will return `Some(payload)`. /// /// This function will increment a reference count on the saved payload commitment, so that /// multiple calls to [`insert`](Self::insert) for the same payload commitment result in /// multiple owning references to the payload commitment. [`remove`](Self::remove) must be /// called once for each reference before the payload commitment will be deallocated. - pub fn insert( - &mut self, - payload_commitment: Commitment, - transaction_commitments: TransactionCommitments, - ) { + pub fn insert(&mut self, payload: PAYLOAD) { self.0 - .entry(payload_commitment) + .entry(payload.commit()) .and_modify(|(_, refcount)| *refcount += 1) - .or_insert((transaction_commitments, 1)); + .or_insert((payload, 1)); } /// Get a saved set of transaction commitments, if available. @@ -396,29 +386,21 @@ impl TransactionStore { /// function will retrieve it. It may return [`None`] if a block with the given commitment has /// not been saved or if the block has been dropped with [`remove`](Self::remove). #[must_use] - pub fn get( - &self, - payload_commitment: Commitment, - ) -> Option<&HashSet::Transaction>>> { - self.0 - .get(&payload_commitment) - .map(|(txn_comm, _)| txn_comm) + pub fn get(&self, payload_commitment: Commitment) -> Option<&PAYLOAD> { + self.0.get(&payload_commitment).map(|(payload, _)| payload) } /// Drop a reference to a saved set of transaction commitments. /// /// If the set exists and this call drops the last reference to it, the set will be returned, /// Otherwise, the return value is [`None`]. - pub fn remove( - &mut self, - payload_commitment: Commitment, - ) -> Option::Transaction>>> { + pub fn remove(&mut self, payload_commitment: Commitment) -> Option { if let Entry::Occupied(mut e) = self.0.entry(payload_commitment) { let (_, refcount) = e.get_mut(); *refcount -= 1; if *refcount == 0 { - let (txn_comm, _) = e.remove(); - return Some(txn_comm); + let (payload, _) = e.remove(); + return Some(payload); } } None diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index 418fbd64b9..7a00900131 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -28,8 +28,8 @@ use hotshot_utils::bincode::bincode_opts; use jf_primitives::pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme}; use rand::Rng; use serde::{Deserialize, Serialize}; +use snafu::Snafu; use std::{ - collections::HashSet, fmt::{Debug, Display}, hash::Hash, }; @@ -315,6 +315,17 @@ pub trait DeltasType: fn fill(&mut self, block: BlockPayload) -> Result<(), Self::Error>; } +/// Error which occurs when [`LeafType::fill_block_payload`] is called with a payload commitment +/// that does not match the internal payload commitment of the leaf. +#[derive(Clone, Copy, Debug, Snafu)] +#[snafu(display("the block payload {:?} has commitment {} (expected {})", payload, payload.commit(), commitment))] +pub struct InconsistentPayloadCommitmentError { + /// The block payload with the wrong commitment. + payload: PAYLOAD, + /// The expected commitment. + commitment: Commitment, +} + /// An item which is appended to a blockchain. pub trait LeafType: Debug @@ -362,20 +373,22 @@ pub trait LeafType: /// Commitment to this leaf's parent. fn get_parent_commitment(&self) -> Commitment; /// The block header contained in this leaf. - fn get_block_header(&self) -> ::BlockHeader; + fn get_block_header(&self) -> &::BlockHeader; /// A commitment to the block payload contained in this leaf. fn get_payload_commitment(&self) -> Commitment> { self.get_block_header().payload_commitment() } - /// Fill the transaciton commitments of this leaf with the corresponding block payload. - fn fill_transaction_commitments( + /// Fill this leaf with the block payload. + /// + /// # Errors + /// + /// Fails if the payload commitment doesn't match `self.block_header.payload_commitment()`. + fn fill_block_payload( &mut self, - transaction_commitments: HashSet::Transaction>>, - ); - /// Optional set of commitments to the transactions. - fn get_transanction_commitments( - &self, - ) -> HashSet::Transaction>>; + block_payload: ::BlockPayload, + ) -> Result<(), InconsistentPayloadCommitmentError<::BlockPayload>>; + /// Optional block payload. + fn get_block_payload(&self) -> Option<::BlockPayload>; /// The blockchain state after appending this leaf. fn get_state(&self) -> Self::MaybeState; /// Transactions rejected or invalidated by the application of this leaf. @@ -473,11 +486,10 @@ pub struct Leaf { /// Block header. pub block_header: TYPES::BlockHeader, - /// Set of commitments to the contained transactions. + /// Optional block payload. /// /// It may be empty for nodes not in the DA committee. - pub transaction_commitments: - HashSet::Transaction>>, + pub block_payload: Option, /// Transactions that were marked for rejection while collecting the block. pub rejected: Vec<::Transaction>, @@ -496,7 +508,6 @@ impl PartialEq for Leaf { && self.justify_qc == other.justify_qc && self.parent_commitment == other.parent_commitment && self.block_header == other.block_header - && self.transaction_commitments == other.transaction_commitments && self.rejected == other.rejected } } @@ -507,13 +518,9 @@ impl Hash for Leaf { self.justify_qc.hash(state); self.parent_commitment.hash(state); self.block_header.hash(state); - for com in &self.transaction_commitments { - com.hash(state); - } self.rejected.hash(state); } } - impl Display for ValidatingLeaf { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( @@ -526,7 +533,6 @@ impl Display for ValidatingLeaf { impl LeafType for ValidatingLeaf { type NodeType = TYPES; - // type DeltasType = TYPES::BlockPayload; type MaybeState = TYPES::StateType; fn new( @@ -564,20 +570,19 @@ impl LeafType for ValidatingLeaf { self.parent_commitment } - fn get_block_header(&self) -> ::BlockHeader { + fn get_block_header(&self) -> &::BlockHeader { unimplemented!("Unimplemented for validating consensus which will be removed."); } - fn fill_transaction_commitments( + fn fill_block_payload( &mut self, - _transaction_commitments: HashSet::Transaction>>, - ) { + _block_payload: ::BlockPayload, + ) -> Result<(), InconsistentPayloadCommitmentError<::BlockPayload>> + { unimplemented!("Unimplemented for validating consensus which will be removed."); } - fn get_transanction_commitments( - &self, - ) -> HashSet::Transaction>> { + fn get_block_payload(&self) -> Option<::BlockPayload> { unimplemented!("Unimplemented for validating consensus which will be removed."); } @@ -650,7 +655,7 @@ impl LeafType for Leaf { justify_qc, parent_commitment: fake_commitment(), block_header: TYPES::BlockHeader::genesis(payload.clone()), - transaction_commitments: payload.transaction_commitments(), + block_payload: Some(payload), rejected: Vec::new(), timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id: genesis_proposer_id(), @@ -673,21 +678,27 @@ impl LeafType for Leaf { self.parent_commitment } - fn get_block_header(&self) -> ::BlockHeader { - self.block_header.clone() + fn get_block_header(&self) -> &::BlockHeader { + &self.block_header } - fn fill_transaction_commitments( + fn fill_block_payload( &mut self, - transaction_commitments: HashSet::Transaction>>, - ) { - self.transaction_commitments = transaction_commitments; + block_payload: ::BlockPayload, + ) -> Result<(), InconsistentPayloadCommitmentError<::BlockPayload>> + { + if block_payload.commit() != self.block_header.payload_commitment() { + return Err(InconsistentPayloadCommitmentError { + payload: block_payload, + commitment: self.block_header.payload_commitment(), + }); + } + self.block_payload = Some(block_payload); + Ok(()) } - fn get_transanction_commitments( - &self, - ) -> HashSet::Transaction>> { - self.transaction_commitments.clone() + fn get_block_payload(&self) -> Option<::BlockPayload> { + self.block_payload.clone() } // The Sequencing Leaf doesn't have a state. @@ -711,7 +722,7 @@ impl LeafType for Leaf { justify_qc: stored_view.justify_qc, parent_commitment: stored_view.parent, block_header: stored_view.block_header, - transaction_commitments: stored_view.transaction_commitments, + block_payload: stored_view.block_payload, rejected: stored_view.rejected, timestamp: stored_view.timestamp, proposer_id: stored_view.proposer_id, @@ -886,8 +897,8 @@ where parent: leaf.get_parent_commitment(), justify_qc: leaf.get_justify_qc(), state: leaf.get_state(), - block_header: leaf.get_block_header(), - transaction_commitments: leaf.get_transanction_commitments(), + block_header: leaf.get_block_header().clone(), + block_payload: leaf.get_block_payload(), rejected: leaf.get_rejected(), timestamp: leaf.get_timestamp(), proposer_id: leaf.get_proposer_id(), diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index 80df3b9841..24c7e0459f 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -10,7 +10,7 @@ use async_trait::async_trait; use commit::Commitment; use derivative::Derivative; use snafu::Snafu; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet}; /// Errors that can occur in the storage layer. #[derive(Clone, Debug, Snafu)] #[snafu(visibility(pub))] @@ -135,11 +135,10 @@ pub struct StoredView> { pub state: LEAF::MaybeState, /// Block header. pub block_header: TYPES::BlockHeader, - /// Set of commitments to the contained transactions. + /// Optional block payload. /// /// It may be empty for nodes not in the DA committee. - pub transaction_commitments: - HashSet::Transaction>>, + pub block_payload: Option, /// transactions rejected in this view pub rejected: Vec, /// the timestamp this view was recv-ed in nanonseconds @@ -155,15 +154,14 @@ where TYPES: NodeType, LEAF: LeafType, { - /// Create a new `StoredView` from the given QC, `BlockPayload` and State. + /// Create a new `StoredView` from the given QC, `BlockHeader`, `BlockPayload` and State. /// - /// Note that this will set the `parent` to `LeafHash::default()`, so this will not have a parent. + /// Note that this will set the `parent` to `LeafHash::default()`, so this will not have a + /// parent. pub fn from_qc_block_and_state( qc: QuorumCertificate>, block_header: TYPES::BlockHeader, - transaction_commitments: HashSet< - Commitment<::Transaction>, - >, + block_payload: Option, state: LEAF::MaybeState, parent_commitment: Commitment, rejected: Vec<::Transaction>, @@ -175,7 +173,7 @@ where justify_qc: qc, state, block_header, - transaction_commitments, + block_payload, rejected, timestamp: time::OffsetDateTime::now_utc().unix_timestamp_nanos(), proposer_id,