diff --git a/Cargo.lock b/Cargo.lock index 737e4d42c0..47b4eec995 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3212,6 +3212,7 @@ dependencies = [ "rand_chacha 0.3.1", "serde", "serde-inline-default", + "serde_bytes", "serde_json", "sha2 0.10.8", "surf-disco", @@ -6450,6 +6451,15 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.215" diff --git a/Cargo.toml b/Cargo.toml index 98018b9d3c..e2be87732b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ portpicker = "0.1" rand = { version = "0.8", features = ["small_rng"] } rand_chacha = { version = "0.3", default-features = false } serde = { version = "1", features = ["derive"] } +serde-inline-default = "0.2" serde_bytes = { version = "0.11" } serde_json = { version = "1.0" } sha2 = "0.10" diff --git a/crates/example-types/src/block_types.rs b/crates/example-types/src/block_types.rs index add9d76269..b7e763546d 100644 --- a/crates/example-types/src/block_types.rs +++ b/crates/example-types/src/block_types.rs @@ -13,7 +13,7 @@ use std::{ use async_trait::async_trait; use committable::{Commitment, Committable, RawCommitmentBuilder}; use hotshot_types::{ - data::{BlockError, Leaf}, + data::{BlockError, Leaf2}, traits::{ block_contents::{BlockHeader, BuilderFee, EncodeBytes, TestableBlock, Transaction}, node_implementation::NodeType, @@ -272,7 +272,7 @@ pub struct TestBlockHeader { impl TestBlockHeader { pub fn new>( - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: TestMetadata, @@ -312,7 +312,7 @@ impl< async fn new_legacy( _parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, @@ -332,7 +332,7 @@ impl< async fn new_marketplace( _parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, diff --git a/crates/example-types/src/state_types.rs b/crates/example-types/src/state_types.rs index c5fde414bf..9b52e0c662 100644 --- a/crates/example-types/src/state_types.rs +++ b/crates/example-types/src/state_types.rs @@ -10,7 +10,7 @@ use std::fmt::Debug; use async_trait::async_trait; use committable::{Commitment, Committable}; use hotshot_types::{ - data::{fake_commitment, BlockError, Leaf, ViewNumber}, + data::{fake_commitment, BlockError, Leaf2, ViewNumber}, traits::{ block_contents::BlockHeader, node_implementation::NodeType, @@ -103,7 +103,7 @@ impl ValidatedState for TestValidatedState { async fn validate_and_apply_header( &self, instance: &Self::Instance, - _parent_leaf: &Leaf, + _parent_leaf: &Leaf2, _proposed_header: &TYPES::BlockHeader, _vid_common: VidCommon, _version: Version, diff --git a/crates/example-types/src/storage_types.rs b/crates/example-types/src/storage_types.rs index c4be058fe4..be94cec84d 100644 --- a/crates/example-types/src/storage_types.rs +++ b/crates/example-types/src/storage_types.rs @@ -14,10 +14,10 @@ use async_lock::RwLock; use async_trait::async_trait; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, Leaf, QuorumProposal, VidDisperseShare}, + data::{DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate2, UpgradeCertificate}, traits::{ node_implementation::{ConsensusTime, NodeType}, storage::Storage, @@ -34,13 +34,14 @@ type VidShares = HashMap< ::View, HashMap<::SignatureKey, Proposal>>, >; - #[derive(Clone, Debug)] pub struct TestStorageState { vids: VidShares, das: HashMap>>, proposals: BTreeMap>>, + proposals2: BTreeMap>>, high_qc: Option>, + high_qc2: Option>, action: TYPES::View, epoch: TYPES::Epoch, } @@ -51,7 +52,9 @@ impl Default for TestStorageState { vids: HashMap::new(), das: HashMap::new(), proposals: BTreeMap::new(), + proposals2: BTreeMap::new(), high_qc: None, + high_qc2: None, action: TYPES::View::genesis(), epoch: TYPES::Epoch::genesis(), } @@ -91,11 +94,11 @@ impl TestableDelay for TestStorage { impl TestStorage { pub async fn proposals_cloned( &self, - ) -> BTreeMap>> { - self.inner.read().await.proposals.clone() + ) -> BTreeMap>> { + self.inner.read().await.proposals2.clone() } - pub async fn high_qc_cloned(&self) -> Option> { - self.inner.read().await.high_qc.clone() + pub async fn high_qc_cloned(&self) -> Option> { + self.inner.read().await.high_qc2.clone() } pub async fn decided_upgrade_certificate(&self) -> Option> { self.decided_upgrade_certificate.read().await.clone() @@ -153,6 +156,20 @@ impl Storage for TestStorage { .insert(proposal.data.view_number, proposal.clone()); Ok(()) } + async fn append_proposal2( + &self, + proposal: &Proposal>, + ) -> Result<()> { + if self.should_return_err { + bail!("Failed to append VID proposal to storage"); + } + Self::run_delay_settings_from_config(&self.delay_config).await; + let mut inner = self.inner.write().await; + inner + .proposals2 + .insert(proposal.data.view_number, proposal.clone()); + Ok(()) + } async fn record_action( &self, @@ -188,6 +205,25 @@ impl Storage for TestStorage { } Ok(()) } + + async fn update_high_qc2( + &self, + new_high_qc: hotshot_types::simple_certificate::QuorumCertificate2, + ) -> Result<()> { + if self.should_return_err { + bail!("Failed to update high qc to storage"); + } + Self::run_delay_settings_from_config(&self.delay_config).await; + let mut inner = self.inner.write().await; + if let Some(ref current_high_qc) = inner.high_qc2 { + if new_high_qc.view_number() > current_high_qc.view_number() { + inner.high_qc2 = Some(new_high_qc); + } + } else { + inner.high_qc2 = Some(new_high_qc); + } + Ok(()) + } async fn update_undecided_state( &self, _leafs: CommitmentMap>, @@ -199,6 +235,17 @@ impl Storage for TestStorage { Self::run_delay_settings_from_config(&self.delay_config).await; Ok(()) } + async fn update_undecided_state2( + &self, + _leafs: CommitmentMap>, + _state: BTreeMap>, + ) -> Result<()> { + if self.should_return_err { + bail!("Failed to update high qc to storage"); + } + Self::run_delay_settings_from_config(&self.delay_config).await; + Ok(()) + } async fn update_decided_upgrade_certificate( &self, decided_upgrade_certificate: Option>, @@ -207,4 +254,22 @@ impl Storage for TestStorage { Ok(()) } + + async fn migrate_consensus( + &self, + _convert_leaf: fn(Leaf) -> Leaf2, + convert_proposal: fn( + Proposal>, + ) -> Proposal>, + ) -> Result<()> { + let mut storage_writer = self.inner.write().await; + + for (view, proposal) in storage_writer.proposals.clone().iter() { + storage_writer + .proposals2 + .insert(*view, convert_proposal(proposal.clone())); + } + + Ok(()) + } } diff --git a/crates/examples/push-cdn/all.rs b/crates/examples/push-cdn/all.rs index 2e04d2d72b..12599c36a4 100644 --- a/crates/examples/push-cdn/all.rs +++ b/crates/examples/push-cdn/all.rs @@ -14,7 +14,6 @@ use cdn_broker::{ reexports::{crypto::signature::KeyPair, def::hook::NoMessageHook}, Broker, }; - use cdn_marshal::Marshal; use hotshot::{ helpers::initialize_logging, diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index 05ccea0a0c..9f6d8591e5 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -11,6 +11,7 @@ #[cfg(feature = "docs")] pub mod documentation; +use committable::Committable; use futures::future::{select, Either}; use hotshot_types::{ message::UpgradeLock, @@ -48,10 +49,10 @@ pub use hotshot_types::error::HotShotError; use hotshot_types::{ consensus::{Consensus, ConsensusMetricsValue, OuterConsensus, View, ViewInner}, constants::{EVENT_CHANNEL_SIZE, EXTERNAL_EVENT_CHANNEL_SIZE}, - data::{Leaf, QuorumProposal}, + data::{Leaf, Leaf2, QuorumProposal, QuorumProposal2}, event::{EventType, LeafInfo}, - message::{DataMessage, Message, MessageKind, Proposal}, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + message::{convert_proposal, DataMessage, Message, MessageKind, Proposal}, + simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate}, traits::{ consensus_api::ConsensusApi, election::Membership, @@ -59,6 +60,7 @@ use hotshot_types::{ node_implementation::{ConsensusTime, NodeType}, signature_key::SignatureKey, states::ValidatedState, + storage::Storage, EncodeBytes, }, HotShotConfig, @@ -138,7 +140,7 @@ pub struct SystemContext, V: Versi pub(crate) external_event_stream: (Sender>, InactiveReceiver>), /// Anchored leaf provided by the initializer. - anchored_leaf: Leaf, + anchored_leaf: Leaf2, /// access to the internal event stream, in case we need to, say, shut something down #[allow(clippy::type_complexity)] @@ -195,6 +197,10 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext, ) -> Arc { + #[allow(clippy::panic)] + match storage + .migrate_consensus( + Into::>::into, + convert_proposal::, QuorumProposal2>, + ) + .await + { + Ok(()) => {} + Err(e) => { + panic!("Failed to migrate consensus storage: {e}"); + } + } + let interal_chan = broadcast(EVENT_CHANNEL_SIZE); let external_chan = broadcast(EXTERNAL_EVENT_CHANNEL_SIZE); @@ -225,7 +245,6 @@ impl, V: Versions> SystemContext`] with the given configuration options. @@ -236,7 +255,7 @@ impl, V: Versions> SystemContext::PrivateKey, nonce: u64, @@ -287,7 +306,7 @@ impl, V: Versions> SystemContext, V: Versions> SystemContext, V: Versions> SystemContext, V: Versions> SystemContext, V: Versions> SystemContext(&validated_state, self.instance_state.as_ref()) - .await, + .await + .to_qc2(), ); broadcast_event( @@ -532,7 +549,7 @@ impl, V: Versions> SystemContext Leaf { + pub async fn decided_leaf(&self) -> Leaf2 { self.consensus.read().await.decided_leaf() } @@ -543,7 +560,7 @@ impl, V: Versions> SystemContext Option> { + pub fn try_decided_leaf(&self) -> Option> { self.consensus.try_read().map(|guard| guard.decided_leaf()) } @@ -957,7 +974,7 @@ impl, V: Versions> ConsensusApi { /// the leaf specified initialization - inner: Leaf, + inner: Leaf2, /// Instance-level state. instance_state: TYPES::InstanceState, @@ -983,16 +1000,16 @@ pub struct HotShotInitializer { /// Highest QC that was seen, for genesis it's the genesis QC. It should be for a view greater /// than `inner`s view number for the non genesis case because we must have seen higher QCs /// to decide on the leaf. - high_qc: QuorumCertificate, + high_qc: QuorumCertificate2, /// Previously decided upgrade certificate; this is necessary if an upgrade has happened and we are not restarting with the new version decided_upgrade_certificate: Option>, /// Undecided leafs that were seen, but not yet decided on. These allow a restarting node /// to vote and propose right away if they didn't miss anything while down. - undecided_leafs: Vec>, + undecided_leafs: Vec>, /// Not yet decided state undecided_state: BTreeMap>, /// Proposals we have sent out to provide to others for catchup - saved_proposals: BTreeMap>>, + saved_proposals: BTreeMap>>, } impl HotShotInitializer { @@ -1003,10 +1020,14 @@ impl HotShotInitializer { instance_state: TYPES::InstanceState, ) -> Result> { let (validated_state, state_delta) = TYPES::ValidatedState::genesis(&instance_state); - let high_qc = QuorumCertificate::genesis::(&validated_state, &instance_state).await; + let high_qc = QuorumCertificate::genesis::(&validated_state, &instance_state) + .await + .to_qc2(); Ok(Self { - inner: Leaf::genesis(&validated_state, &instance_state).await, + inner: Leaf::genesis(&validated_state, &instance_state) + .await + .into(), validated_state: Some(Arc::new(validated_state)), state_delta: Some(Arc::new(state_delta)), start_view: TYPES::View::new(0), @@ -1030,16 +1051,16 @@ impl HotShotInitializer { /// `SystemContext`. #[allow(clippy::too_many_arguments)] pub fn from_reload( - anchor_leaf: Leaf, + anchor_leaf: Leaf2, instance_state: TYPES::InstanceState, validated_state: Option>, start_view: TYPES::View, start_epoch: TYPES::Epoch, actioned_view: TYPES::View, - saved_proposals: BTreeMap>>, - high_qc: QuorumCertificate, + saved_proposals: BTreeMap>>, + high_qc: QuorumCertificate2, decided_upgrade_certificate: Option>, - undecided_leafs: Vec>, + undecided_leafs: Vec>, undecided_state: BTreeMap>, ) -> Self { Self { diff --git a/crates/hotshot/src/tasks/mod.rs b/crates/hotshot/src/tasks/mod.rs index 6ddd00b252..bb9f7ed630 100644 --- a/crates/hotshot/src/tasks/mod.rs +++ b/crates/hotshot/src/tasks/mod.rs @@ -10,11 +10,6 @@ pub mod task_state; use std::{collections::BTreeMap, fmt::Debug, num::NonZeroUsize, sync::Arc, time::Duration}; -use crate::{ - tasks::task_state::CreateTaskState, types::SystemContextHandle, ConsensusApi, - ConsensusMetricsValue, ConsensusTaskRegistry, HotShotConfig, HotShotInitializer, - MarketplaceConfig, Memberships, NetworkTaskRegistry, SignatureKey, SystemContext, Versions, -}; use async_broadcast::{broadcast, RecvError}; use async_lock::RwLock; use async_trait::async_trait; @@ -48,6 +43,12 @@ use hotshot_types::{ use tokio::{spawn, time::sleep}; use vbs::version::StaticVersionType; +use crate::{ + tasks::task_state::CreateTaskState, types::SystemContextHandle, ConsensusApi, + ConsensusMetricsValue, ConsensusTaskRegistry, HotShotConfig, HotShotInitializer, + MarketplaceConfig, Memberships, NetworkTaskRegistry, SignatureKey, SystemContext, Versions, +}; + /// event for global event stream #[derive(Clone, Debug)] pub enum GlobalEvent { diff --git a/crates/hotshot/src/types/handle.rs b/crates/hotshot/src/types/handle.rs index 2ab7874b8f..19fcb97f1d 100644 --- a/crates/hotshot/src/types/handle.rs +++ b/crates/hotshot/src/types/handle.rs @@ -20,7 +20,7 @@ use hotshot_task::{ use hotshot_task_impls::{events::HotShotEvent, helpers::broadcast_event}; use hotshot_types::{ consensus::Consensus, - data::{Leaf, QuorumProposal}, + data::{Leaf2, QuorumProposal2}, error::HotShotError, message::{Message, MessageKind, Proposal, RecipientList}, request_response::ProposalRequestPayload, @@ -141,8 +141,9 @@ impl + 'static, V: Versions> &self, view: TYPES::View, epoch: TYPES::Epoch, - leaf_commitment: Commitment>, - ) -> Result>>>> { + leaf_commitment: Commitment>, + ) -> Result>>>> + { // We need to be able to sign this request before submitting it to the network. Compute the // payload first. let signed_proposal_request = ProposalRequestPayload { @@ -157,7 +158,6 @@ impl + 'static, V: Versions> )?; let mem = self.memberships.quorum_membership.clone(); - let upgrade_lock = self.hotshot.upgrade_lock.clone(); let receiver = self.internal_event_stream.1.activate_cloned(); let sender = self.internal_event_stream.0.clone(); Ok(async move { @@ -188,15 +188,12 @@ impl + 'static, V: Versions> if let HotShotEvent::QuorumProposalResponseRecv(quorum_proposal) = hs_event.as_ref() { // Make sure that the quorum_proposal is valid - if let Err(err) = quorum_proposal - .validate_signature(&mem, epoch, &upgrade_lock) - .await - { + if let Err(err) = quorum_proposal.validate_signature(&mem, epoch) { tracing::warn!("Invalid Proposal Received after Request. Err {:?}", err); continue; } - let proposed_leaf = Leaf::from_quorum_proposal(&quorum_proposal.data); - let commit = proposed_leaf.commit(&upgrade_lock).await; + let proposed_leaf = Leaf2::from_quorum_proposal(&quorum_proposal.data); + let commit = proposed_leaf.commit(); if commit == leaf_commitment { return Ok(quorum_proposal.clone()); } @@ -255,7 +252,7 @@ impl + 'static, V: Versions> /// /// # Panics /// If the internal consensus is in an inconsistent state. - pub async fn decided_leaf(&self) -> Leaf { + pub async fn decided_leaf(&self) -> Leaf2 { self.hotshot.decided_leaf().await } @@ -265,7 +262,7 @@ impl + 'static, V: Versions> /// # Panics /// Panics if internal consensus is in an inconsistent state. #[must_use] - pub fn try_decided_leaf(&self) -> Option> { + pub fn try_decided_leaf(&self) -> Option> { self.hotshot.try_decided_leaf() } diff --git a/crates/task-impls/src/consensus/handlers.rs b/crates/task-impls/src/consensus/handlers.rs index 287e14e03e..1dd319fed9 100644 --- a/crates/task-impls/src/consensus/handlers.rs +++ b/crates/task-impls/src/consensus/handlers.rs @@ -10,7 +10,7 @@ use async_broadcast::Sender; use chrono::Utc; use hotshot_types::{ event::{Event, EventType}, - simple_vote::{QuorumVote, TimeoutData, TimeoutVote}, + simple_vote::{QuorumVote2, TimeoutData, TimeoutVote}, traits::{ election::Membership, node_implementation::{ConsensusTime, NodeImplementation, NodeType}, @@ -20,13 +20,13 @@ use hotshot_types::{ use tokio::{spawn, time::sleep}; use tracing::instrument; use utils::anytrace::*; +use vbs::version::StaticVersionType; use super::ConsensusTaskState; use crate::{ consensus::Versions, events::HotShotEvent, helpers::broadcast_event, vote_collection::handle_vote, }; -use vbs::version::StaticVersionType; /// Handle a `QuorumVoteRecv` event. pub(crate) async fn handle_quorum_vote_recv< @@ -34,7 +34,7 @@ pub(crate) async fn handle_quorum_vote_recv< I: NodeImplementation, V: Versions, >( - vote: &QuorumVote, + vote: &QuorumVote2, event: Arc>, sender: &Sender>>, task_state: &mut ConsensusTaskState, diff --git a/crates/task-impls/src/consensus/mod.rs b/crates/task-impls/src/consensus/mod.rs index 15d6bc6ec8..6def6ebd82 100644 --- a/crates/task-impls/src/consensus/mod.rs +++ b/crates/task-impls/src/consensus/mod.rs @@ -14,8 +14,8 @@ use hotshot_types::{ consensus::OuterConsensus, event::Event, message::UpgradeLock, - simple_certificate::{QuorumCertificate, TimeoutCertificate}, - simple_vote::{QuorumVote, TimeoutVote}, + simple_certificate::{QuorumCertificate2, TimeoutCertificate}, + simple_vote::{QuorumVote2, TimeoutVote}, traits::{ node_implementation::{NodeImplementation, NodeType, Versions}, signature_key::SignatureKey, @@ -57,7 +57,7 @@ pub struct ConsensusTaskState, V: pub committee_membership: Arc, /// A map of `QuorumVote` collector tasks. - pub vote_collectors: VoteCollectorsMap, QuorumCertificate, V>, + pub vote_collectors: VoteCollectorsMap, QuorumCertificate2, V>, /// A map of `TimeoutVote` collector tasks. pub timeout_vote_collectors: diff --git a/crates/task-impls/src/events.rs b/crates/task-impls/src/events.rs index 6744208fa5..5153028e99 100644 --- a/crates/task-impls/src/events.rs +++ b/crates/task-impls/src/events.rs @@ -11,17 +11,18 @@ use either::Either; use hotshot_task::task::TaskEvent; use hotshot_types::{ data::{ - DaProposal, Leaf, PackedBundle, QuorumProposal, UpgradeProposal, VidDisperse, + DaProposal, Leaf2, PackedBundle, QuorumProposal2, UpgradeProposal, VidDisperse, VidDisperseShare, }, message::Proposal, request_response::ProposalRequestPayload, simple_certificate::{ - DaCertificate, QuorumCertificate, TimeoutCertificate, UpgradeCertificate, - ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, ViewSyncPreCommitCertificate2, + DaCertificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, + UpgradeCertificate, ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, + ViewSyncPreCommitCertificate2, }, simple_vote::{ - DaVote, QuorumVote, TimeoutVote, UpgradeVote, ViewSyncCommitVote, ViewSyncFinalizeVote, + DaVote, QuorumVote2, TimeoutVote, UpgradeVote, ViewSyncCommitVote, ViewSyncFinalizeVote, ViewSyncPreCommitVote, }, traits::{ @@ -49,7 +50,7 @@ pub struct ProposalMissing { /// View of missing proposal pub view: TYPES::View, /// Channel to send the response back to - pub response_chan: Sender>>>, + pub response_chan: Sender>>>, } impl PartialEq for ProposalMissing { @@ -71,9 +72,9 @@ pub enum HotShotEvent { /// Shutdown the task Shutdown, /// A quorum proposal has been received from the network; handled by the consensus task - QuorumProposalRecv(Proposal>, TYPES::SignatureKey), + QuorumProposalRecv(Proposal>, TYPES::SignatureKey), /// A quorum vote has been received from the network; handled by the consensus task - QuorumVoteRecv(QuorumVote), + QuorumVoteRecv(QuorumVote2), /// A timeout vote received from the network; handled by consensus task TimeoutVoteRecv(TimeoutVote), /// Send a timeout vote to the network; emitted by consensus task replicas @@ -89,18 +90,18 @@ pub enum HotShotEvent { /// A DAC is validated. DaCertificateValidated(DaCertificate), /// Send a quorum proposal to the network; emitted by the leader in the consensus task - QuorumProposalSend(Proposal>, TYPES::SignatureKey), + QuorumProposalSend(Proposal>, TYPES::SignatureKey), /// Send a quorum vote to the next leader; emitted by a replica in the consensus task after seeing a valid quorum proposal - QuorumVoteSend(QuorumVote), + QuorumVoteSend(QuorumVote2), /// Broadcast a quorum vote to form an eQC; emitted by a replica in the consensus task after seeing a valid quorum proposal - ExtendedQuorumVoteSend(QuorumVote), + ExtendedQuorumVoteSend(QuorumVote2), /// A quorum proposal with the given parent leaf is validated. /// The full validation checks include: /// 1. The proposal is not for an old view /// 2. The proposal has been correctly signed by the leader of the current view /// 3. The justify QC is valid /// 4. The proposal passes either liveness or safety check. - QuorumProposalValidated(Proposal>, Leaf), + QuorumProposalValidated(Proposal>, Leaf2), /// A quorum proposal is missing for a view that we need. QuorumProposalRequestSend( ProposalRequestPayload, @@ -112,15 +113,17 @@ pub enum HotShotEvent { ::PureAssembledSignatureType, ), /// A quorum proposal was missing for a view. As the leader, we send a reply to the recipient with their key. - QuorumProposalResponseSend(TYPES::SignatureKey, Proposal>), + QuorumProposalResponseSend(TYPES::SignatureKey, Proposal>), /// A quorum proposal was requested by a node for a view. - QuorumProposalResponseRecv(Proposal>), + QuorumProposalResponseRecv(Proposal>), /// Send a DA proposal to the DA committee; emitted by the DA leader (which is the same node as the leader of view v + 1) in the DA task DaProposalSend(Proposal>, TYPES::SignatureKey), /// Send a DA vote to the DA leader; emitted by DA committee members in the DA task after seeing a valid DA proposal DaVoteSend(DaVote), /// The next leader has collected enough votes to form a QC; emitted by the next leader in the consensus task; an internal event only QcFormed(Either, TimeoutCertificate>), + /// The next leader has collected enough votes to form a QC; emitted by the next leader in the consensus task; an internal event only + Qc2Formed(Either, TimeoutCertificate>), /// The DA leader has collected enough votes to form a DAC; emitted by the DA leader in the DA task; sent to the entire network via the networking task DacSend(DaCertificate, TYPES::SignatureKey), /// The current view has changed; emitted by the replica in the consensus task or replica in the view sync task; received by almost all other tasks @@ -175,7 +178,6 @@ pub enum HotShotEvent { ), /// Event when the transactions task has sequenced transactions. Contains the encoded transactions, the metadata, and the view number BlockRecv(PackedBundle), - /// Send VID shares to VID storage nodes; emitted by the DA leader /// /// Like [`HotShotEvent::DaProposalSend`]. @@ -199,13 +201,12 @@ pub enum HotShotEvent { UpgradeVoteSend(UpgradeVote), /// Upgrade certificate has been sent to the network UpgradeCertificateFormed(UpgradeCertificate), - /// A quorum proposal has been preliminarily validated. /// The preliminary checks include: /// 1. The proposal is not for an old view /// 2. The proposal has been correctly signed by the leader of the current view /// 3. The justify QC is valid - QuorumProposalPreliminarilyValidated(Proposal>), + QuorumProposalPreliminarilyValidated(Proposal>), /// Send a VID request to the network; emitted to on of the members of DA committee. /// Includes the data request, node's public key and signature as well as public key of DA committee who we want to send to. @@ -238,10 +239,10 @@ pub enum HotShotEvent { ), /// A replica send us a High QC - HighQcRecv(QuorumCertificate, TYPES::SignatureKey), + HighQcRecv(QuorumCertificate2, TYPES::SignatureKey), - /// Send our HighQC to the next leader, should go to the same leader as our vote - HighQcSend(QuorumCertificate, TYPES::SignatureKey), + /// Send our HighQc to the next leader, should go to the same leader as our vote + HighQcSend(QuorumCertificate2, TYPES::SignatureKey), } impl HotShotEvent { @@ -256,8 +257,8 @@ impl HotShotEvent { HotShotEvent::QuorumProposalRecv(proposal, _) | HotShotEvent::QuorumProposalSend(proposal, _) | HotShotEvent::QuorumProposalValidated(proposal, _) - | HotShotEvent::QuorumProposalResponseSend(_, proposal) | HotShotEvent::QuorumProposalResponseRecv(proposal) + | HotShotEvent::QuorumProposalResponseSend(_, proposal) | HotShotEvent::QuorumProposalPreliminarilyValidated(proposal) => { Some(proposal.data.view_number()) } @@ -274,6 +275,10 @@ impl HotShotEvent { either::Left(qc) => Some(qc.view_number()), either::Right(tc) => Some(tc.view_number()), }, + HotShotEvent::Qc2Formed(cert) => match cert { + either::Left(qc) => Some(qc.view_number()), + either::Right(tc) => Some(tc.view_number()), + }, HotShotEvent::ViewSyncCommitVoteSend(vote) | HotShotEvent::ViewSyncCommitVoteRecv(vote) => Some(vote.view_number()), HotShotEvent::ViewSyncPreCommitVoteRecv(vote) @@ -396,6 +401,10 @@ impl Display for HotShotEvent { either::Left(qc) => write!(f, "QcFormed(view_number={:?})", qc.view_number()), either::Right(tc) => write!(f, "QcFormed(view_number={:?})", tc.view_number()), }, + HotShotEvent::Qc2Formed(cert) => match cert { + either::Left(qc) => write!(f, "QcFormed(view_number={:?})", qc.view_number()), + either::Right(tc) => write!(f, "QcFormed(view_number={:?})", tc.view_number()), + }, HotShotEvent::DacSend(cert, _) => { write!(f, "DacSend(view_number={:?})", cert.view_number()) } diff --git a/crates/task-impls/src/helpers.rs b/crates/task-impls/src/helpers.rs index bcabd40e9e..19753c329b 100644 --- a/crates/task-impls/src/helpers.rs +++ b/crates/task-impls/src/helpers.rs @@ -13,14 +13,13 @@ use async_broadcast::{Receiver, SendError, Sender}; use async_lock::RwLock; use committable::{Commitment, Committable}; use hotshot_task::dependency::{Dependency, EventDependency}; -use hotshot_types::utils::epoch_from_block_number; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal, ViewChangeEvidence}, + data::{Leaf2, QuorumProposal2, ViewChangeEvidence}, event::{Event, EventType, LeafInfo}, message::{Proposal, UpgradeLock}, request_response::ProposalRequestPayload, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate2, UpgradeCertificate}, traits::{ block_contents::BlockHeader, election::Membership, @@ -28,7 +27,7 @@ use hotshot_types::{ signature_key::SignatureKey, BlockPayload, ValidatedState, }, - utils::{Terminator, View, ViewInner}, + utils::{epoch_from_block_number, Terminator, View, ViewInner}, vote::{Certificate, HasViewNumber}, }; use tokio::time::timeout; @@ -49,7 +48,7 @@ pub(crate) async fn fetch_proposal( sender_public_key: TYPES::SignatureKey, sender_private_key: ::PrivateKey, upgrade_lock: &UpgradeLock, -) -> Result<(Leaf, View)> { +) -> Result<(Leaf2, View)> { // We need to be able to sign this request before submitting it to the network. Compute the // payload first. let signed_proposal_request = ProposalRequestPayload { @@ -105,7 +104,7 @@ pub(crate) async fn fetch_proposal( hs_event.as_ref() { // Make sure that the quorum_proposal is valid - if quorum_proposal.validate_signature(&mem, cur_epoch, upgrade_lock).await.is_ok() { + if quorum_proposal.validate_signature(&mem, cur_epoch).is_ok() { proposal = Some(quorum_proposal.clone()); } @@ -132,20 +131,17 @@ pub(crate) async fn fetch_proposal( bail!("Invalid justify_qc in proposal for view {}", *view_number); } let mut consensus_writer = consensus.write().await; - let leaf = Leaf::from_quorum_proposal(&proposal.data); + let leaf = Leaf2::from_quorum_proposal(&proposal.data); let state = Arc::new( >::from_header(&proposal.data.block_header), ); - if let Err(e) = consensus_writer - .update_leaf(leaf.clone(), Arc::clone(&state), None, upgrade_lock) - .await - { + if let Err(e) = consensus_writer.update_leaf(leaf.clone(), Arc::clone(&state), None) { tracing::trace!("{e:?}"); } let view = View { view_inner: ViewInner::Leaf { - leaf: leaf.commit(upgrade_lock).await, + leaf: leaf.commit(), state, delta: None, }, @@ -163,7 +159,7 @@ pub struct LeafChainTraversalOutcome { pub new_decided_view_number: Option, /// The qc for the decided chain. - pub new_decide_qc: Option>, + pub new_decide_qc: Option>, /// The decided leaves with corresponding validated state and VID info. pub leaf_views: Vec>, @@ -197,14 +193,14 @@ impl Default for LeafChainTraversalOutcome { /// # Panics /// Can't actually panic pub async fn decide_from_proposal_2( - proposal: &QuorumProposal, + proposal: &QuorumProposal2, consensus: OuterConsensus, existing_upgrade_cert: Arc>>>, public_key: &TYPES::SignatureKey, ) -> LeafChainTraversalOutcome { let mut res = LeafChainTraversalOutcome::default(); let consensus_reader = consensus.read().await; - let proposed_leaf = Leaf::from_quorum_proposal(proposal); + let proposed_leaf = Leaf2::from_quorum_proposal(proposal); res.new_locked_view_number = Some(proposed_leaf.justify_qc().view_number()); // If we don't have the proposals parent return early @@ -303,7 +299,7 @@ pub async fn decide_from_proposal_2( /// Upon receipt then of a proposal for view 9, assuming it is valid, this entire process will repeat, and /// the anchor view will be set to view 6, with the locked view as view 7. pub async fn decide_from_proposal( - proposal: &QuorumProposal, + proposal: &QuorumProposal2, consensus: OuterConsensus, existing_upgrade_cert: Arc>>>, public_key: &TYPES::SignatureKey, @@ -434,7 +430,7 @@ pub(crate) async fn parent_leaf_and_state( consensus: OuterConsensus, upgrade_lock: &UpgradeLock, parent_view_number: TYPES::View, -) -> Result<(Leaf, Arc<::ValidatedState>)> { +) -> Result<(Leaf2, Arc<::ValidatedState>)> { let consensus_reader = consensus.read().await; let cur_epoch = consensus_reader.cur_epoch(); ensure!( @@ -504,18 +500,17 @@ pub async fn validate_proposal_safety_and_liveness< I: NodeImplementation, V: Versions, >( - proposal: Proposal>, - parent_leaf: Leaf, + proposal: Proposal>, + parent_leaf: Leaf2, validation_info: &ValidationInfo, event_stream: Sender>>, sender: TYPES::SignatureKey, ) -> Result<()> { let view_number = proposal.data.view_number(); - let proposed_leaf = Leaf::from_quorum_proposal(&proposal.data); + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal.data); ensure!( - proposed_leaf.parent_commitment() - == parent_leaf.commit(&validation_info.upgrade_lock).await, + proposed_leaf.parent_commitment() == parent_leaf.commit(), "Proposed leaf does not extend the parent leaf." ); @@ -525,15 +520,7 @@ pub async fn validate_proposal_safety_and_liveness< { let mut consensus_writer = validation_info.consensus.write().await; - if let Err(e) = consensus_writer - .update_leaf( - proposed_leaf.clone(), - state, - None, - &validation_info.upgrade_lock, - ) - .await - { + if let Err(e) = consensus_writer.update_leaf(proposed_leaf.clone(), state, None) { tracing::trace!("{e:?}"); } @@ -657,7 +644,7 @@ pub(crate) async fn validate_proposal_view_and_certs< I: NodeImplementation, V: Versions, >( - proposal: &Proposal>, + proposal: &Proposal>, validation_info: &ValidationInfo, ) -> Result<()> { let view_number = proposal.data.view_number(); @@ -668,18 +655,15 @@ pub(crate) async fn validate_proposal_view_and_certs< ); // Validate the proposal's signature. This should also catch if the leaf_commitment does not equal our calculated parent commitment - proposal - .validate_signature( - &validation_info.quorum_membership, - validation_info.cur_epoch, - &validation_info.upgrade_lock, - ) - .await?; + proposal.validate_signature( + &validation_info.quorum_membership, + validation_info.cur_epoch, + )?; // Verify a timeout certificate OR a view sync certificate exists and is valid. if proposal.data.justify_qc.view_number() != view_number - 1 { let received_proposal_cert = - proposal.data.proposal_certificate.clone().context(debug!( + proposal.data.view_change_evidence.clone().context(debug!( "Quorum proposal for view {} needed a timeout or view sync certificate, but did not have one", *view_number ))?; diff --git a/crates/task-impls/src/network.rs b/crates/task-impls/src/network.rs index a80c34759b..a4c972b95f 100644 --- a/crates/task-impls/src/network.rs +++ b/crates/task-impls/src/network.rs @@ -19,8 +19,8 @@ use hotshot_types::{ data::{VidDisperse, VidDisperseShare}, event::{Event, EventType, HotShotAction}, message::{ - DaConsensusMessage, DataMessage, GeneralConsensusMessage, Message, MessageKind, Proposal, - SequencingMessage, UpgradeLock, + convert_proposal, DaConsensusMessage, DataMessage, GeneralConsensusMessage, Message, + MessageKind, Proposal, SequencingMessage, UpgradeLock, }, traits::{ election::Membership, @@ -72,16 +72,16 @@ impl NetworkMessageTaskState { let event = match consensus_message { SequencingMessage::General(general_message) => match general_message { GeneralConsensusMessage::Proposal(proposal) => { - HotShotEvent::QuorumProposalRecv(proposal, sender) + HotShotEvent::QuorumProposalRecv(convert_proposal(proposal), sender) } GeneralConsensusMessage::ProposalRequested(req, sig) => { HotShotEvent::QuorumProposalRequestRecv(req, sig) } GeneralConsensusMessage::ProposalResponse(proposal) => { - HotShotEvent::QuorumProposalResponseRecv(proposal) + HotShotEvent::QuorumProposalResponseRecv(convert_proposal(proposal)) } GeneralConsensusMessage::Vote(vote) => { - HotShotEvent::QuorumVoteRecv(vote.clone()) + HotShotEvent::QuorumVoteRecv(vote.to_vote2()) } GeneralConsensusMessage::ViewSyncPreCommitVote(view_sync_message) => { HotShotEvent::ViewSyncPreCommitVoteRecv(view_sync_message) @@ -114,7 +114,9 @@ impl NetworkMessageTaskState { tracing::error!("Received upgrade vote!"); HotShotEvent::UpgradeVoteRecv(message) } - GeneralConsensusMessage::HighQC(qc) => HotShotEvent::HighQcRecv(qc, sender), + GeneralConsensusMessage::HighQc(qc) => { + HotShotEvent::HighQcRecv(qc.to_qc2(), sender) + } }, SequencingMessage::Da(da_message) => match da_message { DaConsensusMessage::DaProposal(proposal) => { @@ -379,7 +381,7 @@ impl< Some(( sender, MessageKind::::from_consensus_message(SequencingMessage::General( - GeneralConsensusMessage::Proposal(proposal), + GeneralConsensusMessage::Proposal(convert_proposal(proposal)), )), TransmitType::Broadcast, )) @@ -404,7 +406,7 @@ impl< Some(( vote.signing_key(), MessageKind::::from_consensus_message(SequencingMessage::General( - GeneralConsensusMessage::Vote(vote.clone()), + GeneralConsensusMessage::Vote(vote.clone().to_vote()), )), TransmitType::Direct(leader), )) @@ -414,7 +416,7 @@ impl< Some(( vote.signing_key(), MessageKind::::from_consensus_message(SequencingMessage::General( - GeneralConsensusMessage::Vote(vote.clone()), + GeneralConsensusMessage::Vote(vote.clone().to_vote()), )), TransmitType::Broadcast, )) @@ -429,7 +431,7 @@ impl< HotShotEvent::QuorumProposalResponseSend(sender_key, proposal) => Some(( sender_key.clone(), MessageKind::::from_consensus_message(SequencingMessage::General( - GeneralConsensusMessage::ProposalResponse(proposal), + GeneralConsensusMessage::ProposalResponse(convert_proposal(proposal)), )), TransmitType::Direct(sender_key), )), diff --git a/crates/task-impls/src/quorum_proposal/handlers.rs b/crates/task-impls/src/quorum_proposal/handlers.rs index cfc46f4201..94f2b12207 100644 --- a/crates/task-impls/src/quorum_proposal/handlers.rs +++ b/crates/task-impls/src/quorum_proposal/handlers.rs @@ -13,20 +13,16 @@ use std::{ time::{Duration, Instant}, }; -use crate::{ - events::HotShotEvent, - helpers::{broadcast_event, parent_leaf_and_state}, - quorum_proposal::{UpgradeLock, Versions}, -}; use anyhow::{ensure, Context, Result}; use async_broadcast::{Receiver, Sender}; use async_lock::RwLock; +use committable::Committable; use hotshot_task::dependency_task::HandleDepOutput; use hotshot_types::{ consensus::{CommitmentAndMetadata, OuterConsensus}, - data::{Leaf, QuorumProposal, VidDisperse, ViewChangeEvidence}, + data::{Leaf2, QuorumProposal, VidDisperse, ViewChangeEvidence}, message::Proposal, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate2, UpgradeCertificate}, traits::{ block_contents::BlockHeader, node_implementation::{ConsensusTime, NodeType}, @@ -38,19 +34,25 @@ use tracing::instrument; use utils::anytrace::*; use vbs::version::StaticVersionType; +use crate::{ + events::HotShotEvent, + helpers::{broadcast_event, parent_leaf_and_state}, + quorum_proposal::{UpgradeLock, Versions}, +}; + /// Proposal dependency types. These types represent events that precipitate a proposal. #[derive(PartialEq, Debug)] pub(crate) enum ProposalDependency { /// For the `SendPayloadCommitmentAndMetadata` event. PayloadAndMetadata, - /// For the `QcFormed` event. + /// For the `Qc2Formed` event. Qc, /// For the `ViewSyncFinalizeCertificate2Recv` event. ViewSyncCert, - /// For the `QcFormed` event timeout branch. + /// For the `Qc2Formed` event timeout branch. TimeoutCert, /// For the `QuroumProposalRecv` event. @@ -109,15 +111,15 @@ pub struct ProposalDependencyHandle { pub view_start_time: Instant, /// The higest_qc we've seen at the start of this task - pub highest_qc: QuorumCertificate, + pub highest_qc: QuorumCertificate2, } impl ProposalDependencyHandle { - /// Return the next HighQC we get from the event stream + /// Return the next HighQc we get from the event stream async fn wait_for_qc_event( &self, rx: &mut Receiver>>, - ) -> Option> { + ) -> Option> { while let Ok(event) = rx.recv_direct().await { if let HotShotEvent::HighQcRecv(qc, _sender) = event.as_ref() { if qc @@ -134,7 +136,7 @@ impl ProposalDependencyHandle { } None } - /// Waits for the ocnfigured timeout for nodes to send HighQC messages to us. We'll + /// Waits for the ocnfigured timeout for nodes to send HighQc messages to us. We'll /// then propose with the higest QC from among these proposals. async fn wait_for_highest_qc(&mut self) { tracing::error!("waiting for QC"); @@ -188,7 +190,7 @@ impl ProposalDependencyHandle { view_change_evidence: Option>, formed_upgrade_certificate: Option>, decided_upgrade_certificate: Arc>>>, - parent_qc: QuorumCertificate, + parent_qc: QuorumCertificate2, ) -> Result<()> { let (parent_leaf, state) = parent_leaf_and_state( self.view_number, @@ -293,23 +295,22 @@ impl ProposalDependencyHandle { let proposal = QuorumProposal { block_header, view_number: self.view_number, - justify_qc: parent_qc, + justify_qc: parent_qc.to_qc(), upgrade_certificate, proposal_certificate, - }; + } + .into(); - let proposed_leaf = Leaf::from_quorum_proposal(&proposal); + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal); ensure!( - proposed_leaf.parent_commitment() == parent_leaf.commit(&self.upgrade_lock).await, + proposed_leaf.parent_commitment() == parent_leaf.commit(), "Proposed leaf parent does not equal high qc" ); - let signature = TYPES::SignatureKey::sign( - &self.private_key, - proposed_leaf.commit(&self.upgrade_lock).await.as_ref(), - ) - .wrap() - .context(error!("Failed to compute proposed_leaf.commit()"))?; + let signature = + TYPES::SignatureKey::sign(&self.private_key, proposed_leaf.commit().as_ref()) + .wrap() + .context(error!("Failed to compute proposed_leaf.commit()"))?; let message = Proposal { data: proposal, @@ -363,7 +364,7 @@ impl HandleDepOutput for ProposalDependencyHandle< auction_result: auction_result.clone(), }); } - HotShotEvent::QcFormed(cert) => match cert { + HotShotEvent::Qc2Formed(cert) => match cert { either::Right(timeout) => { timeout_certificate = Some(timeout.clone()); } diff --git a/crates/task-impls/src/quorum_proposal/mod.rs b/crates/task-impls/src/quorum_proposal/mod.rs index a6b641e602..9951f63e6d 100644 --- a/crates/task-impls/src/quorum_proposal/mod.rs +++ b/crates/task-impls/src/quorum_proposal/mod.rs @@ -19,7 +19,7 @@ use hotshot_types::{ consensus::OuterConsensus, event::Event, message::UpgradeLock, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate2, UpgradeCertificate}, traits::{ election::Membership, node_implementation::{ConsensusTime, NodeImplementation, NodeType, Versions}, @@ -93,7 +93,7 @@ pub struct QuorumProposalTaskState pub epoch_height: u64, /// The higest_qc we've seen at the start of this task - pub highest_qc: QuorumCertificate, + pub highest_qc: QuorumCertificate2, } impl, V: Versions> @@ -113,14 +113,14 @@ impl, V: Versions> let event = event.as_ref(); let event_view = match dependency_type { ProposalDependency::Qc => { - if let HotShotEvent::QcFormed(either::Left(qc)) = event { + if let HotShotEvent::Qc2Formed(either::Left(qc)) = event { qc.view_number() + 1 } else { return false; } } ProposalDependency::TimeoutCert => { - if let HotShotEvent::QcFormed(either::Right(timeout)) = event { + if let HotShotEvent::Qc2Formed(either::Right(timeout)) = event { timeout.view_number() + 1 } else { return false; @@ -227,7 +227,7 @@ impl, V: Versions> HotShotEvent::QuorumProposalPreliminarilyValidated(..) => { proposal_dependency.mark_as_completed(event); } - HotShotEvent::QcFormed(quorum_certificate) => match quorum_certificate { + HotShotEvent::Qc2Formed(quorum_certificate) => match quorum_certificate { Either::Right(_) => { timeout_dependency.mark_as_completed(event); } @@ -251,7 +251,7 @@ impl, V: Versions> // 2. A view sync cert was received. AndDependency::from_deps(vec![view_sync_dependency]), ]; - // 3. A `QcFormed`` event (and `QuorumProposalRecv` event) + // 3. A `Qc2Formed`` event (and `QuorumProposalRecv` event) if *view_number > 1 { secondary_deps.push(AndDependency::from_deps(vec![ qc_dependency, @@ -381,7 +381,7 @@ impl, V: Versions> self.formed_upgrade_certificate = Some(cert.clone()); } } - HotShotEvent::QcFormed(cert) => match cert.clone() { + HotShotEvent::Qc2Formed(cert) => match cert.clone() { either::Right(timeout_cert) => { let view_number = timeout_cert.view_number + 1; let epoch_number = self.consensus.read().await.cur_epoch(); @@ -413,7 +413,7 @@ impl, V: Versions> self.storage .write() .await - .update_high_qc(qc.clone()) + .update_high_qc2(qc.clone()) .await .wrap() .context(error!("Failed to update high QC in storage!"))?; diff --git a/crates/task-impls/src/quorum_proposal_recv/handlers.rs b/crates/task-impls/src/quorum_proposal_recv/handlers.rs index d7ce8aefc0..f5fe251367 100644 --- a/crates/task-impls/src/quorum_proposal_recv/handlers.rs +++ b/crates/task-impls/src/quorum_proposal_recv/handlers.rs @@ -11,13 +11,13 @@ use std::sync::Arc; use async_broadcast::{broadcast, Receiver, Sender}; use async_lock::RwLockUpgradableReadGuard; use committable::Committable; -use hotshot_types::traits::block_contents::BlockHeader; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal}, + data::{Leaf2, QuorumProposal, QuorumProposal2}, message::Proposal, simple_certificate::QuorumCertificate, traits::{ + block_contents::BlockHeader, election::Membership, node_implementation::{ConsensusTime, NodeImplementation, NodeType}, signature_key::SignatureKey, @@ -30,6 +30,7 @@ use hotshot_types::{ use tokio::spawn; use tracing::instrument; use utils::anytrace::*; +use vbs::version::StaticVersionType; use super::{QuorumProposalRecvTaskState, ValidationInfo}; use crate::{ @@ -40,25 +41,21 @@ use crate::{ }, quorum_proposal_recv::{UpgradeLock, Versions}, }; -use vbs::version::StaticVersionType; /// Update states in the event that the parent state is not found for a given `proposal`. #[instrument(skip_all)] async fn validate_proposal_liveness, V: Versions>( - proposal: &Proposal>, + proposal: &Proposal>, validation_info: &ValidationInfo, ) -> Result<()> { let mut consensus_writer = validation_info.consensus.write().await; - let leaf = Leaf::from_quorum_proposal(&proposal.data); + let leaf = Leaf2::from_quorum_proposal(&proposal.data); let state = Arc::new( >::from_header(&proposal.data.block_header), ); - if let Err(e) = consensus_writer - .update_leaf(leaf.clone(), state, None, &validation_info.upgrade_lock) - .await - { + if let Err(e) = consensus_writer.update_leaf(leaf.clone(), state, None) { tracing::trace!("{e:?}"); } @@ -66,7 +63,7 @@ async fn validate_proposal_liveness, V: Versions, >( - proposal: &Proposal>, + proposal: &Proposal>, quorum_proposal_sender_key: &TYPES::SignatureKey, event_sender: &Sender>>, event_receiver: &Receiver>>, @@ -218,7 +215,7 @@ pub(crate) async fn handle_quorum_proposal_recv< .storage .write() .await - .update_high_qc(justify_qc.clone()) + .update_high_qc2(justify_qc.clone()) .await { bail!("Failed to store High QC, not voting; error = {:?}", e); diff --git a/crates/task-impls/src/quorum_proposal_recv/mod.rs b/crates/task-impls/src/quorum_proposal_recv/mod.rs index 332ecaf241..db62d33e5d 100644 --- a/crates/task-impls/src/quorum_proposal_recv/mod.rs +++ b/crates/task-impls/src/quorum_proposal_recv/mod.rs @@ -8,11 +8,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use self::handlers::handle_quorum_proposal_recv; -use crate::{ - events::{HotShotEvent, ProposalMissing}, - helpers::{broadcast_event, fetch_proposal}, -}; use async_broadcast::{broadcast, Receiver, Sender}; use async_lock::RwLock; use async_trait::async_trait; @@ -35,6 +30,12 @@ use tokio::task::JoinHandle; use tracing::{debug, error, info, instrument, warn}; use utils::anytrace::{bail, Result}; use vbs::version::Version; + +use self::handlers::handle_quorum_proposal_recv; +use crate::{ + events::{HotShotEvent, ProposalMissing}, + helpers::{broadcast_event, fetch_proposal, parent_leaf_and_state}, +}; /// Event handlers for this task. mod handlers; diff --git a/crates/task-impls/src/quorum_vote/handlers.rs b/crates/task-impls/src/quorum_vote/handlers.rs index 198d075de5..d585259385 100644 --- a/crates/task-impls/src/quorum_vote/handlers.rs +++ b/crates/task-impls/src/quorum_vote/handlers.rs @@ -9,12 +9,13 @@ use std::sync::Arc; use async_broadcast::{InactiveReceiver, Sender}; use async_lock::RwLock; use chrono::Utc; +use committable::Committable; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal, VidDisperseShare}, + data::{Leaf2, QuorumProposal2, VidDisperseShare}, event::{Event, EventType}, message::{Proposal, UpgradeLock}, - simple_vote::{QuorumData, QuorumVote}, + simple_vote::{QuorumData2, QuorumVote2}, traits::{ election::Membership, node_implementation::{ConsensusTime, NodeImplementation, NodeType}, @@ -45,7 +46,7 @@ pub(crate) async fn handle_quorum_proposal_validated< I: NodeImplementation, V: Versions, >( - proposal: &QuorumProposal, + proposal: &QuorumProposal2, task_state: &mut QuorumVoteTaskState, ) -> Result<()> { let version = task_state @@ -172,7 +173,7 @@ pub(crate) async fn update_shared_state< view_number: TYPES::View, instance_state: Arc, storage: Arc>, - proposed_leaf: &Leaf, + proposed_leaf: &Leaf2, vid_share: &Proposal>, parent_view_number: Option, ) -> Result<()> { @@ -259,15 +260,11 @@ pub(crate) async fn update_shared_state< // Now that we've rounded everyone up, we need to update the shared state let mut consensus_writer = consensus.write().await; - if let Err(e) = consensus_writer - .update_leaf( - proposed_leaf.clone(), - Arc::clone(&state), - Some(Arc::clone(&delta)), - &upgrade_lock, - ) - .await - { + if let Err(e) = consensus_writer.update_leaf( + proposed_leaf.clone(), + Arc::clone(&state), + Some(Arc::clone(&delta)), + ) { tracing::trace!("{e:?}"); } @@ -280,7 +277,7 @@ pub(crate) async fn update_shared_state< storage .write() .await - .update_undecided_state(new_leaves, new_state) + .update_undecided_state2(new_leaves, new_state) .await .wrap() .context(error!("Failed to update undecided state"))?; @@ -300,7 +297,7 @@ pub(crate) async fn submit_vote, V view_number: TYPES::View, epoch_number: TYPES::Epoch, storage: Arc>, - leaf: Leaf, + leaf: Leaf2, vid_share: Proposal>, extended_vote: bool, ) -> Result<()> { @@ -313,9 +310,9 @@ pub(crate) async fn submit_vote, V ); // Create and send the vote. - let vote = QuorumVote::::create_signed_vote( - QuorumData { - leaf_commit: leaf.commit(&upgrade_lock).await, + let vote = QuorumVote2::::create_signed_vote( + QuorumData2 { + leaf_commit: leaf.commit(), }, view_number, &public_key, diff --git a/crates/task-impls/src/quorum_vote/mod.rs b/crates/task-impls/src/quorum_vote/mod.rs index 6ac5fe37dd..393488402f 100644 --- a/crates/task-impls/src/quorum_vote/mod.rs +++ b/crates/task-impls/src/quorum_vote/mod.rs @@ -6,14 +6,10 @@ use std::{collections::BTreeMap, sync::Arc}; -use crate::{ - events::HotShotEvent, - helpers::broadcast_event, - quorum_vote::handlers::{handle_quorum_proposal_validated, submit_vote, update_shared_state}, -}; use async_broadcast::{InactiveReceiver, Receiver, Sender}; use async_lock::RwLock; use async_trait::async_trait; +use committable::Committable; use hotshot_task::{ dependency::{AndDependency, EventDependency}, dependency_task::{DependencyTask, HandleDepOutput}, @@ -21,7 +17,7 @@ use hotshot_task::{ }; use hotshot_types::{ consensus::OuterConsensus, - data::{Leaf, QuorumProposal}, + data::{Leaf2, QuorumProposal2}, event::Event, message::{Proposal, UpgradeLock}, traits::{ @@ -41,6 +37,12 @@ use tracing::instrument; use utils::anytrace::*; use vbs::version::StaticVersionType; +use crate::{ + events::HotShotEvent, + helpers::broadcast_event, + quorum_vote::handlers::{handle_quorum_proposal_validated, submit_vote, update_shared_state}, +}; + /// Event handlers for `QuorumProposalValidated`. mod handlers; @@ -107,8 +109,8 @@ impl + 'static, V: Versions> Handl } }; let proposal_payload_comm = proposal.data.block_header.payload_commitment(); - let parent_commitment = parent_leaf.commit(&self.upgrade_lock).await; - let proposed_leaf = Leaf::from_quorum_proposal(&proposal.data); + let parent_commitment = parent_leaf.commit(); + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal.data); if version >= V::Epochs::VERSION && self @@ -134,7 +136,7 @@ impl + 'static, V: Versions> Handl } // Update our persistent storage of the proposal. If we cannot store the proposal return // and error so we don't vote - if let Err(e) = self.storage.write().await.append_proposal(proposal).await { + if let Err(e) = self.storage.write().await.append_proposal2(proposal).await { tracing::error!("failed to store proposal, not voting. error = {e:#}"); return; } @@ -222,11 +224,7 @@ impl + 'static, V: Versions> Handl ) .await; - let is_vote_leaf_extended = self - .consensus - .read() - .await - .is_leaf_extended(leaf.commit(&self.upgrade_lock).await); + let is_vote_leaf_extended = self.consensus.read().await.is_leaf_extended(leaf.commit()); if let Err(e) = submit_vote::( self.sender.clone(), Arc::clone(&self.quorum_membership), @@ -588,14 +586,14 @@ impl, V: Versions> QuorumVoteTaskS #[allow(clippy::too_many_lines)] async fn handle_eqc_voting( &self, - proposal: &Proposal>, - parent_leaf: &Leaf, + proposal: &Proposal>, + parent_leaf: &Leaf2, event_sender: Sender>>, event_receiver: Receiver>>, ) { tracing::info!("Reached end of epoch. Justify QC is for the last block in the epoch."); - let proposed_leaf = Leaf::from_quorum_proposal(&proposal.data); - let parent_commitment = parent_leaf.commit(&self.upgrade_lock).await; + let proposed_leaf = Leaf2::from_quorum_proposal(&proposal.data); + let parent_commitment = parent_leaf.commit(); if proposed_leaf.height() != parent_leaf.height() || proposed_leaf.payload_commitment() != parent_leaf.payload_commitment() { @@ -634,7 +632,7 @@ impl, V: Versions> QuorumVoteTaskS } // Update our persistent storage of the proposal. If we cannot store the proposal return // and error so we don't vote - if let Err(e) = self.storage.write().await.append_proposal(proposal).await { + if let Err(e) = self.storage.write().await.append_proposal2(proposal).await { tracing::error!("failed to store proposal, not voting. error = {e:#}"); return; } diff --git a/crates/task-impls/src/vote_collection.rs b/crates/task-impls/src/vote_collection.rs index c9266ae808..6023f5cdfb 100644 --- a/crates/task-impls/src/vote_collection.rs +++ b/crates/task-impls/src/vote_collection.rs @@ -17,12 +17,13 @@ use either::Either::{self, Left, Right}; use hotshot_types::{ message::UpgradeLock, simple_certificate::{ - DaCertificate, QuorumCertificate, TimeoutCertificate, UpgradeCertificate, - ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, ViewSyncPreCommitCertificate2, + DaCertificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, + UpgradeCertificate, ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, + ViewSyncPreCommitCertificate2, }, simple_vote::{ - DaVote, QuorumVote, TimeoutVote, UpgradeVote, ViewSyncCommitVote, ViewSyncFinalizeVote, - ViewSyncPreCommitVote, + DaVote, QuorumVote, QuorumVote2, TimeoutVote, UpgradeVote, ViewSyncCommitVote, + ViewSyncFinalizeVote, ViewSyncPreCommitVote, }, traits::{ election::Membership, @@ -300,7 +301,7 @@ where /// Alias for Quorum vote accumulator type QuorumVoteState = - VoteCollectionTaskState, QuorumCertificate, V>; + VoteCollectionTaskState, QuorumCertificate2, V>; /// Alias for DA vote accumulator type DaVoteState = VoteCollectionTaskState, DaCertificate, V>; /// Alias for Timeout vote accumulator @@ -345,6 +346,24 @@ impl AggregatableVote, QuorumCertifica } } +impl AggregatableVote, QuorumCertificate2> + for QuorumVote2 +{ + fn leader( + &self, + membership: &TYPES::Membership, + epoch: TYPES::Epoch, + ) -> Result { + membership.leader(self.view_number() + 1, epoch) + } + fn make_cert_event( + certificate: QuorumCertificate2, + _key: &TYPES::SignatureKey, + ) -> HotShotEvent { + HotShotEvent::Qc2Formed(Left(certificate)) + } +} + impl AggregatableVote, UpgradeCertificate> for UpgradeVote { @@ -395,7 +414,7 @@ impl AggregatableVote, TimeoutCertifi certificate: TimeoutCertificate, _key: &TYPES::SignatureKey, ) -> HotShotEvent { - HotShotEvent::QcFormed(Right(certificate)) + HotShotEvent::Qc2Formed(Right(certificate)) } } @@ -459,14 +478,14 @@ impl // Handlers for all vote accumulators #[async_trait] impl - HandleVoteEvent, QuorumCertificate> + HandleVoteEvent, QuorumCertificate2> for QuorumVoteState { async fn handle_vote_event( &mut self, event: Arc>, sender: &Sender>>, - ) -> Result>> { + ) -> Result>> { match event.as_ref() { HotShotEvent::QuorumVoteRecv(vote) => self.accumulate_vote(vote, sender).await, _ => Ok(None), diff --git a/crates/testing/src/byzantine/byzantine_behaviour.rs b/crates/testing/src/byzantine/byzantine_behaviour.rs index 1ab38f1f29..0ffc97df37 100644 --- a/crates/testing/src/byzantine/byzantine_behaviour.rs +++ b/crates/testing/src/byzantine/byzantine_behaviour.rs @@ -19,9 +19,9 @@ use hotshot_task_impls::{ }; use hotshot_types::{ consensus::{Consensus, OuterConsensus}, - data::QuorumProposal, + data::QuorumProposal2, message::{Proposal, UpgradeLock}, - simple_vote::QuorumVote, + simple_vote::QuorumVote2, traits::node_implementation::{ConsensusTime, NodeImplementation, NodeType, Versions}, }; @@ -106,7 +106,7 @@ impl, V: Versions> EventTransforme /// An `EventHandlerState` that modifies justify_qc on `QuorumProposalSend` to that of a previous view to mock dishonest leader pub struct DishonestLeader { /// Store events from previous views - pub validated_proposals: Vec>, + pub validated_proposals: Vec>, /// How many times current node has been elected leader and sent proposal pub total_proposals_from_node: u64, /// Which proposals to be dishonest at @@ -126,7 +126,7 @@ impl DishonestLeader { async fn handle_proposal_send_event( &self, event: &HotShotEvent, - proposal: &Proposal>, + proposal: &Proposal>, sender: &TYPES::SignatureKey, ) -> HotShotEvent { let length = self.validated_proposals.len(); @@ -319,7 +319,7 @@ impl + std::fmt::Debug, V: Version ) -> Vec> { if let HotShotEvent::QuorumVoteSend(vote) = event { let new_view = vote.view_number + self.view_increment; - let spoofed_vote = QuorumVote::::create_signed_vote( + let spoofed_vote = QuorumVote2::::create_signed_vote( vote.data.clone(), new_view, public_key, @@ -373,7 +373,7 @@ impl std::fmt::Debug for DishonestVoting { /// An `EventHandlerState` that will send a vote for a bad proposal pub struct DishonestVoter { /// Collect all votes the node sends - pub votes_sent: Vec>, + pub votes_sent: Vec>, /// Shared state with views numbers that leaders were dishonest at pub dishonest_proposal_view_numbers: Arc>>, } @@ -402,7 +402,7 @@ impl + std::fmt::Debug, V: Version // Create a vote using data from most recent vote and the current event number // We wont update internal consensus state for this Byzantine replica but we are at least // Going to send a vote to the next honest leader - let vote = QuorumVote::::create_signed_vote( + let vote = QuorumVote2::::create_signed_vote( self.votes_sent.last().unwrap().data.clone(), event.view_number().unwrap(), public_key, diff --git a/crates/testing/src/consistency_task.rs b/crates/testing/src/consistency_task.rs index 0d78c4d12c..9d51d50a78 100644 --- a/crates/testing/src/consistency_task.rs +++ b/crates/testing/src/consistency_task.rs @@ -9,9 +9,10 @@ use std::{collections::BTreeMap, marker::PhantomData}; use anyhow::{bail, ensure, Context, Result}; use async_trait::async_trait; +use committable::Committable; use hotshot_example_types::block_types::TestBlockHeader; use hotshot_types::{ - data::Leaf, + data::Leaf2, event::{Event, EventType}, message::UpgradeLock, traits::node_implementation::{ConsensusTime, NodeType, Versions}, @@ -24,10 +25,10 @@ use crate::{ }; /// Map from views to leaves for a single node, allowing multiple leaves for each view (because the node may a priori send us multiple leaves for a given view). -pub type NodeMap = BTreeMap<::View, Vec>>; +pub type NodeMap = BTreeMap<::View, Vec>>; /// A sanitized map from views to leaves for a single node, with only a single leaf per view. -pub type NodeMapSanitized = BTreeMap<::View, Leaf>; +pub type NodeMapSanitized = BTreeMap<::View, Leaf2>; /// Validate that the `NodeMap` only has a single leaf per view. fn sanitize_node_map( @@ -104,7 +105,7 @@ async fn validate_node_map( // We want to make sure the commitment matches, // but allow for the possibility that we may have skipped views in between. if child.justify_qc().view_number == parent.view_number() - && child.justify_qc().data.leaf_commit != parent.commit(&upgrade_lock).await + && child.justify_qc().data.leaf_commit != parent.commit() { bail!("The node has provided leaf:\n\n{child:?}\n\nwhich points to:\n\n{parent:?}\n\nbut the commits do not match."); } @@ -144,7 +145,7 @@ fn sanitize_network_map( Ok(result) } -pub type ViewMap = BTreeMap<::View, BTreeMap>>; +pub type ViewMap = BTreeMap<::View, BTreeMap>>; // Invert the network map by interchanging the roles of the node_id and view number. // @@ -171,7 +172,7 @@ async fn invert_network_map( } /// A view map, sanitized to have exactly one leaf per view. -pub type ViewMapSanitized = BTreeMap<::View, Leaf>; +pub type ViewMapSanitized = BTreeMap<::View, Leaf2>; fn sanitize_view_map( view_map: &ViewMap, diff --git a/crates/testing/src/helpers.rs b/crates/testing/src/helpers.rs index 40760372c6..49f28e2504 100644 --- a/crates/testing/src/helpers.rs +++ b/crates/testing/src/helpers.rs @@ -26,7 +26,7 @@ use hotshot_example_types::{ use hotshot_task_impls::events::HotShotEvent; use hotshot_types::{ consensus::ConsensusMetricsValue, - data::{Leaf, QuorumProposal, VidDisperse, VidDisperseShare}, + data::{Leaf, Leaf2, QuorumProposal, VidDisperse, VidDisperseShare}, message::{GeneralConsensusMessage, Proposal, UpgradeLock}, simple_certificate::DaCertificate, simple_vote::{DaData, DaVote, QuorumData, QuorumVote, SimpleVote, VersionedVoteData}, @@ -437,7 +437,7 @@ where /// This function will create a fake [`View`] from a provided [`Leaf`]. pub async fn build_fake_view_with_leaf( - leaf: Leaf, + leaf: Leaf2, upgrade_lock: &UpgradeLock, ) -> View { build_fake_view_with_leaf_and_state(leaf, TestValidatedState::default(), upgrade_lock).await @@ -445,13 +445,13 @@ pub async fn build_fake_view_with_leaf( /// This function will create a fake [`View`] from a provided [`Leaf`] and `state`. pub async fn build_fake_view_with_leaf_and_state( - leaf: Leaf, + leaf: Leaf2, state: TestValidatedState, - upgrade_lock: &UpgradeLock, + _upgrade_lock: &UpgradeLock, ) -> View { View { view_inner: ViewInner::Leaf { - leaf: leaf.commit(upgrade_lock).await, + leaf: leaf.commit(), state: state.into(), delta: None, }, diff --git a/crates/testing/src/overall_safety_task.rs b/crates/testing/src/overall_safety_task.rs index ba7136c90f..c82315d18d 100644 --- a/crates/testing/src/overall_safety_task.rs +++ b/crates/testing/src/overall_safety_task.rs @@ -15,10 +15,10 @@ use async_lock::RwLock; use async_trait::async_trait; use hotshot::{traits::TestableNodeImplementation, HotShotError}; use hotshot_types::{ - data::Leaf, + data::Leaf2, error::RoundTimedoutState, event::{Event, EventType, LeafChain}, - simple_certificate::QuorumCertificate, + simple_certificate::QuorumCertificate2, traits::node_implementation::{ConsensusTime, NodeType, Versions}, vid::VidCommitment, }; @@ -311,7 +311,7 @@ pub struct RoundResult { /// Nodes that committed this round /// id -> (leaf, qc) - success_nodes: HashMap, QuorumCertificate)>, + success_nodes: HashMap, QuorumCertificate2)>, /// Nodes that failed to commit this round pub failed_nodes: HashMap>>, @@ -322,7 +322,7 @@ pub struct RoundResult { /// NOTE: technically a map is not needed /// left one anyway for ease of viewing /// leaf -> # entries decided on that leaf - pub leaf_map: HashMap, usize>, + pub leaf_map: HashMap, usize>, /// block -> # entries decided on that block pub block_map: HashMap, @@ -403,9 +403,9 @@ impl RoundResult { pub fn insert_into_result( &mut self, idx: usize, - result: (LeafChain, QuorumCertificate), + result: (LeafChain, QuorumCertificate2), maybe_block_size: Option, - ) -> Option> { + ) -> Option> { self.success_nodes.insert(idx as u64, result.clone()); let maybe_leaf = result.0.first(); @@ -458,7 +458,7 @@ impl RoundResult { &mut self, threshold: usize, total_num_nodes: usize, - key: &Leaf, + key: &Leaf2, check_leaf: bool, check_block: bool, transaction_threshold: u64, @@ -530,8 +530,8 @@ impl RoundResult { /// generate leaves #[must_use] - pub fn gen_leaves(&self) -> HashMap, usize> { - let mut leaves = HashMap::, usize>::new(); + pub fn gen_leaves(&self) -> HashMap, usize> { + let mut leaves = HashMap::, usize>::new(); for (leaf_vec, _) in self.success_nodes.values() { let most_recent_leaf = leaf_vec.iter().last(); diff --git a/crates/testing/src/spinning_task.rs b/crates/testing/src/spinning_task.rs index e2a299a171..e98726ff51 100644 --- a/crates/testing/src/spinning_task.rs +++ b/crates/testing/src/spinning_task.rs @@ -26,9 +26,9 @@ use hotshot_example_types::{ }; use hotshot_types::{ constants::EVENT_CHANNEL_SIZE, - data::Leaf, + data::Leaf2, event::Event, - simple_certificate::QuorumCertificate, + simple_certificate::{QuorumCertificate, QuorumCertificate2}, traits::{ network::{AsyncGenerator, ConnectedNetwork}, node_implementation::{ConsensusTime, NodeImplementation, NodeType, Versions}, @@ -62,9 +62,9 @@ pub struct SpinningTask< /// most recent view seen by spinning task pub(crate) latest_view: Option, /// Last decided leaf that can be used as the anchor leaf to initialize the node. - pub(crate) last_decided_leaf: Leaf, + pub(crate) last_decided_leaf: Leaf2, /// Highest qc seen in the test for restarting nodes - pub(crate) high_qc: QuorumCertificate, + pub(crate) high_qc: QuorumCertificate2, /// Add specified delay to async calls pub(crate) async_delay_config: DelayConfig, /// Context stored for nodes to be restarted with @@ -247,7 +247,8 @@ where &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .to_qc2(), ), read_storage.decided_upgrade_certificate().await, Vec::new(), diff --git a/crates/testing/src/test_runner.rs b/crates/testing/src/test_runner.rs index 2a08fda5e4..3460795db0 100644 --- a/crates/testing/src/test_runner.rs +++ b/crates/testing/src/test_runner.rs @@ -183,12 +183,14 @@ where &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .into(), high_qc: QuorumCertificate::genesis::( &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .to_qc2(), async_delay_config: launcher.metadata.async_delay_config, restart_contexts: HashMap::new(), channel_generator: launcher.resource_generator.channel_generator, @@ -645,7 +647,6 @@ where internal_channel, external_channel, ) - .await } } diff --git a/crates/testing/src/view_generator.rs b/crates/testing/src/view_generator.rs index 87a688b74b..53a0d74b7e 100644 --- a/crates/testing/src/view_generator.rs +++ b/crates/testing/src/view_generator.rs @@ -12,6 +12,7 @@ use std::{ task::{Context, Poll}, }; +use committable::Committable; use futures::{FutureExt, Stream}; use hotshot::types::{BLSPubKey, SignatureKey, SystemContextHandle}; use hotshot_example_types::{ @@ -21,16 +22,16 @@ use hotshot_example_types::{ }; use hotshot_types::{ data::{ - DaProposal, EpochNumber, Leaf, QuorumProposal, VidDisperse, VidDisperseShare, + DaProposal, EpochNumber, Leaf, Leaf2, QuorumProposal2, VidDisperse, VidDisperseShare, ViewChangeEvidence, ViewNumber, }, message::{Proposal, UpgradeLock}, simple_certificate::{ - DaCertificate, QuorumCertificate, TimeoutCertificate, UpgradeCertificate, - ViewSyncFinalizeCertificate2, + DaCertificate, QuorumCertificate, QuorumCertificate2, TimeoutCertificate, + UpgradeCertificate, ViewSyncFinalizeCertificate2, }, simple_vote::{ - DaData, DaVote, QuorumData, QuorumVote, TimeoutData, TimeoutVote, UpgradeProposalData, + DaData, DaVote, QuorumData2, QuorumVote2, TimeoutData, TimeoutVote, UpgradeProposalData, UpgradeVote, ViewSyncFinalizeData, ViewSyncFinalizeVote, }, traits::{ @@ -49,8 +50,8 @@ use crate::helpers::{ #[derive(Clone)] pub struct TestView { pub da_proposal: Proposal>, - pub quorum_proposal: Proposal>, - pub leaf: Leaf, + pub quorum_proposal: Proposal>, + pub leaf: Leaf2, pub view_number: ViewNumber, pub epoch_number: EpochNumber, pub quorum_membership: ::Membership, @@ -130,22 +131,26 @@ impl TestView { &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .into(), payload_commitment, builder_commitment, metadata, ); - let quorum_proposal_inner = QuorumProposal:: { + let quorum_proposal_inner = QuorumProposal2:: { block_header: block_header.clone(), view_number: genesis_view, justify_qc: QuorumCertificate::genesis::( &TestValidatedState::default(), &TestInstanceState::default(), ) - .await, + .await + .to_qc2(), upgrade_certificate: None, - proposal_certificate: None, + view_change_evidence: None, + drb_result: [0; 32], + drb_seed: [0; 96], }; let encoded_transactions = Arc::from(TestTransaction::encode(&transactions)); @@ -166,16 +171,13 @@ impl TestView { _pd: PhantomData, }; - let mut leaf = Leaf::from_quorum_proposal(&quorum_proposal_inner); + let mut leaf = Leaf2::from_quorum_proposal(&quorum_proposal_inner); leaf.fill_block_payload_unchecked(TestBlockPayload { transactions: transactions.clone(), }); - let signature = ::sign( - &private_key, - leaf.commit(&upgrade_lock).await.as_ref(), - ) - .expect("Failed to sign leaf commitment!"); + let signature = ::sign(&private_key, leaf.commit().as_ref()) + .expect("Failed to sign leaf commitment!"); let quorum_proposal = Proposal { data: quorum_proposal_inner, @@ -222,8 +224,8 @@ impl TestView { let transactions = &self.transactions; - let quorum_data = QuorumData { - leaf_commit: old.leaf.commit(&self.upgrade_lock).await, + let quorum_data = QuorumData2 { + leaf_commit: old.leaf.commit(), }; let (old_private_key, old_public_key) = key_pair_for_id::(*old_view); @@ -274,9 +276,9 @@ impl TestView { let quorum_certificate = build_cert::< TestTypes, TestVersions, - QuorumData, - QuorumVote, - QuorumCertificate, + QuorumData2, + QuorumVote2, + QuorumCertificate2, >( quorum_data, quorum_membership, @@ -357,7 +359,7 @@ impl TestView { None }; - let proposal_certificate = if let Some(tc) = timeout_certificate { + let view_change_evidence = if let Some(tc) = timeout_certificate { Some(ViewChangeEvidence::Timeout(tc)) } else { view_sync_certificate.map(ViewChangeEvidence::ViewSync) @@ -374,24 +376,23 @@ impl TestView { random, }; - let proposal = QuorumProposal:: { + let proposal = QuorumProposal2:: { block_header: block_header.clone(), view_number: next_view, justify_qc: quorum_certificate.clone(), upgrade_certificate: upgrade_certificate.clone(), - proposal_certificate, + view_change_evidence, + drb_result: [0; 32], + drb_seed: [0; 96], }; - let mut leaf = Leaf::from_quorum_proposal(&proposal); + let mut leaf = Leaf2::from_quorum_proposal(&proposal); leaf.fill_block_payload_unchecked(TestBlockPayload { transactions: transactions.clone(), }); - let signature = ::sign( - &private_key, - leaf.commit(&self.upgrade_lock).await.as_ref(), - ) - .expect("Failed to sign leaf commitment."); + let signature = ::sign(&private_key, leaf.commit().as_ref()) + .expect("Failed to sign leaf commitment."); let quorum_proposal = Proposal { data: proposal, @@ -451,10 +452,10 @@ impl TestView { pub async fn create_quorum_vote( &self, handle: &SystemContextHandle, - ) -> QuorumVote { - QuorumVote::::create_signed_vote( - QuorumData { - leaf_commit: self.leaf.commit(&handle.hotshot.upgrade_lock).await, + ) -> QuorumVote2 { + QuorumVote2::::create_signed_vote( + QuorumData2 { + leaf_commit: self.leaf.commit(), }, self.view_number, &handle.public_key(), diff --git a/crates/testing/tests/tests_1/message.rs b/crates/testing/tests/tests_1/message.rs index 92c8cb1da7..5e0ff49eaa 100644 --- a/crates/testing/tests/tests_1/message.rs +++ b/crates/testing/tests/tests_1/message.rs @@ -57,3 +57,77 @@ fn version_number_at_start_of_serialization() { assert_eq!(version.major, version_read.major); assert_eq!(version.minor, version_read.minor); } + +#[cfg(test)] +#[tokio::test(flavor = "multi_thread")] +async fn test_certificate2_validity() { + use futures::StreamExt; + use hotshot_example_types::node_types::{MemoryImpl, TestTypes, TestVersions}; + use hotshot_testing::{helpers::build_system_handle, view_generator::TestViewGenerator}; + use hotshot_types::{ + data::{EpochNumber, Leaf, Leaf2}, + traits::node_implementation::ConsensusTime, + vote::Certificate, + }; + + hotshot::helpers::initialize_logging(); + + let node_id = 1; + let handle = build_system_handle::(node_id) + .await + .0; + let quorum_membership = handle.hotshot.memberships.quorum_membership.clone(); + let da_membership = handle.hotshot.memberships.da_membership.clone(); + + let mut generator = TestViewGenerator::generate(quorum_membership.clone(), da_membership); + + let mut proposals = Vec::new(); + let mut leaders = Vec::new(); + let mut leaves = Vec::new(); + let mut vids = Vec::new(); + let mut vid_dispersals = Vec::new(); + + for view in (&mut generator).take(4).collect::>().await { + proposals.push(view.quorum_proposal.clone()); + leaders.push(view.leader_public_key); + leaves.push(view.leaf.clone()); + vids.push(view.vid_proposal.clone()); + vid_dispersals.push(view.vid_disperse.clone()); + } + + let proposal = proposals[3].clone(); + let parent_proposal = proposals[2].clone(); + + // ensure that we don't break certificate validation + let qc2 = proposal.data.justify_qc.clone(); + let qc = qc2.clone().to_qc(); + + assert!( + qc.is_valid_cert( + &quorum_membership, + EpochNumber::new(0), + &handle.hotshot.upgrade_lock + ) + .await + ); + + assert!( + qc2.is_valid_cert( + &quorum_membership, + EpochNumber::new(0), + &handle.hotshot.upgrade_lock + ) + .await + ); + + // ensure that we don't break the leaf commitment chain + let leaf2 = Leaf2::from_quorum_proposal(&proposal.data); + let parent_leaf2 = Leaf2::from_quorum_proposal(&parent_proposal.data); + + let leaf = Leaf::from_quorum_proposal(&proposal.data.into()); + let parent_leaf = Leaf::from_quorum_proposal(&parent_proposal.data.into()); + + assert!(leaf.parent_commitment() == parent_leaf.commit(&handle.hotshot.upgrade_lock).await); + + assert!(leaf2.parent_commitment() == parent_leaf2.commit()); +} diff --git a/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs b/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs index dc42dfc723..7a1636d9c6 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_recv_task.rs @@ -27,9 +27,8 @@ use hotshot_testing::{ serial, view_generator::TestViewGenerator, }; -use hotshot_types::data::EpochNumber; use hotshot_types::{ - data::{Leaf, ViewNumber}, + data::{EpochNumber, Leaf2, ViewNumber}, request_response::ProposalRequestPayload, traits::{ consensus_api::ConsensusApi, @@ -78,12 +77,10 @@ async fn test_quorum_proposal_recv_task() { // to that, we'll just put them in here. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); @@ -125,7 +122,7 @@ async fn test_quorum_proposal_recv_task_liveness_check() { helpers::{build_fake_view_with_leaf, build_fake_view_with_leaf_and_state}, script::{Expectations, TaskScript}, }; - use hotshot_types::{data::Leaf, vote::HasViewNumber}; + use hotshot_types::{data::Leaf2, vote::HasViewNumber}; hotshot::helpers::initialize_logging(); diff --git a/crates/testing/tests/tests_1/quorum_proposal_task.rs b/crates/testing/tests/tests_1/quorum_proposal_task.rs index d8ae3e2b2b..f522d28f7c 100644 --- a/crates/testing/tests/tests_1/quorum_proposal_task.rs +++ b/crates/testing/tests/tests_1/quorum_proposal_task.rs @@ -25,7 +25,7 @@ use hotshot_testing::{ view_generator::TestViewGenerator, }; use hotshot_types::{ - data::{null_block, EpochNumber, Leaf, ViewChangeEvidence, ViewNumber}, + data::{null_block, EpochNumber, Leaf2, ViewChangeEvidence, ViewNumber}, simple_vote::{TimeoutData, ViewSyncFinalizeData}, traits::{ election::Membership, @@ -79,12 +79,10 @@ async fn test_quorum_proposal_task_quorum_proposal_view_1() { // to make sure they show up during tests. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } @@ -105,7 +103,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_1() { handle.public_key() )], random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( payload_commitment, builder_commitment, @@ -169,12 +167,10 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { // to make sure they show up during tests. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } @@ -194,7 +190,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { let inputs = vec![ random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -213,7 +209,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[0].clone()), - QcFormed(either::Left(proposals[1].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[1].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -230,7 +226,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[1].clone()), - QcFormed(either::Left(proposals[2].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[2].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -247,7 +243,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[2].clone()), - QcFormed(either::Left(proposals[3].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[3].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -264,7 +260,7 @@ async fn test_quorum_proposal_task_quorum_proposal_view_gt_1() { ], random![ QuorumProposalPreliminarilyValidated(proposals[3].clone()), - QcFormed(either::Left(proposals[4].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[4].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -349,13 +345,13 @@ async fn test_quorum_proposal_task_qc_timeout() { } // Get the proposal cert out for the view sync input - let cert = match proposals[1].data.proposal_certificate.clone().unwrap() { + let cert = match proposals[1].data.view_change_evidence.clone().unwrap() { ViewChangeEvidence::Timeout(tc) => tc, _ => panic!("Found a View Sync Cert when there should have been a Timeout cert"), }; let inputs = vec![random![ - QcFormed(either::Right(cert.clone())), + Qc2Formed(either::Right(cert.clone())), SendPayloadCommitmentAndMetadata( payload_commitment, builder_commitment, @@ -439,7 +435,7 @@ async fn test_quorum_proposal_task_view_sync() { } // Get the proposal cert out for the view sync input - let cert = match proposals[1].data.proposal_certificate.clone().unwrap() { + let cert = match proposals[1].data.view_change_evidence.clone().unwrap() { ViewChangeEvidence::ViewSync(vsc) => vsc, _ => panic!("Found a TC when there should have been a view sync cert"), }; @@ -511,12 +507,10 @@ async fn test_quorum_proposal_task_liveness_check() { // to make sure they show up during tests. consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); @@ -535,7 +529,7 @@ async fn test_quorum_proposal_task_liveness_check() { let inputs = vec![ random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -554,7 +548,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[0].clone()), - QcFormed(either::Left(proposals[1].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[1].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -571,7 +565,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[1].clone()), - QcFormed(either::Left(proposals[2].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[2].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -588,7 +582,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[2].clone()), - QcFormed(either::Left(proposals[3].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[3].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -605,7 +599,7 @@ async fn test_quorum_proposal_task_liveness_check() { ], random![ QuorumProposalPreliminarilyValidated(proposals[3].clone()), - QcFormed(either::Left(proposals[4].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[4].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, diff --git a/crates/testing/tests/tests_1/quorum_vote_task.rs b/crates/testing/tests/tests_1/quorum_vote_task.rs index b4350abe3c..f5c5940b20 100644 --- a/crates/testing/tests/tests_1/quorum_vote_task.rs +++ b/crates/testing/tests/tests_1/quorum_vote_task.rs @@ -23,7 +23,7 @@ use hotshot_testing::{ script::{Expectations, InputOrder, TaskScript}, }; use hotshot_types::{ - data::{EpochNumber, Leaf, ViewNumber}, + data::{EpochNumber, Leaf2, ViewNumber}, traits::node_implementation::ConsensusTime, }; @@ -64,12 +64,10 @@ async fn test_quorum_vote_task_success() { vids.push(view.vid_proposal.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); @@ -136,12 +134,10 @@ async fn test_quorum_vote_task_miss_dependency() { consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs index ac4b338f37..0cef44ab81 100644 --- a/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs +++ b/crates/testing/tests/tests_1/upgrade_task_with_proposal.rs @@ -30,7 +30,7 @@ use hotshot_testing::{ view_generator::TestViewGenerator, }; use hotshot_types::{ - data::{null_block, EpochNumber, Leaf, ViewNumber}, + data::{null_block, EpochNumber, Leaf2, ViewNumber}, simple_vote::UpgradeProposalData, traits::{ election::Membership, @@ -96,12 +96,10 @@ async fn test_upgrade_task_with_proposal() { views.push(view.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } @@ -117,12 +115,10 @@ async fn test_upgrade_task_with_proposal() { views.push(view.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); @@ -155,7 +151,7 @@ async fn test_upgrade_task_with_proposal() { let inputs = vec![ random![ - QcFormed(either::Left(genesis_cert.clone())), + Qc2Formed(either::Left(genesis_cert.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -174,7 +170,7 @@ async fn test_upgrade_task_with_proposal() { ], random![ QuorumProposalPreliminarilyValidated(proposals[0].clone()), - QcFormed(either::Left(proposals[1].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[1].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, @@ -192,7 +188,7 @@ async fn test_upgrade_task_with_proposal() { InputOrder::Random(upgrade_vote_recvs), random![ QuorumProposalPreliminarilyValidated(proposals[1].clone()), - QcFormed(either::Left(proposals[2].data.justify_qc.clone())), + Qc2Formed(either::Left(proposals[2].data.justify_qc.clone())), SendPayloadCommitmentAndMetadata( build_payload_commitment::( &quorum_membership, diff --git a/crates/testing/tests/tests_1/upgrade_task_with_vote.rs b/crates/testing/tests/tests_1/upgrade_task_with_vote.rs index 72790bec4d..cc942451f1 100644 --- a/crates/testing/tests/tests_1/upgrade_task_with_vote.rs +++ b/crates/testing/tests/tests_1/upgrade_task_with_vote.rs @@ -29,7 +29,7 @@ use hotshot_testing::{ view_generator::TestViewGenerator, }; use hotshot_types::{ - data::{null_block, EpochNumber, Leaf, ViewNumber}, + data::{null_block, EpochNumber, Leaf2, ViewNumber}, simple_vote::UpgradeProposalData, traits::{election::Membership, node_implementation::ConsensusTime}, vote::HasViewNumber, @@ -82,12 +82,10 @@ async fn test_upgrade_task_with_vote() { leaves.push(view.leaf.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/testing/tests/tests_1/view_sync_task.rs b/crates/testing/tests/tests_1/view_sync_task.rs index 85827dbbdb..ec313ec2a7 100644 --- a/crates/testing/tests/tests_1/view_sync_task.rs +++ b/crates/testing/tests/tests_1/view_sync_task.rs @@ -10,9 +10,9 @@ use hotshot_task_impls::{ events::HotShotEvent, harness::run_harness, view_sync::ViewSyncTaskState, }; use hotshot_testing::helpers::build_system_handle; -use hotshot_types::data::EpochNumber; use hotshot_types::{ - data::ViewNumber, simple_vote::ViewSyncPreCommitData, + data::{EpochNumber, ViewNumber}, + simple_vote::ViewSyncPreCommitData, traits::node_implementation::ConsensusTime, }; diff --git a/crates/testing/tests/tests_1/vote_dependency_handle.rs b/crates/testing/tests/tests_1/vote_dependency_handle.rs index 9a58e4046c..f22d658d34 100644 --- a/crates/testing/tests/tests_1/vote_dependency_handle.rs +++ b/crates/testing/tests/tests_1/vote_dependency_handle.rs @@ -15,7 +15,7 @@ use hotshot_testing::{ }; use hotshot_types::{ consensus::OuterConsensus, - data::{EpochNumber, Leaf, ViewNumber}, + data::{EpochNumber, Leaf2, ViewNumber}, traits::{consensus_api::ConsensusApi, node_implementation::ConsensusTime}, }; use itertools::Itertools; @@ -55,12 +55,10 @@ async fn test_vote_dependency_handle() { vids.push(view.vid_proposal.clone()); consensus_writer .update_leaf( - Leaf::from_quorum_proposal(&view.quorum_proposal.data), + Leaf2::from_quorum_proposal(&view.quorum_proposal.data), Arc::new(TestValidatedState::default()), None, - &handle.hotshot.upgrade_lock, ) - .await .unwrap(); } drop(consensus_writer); diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 4be463a1cb..c15d4865a3 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -25,7 +25,6 @@ either = { workspace = true } ethereum-types = { workspace = true } futures = { workspace = true } cdn-proto = { workspace = true } -serde-inline-default = "0.2" lazy_static = { workspace = true } memoize = { workspace = true } rand = { workspace = true } @@ -41,6 +40,8 @@ jf-signature = { workspace = true, features = ["bls", "schnorr"] } jf-utils = { workspace = true } rand_chacha = { workspace = true } serde = { workspace = true } +serde-inline-default = { workspace = true } +serde_bytes = { workspace = true } tagged-base64 = { workspace = true } vbs = { workspace = true } displaydoc = { version = "0.2.5", default-features = false } diff --git a/crates/types/src/consensus.rs b/crates/types/src/consensus.rs index 499430c0ca..a27220a7ed 100644 --- a/crates/types/src/consensus.rs +++ b/crates/types/src/consensus.rs @@ -21,15 +21,15 @@ use vec1::Vec1; pub use crate::utils::{View, ViewInner}; use crate::{ - data::{Leaf, QuorumProposal, VidDisperse, VidDisperseShare}, + data::{Leaf2, QuorumProposal2, VidDisperse, VidDisperseShare}, error::HotShotError, event::{HotShotAction, LeafInfo}, - message::{Proposal, UpgradeLock}, - simple_certificate::{DaCertificate, QuorumCertificate}, + message::Proposal, + simple_certificate::{DaCertificate, QuorumCertificate2}, traits::{ block_contents::BuilderFee, metrics::{Counter, Gauge, Histogram, Metrics, NoMetrics}, - node_implementation::{ConsensusTime, NodeType, Versions}, + node_implementation::{ConsensusTime, NodeType}, signature_key::SignatureKey, BlockPayload, ValidatedState, }, @@ -291,7 +291,7 @@ pub struct Consensus { /// Last proposals we sent out, None if we haven't proposed yet. /// Prevents duplicate proposals, and can be served to those trying to catchup - last_proposals: BTreeMap>>, + last_proposals: BTreeMap>>, /// last view had a successful decide event last_decided_view: TYPES::View, @@ -302,7 +302,7 @@ pub struct Consensus { /// Map of leaf hash -> leaf /// - contains undecided leaves /// - includes the MOST RECENT decided leaf - saved_leaves: CommitmentMap>, + saved_leaves: CommitmentMap>, /// Bundle of views which we performed the most recent action /// visibible to the network. Actions are votes and proposals @@ -315,7 +315,7 @@ pub struct Consensus { saved_payloads: BTreeMap>, /// the highqc per spec - high_qc: QuorumCertificate, + high_qc: QuorumCertificate2, /// A reference to the metrics trait pub metrics: Arc, @@ -405,10 +405,10 @@ impl Consensus { locked_view: TYPES::View, last_decided_view: TYPES::View, last_actioned_view: TYPES::View, - last_proposals: BTreeMap>>, - saved_leaves: CommitmentMap>, + last_proposals: BTreeMap>>, + saved_leaves: CommitmentMap>, saved_payloads: BTreeMap>, - high_qc: QuorumCertificate, + high_qc: QuorumCertificate2, metrics: Arc, epoch_height: u64, ) -> Self { @@ -451,7 +451,7 @@ impl Consensus { } /// Get the high QC. - pub fn high_qc(&self) -> &QuorumCertificate { + pub fn high_qc(&self) -> &QuorumCertificate2 { &self.high_qc } @@ -461,7 +461,7 @@ impl Consensus { } /// Get the saved leaves. - pub fn saved_leaves(&self) -> &CommitmentMap> { + pub fn saved_leaves(&self) -> &CommitmentMap> { &self.saved_leaves } @@ -481,7 +481,9 @@ impl Consensus { } /// Get the map of our recent proposals - pub fn last_proposals(&self) -> &BTreeMap>> { + pub fn last_proposals( + &self, + ) -> &BTreeMap>> { &self.last_proposals } @@ -501,7 +503,7 @@ impl Consensus { /// Returns None if we don't have the data in out state pub fn parent_leaf_info( &self, - leaf: &Leaf, + leaf: &Leaf2, public_key: &TYPES::SignatureKey, ) -> Option> { let parent_view_number = leaf.justify_qc().view_number(); @@ -578,7 +580,7 @@ impl Consensus { /// Can return an error when the new view_number is not higher than the existing proposed view number. pub fn update_proposed_view( &mut self, - proposal: Proposal>, + proposal: Proposal>, ) -> Result<()> { ensure!( proposal.data.view_number() @@ -640,23 +642,22 @@ impl Consensus { /// # Errors /// Can return an error when the new view contains less information than the exisiting view /// with the same view number. - pub async fn update_leaf( + pub fn update_leaf( &mut self, - leaf: Leaf, + leaf: Leaf2, state: Arc, delta: Option>::Delta>>, - upgrade_lock: &UpgradeLock, ) -> Result<()> { let view_number = leaf.view_number(); let view = View { view_inner: ViewInner::Leaf { - leaf: leaf.commit(upgrade_lock).await, + leaf: leaf.commit(), state, delta, }, }; self.update_validated_state_map(view_number, view)?; - self.update_saved_leaves(leaf, upgrade_lock).await; + self.update_saved_leaves(leaf); Ok(()) } @@ -695,13 +696,8 @@ impl Consensus { } /// Update the saved leaves with a new leaf. - async fn update_saved_leaves( - &mut self, - leaf: Leaf, - upgrade_lock: &UpgradeLock, - ) { - self.saved_leaves - .insert(leaf.commit(upgrade_lock).await, leaf); + fn update_saved_leaves(&mut self, leaf: Leaf2) { + self.saved_leaves.insert(leaf.commit(), leaf); } /// Update the saved payloads with a new encoded transaction. @@ -724,7 +720,7 @@ impl Consensus { /// Update the high QC if given a newer one. /// # Errors /// Can return an error when the provided high_qc is not newer than the existing entry. - pub fn update_high_qc(&mut self, high_qc: QuorumCertificate) -> Result<()> { + pub fn update_high_qc(&mut self, high_qc: QuorumCertificate2) -> Result<()> { ensure!( high_qc.view_number > self.high_qc.view_number || high_qc == self.high_qc, debug!("High QC with an equal or higher view exists.") @@ -764,7 +760,7 @@ impl Consensus { ) -> std::result::Result<(), HotShotError> where F: FnMut( - &Leaf, + &Leaf2, Arc<::ValidatedState>, Option::ValidatedState as ValidatedState>::Delta>>, ) -> bool, @@ -851,7 +847,7 @@ impl Consensus { /// if the last decided view's leaf does not exist in the state map or saved leaves, which /// should never happen. #[must_use] - pub fn decided_leaf(&self) -> Leaf { + pub fn decided_leaf(&self) -> Leaf2 { let decided_view_num = self.last_decided_view; let view = self.validated_state_map.get(&decided_view_num).unwrap(); let leaf = view @@ -918,7 +914,7 @@ impl Consensus { /// Return true if the QC takes part in forming an eQC, i.e. /// it is one of the 3-chain certificates but not the eQC itself - pub fn is_qc_forming_eqc(&self, qc: &QuorumCertificate) -> bool { + pub fn is_qc_forming_eqc(&self, qc: &QuorumCertificate2) -> bool { let high_qc_leaf_commit = qc.data.leaf_commit; let is_high_qc_extended = self.is_leaf_extended(high_qc_leaf_commit); if is_high_qc_extended { @@ -1007,14 +1003,14 @@ impl Consensus { } /// Returns true if the `parent_leaf` formed an eQC for the previous epoch to the `proposed_leaf` - pub fn check_eqc(&self, proposed_leaf: &Leaf, parent_leaf: &Leaf) -> bool { + pub fn check_eqc(&self, proposed_leaf: &Leaf2, parent_leaf: &Leaf2) -> bool { if parent_leaf.view_number() == TYPES::View::genesis() { return true; } let new_epoch = epoch_from_block_number(proposed_leaf.height(), self.epoch_height); let old_epoch = epoch_from_block_number(parent_leaf.height(), self.epoch_height); - let parent_leaf_commit = as Committable>::commit(parent_leaf); - new_epoch - 1 == old_epoch && self.is_leaf_extended(parent_leaf_commit) + + new_epoch - 1 == old_epoch && self.is_leaf_extended(parent_leaf.commit()) } } diff --git a/crates/types/src/data.rs b/crates/types/src/data.rs index fc5506ff25..b97c84a62c 100644 --- a/crates/types/src/data.rs +++ b/crates/types/src/data.rs @@ -33,7 +33,8 @@ use vec1::Vec1; use crate::{ message::{Proposal, UpgradeLock}, simple_certificate::{ - QuorumCertificate, TimeoutCertificate, UpgradeCertificate, ViewSyncFinalizeCertificate2, + QuorumCertificate, QuorumCertificate2, TimeoutCertificate, UpgradeCertificate, + ViewSyncFinalizeCertificate2, }, simple_vote::{QuorumData, UpgradeProposalData, VersionedVoteData}, traits::{ @@ -372,6 +373,78 @@ pub struct QuorumProposal { pub proposal_certificate: Option>, } +/// Proposal to append a block. +#[derive(custom_debug::Debug, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] +#[serde(bound(deserialize = ""))] +pub struct QuorumProposal2 { + /// The block header to append + pub block_header: TYPES::BlockHeader, + + /// view number for the proposal + pub view_number: TYPES::View, + + /// certificate that the proposal is chaining from + pub justify_qc: QuorumCertificate2, + + /// Possible upgrade certificate, which the leader may optionally attach. + pub upgrade_certificate: Option>, + + /// Possible timeout or view sync certificate. If the `justify_qc` is not for a proposal in the immediately preceding view, then either a timeout or view sync certificate must be attached. + pub view_change_evidence: Option>, + + /// the DRB seed currently being calculated + #[serde(with = "serde_bytes")] + pub drb_seed: [u8; 96], + + /// the result of the DRB calculation + #[serde(with = "serde_bytes")] + pub drb_result: [u8; 32], +} + +impl From> for QuorumProposal2 { + fn from(quorum_proposal: QuorumProposal) -> Self { + Self { + block_header: quorum_proposal.block_header, + view_number: quorum_proposal.view_number, + justify_qc: quorum_proposal.justify_qc.to_qc2(), + upgrade_certificate: quorum_proposal.upgrade_certificate, + view_change_evidence: quorum_proposal.proposal_certificate, + drb_seed: [0; 96], + drb_result: [0; 32], + } + } +} + +impl From> for QuorumProposal { + fn from(quorum_proposal: QuorumProposal2) -> Self { + Self { + block_header: quorum_proposal.block_header, + view_number: quorum_proposal.view_number, + justify_qc: quorum_proposal.justify_qc.to_qc(), + upgrade_certificate: quorum_proposal.upgrade_certificate, + proposal_certificate: quorum_proposal.view_change_evidence, + } + } +} + +impl From> for Leaf2 { + fn from(leaf: Leaf) -> Self { + let bytes: [u8; 32] = leaf.parent_commitment.into(); + + Self { + view_number: leaf.view_number, + justify_qc: leaf.justify_qc.to_qc2(), + parent_commitment: Commitment::from_raw(bytes), + block_header: leaf.block_header, + upgrade_certificate: leaf.upgrade_certificate, + block_payload: leaf.block_payload, + view_change_evidence: None, + drb_seed: [0; 96], + drb_result: [0; 32], + } + } +} + impl HasViewNumber for DaProposal { fn view_number(&self) -> TYPES::View { self.view_number @@ -396,6 +469,12 @@ impl HasViewNumber for QuorumProposal { } } +impl HasViewNumber for QuorumProposal2 { + fn view_number(&self) -> TYPES::View { + self.view_number + } +} + impl HasViewNumber for UpgradeProposal { fn view_number(&self) -> TYPES::View { self.view_number @@ -455,6 +534,183 @@ pub struct Leaf { block_payload: Option, } +/// This is the consensus-internal analogous concept to a block, and it contains the block proper, +/// as well as the hash of its parent `Leaf`. +#[derive(Serialize, Deserialize, Clone, Debug, Derivative, Eq)] +#[serde(bound(deserialize = ""))] +pub struct Leaf2 { + /// CurView from leader when proposing leaf + view_number: TYPES::View, + + /// Per spec, justification + justify_qc: QuorumCertificate2, + + /// The hash of the parent `Leaf` + /// So we can ask if it extends + parent_commitment: Commitment, + + /// Block header. + block_header: TYPES::BlockHeader, + + /// Optional upgrade certificate, if one was attached to the quorum proposal for this view. + upgrade_certificate: Option>, + + /// Optional block payload. + /// + /// It may be empty for nodes not in the DA committee. + block_payload: Option, + + /// Possible timeout or view sync certificate. If the `justify_qc` is not for a proposal in the immediately preceding view, then either a timeout or view sync certificate must be attached. + pub view_change_evidence: Option>, + + /// the DRB seed currently being calculated + #[serde(with = "serde_bytes")] + pub drb_seed: [u8; 96], + + /// the result of the DRB calculation + #[serde(with = "serde_bytes")] + pub drb_result: [u8; 32], +} + +impl Leaf2 { + /// Time when this leaf was created. + pub fn view_number(&self) -> TYPES::View { + self.view_number + } + /// Height of this leaf in the chain. + /// + /// Equivalently, this is the number of leaves before this one in the chain. + pub fn height(&self) -> u64 { + self.block_header.block_number() + } + /// The QC linking this leaf to its parent in the chain. + pub fn justify_qc(&self) -> QuorumCertificate2 { + self.justify_qc.clone() + } + /// The QC linking this leaf to its parent in the chain. + pub fn upgrade_certificate(&self) -> Option> { + self.upgrade_certificate.clone() + } + /// Commitment to this leaf's parent. + pub fn parent_commitment(&self) -> Commitment { + self.parent_commitment + } + /// The block header contained in this leaf. + pub fn block_header(&self) -> &::BlockHeader { + &self.block_header + } + + /// Get a mutable reference to the block header contained in this leaf. + pub fn block_header_mut(&mut self) -> &mut ::BlockHeader { + &mut self.block_header + } + /// Fill this leaf with the block payload. + /// + /// # Errors + /// + /// Fails if the payload commitment doesn't match `self.block_header.payload_commitment()` + /// or if the transactions are of invalid length + pub fn fill_block_payload( + &mut self, + block_payload: TYPES::BlockPayload, + num_storage_nodes: usize, + ) -> std::result::Result<(), BlockError> { + let encoded_txns = block_payload.encode(); + let commitment = vid_commitment(&encoded_txns, num_storage_nodes); + if commitment != self.block_header.payload_commitment() { + return Err(BlockError::InconsistentPayloadCommitment); + } + self.block_payload = Some(block_payload); + Ok(()) + } + + /// Take the block payload from the leaf and return it if it is present + pub fn unfill_block_payload(&mut self) -> Option { + self.block_payload.take() + } + + /// Fill this leaf with the block payload, without checking + /// header and payload consistency + pub fn fill_block_payload_unchecked(&mut self, block_payload: TYPES::BlockPayload) { + self.block_payload = Some(block_payload); + } + + /// Optional block payload. + pub fn block_payload(&self) -> Option { + self.block_payload.clone() + } + + /// A commitment to the block payload contained in this leaf. + pub fn payload_commitment(&self) -> VidCommitment { + self.block_header().payload_commitment() + } + + /// Validate that a leaf has the right upgrade certificate to be the immediate child of another leaf + /// + /// This may not be a complete function. Please double-check that it performs the checks you expect before subtituting validation logic with it. + /// + /// # Errors + /// Returns an error if the certificates are not identical, or that when we no longer see a + /// cert, it's for the right reason. + pub async fn extends_upgrade( + &self, + parent: &Self, + decided_upgrade_certificate: &Arc>>>, + ) -> Result<()> { + match (self.upgrade_certificate(), parent.upgrade_certificate()) { + // Easiest cases are: + // - no upgrade certificate on either: this is the most common case, and is always fine. + // - if the parent didn't have a certificate, but we see one now, it just means that we have begun an upgrade: again, this is always fine. + (None | Some(_), None) => {} + // If we no longer see a cert, we have to make sure that we either: + // - no longer care because we have passed new_version_first_view, or + // - no longer care because we have passed `decide_by` without deciding the certificate. + (None, Some(parent_cert)) => { + let decided_upgrade_certificate_read = decided_upgrade_certificate.read().await; + ensure!(self.view_number() > parent_cert.data.new_version_first_view + || (self.view_number() > parent_cert.data.decide_by && decided_upgrade_certificate_read.is_none()), + "The new leaf is missing an upgrade certificate that was present in its parent, and should still be live." + ); + } + // If we both have a certificate, they should be identical. + // Technically, this prevents us from initiating a new upgrade in the view immediately following an upgrade. + // I think this is a fairly lax restriction. + (Some(cert), Some(parent_cert)) => { + ensure!(cert == parent_cert, "The new leaf does not extend the parent leaf, because it has attached a different upgrade certificate."); + } + } + + // This check should be added once we sort out the genesis leaf/justify_qc issue. + // ensure!(self.parent_commitment() == parent_leaf.commit(), "The commitment of the parent leaf does not match the specified parent commitment."); + + Ok(()) + } +} + +impl Committable for Leaf2 { + fn commit(&self) -> committable::Commitment { + if self.drb_seed == [0; 96] && self.drb_result == [0; 32] { + RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", self.justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .finalize() + } else { + RawCommitmentBuilder::new("leaf commitment") + .u64_field("view number", *self.view_number) + .field("parent leaf commitment", self.parent_commitment) + .field("block header", self.block_header.commit()) + .field("justify qc", self.justify_qc.commit()) + .optional("upgrade certificate", &self.upgrade_certificate) + .fixed_size_bytes(&self.drb_seed) + .fixed_size_bytes(&self.drb_result) + .finalize() + } + } +} + impl Leaf { #[allow(clippy::unused_async)] /// Calculate the leaf commitment, @@ -476,6 +732,31 @@ impl PartialEq for Leaf { } } +impl PartialEq for Leaf2 { + fn eq(&self, other: &Self) -> bool { + let Leaf2 { + view_number, + justify_qc, + parent_commitment, + block_header, + upgrade_certificate, + block_payload: _, + view_change_evidence, + drb_seed, + drb_result, + } = self; + + *view_number == other.view_number + && *justify_qc == other.justify_qc + && *parent_commitment == other.parent_commitment + && *block_header == other.block_header + && *upgrade_certificate == other.upgrade_certificate + && *view_change_evidence == other.view_change_evidence + && *drb_seed == other.drb_seed + && *drb_result == other.drb_result + } +} + impl Hash for Leaf { fn hash(&self, state: &mut H) { self.view_number.hash(state); @@ -485,6 +766,16 @@ impl Hash for Leaf { } } +impl Hash for Leaf2 { + fn hash(&self, state: &mut H) { + self.commit().hash(state); + self.view_number.hash(state); + self.justify_qc.hash(state); + self.parent_commitment.hash(state); + self.block_header.hash(state); + } +} + impl Display for Leaf { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( @@ -712,6 +1003,22 @@ where TYPES::ValidatedState::create_random_transaction(None, rng, padding) } } +impl TestableLeaf for Leaf2 +where + TYPES::ValidatedState: TestableState, + TYPES::BlockPayload: TestableBlock, +{ + type NodeType = TYPES; + + fn create_random_transaction( + &self, + rng: &mut dyn rand::RngCore, + padding: u64, + ) -> <::BlockPayload as BlockPayload>::Transaction + { + TYPES::ValidatedState::create_random_transaction(None, rng, padding) + } +} /// Fake the thing a genesis block points to. Needed to avoid infinite recursion #[must_use] pub fn fake_commitment() -> Commitment { @@ -763,6 +1070,35 @@ impl Committable for Leaf { } } +impl Leaf2 { + /// Constructs a leaf from a given quorum proposal. + pub fn from_quorum_proposal(quorum_proposal: &QuorumProposal2) -> Self { + // WARNING: Do NOT change this to a wildcard match, or reference the fields directly in the construction of the leaf. + // The point of this match is that we will get a compile-time error if we add a field without updating this. + let QuorumProposal2 { + view_number, + justify_qc, + block_header, + upgrade_certificate, + view_change_evidence, + drb_seed, + drb_result, + } = quorum_proposal; + + Self { + view_number: *view_number, + justify_qc: justify_qc.clone(), + parent_commitment: justify_qc.data().leaf_commit, + block_header: block_header.clone(), + upgrade_certificate: upgrade_certificate.clone(), + block_payload: None, + view_change_evidence: view_change_evidence.clone(), + drb_seed: *drb_seed, + drb_result: *drb_result, + } + } +} + impl Leaf { /// Constructs a leaf from a given quorum proposal. pub fn from_quorum_proposal(quorum_proposal: &QuorumProposal) -> Self { @@ -775,7 +1111,8 @@ impl Leaf { upgrade_certificate, proposal_certificate: _, } = quorum_proposal; - Leaf { + + Self { view_number: *view_number, justify_qc: justify_qc.clone(), parent_commitment: justify_qc.data().leaf_commit, diff --git a/crates/types/src/error.rs b/crates/types/src/error.rs index f6c25e376f..5357bb8bbd 100644 --- a/crates/types/src/error.rs +++ b/crates/types/src/error.rs @@ -13,7 +13,7 @@ use committable::Commitment; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::{data::Leaf, traits::node_implementation::NodeType}; +use crate::{data::Leaf2, traits::node_implementation::NodeType}; /// Error type for `HotShot` #[derive(Debug, Error)] @@ -25,7 +25,7 @@ pub enum HotShotError { /// Leaf was not present in storage #[error("Missing leaf with commitment: {0}")] - MissingLeaf(Commitment>), + MissingLeaf(Commitment>), /// Failed to serialize data #[error("Failed to serialize: {0}")] @@ -49,8 +49,8 @@ pub enum HotShotError { #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] pub enum RoundTimedoutState { - /// Leader is in a Prepare phase and is waiting for a HighQC - LeaderWaitingForHighQC, + /// Leader is in a Prepare phase and is waiting for a HighQc + LeaderWaitingForHighQc, /// Leader is in a Prepare phase and timed out before the round min time is reached LeaderMinRoundTimeNotReached, /// Leader is waiting for prepare votes diff --git a/crates/types/src/event.rs b/crates/types/src/event.rs index f9d48779e8..02d7f07ac2 100644 --- a/crates/types/src/event.rs +++ b/crates/types/src/event.rs @@ -11,10 +11,10 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use crate::{ - data::{DaProposal, Leaf, QuorumProposal, UpgradeProposal, VidDisperseShare}, + data::{DaProposal, Leaf2, QuorumProposal2, UpgradeProposal, VidDisperseShare}, error::HotShotError, message::Proposal, - simple_certificate::QuorumCertificate, + simple_certificate::QuorumCertificate2, traits::{node_implementation::NodeType, ValidatedState}, }; /// A status event emitted by a `HotShot` instance @@ -35,7 +35,7 @@ pub struct Event { #[serde(bound(deserialize = "TYPES: NodeType"))] pub struct LeafInfo { /// Decided leaf. - pub leaf: Leaf, + pub leaf: Leaf2, /// Validated state. pub state: Arc<::ValidatedState>, /// Optional application-specific state delta. @@ -47,7 +47,7 @@ pub struct LeafInfo { impl LeafInfo { /// Constructor. pub fn new( - leaf: Leaf, + leaf: Leaf2, state: Arc<::ValidatedState>, delta: Option::ValidatedState as ValidatedState>::Delta>>, vid_share: Option>, @@ -100,6 +100,7 @@ pub mod error_adaptor { #[non_exhaustive] #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(bound(deserialize = "TYPES: NodeType"))] +#[allow(clippy::large_enum_variant)] pub enum EventType { /// A view encountered an error and was interrupted Error { @@ -121,7 +122,7 @@ pub enum EventType { /// /// Note that the QC for each additional leaf in the chain can be obtained from the leaf /// before it using - qc: Arc>, + qc: Arc>, /// Optional information of the number of transactions in the block, for logging purposes. block_size: Option, }, @@ -158,7 +159,7 @@ pub enum EventType { /// or submitted to the network by us QuorumProposal { /// Contents of the proposal - proposal: Proposal>, + proposal: Proposal>, /// Public key of the leader submitting the proposal sender: TYPES::SignatureKey, }, diff --git a/crates/types/src/message.rs b/crates/types/src/message.rs index bf31005c9a..f57b41ec0d 100644 --- a/crates/types/src/message.rs +++ b/crates/types/src/message.rs @@ -17,6 +17,7 @@ use std::{ use async_lock::RwLock; use cdn_proto::util::mnemonic; +use committable::Committable; use derivative::Derivative; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use utils::anytrace::*; @@ -26,7 +27,9 @@ use vbs::{ }; use crate::{ - data::{DaProposal, Leaf, QuorumProposal, UpgradeProposal, VidDisperseShare}, + data::{ + DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, UpgradeProposal, VidDisperseShare, + }, request_response::ProposalRequestPayload, simple_certificate::{ DaCertificate, QuorumCertificate, UpgradeCertificate, ViewSyncCommitCertificate2, @@ -208,7 +211,7 @@ pub enum GeneralConsensusMessage { ProposalResponse(Proposal>), /// Message for the next leader containing our highest QC - HighQC(QuorumCertificate), + HighQc(QuorumCertificate), } #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Hash, Eq)] @@ -274,7 +277,7 @@ impl SequencingMessage { } GeneralConsensusMessage::UpgradeProposal(message) => message.data.view_number(), GeneralConsensusMessage::UpgradeVote(message) => message.view_number(), - GeneralConsensusMessage::HighQC(qc) => qc.view_number(), + GeneralConsensusMessage::HighQc(qc) => qc.view_number(), } } SequencingMessage::Da(da_message) => { @@ -323,6 +326,22 @@ pub struct Proposal + Deserializ pub _pd: PhantomData, } +/// Convert a `Proposal` by converting the underlying proposal type +pub fn convert_proposal( + proposal: Proposal, +) -> Proposal +where + TYPES: NodeType, + PROPOSAL: HasViewNumber + DeserializeOwned, + PROPOSAL2: HasViewNumber + DeserializeOwned + From, +{ + Proposal { + data: proposal.data.into(), + signature: proposal.signature, + _pd: proposal._pd, + } +} + impl Proposal> where TYPES: NodeType, @@ -352,6 +371,31 @@ where } } +impl Proposal> +where + TYPES: NodeType, +{ + /// Checks that the signature of the quorum proposal is valid. + /// # Errors + /// Returns an error when the proposal signature is invalid. + pub fn validate_signature( + &self, + quorum_membership: &TYPES::Membership, + epoch: TYPES::Epoch, + ) -> Result<()> { + let view_number = self.data.view_number(); + let view_leader_key = quorum_membership.leader(view_number, epoch)?; + let proposed_leaf = Leaf2::from_quorum_proposal(&self.data); + + ensure!( + view_leader_key.validate(&self.signature, proposed_leaf.commit().as_ref()), + "Proposal signature is invalid." + ); + + Ok(()) + } +} + #[derive(Clone, Debug)] /// A lock for an upgrade certificate decided by HotShot, which doubles as `PhantomData` for an instance of the `Versions` trait. pub struct UpgradeLock { diff --git a/crates/types/src/simple_certificate.rs b/crates/types/src/simple_certificate.rs index f7f58fd4e1..9c75b294a3 100644 --- a/crates/types/src/simple_certificate.rs +++ b/crates/types/src/simple_certificate.rs @@ -23,7 +23,7 @@ use crate::{ data::serialize_signature2, message::UpgradeLock, simple_vote::{ - DaData, QuorumData, TimeoutData, UpgradeProposalData, VersionedVoteData, + DaData, QuorumData, QuorumData2, TimeoutData, UpgradeProposalData, VersionedVoteData, ViewSyncCommitData, ViewSyncFinalizeData, ViewSyncPreCommitData, Voteable, }, traits::{ @@ -249,8 +249,52 @@ impl UpgradeCertificate { } } -/// Type alias for a `QuorumCertificate`, which is a `SimpleCertificate` of `QuorumVotes` +impl QuorumCertificate { + /// Convert a `QuorumCertificate` into a `QuorumCertificate2` + pub fn to_qc2(self) -> QuorumCertificate2 { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + let data = QuorumData2 { + leaf_commit: Commitment::from_raw(bytes), + }; + + let bytes: [u8; 32] = self.vote_commitment.into(); + let vote_commitment = Commitment::from_raw(bytes); + + SimpleCertificate { + data, + vote_commitment, + view_number: self.view_number, + signatures: self.signatures.clone(), + _pd: PhantomData, + } + } +} + +impl QuorumCertificate2 { + /// Convert a `QuorumCertificate2` into a `QuorumCertificate` + pub fn to_qc(self) -> QuorumCertificate { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + let data = QuorumData { + leaf_commit: Commitment::from_raw(bytes), + }; + + let bytes: [u8; 32] = self.vote_commitment.into(); + let vote_commitment = Commitment::from_raw(bytes); + + SimpleCertificate { + data, + vote_commitment, + view_number: self.view_number, + signatures: self.signatures.clone(), + _pd: PhantomData, + } + } +} + +/// Type alias for a `QuorumCertificate`, which is a `SimpleCertificate` over `QuorumData` pub type QuorumCertificate = SimpleCertificate, SuccessThreshold>; +/// Type alias for a `QuorumCertificate2`, which is a `SimpleCertificate` over `QuorumData2` +pub type QuorumCertificate2 = SimpleCertificate, SuccessThreshold>; /// Type alias for a DA certificate over `DaData` pub type DaCertificate = SimpleCertificate; /// Type alias for a Timeout certificate over a view number diff --git a/crates/types/src/simple_vote.rs b/crates/types/src/simple_vote.rs index f5111084b8..013653a959 100644 --- a/crates/types/src/simple_vote.rs +++ b/crates/types/src/simple_vote.rs @@ -14,7 +14,7 @@ use utils::anytrace::*; use vbs::version::Version; use crate::{ - data::Leaf, + data::{Leaf, Leaf2}, message::UpgradeLock, traits::{ node_implementation::{NodeType, Versions}, @@ -32,6 +32,13 @@ pub struct QuorumData { pub leaf_commit: Commitment>, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Hash, Eq)] +/// Data used for a yes vote. +#[serde(bound(deserialize = ""))] +pub struct QuorumData2 { + /// Commitment to the leaf + pub leaf_commit: Commitment>, +} +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Hash, Eq)] /// Data used for a DA vote. pub struct DaData { /// Commitment to a block payload @@ -257,6 +264,14 @@ impl Committable for QuorumData { } } +impl Committable for QuorumData2 { + fn commit(&self) -> Commitment { + committable::RawCommitmentBuilder::new("Quorum data") + .var_size_bytes(self.leaf_commit.as_ref()) + .finalize() + } +} + impl Committable for TimeoutData { fn commit(&self) -> Commitment { committable::RawCommitmentBuilder::new("Timeout data") @@ -331,9 +346,50 @@ impl QuorumVote { + /// Convert a `QuorumVote` to a `QuorumVote2` + pub fn to_vote2(self) -> QuorumVote2 { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + + let signature = self.signature; + let data = QuorumData2 { + leaf_commit: Commitment::from_raw(bytes), + }; + let view_number = self.view_number; + + SimpleVote { + signature, + data, + view_number, + } + } +} + +impl QuorumVote2 { + /// Convert a `QuorumVote2` to a `QuorumVote` + pub fn to_vote(self) -> QuorumVote { + let bytes: [u8; 32] = self.data.leaf_commit.into(); + + let signature = self.signature; + let data = QuorumData { + leaf_commit: Commitment::from_raw(bytes), + }; + let view_number = self.view_number; + + SimpleVote { + signature, + data, + view_number, + } + } +} + // Type aliases for simple use of all the main votes. We should never see `SimpleVote` outside this file /// Quorum vote Alias pub type QuorumVote = SimpleVote>; +// Type aliases for simple use of all the main votes. We should never see `SimpleVote` outside this file +/// Quorum vote Alias +pub type QuorumVote2 = SimpleVote>; /// DA vote type alias pub type DaVote = SimpleVote; /// Timeout Vote type alias diff --git a/crates/types/src/traits/block_contents.rs b/crates/types/src/traits/block_contents.rs index 5901b7242d..febf32798f 100644 --- a/crates/types/src/traits/block_contents.rs +++ b/crates/types/src/traits/block_contents.rs @@ -25,7 +25,7 @@ use vbs::version::Version; use super::signature_key::BuilderSignatureKey; use crate::{ - data::Leaf, + data::Leaf2, traits::{node_implementation::NodeType, states::InstanceState, ValidatedState}, utils::BuilderCommitment, vid::{vid_scheme, VidCommitment, VidCommon, VidSchemeType}, @@ -199,7 +199,7 @@ pub trait BlockHeader: fn new_legacy( parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, @@ -215,7 +215,7 @@ pub trait BlockHeader: fn new_marketplace( parent_state: &TYPES::ValidatedState, instance_state: &>::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, metadata: >::Metadata, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index d05652b783..32093b8714 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -35,7 +35,7 @@ use super::{ ValidatedState, }; use crate::{ - data::{Leaf, TestableLeaf}, + data::{Leaf2, TestableLeaf}, traits::{ election::Membership, signature_key::SignatureKey, states::InstanceState, BlockPayload, }, @@ -87,7 +87,7 @@ pub trait TestableNodeImplementation: NodeImplementation /// otherwise panics /// `padding` is the bytes of padding to add to the transaction fn leaf_create_random_transaction( - leaf: &Leaf, + leaf: &Leaf2, rng: &mut dyn rand::RngCore, padding: u64, ) -> >::Transaction; @@ -126,11 +126,11 @@ where } fn leaf_create_random_transaction( - leaf: &Leaf, + leaf: &Leaf2, rng: &mut dyn rand::RngCore, padding: u64, ) -> >::Transaction { - Leaf::create_random_transaction(leaf, rng, padding) + Leaf2::create_random_transaction(leaf, rng, padding) } fn block_genesis() -> TYPES::BlockPayload { diff --git a/crates/types/src/traits/states.rs b/crates/types/src/traits/states.rs index 956fd22d02..00a5a6aab1 100644 --- a/crates/types/src/traits/states.rs +++ b/crates/types/src/traits/states.rs @@ -17,7 +17,7 @@ use vbs::version::Version; use super::block_contents::TestableBlock; use crate::{ - data::Leaf, + data::Leaf2, traits::{ node_implementation::{ConsensusTime, NodeType}, BlockPayload, @@ -69,7 +69,7 @@ pub trait ValidatedState: fn validate_and_apply_header( &self, instance: &Self::Instance, - parent_leaf: &Leaf, + parent_leaf: &Leaf2, proposed_header: &TYPES::BlockHeader, vid_common: VidCommon, version: Version, diff --git a/crates/types/src/traits/storage.rs b/crates/types/src/traits/storage.rs index 81ba2f66f8..4ce9fdeac4 100644 --- a/crates/types/src/traits/storage.rs +++ b/crates/types/src/traits/storage.rs @@ -18,10 +18,10 @@ use jf_vid::VidScheme; use super::node_implementation::NodeType; use crate::{ consensus::{CommitmentMap, View}, - data::{DaProposal, Leaf, QuorumProposal, VidDisperseShare}, + data::{DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare}, event::HotShotAction, message::Proposal, - simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate}, vid::VidSchemeType, }; @@ -41,10 +41,17 @@ pub trait Storage: Send + Sync + Clone { &self, proposal: &Proposal>, ) -> Result<()>; + /// Add a proposal we sent to the store + async fn append_proposal2( + &self, + proposal: &Proposal>, + ) -> Result<()>; /// Record a HotShotAction taken. async fn record_action(&self, view: TYPES::View, action: HotShotAction) -> Result<()>; /// Update the current high QC in storage. async fn update_high_qc(&self, high_qc: QuorumCertificate) -> Result<()>; + /// Update the current high QC in storage. + async fn update_high_qc2(&self, high_qc: QuorumCertificate2) -> Result<()>; /// Update the currently undecided state of consensus. This includes the undecided leaf chain, /// and the undecided state. async fn update_undecided_state( @@ -52,9 +59,24 @@ pub trait Storage: Send + Sync + Clone { leafs: CommitmentMap>, state: BTreeMap>, ) -> Result<()>; + /// Update the currently undecided state of consensus. This includes the undecided leaf chain, + /// and the undecided state. + async fn update_undecided_state2( + &self, + leafs: CommitmentMap>, + state: BTreeMap>, + ) -> Result<()>; /// Upgrade the current decided upgrade certificate in storage. async fn update_decided_upgrade_certificate( &self, decided_upgrade_certificate: Option>, ) -> Result<()>; + /// Migrate leaves from `Leaf` to `Leaf2`, and proposals from `QuorumProposal` to `QuorumProposal2` + async fn migrate_consensus( + &self, + convert_leaf: fn(Leaf) -> Leaf2, + convert_proposal: fn( + Proposal>, + ) -> Proposal>, + ) -> Result<()>; } diff --git a/crates/types/src/utils.rs b/crates/types/src/utils.rs index c3f8780575..aacbafa652 100644 --- a/crates/types/src/utils.rs +++ b/crates/types/src/utils.rs @@ -24,7 +24,7 @@ use tagged_base64::tagged; use typenum::Unsigned; use crate::{ - data::Leaf, + data::Leaf2, traits::{node_implementation::NodeType, ValidatedState}, vid::VidCommitment, }; @@ -70,7 +70,7 @@ impl Clone for ViewInner { } } /// The hash of a leaf. -pub type LeafCommitment = Commitment>; +pub type LeafCommitment = Commitment>; /// Optional validated state and state delta. pub type StateAndDelta = (