From bd2a5525ce61b80fa087a62b4360d7f99f381f26 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Wed, 25 Oct 2023 08:15:27 -0400 Subject: [PATCH 01/18] feat: libp2p byzantine completed --- crates/hotshot/examples/infra/modDA.rs | 1 + .../src/traits/networking/libp2p_network.rs | 71 ++++++++++++++++++- .../src/traits/networking/memory_network.rs | 2 +- .../src/network/node/handle.rs | 18 ++++- crates/types/src/traits/network.rs | 3 + 5 files changed, 88 insertions(+), 7 deletions(-) diff --git a/crates/hotshot/examples/infra/modDA.rs b/crates/hotshot/examples/infra/modDA.rs index a06f023159..dfeec6e5b9 100644 --- a/crates/hotshot/examples/infra/modDA.rs +++ b/crates/hotshot/examples/infra/modDA.rs @@ -706,6 +706,7 @@ where // function all_keys, da_keys, + None, ) .await .unwrap(); diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 20e6718955..1c21eca294 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -21,7 +21,7 @@ use hotshot_types::{ network::{ CommunicationChannel, ConnectedNetwork, ConsensusIntentEvent, FailedToSerializeSnafu, NetworkError, NetworkMsg, TestableChannelImplementation, - TestableNetworkingImplementation, TransmitType, ViewMessage, + TestableNetworkingImplementation, TransmitType, ViewMessage, NetworkReliability, }, node_implementation::NodeType, signature_key::SignatureKey, @@ -113,6 +113,8 @@ struct Libp2pNetworkInner { /// NOTE: supposed to represent a ViewNumber but we /// haven't made that atomic yet and we prefer lock-free latest_seen_view: Arc, + /// reliability_config + reliability_config: Option>> } /// Networking implementation that uses libp2p @@ -233,6 +235,7 @@ where node_id as usize, keys, da, + None, ) .await .unwrap() @@ -281,6 +284,7 @@ impl Libp2pNetwork { // HACK committee_pks: BTreeSet, da_pks: BTreeSet, + reliability_config: Option>>, ) -> Result, NetworkError> { assert!(bootstrap_addrs_len > 4, "Need at least 5 bootstrap nodes"); let network_handle = Arc::new( @@ -336,6 +340,8 @@ impl Libp2pNetwork { // proposals on". We need this because to have consensus info injected we need a working // network already. In the worst case, we send a few lookups we don't need. latest_seen_view: Arc::new(AtomicU64::new(0)), + reliability_config + }), }; @@ -596,6 +602,37 @@ impl ConnectedNetwork for Libp2p .map_err(|_| NetworkError::ShutDown)?; } + // TODO maybe we should lift the metrics mutex up a level and copy the inner pattern + // ask during pair programming + // or maybe channels would be better? + let metrics = self.inner.metrics.clone(); + if let Some(config) = &self.inner.reliability_config { + let handle = self.inner.handle.clone(); + + let serialized_msg = bincode_opts().serialize(&message).context(FailedToSerializeSnafu)?; + let fut = config.read().await.chaos_send_msg( + serialized_msg, + Arc::new(move |msg: Vec| { + let topic_2 = topic.clone(); + let handle_2 = handle.clone(); + let metrics_2 = metrics.clone(); + boxed_sync(async move { + match handle_2.gossip_no_serialize(topic_2, msg).await { + Err(e) => { + metrics_2.message_failed_to_send.add(1); + warn!("Failed to broadcast to libp2p: {:?}", e) + }, + Ok(_) => { + metrics_2.outgoing_direct_message_count.add(1); + + } + } + }) + })); + async_spawn(async move {fut.await}); + return Ok(()); + } + match self.inner.handle.gossip(topic, &message).await { Ok(()) => { self.inner.metrics.outgoing_broadcast_message_count.add(1); @@ -644,13 +681,41 @@ impl ConnectedNetwork for Libp2p } }; + // TODO maybe we should lift the metrics mutex up a level and copy the inner pattern + // ask during pair programming + // or maybe channels would be better? + let metrics = self.inner.metrics.clone(); + if let Some(config) = &self.inner.reliability_config { + let handle = self.inner.handle.clone(); + + let serialized_msg = bincode_opts().serialize(&message).context(FailedToSerializeSnafu)?; + let fut = config.read().await.chaos_send_msg( + serialized_msg, + Arc::new(move |msg: Vec| { + let handle_2 = handle.clone(); + let metrics_2 = metrics.clone(); + boxed_sync(async move { + match handle_2.direct_request_no_serialize(pid, msg).await { + Err(e) => { + metrics_2.message_failed_to_send.add(1); + warn!("Failed to broadcast to libp2p: {:?}", e) + }, + Ok(_) => { + metrics_2.outgoing_direct_message_count.add(1); + + } + } + }) + })); + async_spawn(async move {fut.await}); + return Ok(()); + } + match self.inner.handle.direct_request(pid, &message).await { Ok(()) => { - self.inner.metrics.outgoing_direct_message_count.add(1); Ok(()) } Err(e) => { - self.inner.metrics.message_failed_to_send.add(1); Err(e.into()) } } diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 3ea28fe7e6..47edc766ea 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -361,7 +361,7 @@ impl ConnectedNetwork for Memory Arc::new(move |msg: Vec| { let node2 = node.clone(); boxed_sync(async move { - let _res = node2.broadcast_input(msg).await; + let _res = node2.direct_input(msg).await; // NOTE we're dropping metrics here but this is only for testing // purposes. I think that should be okay }) diff --git a/crates/libp2p-networking/src/network/node/handle.rs b/crates/libp2p-networking/src/network/node/handle.rs index 39c763ac31..76bdfdab21 100644 --- a/crates/libp2p-networking/src/network/node/handle.rs +++ b/crates/libp2p-networking/src/network/node/handle.rs @@ -452,9 +452,17 @@ impl NetworkNodeHandle { msg: &impl Serialize, ) -> Result<(), NetworkNodeHandleError> { let serialized_msg = bincode_opts().serialize(msg).context(SerializationSnafu)?; + self.direct_request_no_serialize(pid, serialized_msg).await + } + + pub async fn direct_request_no_serialize( + &self, + pid: PeerId, + contents: Vec, + ) -> Result<(), NetworkNodeHandleError> { let req = ClientRequest::DirectRequest { pid, - contents: serialized_msg, + contents, retry_count: 1, }; self.send_request(req).await @@ -474,7 +482,7 @@ impl NetworkNodeHandle { self.send_request(req).await } - /// Forcefully disconnet from a peer + /// Forcefully disconnect from a peer /// # Errors /// If the channel is closed somehow /// Shouldnt' happen. @@ -496,7 +504,11 @@ impl NetworkNodeHandle { msg: &impl Serialize, ) -> Result<(), NetworkNodeHandleError> { let serialized_msg = bincode_opts().serialize(msg).context(SerializationSnafu)?; - let req = ClientRequest::GossipMsg(topic, serialized_msg); + self.gossip_no_serialize(topic, serialized_msg).await + } + + pub async fn gossip_no_serialize(&self, topic: String, msg: Vec) -> Result<(), NetworkNodeHandleError>{ + let req = ClientRequest::GossipMsg(topic, msg); self.send_request(req).await } diff --git a/crates/types/src/traits/network.rs b/crates/types/src/traits/network.rs index 1729636836..1a6414fd67 100644 --- a/crates/types/src/traits/network.rs +++ b/crates/types/src/traits/network.rs @@ -405,6 +405,9 @@ pub trait NetworkReliability: Debug + Sync + std::marker::Send { /// whether or not to send duplicates /// and whether or not to include noise with the message /// then send the message + /// note: usually self is stored in a rwlock + /// so instead of doing the sending part, we just fiddle with the message + /// then return a future that does the sending and delaying fn chaos_send_msg( &self, msg: Vec, From 192db0df8a01e6e7f61f268b958a0e434982d88f Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Mon, 6 Nov 2023 08:34:20 -0500 Subject: [PATCH 02/18] feat: byzantine network --- .../src/traits/networking/combined_network.rs | 13 +- .../src/traits/networking/libp2p_network.rs | 75 ++-- .../src/traits/networking/memory_network.rs | 22 +- .../traits/networking/web_server_network.rs | 6 +- .../src/network/node/handle.rs | 14 +- crates/testing/Cargo.toml | 2 +- crates/testing/src/node_types.rs | 15 +- crates/testing/src/test_builder.rs | 8 +- crates/testing/tests/byzantine.rs | 322 ++++++++++++++++++ crates/types/src/traits/network.rs | 59 +++- .../types/src/traits/node_implementation.rs | 5 +- 11 files changed, 476 insertions(+), 65 deletions(-) create mode 100644 crates/testing/tests/byzantine.rs diff --git a/crates/hotshot/src/traits/networking/combined_network.rs b/crates/hotshot/src/traits/networking/combined_network.rs index e717c40b1b..023d4b1a33 100644 --- a/crates/hotshot/src/traits/networking/combined_network.rs +++ b/crates/hotshot/src/traits/networking/combined_network.rs @@ -29,7 +29,7 @@ use hotshot_types::{ traits::{ election::Membership, network::{ - CommunicationChannel, ConnectedNetwork, ConsensusIntentEvent, + CommunicationChannel, ConnectedNetwork, ConsensusIntentEvent, NetworkReliability, TestableChannelImplementation, TestableNetworkingImplementation, TransmitType, ViewMessage, }, @@ -168,6 +168,7 @@ impl, MEMBERSHIP: Membership>, ) -> Box Self + 'static> { let generators = ( , MEMBERSHIP: Membership, TYPES::SignatureKey> as TestableNetworkingImplementation<_, _>>::generator( expected_node_count, num_bootstrap, network_id, da_committee_size, - is_da + is_da, + reliability_config, ) ); Box::new(move |node_id| { @@ -212,6 +215,7 @@ impl, MEMBERSHIP: Membership>, ) -> Box Self + 'static> { let generator = , MEMBERSHIP: Membership { /// haven't made that atomic yet and we prefer lock-free latest_seen_view: Arc, /// reliability_config - reliability_config: Option>> + reliability_config: Option>, } /// Networking implementation that uses libp2p @@ -146,6 +146,7 @@ where network_id: usize, da_committee_size: usize, _is_da: bool, + reliability_config: Option>, ) -> Box Self + 'static> { assert!( da_committee_size <= expected_node_count, @@ -225,6 +226,11 @@ where let bootstrap_addrs_ref = bootstrap_addrs.clone(); let keys = all_keys.clone(); let da = da_keys.clone(); + // let reliability_config_dup = match &reliability_config { + // Some(ref config) => Some(config), + // None => todo!(), + // }; + let relaibility_config_dup = reliability_config.clone(); async_block_on(async move { Libp2pNetwork::new( NetworkingMetricsValue::new(), @@ -235,7 +241,7 @@ where node_id as usize, keys, da, - None, + relaibility_config_dup, ) .await .unwrap() @@ -284,7 +290,7 @@ impl Libp2pNetwork { // HACK committee_pks: BTreeSet, da_pks: BTreeSet, - reliability_config: Option>>, + reliability_config: Option>, ) -> Result, NetworkError> { assert!(bootstrap_addrs_len > 4, "Need at least 5 bootstrap nodes"); let network_handle = Arc::new( @@ -340,8 +346,7 @@ impl Libp2pNetwork { // proposals on". We need this because to have consensus info injected we need a working // network already. In the worst case, we send a few lookups we don't need. latest_seen_view: Arc::new(AtomicU64::new(0)), - reliability_config - + reliability_config, }), }; @@ -606,11 +611,13 @@ impl ConnectedNetwork for Libp2p // ask during pair programming // or maybe channels would be better? let metrics = self.inner.metrics.clone(); - if let Some(config) = &self.inner.reliability_config { + if let Some(ref config) = &self.inner.reliability_config { let handle = self.inner.handle.clone(); - let serialized_msg = bincode_opts().serialize(&message).context(FailedToSerializeSnafu)?; - let fut = config.read().await.chaos_send_msg( + let serialized_msg = bincode_opts() + .serialize(&message) + .context(FailedToSerializeSnafu)?; + let fut = config.clone().chaos_send_msg( serialized_msg, Arc::new(move |msg: Vec| { let topic_2 = topic.clone(); @@ -620,16 +627,16 @@ impl ConnectedNetwork for Libp2p match handle_2.gossip_no_serialize(topic_2, msg).await { Err(e) => { metrics_2.message_failed_to_send.add(1); - warn!("Failed to broadcast to libp2p: {:?}", e) - }, - Ok(_) => { - metrics_2.outgoing_direct_message_count.add(1); - + warn!("Failed to broadcast to libp2p: {:?}", e); + } + Ok(()) => { + metrics_2.outgoing_direct_message_count.add(1); } } }) - })); - async_spawn(async move {fut.await}); + }), + ); + async_spawn(fut); return Ok(()); } @@ -685,11 +692,13 @@ impl ConnectedNetwork for Libp2p // ask during pair programming // or maybe channels would be better? let metrics = self.inner.metrics.clone(); - if let Some(config) = &self.inner.reliability_config { + if let Some(ref config) = &self.inner.reliability_config { let handle = self.inner.handle.clone(); - let serialized_msg = bincode_opts().serialize(&message).context(FailedToSerializeSnafu)?; - let fut = config.read().await.chaos_send_msg( + let serialized_msg = bincode_opts() + .serialize(&message) + .context(FailedToSerializeSnafu)?; + let fut = config.clone().chaos_send_msg( serialized_msg, Arc::new(move |msg: Vec| { let handle_2 = handle.clone(); @@ -698,26 +707,22 @@ impl ConnectedNetwork for Libp2p match handle_2.direct_request_no_serialize(pid, msg).await { Err(e) => { metrics_2.message_failed_to_send.add(1); - warn!("Failed to broadcast to libp2p: {:?}", e) - }, - Ok(_) => { - metrics_2.outgoing_direct_message_count.add(1); - + warn!("Failed to broadcast to libp2p: {:?}", e); + } + Ok(()) => { + metrics_2.outgoing_direct_message_count.add(1); } } }) - })); - async_spawn(async move {fut.await}); + }), + ); + async_spawn(fut); return Ok(()); } match self.inner.handle.direct_request(pid, &message).await { - Ok(()) => { - Ok(()) - } - Err(e) => { - Err(e.into()) - } + Ok(()) => Ok(()), + Err(e) => Err(e.into()), } } @@ -843,6 +848,7 @@ where network_id: usize, da_committee_size: usize, is_da: bool, + reliability_config: Option>, ) -> Box Self + 'static> { let generator = , @@ -852,7 +858,8 @@ where num_bootstrap, network_id, da_committee_size, - is_da + is_da, + reliability_config ); Box::new(move |node_id| Self(generator(node_id).into(), PhantomData)) } diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 47edc766ea..75d9e9c693 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -94,7 +94,7 @@ struct MemoryNetworkInner { metrics: NetworkingMetricsValue, /// config to introduce unreliability to the network - reliability_config: Option>>, + reliability_config: Option>, } /// In memory only network simulator. @@ -125,7 +125,7 @@ impl MemoryNetwork { pub_key: K, metrics: NetworkingMetricsValue, master_map: Arc>, - reliability_config: Option>>, + reliability_config: Option>, ) -> MemoryNetwork { info!("Attaching new MemoryNetwork"); let (broadcast_input, broadcast_task_recv) = bounded(128); @@ -252,12 +252,18 @@ impl> _network_id: usize, _da_committee_size: usize, _is_da: bool, + reliability_config: Option>, ) -> Box Self + 'static> { let master: Arc<_> = MasterMap::new(); Box::new(move |node_id| { let privkey = TYPES::SignatureKey::generated_from_seed_indexed([0u8; 32], node_id).1; let pubkey = TYPES::SignatureKey::from_private(&privkey); - MemoryNetwork::new(pubkey, NetworkingMetricsValue::new(), master.clone(), None) + MemoryNetwork::new( + pubkey, + NetworkingMetricsValue::new(), + master.clone(), + reliability_config.clone(), + ) }) } @@ -309,8 +315,7 @@ impl ConnectedNetwork for Memory continue; } trace!(?key, "Sending message to node"); - if let Some(r) = &self.inner.reliability_config { - let config = r.read().await; + if let Some(ref config) = &self.inner.reliability_config { { let node2 = node.clone(); let fut = config.chaos_send_msg( @@ -353,8 +358,7 @@ impl ConnectedNetwork for Memory trace!("Message bincoded, finding recipient"); if let Some(node) = self.inner.master_map.map.get(&recipient) { let node = node.value().clone(); - if let Some(r) = &self.inner.reliability_config { - let config = r.read().await; + if let Some(ref config) = &self.inner.reliability_config { { let fut = config.chaos_send_msg( vec.clone(), @@ -481,6 +485,7 @@ where network_id: usize, da_committee_size: usize, is_da: bool, + reliability_config: Option>, ) -> Box Self + 'static> { let generator = , @@ -490,7 +495,8 @@ where num_bootstrap, network_id, da_committee_size, - is_da + is_da, + reliability_config, ); Box::new(move |node_id| Self(generator(node_id).into(), PhantomData)) } diff --git a/crates/hotshot/src/traits/networking/web_server_network.rs b/crates/hotshot/src/traits/networking/web_server_network.rs index 5a67880e9f..06b21693a5 100644 --- a/crates/hotshot/src/traits/networking/web_server_network.rs +++ b/crates/hotshot/src/traits/networking/web_server_network.rs @@ -29,7 +29,7 @@ use hotshot_web_server::{self, config}; use rand::random; use serde::{Deserialize, Serialize}; -use hotshot_types::traits::network::ViewMessage; +use hotshot_types::traits::network::{NetworkReliability, ViewMessage}; use std::{ collections::{hash_map::Entry, BTreeSet, HashMap}, marker::PhantomData, @@ -1070,6 +1070,7 @@ impl> _network_id: usize, _da_committee_size: usize, is_da: bool, + _reliability_config: Option>, ) -> Box Self + 'static> { let (server_shutdown_sender, server_shutdown) = oneshot(); let sender = Arc::new(server_shutdown_sender); @@ -1120,6 +1121,7 @@ impl, MEMBERSHIP: Membership>, ) -> Box Self + 'static> { let generator = , @@ -1131,6 +1133,8 @@ impl, MEMBERSHIP: Membership NetworkNodeHandle { self.direct_request_no_serialize(pid, serialized_msg).await } + /// Make a direct request to `peer_id` containing `msg` without serializing + /// # Errors + /// - Will return [`NetworkNodeHandleError::SendError`] when underlying `NetworkNode` has been killed + /// - Will return [`NetworkNodeHandleError::SerializationError`] when unable to serialize `msg` pub async fn direct_request_no_serialize( &self, pid: PeerId, @@ -507,7 +511,15 @@ impl NetworkNodeHandle { self.gossip_no_serialize(topic, serialized_msg).await } - pub async fn gossip_no_serialize(&self, topic: String, msg: Vec) -> Result<(), NetworkNodeHandleError>{ + /// Gossip a message to peers without serializing + /// # Errors + /// - Will return [`NetworkNodeHandleError::SendError`] when underlying `NetworkNode` has been killed + /// - Will return [`NetworkNodeHandleError::SerializationError`] when unable to serialize `msg` + pub async fn gossip_no_serialize( + &self, + topic: String, + msg: Vec, + ) -> Result<(), NetworkNodeHandleError> { let req = ClientRequest::GossipMsg(topic, msg); self.send_request(req).await } diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 935f9e4342..7e8a33a134 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -37,9 +37,9 @@ tracing = { workspace = true } serde = { workspace = true } ethereum-types = { workspace = true } bitvec = { workspace = true } +async-lock = { workspace = true } [dev-dependencies] -async-lock = { workspace = true } bincode = { workspace = true } # GG any better options for serialization? [target.'cfg(all(async_executor_impl = "tokio"))'.dependencies] diff --git a/crates/testing/src/node_types.rs b/crates/testing/src/node_types.rs index 7e5308b35f..dbf141b4f2 100644 --- a/crates/testing/src/node_types.rs +++ b/crates/testing/src/node_types.rs @@ -1,4 +1,4 @@ -use hotshot::traits::implementations::CombinedNetworks; +use hotshot::traits::{implementations::CombinedNetworks, NetworkReliability}; use std::{marker::PhantomData, sync::Arc}; use hotshot::{ @@ -156,6 +156,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, + byzantine_metadata: Option>, ) -> Box< dyn Fn( u64, @@ -186,6 +187,7 @@ impl 0, da_committee_size, false, + byzantine_metadata, )); Box::new(move |id| { @@ -252,6 +254,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, + byzantine_metadata: Option>, ) -> Box< dyn Fn( u64, @@ -282,6 +285,7 @@ impl 0, da_committee_size, false, + byzantine_metadata.clone(), )); let network_da_generator = Arc::new(, @@ -295,6 +299,7 @@ impl 1, da_committee_size, true, + byzantine_metadata, )); Box::new(move |id| { let network = Arc::new(network_generator(id)); @@ -385,6 +390,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, + byzantine_metadata: Option>, ) -> Box< dyn Fn( u64, @@ -403,6 +409,7 @@ impl >>::Networking, ) + 'static, > { + // this is unsupported currently let network_generator = Arc::new(, ::SignatureKey, @@ -416,6 +423,7 @@ impl 0, da_committee_size, false, + byzantine_metadata.clone(), )); let network_da_generator = Arc::new(, @@ -430,6 +438,7 @@ impl 1, da_committee_size, true, + byzantine_metadata.clone(), )); Box::new(move |id| { let network = Arc::new(network_generator(id)); @@ -534,6 +543,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, + byzantine_metadata: Option>, ) -> Box< dyn Fn( u64, @@ -565,6 +575,7 @@ impl 0, da_committee_size, false, + byzantine_metadata.clone(), )); let web_server_network_da_generator = Arc::new(>, } impl Default for TimingData { @@ -173,6 +175,7 @@ impl Default for TestMetadata { duration: Duration::from_millis(10000), }, ), + byzantine_metadata: None, } } } @@ -197,6 +200,7 @@ impl TestMetadata { completion_task_description, overall_safety_properties, spinning_properties, + byzantine_metadata, .. } = self.clone(); @@ -261,7 +265,7 @@ impl TestMetadata { let spinning_task_generator = spinning_properties.build(); TestLauncher { resource_generator: ResourceGenerators { - channel_generator: <>::Exchanges as TestableExchange<_, _, _>>::gen_comm_channels(total_nodes, num_bootstrap_nodes, da_committee_size), + channel_generator: <>::Exchanges as TestableExchange<_, _, _>>::gen_comm_channels(total_nodes, num_bootstrap_nodes, da_committee_size, byzantine_metadata), storage: Box::new(|_| I::construct_tmp_storage().unwrap()), config, }, diff --git a/crates/testing/tests/byzantine.rs b/crates/testing/tests/byzantine.rs new file mode 100644 index 0000000000..4558f1ffa6 --- /dev/null +++ b/crates/testing/tests/byzantine.rs @@ -0,0 +1,322 @@ +use hotshot_types::traits::network::AsynchronousNetwork; +use hotshot_types::traits::network::ChaosNetwork; +use hotshot_types::traits::network::PartiallySynchronousNetwork; +use hotshot_types::traits::network::SynchronousNetwork; +use std::time::Duration; +use std::time::Instant; + +use hotshot_testing::{ + completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, + node_types::{SequencingLibp2pImpl, SequencingTestTypes}, + overall_safety_task::OverallSafetyPropertiesDescription, + test_builder::TestMetadata, +}; +use tracing::instrument; + +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +#[instrument] +async fn libp2p_network_sync() { + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + overall_safety_properties: OverallSafetyPropertiesDescription { + check_leaf: true, + ..Default::default() + }, + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::new(240, 0), + }, + ), + byzantine_metadata: Some(Box::new(SynchronousNetwork { + timeout_ms: 30, + delay_low_ms: 4, + })), + ..TestMetadata::default_multiple_rounds() + }; + + metadata + .gen_launcher::() + .launch() + .run_test() + .await +} + +#[cfg(test)] +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +async fn test_memory_network_sync() { + use hotshot_testing::{ + completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, + node_types::{SequencingMemoryImpl, SequencingTestTypes}, + test_builder::TestMetadata, + }; + use std::time::Duration; + + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + // allow more time to pass in CI + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::from_secs(240), + }, + ), + byzantine_metadata: Some(Box::new(SynchronousNetwork { + timeout_ms: 30, + delay_low_ms: 4, + })), + ..TestMetadata::default() + }; + metadata + .gen_launcher::() + .launch() + .run_test() + .await; +} + +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +#[instrument] +async fn libp2p_network_async() { + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + overall_safety_properties: OverallSafetyPropertiesDescription { + check_leaf: true, + ..Default::default() + }, + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::new(240, 0), + }, + ), + byzantine_metadata: Some(Box::new(AsynchronousNetwork { + keep_numerator: 8, + keep_denominator: 10, + delay_low_ms: 4, + delay_high_ms: 30, + })), + ..TestMetadata::default_multiple_rounds() + }; + + metadata + .gen_launcher::() + .launch() + .run_test() + .await +} + +#[cfg(test)] +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +async fn test_memory_network_async() { + use hotshot_testing::{ + completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, + node_types::{SequencingMemoryImpl, SequencingTestTypes}, + test_builder::TestMetadata, + }; + use std::time::Duration; + + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + // allow more time to pass in CI + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::from_secs(240), + }, + ), + byzantine_metadata: Some(Box::new(AsynchronousNetwork { + keep_numerator: 8, + keep_denominator: 10, + delay_low_ms: 4, + delay_high_ms: 30, + })), + ..TestMetadata::default() + }; + metadata + .gen_launcher::() + .launch() + .run_test() + .await; +} + +#[cfg(test)] +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +async fn test_memory_network_partially_sync() { + use hotshot_testing::{ + completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, + node_types::{SequencingMemoryImpl, SequencingTestTypes}, + test_builder::TestMetadata, + }; + use std::time::Duration; + + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + // allow more time to pass in CI + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::from_secs(240), + }, + ), + byzantine_metadata: Some(Box::new(PartiallySynchronousNetwork { + asynchronous: AsynchronousNetwork { + keep_numerator: 8, + keep_denominator: 10, + delay_low_ms: 4, + delay_high_ms: 30, + }, + synchronous: SynchronousNetwork { + timeout_ms: 30, + delay_low_ms: 4, + }, + gst: std::time::Duration::from_millis(1000), + start: Instant::now(), + })), + ..TestMetadata::default() + }; + metadata + .gen_launcher::() + .launch() + .run_test() + .await; +} + +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +#[instrument] +async fn libp2p_network_partially_sync() { + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + overall_safety_properties: OverallSafetyPropertiesDescription { + check_leaf: true, + ..Default::default() + }, + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::new(240, 0), + }, + ), + byzantine_metadata: Some(Box::new(PartiallySynchronousNetwork { + asynchronous: AsynchronousNetwork { + keep_numerator: 8, + keep_denominator: 10, + delay_low_ms: 4, + delay_high_ms: 30, + }, + synchronous: SynchronousNetwork { + timeout_ms: 30, + delay_low_ms: 4, + }, + gst: std::time::Duration::from_millis(1000), + start: Instant::now(), + })), + ..TestMetadata::default_multiple_rounds() + }; + + metadata + .gen_launcher::() + .launch() + .run_test() + .await +} + +#[cfg(test)] +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +async fn test_memory_network_chaos() { + use hotshot_testing::{ + completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, + node_types::{SequencingMemoryImpl, SequencingTestTypes}, + test_builder::TestMetadata, + }; + use std::time::Duration; + + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + // allow more time to pass in CI + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::from_secs(240), + }, + ), + byzantine_metadata: Some(Box::new(ChaosNetwork { + keep_numerator: 8, + keep_denominator: 10, + delay_low_ms: 4, + delay_high_ms: 30, + repeat_low: 1, + repeat_high: 5, + })), + ..TestMetadata::default() + }; + metadata + .gen_launcher::() + .launch() + .run_test() + .await; +} + +#[cfg_attr( + async_executor_impl = "tokio", + tokio::test(flavor = "multi_thread", worker_threads = 2) +)] +#[cfg_attr(async_executor_impl = "async-std", async_std::test)] +#[instrument] +async fn libp2p_network_chaos() { + async_compatibility_layer::logging::setup_logging(); + async_compatibility_layer::logging::setup_backtrace(); + let metadata = TestMetadata { + overall_safety_properties: OverallSafetyPropertiesDescription { + check_leaf: true, + ..Default::default() + }, + completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( + TimeBasedCompletionTaskDescription { + duration: Duration::new(240, 0), + }, + ), + byzantine_metadata: Some(Box::new(ChaosNetwork { + keep_numerator: 8, + keep_denominator: 10, + delay_low_ms: 4, + delay_high_ms: 30, + repeat_low: 1, + repeat_high: 5, + })), + ..TestMetadata::default_multiple_rounds() + }; + + metadata + .gen_launcher::() + .launch() + .run_test() + .await +} diff --git a/crates/types/src/traits/network.rs b/crates/types/src/traits/network.rs index 1a6414fd67..4b9ce1f83b 100644 --- a/crates/types/src/traits/network.rs +++ b/crates/types/src/traits/network.rs @@ -5,6 +5,7 @@ use async_compatibility_layer::art::async_sleep; #[cfg(async_executor_impl = "async-std")] use async_std::future::TimeoutError; +use dyn_clone::DynClone; use hotshot_task::{boxed_sync, BoxSyncFuture}; use libp2p_networking::network::NetworkNodeHandleError; #[cfg(async_executor_impl = "tokio")] @@ -341,6 +342,7 @@ pub trait TestableNetworkingImplementation { network_id: usize, da_committee_size: usize, is_da: bool, + reliability_config: Option>, ) -> Box Self + 'static>; /// Get the number of messages in-flight. @@ -372,7 +374,7 @@ pub enum NetworkChange { /// interface describing how reliable the network is #[async_trait] -pub trait NetworkReliability: Debug + Sync + std::marker::Send { +pub trait NetworkReliability: Debug + Sync + std::marker::Send + DynClone + 'static { /// Sample from bernoulli distribution to decide whether /// or not to keep a packet /// # Panics @@ -433,6 +435,8 @@ pub trait NetworkReliability: Debug + Sync + std::marker::Send { } } +dyn_clone::clone_trait_object!(NetworkReliability); + /// ideal network #[derive(Clone, Copy, Debug, Default)] pub struct PerfectNetwork {} @@ -444,9 +448,9 @@ impl NetworkReliability for PerfectNetwork {} #[derive(Clone, Copy, Debug, Default)] pub struct SynchronousNetwork { /// Max delay of packet before arrival - timeout_ms: u64, + pub timeout_ms: u64, /// Lowest value in milliseconds that a packet may be delayed - delay_low_ms: u64, + pub delay_low_ms: u64, } impl NetworkReliability for SynchronousNetwork { @@ -470,13 +474,13 @@ impl NetworkReliability for SynchronousNetwork { #[derive(Debug, Clone, Copy)] pub struct AsynchronousNetwork { /// numerator for probability of keeping packets - keep_numerator: u32, + pub keep_numerator: u32, /// denominator for probability of keeping packets - keep_denominator: u32, + pub keep_denominator: u32, /// lowest value in milliseconds that a packet may be delayed - delay_low_ms: u64, + pub delay_low_ms: u64, /// highest value in milliseconds that a packet may be delayed - delay_high_ms: u64, + pub delay_high_ms: u64, } impl NetworkReliability for AsynchronousNetwork { @@ -500,13 +504,13 @@ impl NetworkReliability for AsynchronousNetwork { #[derive(Debug, Clone, Copy)] pub struct PartiallySynchronousNetwork { /// asynchronous portion of network - asynchronous: AsynchronousNetwork, + pub asynchronous: AsynchronousNetwork, /// synchronous portion of network - synchronous: SynchronousNetwork, + pub synchronous: SynchronousNetwork, /// time when GST occurs - gst: std::time::Duration, + pub gst: std::time::Duration, /// when the network was started - start: std::time::Instant, + pub start: std::time::Instant, } impl NetworkReliability for PartiallySynchronousNetwork { @@ -601,6 +605,37 @@ impl PartiallySynchronousNetwork { } /// A chaotic network using all the networking calls +#[derive(Debug, Clone)] pub struct ChaosNetwork { - // TODO + /// numerator for probability of keeping packets + pub keep_numerator: u32, + /// denominator for probability of keeping packets + pub keep_denominator: u32, + /// lowest value in milliseconds that a packet may be delayed + pub delay_low_ms: u64, + /// highest value in milliseconds that a packet may be delayed + pub delay_high_ms: u64, + /// lowest value of repeats for a message + pub repeat_low: usize, + /// highest value of repeats for a message + pub repeat_high: usize, +} + +impl NetworkReliability for ChaosNetwork { + fn sample_keep(&self) -> bool { + Bernoulli::from_ratio(self.keep_numerator, self.keep_denominator) + .unwrap() + .sample(&mut rand::thread_rng()) + } + + fn sample_delay(&self) -> Duration { + Duration::from_millis( + Uniform::new_inclusive(self.delay_low_ms, self.delay_high_ms) + .sample(&mut rand::thread_rng()), + ) + } + + fn sample_repeat(&self) -> usize { + Uniform::new_inclusive(self.repeat_low, self.repeat_high).sample(&mut rand::thread_rng()) + } } diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index 6d27d70103..f7b5845330 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -9,7 +9,9 @@ use super::{ CommitteeExchangeType, ConsensusExchange, ElectionConfig, QuorumExchangeType, TimeoutExchange, TimeoutExchangeType, ViewSyncExchangeType, VoteToken, }, - network::{CommunicationChannel, NetworkMsg, TestableNetworkingImplementation}, + network::{ + CommunicationChannel, NetworkMsg, NetworkReliability, TestableNetworkingImplementation, + }, state::{ConsensusTime, TestableBlock, TestableState}, storage::{StorageError, StorageState, TestableStorage}, State, @@ -209,6 +211,7 @@ pub trait TestableExchange, ME expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, + byzantine_metadata: Option>, ) -> Box< dyn Fn( u64, From b5e1a7cba65c125f9c0f61fd7448bad9a3f1d016 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Mon, 6 Nov 2023 08:50:34 -0500 Subject: [PATCH 03/18] fix: compilation error --- crates/hotshot/examples/infra/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/hotshot/examples/infra/mod.rs b/crates/hotshot/examples/infra/mod.rs index 1309525923..755758a9db 100644 --- a/crates/hotshot/examples/infra/mod.rs +++ b/crates/hotshot/examples/infra/mod.rs @@ -802,6 +802,7 @@ where // NOTE: this introduces an invariant that the keys are assigned using this indexed // function all_keys, + None, da_keys.clone(), da_keys.contains(&pubkey), ) From 69f84ccc9e4483bb8667e59dd6983fc229480a1e Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Mon, 6 Nov 2023 08:54:49 -0500 Subject: [PATCH 04/18] fix: minor renaming --- crates/testing/src/node_types.rs | 24 +++++++++---------- crates/testing/src/test_builder.rs | 10 ++++---- .../{byzantine.rs => unreliable_network.rs} | 16 ++++++------- .../types/src/traits/node_implementation.rs | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) rename crates/testing/tests/{byzantine.rs => unreliable_network.rs} (95%) diff --git a/crates/testing/src/node_types.rs b/crates/testing/src/node_types.rs index 4c79cb65b4..fe1826599e 100644 --- a/crates/testing/src/node_types.rs +++ b/crates/testing/src/node_types.rs @@ -155,7 +155,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, - byzantine_metadata: Option>, + unreliable_network: Option>, ) -> Box< dyn Fn( u64, @@ -190,7 +190,7 @@ impl 0, da_committee_size, false, - byzantine_metadata, + unreliable_network, )); Box::new(move |id| { @@ -264,7 +264,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, - byzantine_metadata: Option>, + unreliable_network: Option>, ) -> Box< dyn Fn( u64, @@ -299,7 +299,7 @@ impl 0, da_committee_size, false, - byzantine_metadata.clone(), + unreliable_network.clone(), )); let network_da_generator = Arc::new(, @@ -313,7 +313,7 @@ impl 1, da_committee_size, true, - byzantine_metadata, + unreliable_network, )); Box::new(move |id| { let network = Arc::new(network_generator(id)); @@ -406,7 +406,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, - byzantine_metadata: Option>, + unreliable_network: Option>, ) -> Box< dyn Fn( u64, @@ -443,7 +443,7 @@ impl 0, da_committee_size, false, - byzantine_metadata.clone(), + unreliable_network.clone(), )); let network_da_generator = Arc::new(, @@ -458,7 +458,7 @@ impl 1, da_committee_size, true, - byzantine_metadata.clone(), + unreliable_network.clone(), )); Box::new(move |id| { let network = Arc::new(network_generator(id)); @@ -575,7 +575,7 @@ impl expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, - byzantine_metadata: Option>, + unreliable_network: Option>, ) -> Box< dyn Fn( u64, @@ -611,7 +611,7 @@ impl 0, da_committee_size, false, - byzantine_metadata.clone(), + unreliable_network.clone(), )); let web_server_network_da_generator = Arc::new(>, + /// unrelabile networking metadata + pub unreliable_network: Option>, } impl Default for TimingData { @@ -175,7 +175,7 @@ impl Default for TestMetadata { duration: Duration::from_millis(10000), }, ), - byzantine_metadata: None, + unreliable_network: None, } } } @@ -200,7 +200,7 @@ impl TestMetadata { completion_task_description, overall_safety_properties, spinning_properties, - byzantine_metadata, + unreliable_network, .. } = self.clone(); @@ -272,7 +272,7 @@ impl TestMetadata { _, _, >>::gen_comm_channels( - total_nodes, num_bootstrap_nodes, da_committee_size, byzantine_metadata + total_nodes, num_bootstrap_nodes, da_committee_size, unreliable_network ), storage: Box::new(|_| I::construct_tmp_storage().unwrap()), config, diff --git a/crates/testing/tests/byzantine.rs b/crates/testing/tests/unreliable_network.rs similarity index 95% rename from crates/testing/tests/byzantine.rs rename to crates/testing/tests/unreliable_network.rs index adaa3e847b..9462b72dec 100644 --- a/crates/testing/tests/byzantine.rs +++ b/crates/testing/tests/unreliable_network.rs @@ -32,7 +32,7 @@ async fn libp2p_network_sync() { duration: Duration::new(240, 0), }, ), - byzantine_metadata: Some(Box::new(SynchronousNetwork { + unreliable_network: Some(Box::new(SynchronousNetwork { timeout_ms: 30, delay_low_ms: 4, })), @@ -69,7 +69,7 @@ async fn test_memory_network_sync() { duration: Duration::from_secs(240), }, ), - byzantine_metadata: Some(Box::new(SynchronousNetwork { + unreliable_network: Some(Box::new(SynchronousNetwork { timeout_ms: 30, delay_low_ms: 4, })), @@ -101,7 +101,7 @@ async fn libp2p_network_async() { duration: Duration::new(240, 0), }, ), - byzantine_metadata: Some(Box::new(AsynchronousNetwork { + unreliable_network: Some(Box::new(AsynchronousNetwork { keep_numerator: 8, keep_denominator: 10, delay_low_ms: 4, @@ -140,7 +140,7 @@ async fn test_memory_network_async() { duration: Duration::from_secs(240), }, ), - byzantine_metadata: Some(Box::new(AsynchronousNetwork { + unreliable_network: Some(Box::new(AsynchronousNetwork { keep_numerator: 8, keep_denominator: 10, delay_low_ms: 4, @@ -178,7 +178,7 @@ async fn test_memory_network_partially_sync() { duration: Duration::from_secs(240), }, ), - byzantine_metadata: Some(Box::new(PartiallySynchronousNetwork { + unreliable_network: Some(Box::new(PartiallySynchronousNetwork { asynchronous: AsynchronousNetwork { keep_numerator: 8, keep_denominator: 10, @@ -220,7 +220,7 @@ async fn libp2p_network_partially_sync() { duration: Duration::new(240, 0), }, ), - byzantine_metadata: Some(Box::new(PartiallySynchronousNetwork { + unreliable_network: Some(Box::new(PartiallySynchronousNetwork { asynchronous: AsynchronousNetwork { keep_numerator: 8, keep_denominator: 10, @@ -267,7 +267,7 @@ async fn test_memory_network_chaos() { duration: Duration::from_secs(240), }, ), - byzantine_metadata: Some(Box::new(ChaosNetwork { + unreliable_network: Some(Box::new(ChaosNetwork { keep_numerator: 8, keep_denominator: 10, delay_low_ms: 4, @@ -303,7 +303,7 @@ async fn libp2p_network_chaos() { duration: Duration::new(240, 0), }, ), - byzantine_metadata: Some(Box::new(ChaosNetwork { + unreliable_network: Some(Box::new(ChaosNetwork { keep_numerator: 8, keep_denominator: 10, delay_low_ms: 4, diff --git a/crates/types/src/traits/node_implementation.rs b/crates/types/src/traits/node_implementation.rs index b4afd3eb79..bd61e6329c 100644 --- a/crates/types/src/traits/node_implementation.rs +++ b/crates/types/src/traits/node_implementation.rs @@ -218,7 +218,7 @@ pub trait TestableExchange, ME expected_node_count: usize, num_bootstrap: usize, da_committee_size: usize, - byzantine_metadata: Option>, + unreliable_network: Option>, ) -> Box< dyn Fn( u64, From d26f0be63d088a203c7571fdf1d4413a67cedb74 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Mon, 6 Nov 2023 09:00:14 -0500 Subject: [PATCH 05/18] fix: unstage modDA, not sure how that got there --- crates/hotshot/examples/infra/modDA.rs | 848 ------------------------- 1 file changed, 848 deletions(-) delete mode 100644 crates/hotshot/examples/infra/modDA.rs diff --git a/crates/hotshot/examples/infra/modDA.rs b/crates/hotshot/examples/infra/modDA.rs deleted file mode 100644 index dfeec6e5b9..0000000000 --- a/crates/hotshot/examples/infra/modDA.rs +++ /dev/null @@ -1,848 +0,0 @@ -use crate::infra::{load_config_from_file, OrchestratorArgs}; - -use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; -use async_lock::RwLock; -use async_trait::async_trait; -use futures::StreamExt; -use hotshot::{ - traits::{ - implementations::{ - Libp2pCommChannel, Libp2pNetwork, MemoryStorage, NetworkingMetricsValue, - WebCommChannel, WebServerNetwork, - }, - NodeImplementation, - }, - types::{SignatureKey, SystemContextHandle}, - HotShotType, SystemContext, -}; -use hotshot_orchestrator::{ - self, - client::{OrchestratorClient, ValidatorArgs}, - config::{NetworkConfig, WebServerConfig}, -}; -use hotshot_task::task::FilterEvent; -use hotshot_types::{ - block_impl::{VIDBlockPayload, VIDTransaction}, - certificate::ViewSyncCertificate, - consensus::ConsensusMetricsValue, - data::{QuorumProposal, SequencingLeaf, TestableLeaf}, - event::{Event, EventType}, - message::{Message, SequencingMessage}, - traits::{ - election::{ - CommitteeExchange, ConsensusExchange, Membership, QuorumExchange, ViewSyncExchange, - }, - network::CommunicationChannel, - node_implementation::{ - CommitteeEx, ExchangesType, NodeType, QuorumEx, SequencingExchanges, - }, - state::{ConsensusTime, TestableBlock, TestableState}, - }, - HotShotConfig, -}; -use libp2p_identity::{ - ed25519::{self, SecretKey}, - Keypair, -}; -use libp2p_networking::{ - network::{MeshParams, NetworkNodeConfigBuilder, NetworkNodeType}, - reexport::Multiaddr, -}; -use rand::rngs::StdRng; -use rand::SeedableRng; -use std::{collections::BTreeSet, sync::Arc}; -use std::{num::NonZeroUsize, str::FromStr}; -// use libp2p::{ -// identity::{ -// ed25519::{Keypair as EdKeypair, SecretKey}, -// Keypair, -// }, -// multiaddr::{self, Protocol}, -// Multiaddr, -// }; -use libp2p_identity::PeerId; -// use libp2p_networking::network::{MeshParams, NetworkNodeConfigBuilder, NetworkNodeType}; -use std::{fmt::Debug, net::Ipv4Addr}; -use std::{ - //collections::{BTreeSet, VecDeque}, - //fs, - mem, - net::IpAddr, - //num::NonZeroUsize, - //str::FromStr, - //sync::Arc, - //time::{Duration, Instant}, - time::Instant, -}; -//use surf_disco::error::ClientError; -//use surf_disco::Client; -use tracing::{debug, error, info, warn}; - -/// Runs the orchestrator -pub async fn run_orchestrator_da< - TYPES: NodeType, - MEMBERSHIP: Membership + Debug, - DANETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - QUORUMNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - VIEWSYNCNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - NODE: NodeImplementation< - TYPES, - Leaf = SequencingLeaf, - Exchanges = SequencingExchanges< - TYPES, - Message, - QuorumExchange< - TYPES, - SequencingLeaf, - QuorumProposal>, - MEMBERSHIP, - QUORUMNETWORK, - Message, - >, - CommitteeExchange>, - ViewSyncExchange< - TYPES, - ViewSyncCertificate, - MEMBERSHIP, - VIEWSYNCNETWORK, - Message, - >, - >, - Storage = MemoryStorage>, - ConsensusMessage = SequencingMessage, - >, ->( - OrchestratorArgs { - host, - port, - config_file, - }: OrchestratorArgs, -) { - error!("Starting orchestrator",); - let run_config = load_config_from_file::(config_file); - let _result = hotshot_orchestrator::run_orchestrator::< - TYPES::SignatureKey, - TYPES::ElectionConfigType, - >(run_config, host, port) - .await; -} - -/// Helper function to calculate the nuymber of transactions to send per node per round -fn calculate_num_tx_per_round( - node_index: u64, - total_num_nodes: usize, - transactions_per_round: usize, -) -> usize { - if node_index == 0 { - transactions_per_round / total_num_nodes + transactions_per_round % total_num_nodes - } else { - transactions_per_round / total_num_nodes - } -} - -/// Defines the behavior of a "run" of the network with a given configuration -#[async_trait] -pub trait RunDA< - TYPES: NodeType, - MEMBERSHIP: Membership + Debug, - DANETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - QUORUMNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - VIEWSYNCNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - NODE: NodeImplementation< - TYPES, - Leaf = SequencingLeaf, - Exchanges = SequencingExchanges< - TYPES, - Message, - QuorumExchange< - TYPES, - SequencingLeaf, - QuorumProposal>, - MEMBERSHIP, - QUORUMNETWORK, - Message, - >, - CommitteeExchange>, - ViewSyncExchange< - TYPES, - ViewSyncCertificate, - MEMBERSHIP, - VIEWSYNCNETWORK, - Message, - >, - >, - Storage = MemoryStorage>, - ConsensusMessage = SequencingMessage, - >, -> where - ::StateType: TestableState, - ::BlockType: TestableBlock, - SequencingLeaf: TestableLeaf, - Self: Sync, - SystemContext: HotShotType, -{ - /// Initializes networking, returns self - async fn initialize_networking( - config: NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - >, - ) -> Self; - - /// Initializes the genesis state and HotShot instance; does not start HotShot consensus - /// # Panics if it cannot generate a genesis block, fails to initialize HotShot, or cannot - /// get the anchored view - /// Note: sequencing leaf does not have state, so does not return state - async fn initialize_state_and_hotshot(&self) -> SystemContextHandle { - let genesis_block = TYPES::BlockType::genesis(); - let initializer = - hotshot::HotShotInitializer::>::from_genesis( - genesis_block, - ) - .expect("Couldn't generate genesis block"); - - let config = self.get_config(); - - // Get KeyPair for certificate Aggregation - let (pk, sk) = - TYPES::SignatureKey::generated_from_seed_indexed(config.seed, config.node_index); - let known_nodes_with_stake = config.config.known_nodes_with_stake.clone(); - let entry = pk.get_stake_table_entry(1u64); - - let da_network = self.get_da_network(); - let quorum_network = self.get_quorum_network(); - let view_sync_network = self.get_view_sync_network(); - - // Since we do not currently pass the election config type in the NetworkConfig, this will always be the default election config - let quorum_election_config = config.config.election_config.clone().unwrap_or_else(|| { - as ConsensusExchange< - TYPES, - Message, - >>::Membership::default_election_config(config.config.total_nodes.get() as u64) - }); - - let committee_election_config = as ConsensusExchange< - TYPES, - Message, - >>::Membership::default_election_config( - config.config.da_committee_size.try_into().unwrap(), - ); - - let exchanges = NODE::Exchanges::create( - known_nodes_with_stake.clone(), - (quorum_election_config, committee_election_config), - ( - quorum_network.clone(), - da_network.clone(), - view_sync_network.clone(), - ), - pk.clone(), - entry.clone(), - sk.clone(), - ); - - SystemContext::init( - pk, - sk, - config.node_index, - config.config, - MemoryStorage::empty(), - exchanges, - initializer, - ConsensusMetricsValue::new(), - ) - .await - .expect("Could not init hotshot") - .0 - } - - /// Starts HotShot consensus, returns when consensus has finished - async fn run_hotshot(&self, mut context: SystemContextHandle) { - let NetworkConfig { - padding, - rounds, - transactions_per_round, - node_index, - config: HotShotConfig { total_nodes, .. }, - .. - } = self.get_config(); - - let size = mem::size_of::(); - let padding = padding.saturating_sub(size); - let mut txn_rng = StdRng::seed_from_u64(node_index); - - debug!("Adjusted padding size is {:?} bytes", padding); - - let mut total_transactions_committed = 0; - let mut total_transactions_sent = 0; - let transactions_to_send_per_round = - calculate_num_tx_per_round(node_index, total_nodes.get(), transactions_per_round); - - info!("Starting hotshot!"); - let start = Instant::now(); - - let (mut event_stream, _streamid) = context.get_event_stream(FilterEvent::default()).await; - let mut anchor_view: TYPES::Time = ::genesis(); - let mut num_successful_commits = 0; - - context.hotshot.start_consensus().await; - - loop { - match event_stream.next().await { - None => { - panic!("Error! Event stream completed before consensus ended."); - } - Some(Event { event, .. }) => { - match event { - EventType::Error { error } => { - error!("Error in consensus: {:?}", error); - // TODO what to do here - } - EventType::Decide { - leaf_chain, - qc: _, - block_size, - } => { - // this might be a obob - if let Some(leaf) = leaf_chain.get(0) { - info!("Decide event for leaf: {}", *leaf.view_number); - - let new_anchor = leaf.view_number; - if new_anchor >= anchor_view { - anchor_view = leaf.view_number; - } - } - - // send transactions - for _ in 0..transactions_to_send_per_round { - let txn = - <::StateType as TestableState>::create_random_transaction( - None, - &mut txn_rng, - padding as u64, - ); - _ = context.submit_transaction(txn).await.unwrap(); - total_transactions_sent += 1; - } - - if let Some(size) = block_size { - total_transactions_committed += size; - } - - num_successful_commits += leaf_chain.len(); - if num_successful_commits >= rounds { - break; - } - - if leaf_chain.len() > 1 { - warn!("Leaf chain is greater than 1 with len {}", leaf_chain.len()); - } - // when we make progress, submit new events - } - EventType::ReplicaViewTimeout { view_number } => { - warn!("Timed out as a replicas in view {:?}", view_number); - } - EventType::NextLeaderViewTimeout { view_number } => { - warn!("Timed out as the next leader in view {:?}", view_number); - } - EventType::ViewFinished { view_number: _ } => {} - _ => unimplemented!(), - } - } - } - } - - // Output run results - let total_time_elapsed = start.elapsed(); - error!("[{node_index}]: {rounds} rounds completed in {total_time_elapsed:?} - Total transactions sent: {total_transactions_sent} - Total transactions committed: {total_transactions_committed} - Total commitments: {num_successful_commits}"); - } - - /// Returns the da network for this run - fn get_da_network(&self) -> DANETWORK; - - /// Returns the quorum network for this run - fn get_quorum_network(&self) -> QUORUMNETWORK; - - ///Returns view sync network for this run - fn get_view_sync_network(&self) -> VIEWSYNCNETWORK; - - /// Returns the config for this run - fn get_config( - &self, - ) -> NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - >; -} - -// WEB SERVER - -/// Represents a web server-based run -pub struct WebServerDARun< - TYPES: NodeType, - I: NodeImplementation, - MEMBERSHIP: Membership, -> { - config: NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - >, - quorum_network: WebCommChannel, - da_network: WebCommChannel, - view_sync_network: WebCommChannel, -} - -#[async_trait] -impl< - TYPES: NodeType, - MEMBERSHIP: Membership + Debug, - NODE: NodeImplementation< - TYPES, - Leaf = SequencingLeaf, - Exchanges = SequencingExchanges< - TYPES, - Message, - QuorumExchange< - TYPES, - SequencingLeaf, - QuorumProposal>, - MEMBERSHIP, - WebCommChannel, - Message, - >, - CommitteeExchange< - TYPES, - MEMBERSHIP, - WebCommChannel, - Message, - >, - ViewSyncExchange< - TYPES, - ViewSyncCertificate, - MEMBERSHIP, - WebCommChannel, - Message, - >, - >, - Storage = MemoryStorage>, - ConsensusMessage = SequencingMessage, - >, - > - RunDA< - TYPES, - MEMBERSHIP, - WebCommChannel, - WebCommChannel, - WebCommChannel, - NODE, - > for WebServerDARun -where - ::StateType: TestableState, - ::BlockType: TestableBlock, - SequencingLeaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - >, - ) -> WebServerDARun { - // Generate our own key - let (pub_key, _priv_key) = - <::SignatureKey as SignatureKey>::generated_from_seed_indexed( - config.seed, - config.node_index, - ); - - // Get the configuration for the web server - let WebServerConfig { - host, - port, - wait_between_polls, - }: WebServerConfig = config.clone().web_server_config.unwrap(); - - let underlying_quorum_network = WebServerNetwork::create( - &host.to_string(), - port, - wait_between_polls, - pub_key.clone(), - false, - ); - - // Create the network - let quorum_network: WebCommChannel = - WebCommChannel::new(underlying_quorum_network.clone().into()); - - let view_sync_network: WebCommChannel = - WebCommChannel::new(underlying_quorum_network.into()); - - let WebServerConfig { - host, - port, - wait_between_polls, - }: WebServerConfig = config.clone().da_web_server_config.unwrap(); - - // Each node runs the DA network so that leaders have access to transactions and DA votes - let da_network: WebCommChannel = WebCommChannel::new( - WebServerNetwork::create(&host.to_string(), port, wait_between_polls, pub_key, true) - .into(), - ); - - WebServerDARun { - config, - quorum_network, - da_network, - view_sync_network, - } - } - - fn get_da_network(&self) -> WebCommChannel { - self.da_network.clone() - } - - fn get_quorum_network(&self) -> WebCommChannel { - self.quorum_network.clone() - } - - fn get_view_sync_network(&self) -> WebCommChannel { - self.view_sync_network.clone() - } - - fn get_config( - &self, - ) -> NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - > { - self.config.clone() - } -} - -// Libp2p - -/// Represents a libp2p-based run -pub struct Libp2pDARun, MEMBERSHIP: Membership> -{ - config: NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - >, - quorum_network: Libp2pCommChannel, - da_network: Libp2pCommChannel, - view_sync_network: Libp2pCommChannel, -} - -#[async_trait] -impl< - TYPES: NodeType, - MEMBERSHIP: Membership + Debug, - NODE: NodeImplementation< - TYPES, - Leaf = SequencingLeaf, - Exchanges = SequencingExchanges< - TYPES, - Message, - QuorumExchange< - TYPES, - SequencingLeaf, - QuorumProposal>, - MEMBERSHIP, - Libp2pCommChannel, - Message, - >, - CommitteeExchange< - TYPES, - MEMBERSHIP, - Libp2pCommChannel, - Message, - >, - ViewSyncExchange< - TYPES, - ViewSyncCertificate, - MEMBERSHIP, - Libp2pCommChannel, - Message, - >, - >, - Storage = MemoryStorage>, - ConsensusMessage = SequencingMessage, - >, - > - RunDA< - TYPES, - MEMBERSHIP, - Libp2pCommChannel, - Libp2pCommChannel, - Libp2pCommChannel, - NODE, - > for Libp2pDARun -where - ::StateType: TestableState, - ::BlockType: TestableBlock, - SequencingLeaf: TestableLeaf, - Self: Sync, -{ - async fn initialize_networking( - config: NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - >, - ) -> Libp2pDARun { - let (pubkey, _privkey) = - <::SignatureKey as SignatureKey>::generated_from_seed_indexed( - config.seed, - config.node_index, - ); - let mut config = config; - let libp2p_config = config - .libp2p_config - .take() - .expect("Configuration is not for a Libp2p network"); - let bs_len = libp2p_config.bootstrap_nodes.len(); - let bootstrap_nodes: Vec<(PeerId, Multiaddr)> = libp2p_config - .bootstrap_nodes - .iter() - .map(|(addr, pair)| { - let kp = Keypair::from_protobuf_encoding(pair).unwrap(); - let peer_id = PeerId::from_public_key(&kp.public()); - let multiaddr = - Multiaddr::from_str(&format!("/ip4/{}/udp/{}/quic-v1", addr.ip(), addr.port())) - .unwrap(); - (peer_id, multiaddr) - }) - .collect(); - let identity = libp2p_generate_indexed_identity(config.seed, config.node_index); - let node_type = if (config.node_index as usize) < bs_len { - NetworkNodeType::Bootstrap - } else { - NetworkNodeType::Regular - }; - let node_index = config.node_index; - let port_index = match libp2p_config.index_ports { - true => node_index, - false => 0, - }; - let bound_addr: Multiaddr = format!( - "/{}/{}/udp/{}/quic-v1", - if libp2p_config.public_ip.is_ipv4() { - "ip4" - } else { - "ip6" - }, - libp2p_config.public_ip, - libp2p_config.base_port as u64 + port_index - ) - .parse() - .unwrap(); - - // generate network - let mut config_builder = NetworkNodeConfigBuilder::default(); - assert!(config.config.total_nodes.get() > 2); - let replicated_nodes = NonZeroUsize::new(config.config.total_nodes.get() - 2).unwrap(); - config_builder.replication_factor(replicated_nodes); - config_builder.identity(identity.clone()); - - config_builder.bound_addr(Some(bound_addr.clone())); - - let to_connect_addrs = bootstrap_nodes - .iter() - .map(|(peer_id, multiaddr)| (Some(*peer_id), multiaddr.clone())) - .collect(); - - config_builder.to_connect_addrs(to_connect_addrs); - - let mesh_params = - // NOTE I'm arbitrarily choosing these. - match node_type { - NetworkNodeType::Bootstrap => MeshParams { - mesh_n_high: libp2p_config.bootstrap_mesh_n_high, - mesh_n_low: libp2p_config.bootstrap_mesh_n_low, - mesh_outbound_min: libp2p_config.bootstrap_mesh_outbound_min, - mesh_n: libp2p_config.bootstrap_mesh_n, - }, - NetworkNodeType::Regular => MeshParams { - mesh_n_high: libp2p_config.mesh_n_high, - mesh_n_low: libp2p_config.mesh_n_low, - mesh_outbound_min: libp2p_config.mesh_outbound_min, - mesh_n: libp2p_config.mesh_n, - }, - NetworkNodeType::Conductor => unreachable!(), - }; - config_builder.mesh_params(Some(mesh_params)); - - let mut all_keys = BTreeSet::new(); - let mut da_keys = BTreeSet::new(); - for i in 0..config.config.total_nodes.get() as u64 { - let privkey = TYPES::SignatureKey::generated_from_seed_indexed([0u8; 32], i).1; - let pubkey = TYPES::SignatureKey::from_private(&privkey); - if i < config.config.da_committee_size as u64 { - da_keys.insert(pubkey.clone()); - } - all_keys.insert(pubkey); - } - - let node_config = config_builder.build().unwrap(); - let underlying_quorum_network = Libp2pNetwork::new( - NetworkingMetricsValue::new(), - node_config, - pubkey.clone(), - Arc::new(RwLock::new( - bootstrap_nodes - .iter() - .map(|(peer_id, addr)| (Some(*peer_id), addr.clone())) - .collect(), - )), - bs_len, - config.node_index as usize, - // NOTE: this introduces an invariant that the keys are assigned using this indexed - // function - all_keys, - da_keys, - None, - ) - .await - .unwrap(); - - underlying_quorum_network.wait_for_ready().await; - - // Create the network - let quorum_network: Libp2pCommChannel = - Libp2pCommChannel::new(underlying_quorum_network.clone().into()); - - let view_sync_network: Libp2pCommChannel = - Libp2pCommChannel::new(underlying_quorum_network.clone().into()); - - let da_network: Libp2pCommChannel = - Libp2pCommChannel::new(underlying_quorum_network.clone().into()); - - Libp2pDARun { - config, - quorum_network, - da_network, - view_sync_network, - } - } - - fn get_da_network(&self) -> Libp2pCommChannel { - self.da_network.clone() - } - - fn get_quorum_network(&self) -> Libp2pCommChannel { - self.quorum_network.clone() - } - - fn get_view_sync_network(&self) -> Libp2pCommChannel { - self.view_sync_network.clone() - } - - fn get_config( - &self, - ) -> NetworkConfig< - TYPES::SignatureKey, - ::StakeTableEntry, - TYPES::ElectionConfigType, - > { - self.config.clone() - } -} - -/// Main entry point for validators -pub async fn main_entry_point< - TYPES: NodeType, - MEMBERSHIP: Membership + Debug, - DANETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - QUORUMNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - VIEWSYNCNETWORK: CommunicationChannel, MEMBERSHIP> + Debug, - NODE: NodeImplementation< - TYPES, - Leaf = SequencingLeaf, - Exchanges = SequencingExchanges< - TYPES, - Message, - QuorumExchange< - TYPES, - SequencingLeaf, - QuorumProposal>, - MEMBERSHIP, - QUORUMNETWORK, - Message, - >, - CommitteeExchange>, - ViewSyncExchange< - TYPES, - ViewSyncCertificate, - MEMBERSHIP, - VIEWSYNCNETWORK, - Message, - >, - >, - Storage = MemoryStorage>, - ConsensusMessage = SequencingMessage, - >, - RUNDA: RunDA, ->( - args: ValidatorArgs, -) where - ::StateType: TestableState, - ::BlockType: TestableBlock, - SequencingLeaf: TestableLeaf, -{ - setup_logging(); - setup_backtrace(); - - info!("Starting validator"); - - let orchestrator_client: OrchestratorClient = - OrchestratorClient::connect_to_orchestrator(args.clone()).await; - - // Identify with the orchestrator - let public_ip = match args.public_ip { - Some(ip) => ip, - None => IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), - }; - info!( - "Identifying with orchestrator using IP address {}", - public_ip.to_string() - ); - let node_index: u16 = orchestrator_client - .identify_with_orchestrator(public_ip.to_string()) - .await; - info!("Finished identifying; our node index is {node_index}"); - info!("Getting config from orchestrator"); - - let mut run_config = orchestrator_client - .get_config_from_orchestrator::(node_index) - .await; - - run_config.node_index = node_index.into(); - //run_config.libp2p_config.as_mut().unwrap().public_ip = args.public_ip.unwrap(); - - info!("Initializing networking"); - let run = RUNDA::initialize_networking(run_config.clone()).await; - let hotshot = run.initialize_state_and_hotshot().await; - - info!("Waiting for start command from orchestrator"); - orchestrator_client - .wait_for_all_nodes_ready(run_config.clone().node_index) - .await; - - info!("All nodes are ready! Starting HotShot"); - run.run_hotshot(hotshot).await; -} - -pub fn libp2p_generate_indexed_identity(seed: [u8; 32], index: u64) -> Keypair { - let mut hasher = blake3::Hasher::new(); - hasher.update(&seed); - hasher.update(&index.to_le_bytes()); - let new_seed = *hasher.finalize().as_bytes(); - let sk_bytes = SecretKey::try_from_bytes(new_seed).unwrap(); - >::from(sk_bytes).into() -} From 5e512c4cdbfe144a447451151f9e7d44cbca4dc9 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sun, 26 Nov 2023 10:33:10 -0500 Subject: [PATCH 06/18] feat: incorporating suggested changes --- .../hotshot/src/traits/networking/libp2p_network.rs | 11 +++++------ .../src/traits/networking/web_server_network.rs | 3 ++- crates/testing/tests/unreliable_network.rs | 8 ++++---- crates/types/src/traits/network.rs | 9 +++++---- justfile | 13 ++++++++++--- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index e4364c97c2..6c00fef7b3 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -113,6 +113,7 @@ struct Libp2pNetworkInner { /// NOTE: supposed to represent a ViewNumber but we /// haven't made that atomic yet and we prefer lock-free latest_seen_view: Arc, + #[cfg(feature = "hotshot-testing")] /// reliability_config reliability_config: Option>, /// if we're a member of the DA committee or not @@ -346,6 +347,7 @@ impl Libp2pNetwork { // proposals on". We need this because to have consensus info injected we need a working // network already. In the worst case, we send a few lookups we don't need. latest_seen_view: Arc::new(AtomicU64::new(0)), + #[cfg(feature = "hotshot-testing")] reliability_config, is_da, }), @@ -614,10 +616,9 @@ impl ConnectedNetwork for Libp2p .map_err(|_| NetworkError::ShutDown)?; } - // TODO maybe we should lift the metrics mutex up a level and copy the inner pattern - // ask during pair programming - // or maybe channels would be better? + // NOTE: metrics is threadsafe, so clone is fine (and lightweight) let metrics = self.inner.metrics.clone(); + #[cfg(feature = "hotshot-testing")] if let Some(ref config) = &self.inner.reliability_config { let handle = self.inner.handle.clone(); @@ -695,10 +696,8 @@ impl ConnectedNetwork for Libp2p } }; - // TODO maybe we should lift the metrics mutex up a level and copy the inner pattern - // ask during pair programming - // or maybe channels would be better? let metrics = self.inner.metrics.clone(); + #[cfg(feature = "hotshot-testing")] if let Some(ref config) = &self.inner.reliability_config { let handle = self.inner.handle.clone(); diff --git a/crates/hotshot/src/traits/networking/web_server_network.rs b/crates/hotshot/src/traits/networking/web_server_network.rs index f1ca099e91..9eb174df03 100644 --- a/crates/hotshot/src/traits/networking/web_server_network.rs +++ b/crates/hotshot/src/traits/networking/web_server_network.rs @@ -1277,7 +1277,8 @@ impl, MEMBERSHIP: Membership Duration { Duration::from_millis( - Uniform::new_inclusive(self.delay_low_ms, self.timeout_ms) + Uniform::new_inclusive(self.delay_low_ms, self.delay_high_ms) .sample(&mut rand::thread_rng()), ) } @@ -580,7 +581,7 @@ impl SynchronousNetwork { #[must_use] pub fn new(timeout: u64, delay_low_ms: u64) -> Self { SynchronousNetwork { - timeout_ms: timeout, + delay_high_ms: timeout, delay_low_ms, } } diff --git a/justfile b/justfile index 07ec21631b..e3addf6e58 100644 --- a/justfile +++ b/justfile @@ -16,14 +16,17 @@ run_ci: lint build test export RUST_MIN_STACK=4194304 RUSTDOCFLAGS='--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std" {{original_rustdocflags}}' RUSTFLAGS='--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std" {{original_rustflags}}' && just {{target}} {{ARGS}} build: - cargo build --workspace --examples --bins --tests --lib --benches + cargo build --workspace --examples --bins --tests --lib --benches --release + +build_release: + cargo build --workspace --exclude hotshot-testing --bins --examples --no-default-features --features="demo, docs, doc-images" example *ARGS: cargo run --profile=release-lto --example {{ARGS}} test: echo Testing - cargo test --verbose --lib --bins --tests --benches --workspace --no-fail-fast -- --test-threads=1 --nocapture + cargo test --verbose --lib --bins --tests --benches --workspace --no-fail-fast test_memory_network_chaos -- --test-threads=1 --nocapture test_basic: test_success test_with_failures test_network_task test_consensus_task test_da_task test_view_sync_task @@ -37,7 +40,7 @@ test_success: test_timeout: echo Testing timeout test - ASYNC_STD_THREAD_COUNT=1 cargo test --lib --bins --tests --benches --workspace --no-fail-fast test_timeout -- --test-threads=1 --nocapture + ASYNC_STD_THREAD_COUNT=1 cargo test --lib --bins --tests --benches --workspace --no-fail-fast test_timeout -- --test-threads=1 --nocapture test_web_server: echo Testing web server @@ -94,6 +97,10 @@ lint: fmt echo linting cargo clippy --workspace --bins --tests --examples -- -D warnings +lint_release: fmt + echo linting + cargo clippy --workspace --exclude hotshot-testing --bins --examples --no-default-features --features="demo, docs, doc-images" -- -D warnings + fmt: echo Running cargo fmt cargo fmt From 743451e9bc62f75de013a350537ee8a49a2b647a Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sun, 26 Nov 2023 11:10:50 -0500 Subject: [PATCH 07/18] chore: fix lints when compiling without hotshot-testing --- crates/hotshot/Cargo.toml | 135 ++++++++--------- crates/hotshot/examples/infra/mod.rs | 1 - crates/hotshot/src/lib.rs | 7 +- .../src/traits/networking/combined_network.rs | 9 +- .../src/traits/networking/libp2p_network.rs | 139 ++++++++++-------- crates/hotshot/src/types/handle.rs | 12 +- 6 files changed, 164 insertions(+), 139 deletions(-) diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index f0a62be572..3eb091a32b 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -28,75 +28,76 @@ hotshot-testing = [] # libp2p [[example]] name = "validator-libp2p" -required-features = ["demo", "libp2p/rsa"] +features = ["demo", "libp2p/rsa"] +excluded-features = ["hotshot-testing"] path = "examples/libp2p/validator.rs" -[[example]] -name = "multi-validator-libp2p" -required-features = ["demo", "libp2p/rsa"] -path = "examples/libp2p/multi-validator.rs" - -[[example]] -name = "orchestrator-libp2p" -required-features = ["demo", "libp2p/rsa"] -path = "examples/libp2p/orchestrator.rs" - -[[example]] -name = "all-libp2p" -required-features = ["demo", "libp2p/rsa"] -path = "examples/libp2p/all.rs" - -# webserver -[[example]] -name = "webserver" -required-features = ["demo", "libp2p/rsa"] -path = "examples/webserver/webserver.rs" - -[[example]] -name = "orchestrator-webserver" -required-features = ["demo", "libp2p/rsa"] -path = "examples/webserver/orchestrator.rs" - -[[example]] -name = "validator-webserver" -required-features = ["demo", "libp2p/rsa"] -path = "examples/webserver/validator.rs" - -[[example]] -name = "multi-validator-webserver" -required-features = ["demo", "libp2p/rsa"] -path = "examples/webserver/multi-validator.rs" - -[[example]] -name = "multi-webserver" -required-features = ["demo", "libp2p/rsa"] -path = "examples/webserver/multi-webserver.rs" - -[[example]] -name = "all-webserver" -required-features = ["demo", "libp2p/rsa"] -path = "examples/webserver/all.rs" - -# combined -[[example]] -name = "all-combined" -required-features = ["demo", "libp2p/rsa"] -path = "examples/combined/all.rs" - -[[example]] -name = "multi-validator-combined" -required-features = ["demo", "libp2p/rsa"] -path = "examples/combined/multi-validator.rs" - -[[example]] -name = "validator-combined" -required-features = ["demo", "libp2p/rsa"] -path = "examples/combined/validator.rs" - -[[example]] -name = "orchestrator-combined" -required-features = ["demo", "libp2p/rsa"] -path = "examples/combined/orchestrator.rs" +# [[example]] +# name = "multi-validator-libp2p" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/libp2p/multi-validator.rs" +# +# [[example]] +# name = "orchestrator-libp2p" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/libp2p/orchestrator.rs" +# +# [[example]] +# name = "all-libp2p" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/libp2p/all.rs" +# +# # webserver +# [[example]] +# name = "webserver" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/webserver/webserver.rs" +# +# [[example]] +# name = "orchestrator-webserver" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/webserver/orchestrator.rs" +# +# [[example]] +# name = "validator-webserver" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/webserver/validator.rs" +# +# [[example]] +# name = "multi-validator-webserver" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/webserver/multi-validator.rs" +# +# [[example]] +# name = "multi-webserver" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/webserver/multi-webserver.rs" +# +# [[example]] +# name = "all-webserver" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/webserver/all.rs" +# +# # combined +# [[example]] +# name = "all-combined" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/combined/all.rs" +# +# [[example]] +# name = "multi-validator-combined" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/combined/multi-validator.rs" +# +# [[example]] +# name = "validator-combined" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/combined/validator.rs" +# +# [[example]] +# name = "orchestrator-combined" +# required-features = ["demo", "libp2p/rsa"] +# path = "examples/combined/orchestrator.rs" [dependencies] async-compatibility-layer = { workspace = true } diff --git a/crates/hotshot/examples/infra/mod.rs b/crates/hotshot/examples/infra/mod.rs index 149ebf3848..94013545c0 100644 --- a/crates/hotshot/examples/infra/mod.rs +++ b/crates/hotshot/examples/infra/mod.rs @@ -291,7 +291,6 @@ async fn libp2p_network_from_config( // NOTE: this introduces an invariant that the keys are assigned using this indexed // function all_keys, - None, da_keys.clone(), da_keys.contains(&pub_key), ) diff --git a/crates/hotshot/src/lib.rs b/crates/hotshot/src/lib.rs index afbc586e1e..5ec6a64be8 100644 --- a/crates/hotshot/src/lib.rs +++ b/crates/hotshot/src/lib.rs @@ -51,6 +51,9 @@ use hotshot_task::{ }; use hotshot_task_impls::{events::HotShotEvent, network::NetworkTaskKind}; +#[cfg(feature = "hotshot-testing")] +use hotshot_types::traits::node_implementation::ChannelMaps; + use hotshot_types::{ consensus::{BlockPayloadStore, Consensus, ConsensusMetricsValue, View, ViewInner, ViewQueue}, data::Leaf, @@ -63,7 +66,7 @@ use hotshot_types::{ traits::{ consensus_api::{ConsensusApi, ConsensusSharedApi}, network::{CommunicationChannel, NetworkError}, - node_implementation::{ChannelMaps, NodeType, SendToTasks}, + node_implementation::{NodeType, SendToTasks}, signature_key::SignatureKey, state::ConsensusTime, storage::StoredView, @@ -158,6 +161,7 @@ pub struct SystemContextInner> { /// Channels for sending/recv-ing proposals and votes for quorum and committee exchanges, the /// latter of which is only applicable for sequencing consensus. + #[cfg(feature = "hotshot-testing")] channel_maps: (ChannelMaps, Option>), // global_registry: GlobalRegistry, @@ -242,6 +246,7 @@ impl> SystemContext { let inner: Arc> = Arc::new(SystemContextInner { id: nonce, + #[cfg(feature = "hotshot-testing")] channel_maps: I::new_channel_maps(start_view), consensus, public_key, diff --git a/crates/hotshot/src/traits/networking/combined_network.rs b/crates/hotshot/src/traits/networking/combined_network.rs index 6b7ab6d4b9..ebe4f2d98e 100644 --- a/crates/hotshot/src/traits/networking/combined_network.rs +++ b/crates/hotshot/src/traits/networking/combined_network.rs @@ -20,15 +20,16 @@ use futures::join; use async_compatibility_layer::channel::UnboundedSendError; use hotshot_task::{boxed_sync, BoxSyncFuture}; +#[cfg(feature = "hotshot-testing")] +use hotshot_types::traits::network::{NetworkReliability, TestableNetworkingImplementation}; use hotshot_types::{ data::ViewNumber, message::Message, traits::{ election::Membership, network::{ - CommunicationChannel, ConnectedNetwork, ConsensusIntentEvent, NetworkReliability, - TestableChannelImplementation, TestableNetworkingImplementation, TransmitType, - ViewMessage, + CommunicationChannel, ConnectedNetwork, ConsensusIntentEvent, + TestableChannelImplementation, TransmitType, ViewMessage, }, node_implementation::NodeType, }, @@ -144,6 +145,7 @@ pub struct CombinedNetworks( pub Libp2pNetwork, TYPES::SignatureKey>, ); +#[cfg(feature = "hotshot-testing")] impl TestableNetworkingImplementation for CombinedNetworks { fn generator( expected_node_count: usize, @@ -184,6 +186,7 @@ impl TestableNetworkingImplementation for CombinedNetwor } } +#[cfg(feature = "hotshot-testing")] impl TestableNetworkingImplementation for CombinedCommChannel { fn generator( expected_node_count: usize, diff --git a/crates/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/src/traits/networking/libp2p_network.rs index 14ad5d847a..a1de58bf8e 100644 --- a/crates/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/src/traits/networking/libp2p_network.rs @@ -2,8 +2,10 @@ //! This module provides a libp2p based networking implementation where each node in the //! network forms a tcp or udp connection to a subset of other nodes in the network use super::NetworkingMetricsValue; +#[cfg(feature = "hotshot-testing")] +use async_compatibility_layer::art::async_block_on; use async_compatibility_layer::{ - art::{async_block_on, async_sleep, async_spawn}, + art::{async_sleep, async_spawn}, channel::{unbounded, UnboundedReceiver, UnboundedSendError, UnboundedSender}, }; use async_lock::RwLock; @@ -12,6 +14,8 @@ use bimap::BiHashMap; use bincode::Options; use hotshot_constants::LOOK_AHEAD; use hotshot_task::{boxed_sync, BoxSyncFuture}; +#[cfg(feature = "hotshot-testing")] +use hotshot_types::traits::network::{NetworkReliability, TestableNetworkingImplementation}; use hotshot_types::{ data::ViewNumber, message::{Message, MessageKind}, @@ -19,34 +23,36 @@ use hotshot_types::{ election::Membership, network::{ CommunicationChannel, ConnectedNetwork, ConsensusIntentEvent, FailedToSerializeSnafu, - NetworkError, NetworkMsg, NetworkReliability, TestableChannelImplementation, - TestableNetworkingImplementation, TransmitType, ViewMessage, + NetworkError, NetworkMsg, TestableChannelImplementation, TransmitType, ViewMessage, }, node_implementation::NodeType, signature_key::SignatureKey, state::ConsensusTime, }, }; + use hotshot_utils::bincode::bincode_opts; use libp2p_identity::PeerId; +#[cfg(feature = "hotshot-testing")] +use libp2p_networking::network::{MeshParams, NetworkNodeConfigBuilder}; + use libp2p_networking::{ network::{ - MeshParams, NetworkEvent::{self, DirectRequest, DirectResponse, GossipMsg}, - NetworkNodeConfig, NetworkNodeConfigBuilder, NetworkNodeHandle, NetworkNodeHandleError, - NetworkNodeType, + NetworkNodeConfig, NetworkNodeHandle, NetworkNodeHandleError, NetworkNodeType, }, reexport::Multiaddr, }; use serde::Serialize; use snafu::ResultExt; +#[cfg(feature = "hotshot-testing")] +use std::{collections::HashSet, num::NonZeroUsize, str::FromStr}; + use std::{ - collections::{BTreeSet, HashSet}, + collections::BTreeSet, fmt::Debug, marker::PhantomData, - num::NonZeroUsize, - str::FromStr, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, Arc, @@ -127,6 +133,7 @@ pub struct Libp2pNetwork { inner: Arc>, } +#[cfg(feature = "hotshot-testing")] impl TestableNetworkingImplementation for Libp2pNetwork, TYPES::SignatureKey> where @@ -240,6 +247,7 @@ where num_bootstrap, node_id as usize, keys, + #[cfg(feature = "hotshot-testing")] reliability_config_dup, da.clone(), da.contains(&pubkey), @@ -295,7 +303,7 @@ impl Libp2pNetwork { id: usize, // HACK committee_pks: BTreeSet, - reliability_config: Option>, + #[cfg(feature = "hotshot-testing")] reliability_config: Option>, da_pks: BTreeSet, is_da: bool, ) -> Result, NetworkError> { @@ -622,35 +630,37 @@ impl ConnectedNetwork for Libp2p } // NOTE: metrics is threadsafe, so clone is fine (and lightweight) - let metrics = self.inner.metrics.clone(); #[cfg(feature = "hotshot-testing")] - if let Some(ref config) = &self.inner.reliability_config { - let handle = self.inner.handle.clone(); - - let serialized_msg = bincode_opts() - .serialize(&message) - .context(FailedToSerializeSnafu)?; - let fut = config.clone().chaos_send_msg( - serialized_msg, - Arc::new(move |msg: Vec| { - let topic_2 = topic.clone(); - let handle_2 = handle.clone(); - let metrics_2 = metrics.clone(); - boxed_sync(async move { - match handle_2.gossip_no_serialize(topic_2, msg).await { - Err(e) => { - metrics_2.message_failed_to_send.add(1); - warn!("Failed to broadcast to libp2p: {:?}", e); - } - Ok(()) => { - metrics_2.outgoing_direct_message_count.add(1); + { + let metrics = self.inner.metrics.clone(); + if let Some(ref config) = &self.inner.reliability_config { + let handle = self.inner.handle.clone(); + + let serialized_msg = bincode_opts() + .serialize(&message) + .context(FailedToSerializeSnafu)?; + let fut = config.clone().chaos_send_msg( + serialized_msg, + Arc::new(move |msg: Vec| { + let topic_2 = topic.clone(); + let handle_2 = handle.clone(); + let metrics_2 = metrics.clone(); + boxed_sync(async move { + match handle_2.gossip_no_serialize(topic_2, msg).await { + Err(e) => { + metrics_2.message_failed_to_send.add(1); + warn!("Failed to broadcast to libp2p: {:?}", e); + } + Ok(()) => { + metrics_2.outgoing_direct_message_count.add(1); + } } - } - }) - }), - ); - async_spawn(fut); - return Ok(()); + }) + }), + ); + async_spawn(fut); + return Ok(()); + } } match self.inner.handle.gossip(topic, &message).await { @@ -701,34 +711,36 @@ impl ConnectedNetwork for Libp2p } }; - let metrics = self.inner.metrics.clone(); #[cfg(feature = "hotshot-testing")] - if let Some(ref config) = &self.inner.reliability_config { - let handle = self.inner.handle.clone(); - - let serialized_msg = bincode_opts() - .serialize(&message) - .context(FailedToSerializeSnafu)?; - let fut = config.clone().chaos_send_msg( - serialized_msg, - Arc::new(move |msg: Vec| { - let handle_2 = handle.clone(); - let metrics_2 = metrics.clone(); - boxed_sync(async move { - match handle_2.direct_request_no_serialize(pid, msg).await { - Err(e) => { - metrics_2.message_failed_to_send.add(1); - warn!("Failed to broadcast to libp2p: {:?}", e); - } - Ok(()) => { - metrics_2.outgoing_direct_message_count.add(1); + { + let metrics = self.inner.metrics.clone(); + if let Some(ref config) = &self.inner.reliability_config { + let handle = self.inner.handle.clone(); + + let serialized_msg = bincode_opts() + .serialize(&message) + .context(FailedToSerializeSnafu)?; + let fut = config.clone().chaos_send_msg( + serialized_msg, + Arc::new(move |msg: Vec| { + let handle_2 = handle.clone(); + let metrics_2 = metrics.clone(); + boxed_sync(async move { + match handle_2.direct_request_no_serialize(pid, msg).await { + Err(e) => { + metrics_2.message_failed_to_send.add(1); + warn!("Failed to broadcast to libp2p: {:?}", e); + } + Ok(()) => { + metrics_2.outgoing_direct_message_count.add(1); + } } - } - }) - }), - ); - async_spawn(fut); - return Ok(()); + }) + }), + ); + async_spawn(fut); + return Ok(()); + } } match self.inner.handle.direct_request(pid, &message).await { @@ -832,6 +844,7 @@ impl Libp2pCommChannel { } } +#[cfg(feature = "hotshot-testing")] impl TestableNetworkingImplementation for Libp2pCommChannel where MessageKind: ViewMessage, diff --git a/crates/hotshot/src/types/handle.rs b/crates/hotshot/src/types/handle.rs index 87975429e4..9c576043ca 100644 --- a/crates/hotshot/src/types/handle.rs +++ b/crates/hotshot/src/types/handle.rs @@ -13,15 +13,19 @@ use hotshot_task::{ BoxSyncFuture, }; use hotshot_task_impls::events::HotShotEvent; +#[cfg(feature = "hotshot-testing")] +use hotshot_types::{ + message::{MessageKind, SequencingMessage}, + traits::election::Membership, +}; + use hotshot_types::simple_vote::QuorumData; + use hotshot_types::{ consensus::Consensus, error::HotShotError, event::EventType, - message::{MessageKind, SequencingMessage}, - traits::{ - election::Membership, node_implementation::NodeType, state::ConsensusTime, storage::Storage, - }, + traits::{node_implementation::NodeType, state::ConsensusTime, storage::Storage}, }; use hotshot_types::{data::Leaf, simple_certificate::QuorumCertificate}; use std::sync::Arc; From 90548598a8bb4a81399fe1b019951be805a58a4a Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sun, 26 Nov 2023 11:17:00 -0500 Subject: [PATCH 08/18] chore: fix lints when compiling normally --- crates/hotshot/Cargo.toml | 135 +++++++++++++++++++------------------- justfile | 4 +- 2 files changed, 69 insertions(+), 70 deletions(-) diff --git a/crates/hotshot/Cargo.toml b/crates/hotshot/Cargo.toml index 3eb091a32b..f0a62be572 100644 --- a/crates/hotshot/Cargo.toml +++ b/crates/hotshot/Cargo.toml @@ -28,76 +28,75 @@ hotshot-testing = [] # libp2p [[example]] name = "validator-libp2p" -features = ["demo", "libp2p/rsa"] -excluded-features = ["hotshot-testing"] +required-features = ["demo", "libp2p/rsa"] path = "examples/libp2p/validator.rs" -# [[example]] -# name = "multi-validator-libp2p" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/libp2p/multi-validator.rs" -# -# [[example]] -# name = "orchestrator-libp2p" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/libp2p/orchestrator.rs" -# -# [[example]] -# name = "all-libp2p" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/libp2p/all.rs" -# -# # webserver -# [[example]] -# name = "webserver" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/webserver/webserver.rs" -# -# [[example]] -# name = "orchestrator-webserver" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/webserver/orchestrator.rs" -# -# [[example]] -# name = "validator-webserver" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/webserver/validator.rs" -# -# [[example]] -# name = "multi-validator-webserver" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/webserver/multi-validator.rs" -# -# [[example]] -# name = "multi-webserver" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/webserver/multi-webserver.rs" -# -# [[example]] -# name = "all-webserver" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/webserver/all.rs" -# -# # combined -# [[example]] -# name = "all-combined" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/combined/all.rs" -# -# [[example]] -# name = "multi-validator-combined" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/combined/multi-validator.rs" -# -# [[example]] -# name = "validator-combined" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/combined/validator.rs" -# -# [[example]] -# name = "orchestrator-combined" -# required-features = ["demo", "libp2p/rsa"] -# path = "examples/combined/orchestrator.rs" +[[example]] +name = "multi-validator-libp2p" +required-features = ["demo", "libp2p/rsa"] +path = "examples/libp2p/multi-validator.rs" + +[[example]] +name = "orchestrator-libp2p" +required-features = ["demo", "libp2p/rsa"] +path = "examples/libp2p/orchestrator.rs" + +[[example]] +name = "all-libp2p" +required-features = ["demo", "libp2p/rsa"] +path = "examples/libp2p/all.rs" + +# webserver +[[example]] +name = "webserver" +required-features = ["demo", "libp2p/rsa"] +path = "examples/webserver/webserver.rs" + +[[example]] +name = "orchestrator-webserver" +required-features = ["demo", "libp2p/rsa"] +path = "examples/webserver/orchestrator.rs" + +[[example]] +name = "validator-webserver" +required-features = ["demo", "libp2p/rsa"] +path = "examples/webserver/validator.rs" + +[[example]] +name = "multi-validator-webserver" +required-features = ["demo", "libp2p/rsa"] +path = "examples/webserver/multi-validator.rs" + +[[example]] +name = "multi-webserver" +required-features = ["demo", "libp2p/rsa"] +path = "examples/webserver/multi-webserver.rs" + +[[example]] +name = "all-webserver" +required-features = ["demo", "libp2p/rsa"] +path = "examples/webserver/all.rs" + +# combined +[[example]] +name = "all-combined" +required-features = ["demo", "libp2p/rsa"] +path = "examples/combined/all.rs" + +[[example]] +name = "multi-validator-combined" +required-features = ["demo", "libp2p/rsa"] +path = "examples/combined/multi-validator.rs" + +[[example]] +name = "validator-combined" +required-features = ["demo", "libp2p/rsa"] +path = "examples/combined/validator.rs" + +[[example]] +name = "orchestrator-combined" +required-features = ["demo", "libp2p/rsa"] +path = "examples/combined/orchestrator.rs" [dependencies] async-compatibility-layer = { workspace = true } diff --git a/justfile b/justfile index 61d712edd8..cec7ff7455 100644 --- a/justfile +++ b/justfile @@ -98,7 +98,7 @@ check: lint: fmt echo linting - cargo clippy --workspace --bins --tests --examples -- -D warnings + cargo clippy --workspace --bins --tests -- -D warnings lint_release: fmt echo linting @@ -129,4 +129,4 @@ lint_imports: gen_key_pair: echo Generating key pair from config file in config/ - cargo test --package hotshot-testing --test gen_key_pair -- tests --nocapture + cargo test --package hotshot-testing --test gen_key_pair -- tests --nocapture From 2003f6bee55206296caa19ffaeeae66245bf6316 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sat, 2 Dec 2023 11:51:01 -0500 Subject: [PATCH 09/18] feat: move examples to build_release only --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 88a65a1764..cbe7fd184e 100644 --- a/justfile +++ b/justfile @@ -16,7 +16,7 @@ run_ci: lint build test export RUST_MIN_STACK=4194304 RUSTDOCFLAGS='--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std" {{original_rustdocflags}}' RUSTFLAGS='--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std" {{original_rustflags}}' && just {{target}} {{ARGS}} build: - cargo build --workspace --examples --bins --tests --lib --benches --release + cargo build --workspace --bins --tests --lib --benches --release build_release: cargo build --workspace --exclude hotshot-testing --bins --examples --no-default-features --features="demo, docs, doc-images" From a376007d8a967da5045d19c95739315d13a3fbb4 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sat, 2 Dec 2023 11:53:55 -0500 Subject: [PATCH 10/18] fix: accidental test breakage --- crates/task/src/task_impls.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/task/src/task_impls.rs b/crates/task/src/task_impls.rs index 4ff8dd1045..768e011775 100644 --- a/crates/task/src/task_impls.rs +++ b/crates/task/src/task_impls.rs @@ -300,6 +300,7 @@ pub mod test { )] #[cfg_attr(async_executor_impl = "async-std", async_std::test)] #[allow(clippy::should_panic_without_expect)] + #[should_panic] async fn test_init_with_event_stream() { setup_logging(); let task = TaskBuilder::::new("Test Task".to_string()); From 9797c8f2a032c259ff9c55d7e7c1eb82c156aae8 Mon Sep 17 00:00:00 2001 From: Keyao Shen Date: Thu, 4 Jan 2024 11:58:37 -0800 Subject: [PATCH 11/18] Fix typo --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 12e1acd17c..9e6bba730a 100644 --- a/justfile +++ b/justfile @@ -136,4 +136,4 @@ lint_imports: gen_key_pair: echo Generating key pair from config file in config/ - cargo test --package hotshot-testing --test gen_key_pair socapture + cargo test --package hotshot-testing --test gen_key_pair -- tests --nocapture From b783e8a3e865dbad8648503683281cb7b733c530 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Tue, 9 Jan 2024 11:39:15 -0500 Subject: [PATCH 12/18] feat: parameter tweak for async tests --- crates/testing/src/overall_safety_task.rs | 2 ++ crates/testing/tests/unreliable_network.rs | 27 +++++++++++++++++++--- justfile | 6 ++--- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/crates/testing/src/overall_safety_task.rs b/crates/testing/src/overall_safety_task.rs index f6b6a96c4f..5d703d941e 100644 --- a/crates/testing/src/overall_safety_task.rs +++ b/crates/testing/src/overall_safety_task.rs @@ -22,6 +22,7 @@ use std::{ collections::{hash_map::Entry, HashMap, HashSet}, sync::Arc, }; +use tracing::error; use crate::{test_launcher::TaskGenerator, test_runner::Node}; pub type StateAndBlock = (Vec, Vec); @@ -254,6 +255,7 @@ impl RoundResult { let remaining_nodes = total_num_nodes - (num_decided + num_failed); if check_leaf && self.leaf_map.len() != 1 { + error!("LEAF MAP (that is mismatched) IS: {:?}", self.leaf_map); self.status = ViewStatus::Err(OverallSafetyTaskErr::MismatchedLeaf); return; } diff --git a/crates/testing/tests/unreliable_network.rs b/crates/testing/tests/unreliable_network.rs index 7c2457c65c..b959185abc 100644 --- a/crates/testing/tests/unreliable_network.rs +++ b/crates/testing/tests/unreliable_network.rs @@ -1,3 +1,4 @@ +use hotshot_testing::test_builder::TimingData; use hotshot_types::traits::network::AsynchronousNetwork; use hotshot_types::traits::network::ChaosNetwork; use hotshot_types::traits::network::PartiallySynchronousNetwork; @@ -87,6 +88,7 @@ async fn test_memory_network_sync() { tokio::test(flavor = "multi_thread", worker_threads = 2) )] #[cfg_attr(async_executor_impl = "async-std", async_std::test)] +#[ignore] #[instrument] async fn libp2p_network_async() { async_compatibility_layer::logging::setup_logging(); @@ -94,6 +96,7 @@ async fn libp2p_network_async() { let metadata = TestMetadata { overall_safety_properties: OverallSafetyPropertiesDescription { check_leaf: true, + num_failed_views: 50, ..Default::default() }, completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( @@ -101,8 +104,13 @@ async fn libp2p_network_async() { duration: Duration::new(240, 0), }, ), + timing_data: TimingData { + timeout_ratio: (1, 1), + next_view_timeout: 1000, + ..TestMetadata::default_multiple_rounds().timing_data + }, unreliable_network: Some(Box::new(AsynchronousNetwork { - keep_numerator: 8, + keep_numerator: 9, keep_denominator: 10, delay_low_ms: 4, delay_high_ms: 30, @@ -122,6 +130,7 @@ async fn libp2p_network_async() { async_executor_impl = "tokio", tokio::test(flavor = "multi_thread", worker_threads = 2) )] +#[ignore] #[cfg_attr(async_executor_impl = "async-std", async_std::test)] async fn test_memory_network_async() { use hotshot_testing::{ @@ -134,15 +143,25 @@ async fn test_memory_network_async() { async_compatibility_layer::logging::setup_logging(); async_compatibility_layer::logging::setup_backtrace(); let metadata = TestMetadata { + overall_safety_properties: OverallSafetyPropertiesDescription { + check_leaf: true, + num_failed_views: 5000, + ..Default::default() + }, // allow more time to pass in CI completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( TimeBasedCompletionTaskDescription { duration: Duration::from_secs(240), }, ), + timing_data: TimingData { + timeout_ratio: (1, 1), + next_view_timeout: 1000, + ..TestMetadata::default_multiple_rounds().timing_data + }, unreliable_network: Some(Box::new(AsynchronousNetwork { - keep_numerator: 8, - keep_denominator: 10, + keep_numerator: 95, + keep_denominator: 100, delay_low_ms: 4, delay_high_ms: 30, })), @@ -249,6 +268,7 @@ async fn libp2p_network_partially_sync() { async_executor_impl = "tokio", tokio::test(flavor = "multi_thread", worker_threads = 2) )] +#[ignore] #[cfg_attr(async_executor_impl = "async-std", async_std::test)] async fn test_memory_network_chaos() { use hotshot_testing::{ @@ -289,6 +309,7 @@ async fn test_memory_network_chaos() { tokio::test(flavor = "multi_thread", worker_threads = 2) )] #[cfg_attr(async_executor_impl = "async-std", async_std::test)] +#[ignore] #[instrument] async fn libp2p_network_chaos() { async_compatibility_layer::logging::setup_logging(); diff --git a/justfile b/justfile index 9e6bba730a..bdf192aca9 100644 --- a/justfile +++ b/justfile @@ -24,9 +24,9 @@ build_release: example *ARGS: cargo run --profile=release-lto --example {{ARGS}} -test: - echo Testing - cargo test --verbose --lib --bins --tests --benches --workspace --no-fail-fast -- --test-threads=1 --nocapture --skip crypto_test +test *ARGS: + echo Testing {{ARGS}} + cargo test --verbose --lib --bins --tests --benches --workspace --no-fail-fast {{ARGS}} -- --test-threads=1 --nocapture --skip crypto_test test_basic: test_success test_with_failures test_network_task test_consensus_task test_da_task test_vid_task test_view_sync_task From 3320b44153c5dd6b81eb24f8a190eaad11c3b7fe Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Thu, 11 Jan 2024 11:29:37 -0500 Subject: [PATCH 13/18] feat: switch to release target --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7380b67db0..17ceb54c6d 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -46,7 +46,7 @@ jobs: sudo cp just /usr/bin/just - name: Build all crates in workspace - run: just ${{ matrix.just_variants }} build + run: just ${{ matrix.just_variants }} build_release - name: Unit and integration tests for all crates in workspace run: | @@ -91,7 +91,7 @@ jobs: prefix-key: arm-${{ matrix.just_variants }} - name: Build all crates in workspace - run: just ${{ matrix.just_variants }} build + run: just ${{ matrix.just_variants }} build_release - name: Upload Binaries uses: actions/upload-artifact@v3 From cd60310eef65ffe7717b07cb9bbf5a2d3f2edb0f Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Thu, 11 Jan 2024 12:07:46 -0500 Subject: [PATCH 14/18] fix: missing ff --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 5ca74a5058..64a873d956 100644 --- a/justfile +++ b/justfile @@ -19,7 +19,7 @@ build: cargo build --workspace --bins --tests --lib --benches --release build_release: - cargo build --workspace --exclude hotshot-testing --bins --examples --no-default-features --features="demo, docs, doc-images" + cargo build --workspace --exclude hotshot-testing --bins --examples --no-default-features --features="docs, doc-images" example *ARGS: cargo run --profile=release-lto --example {{ARGS}} From d863ce2f18f141a18692f96dd5bae0f09137b5e5 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sun, 21 Jan 2024 12:30:59 -0500 Subject: [PATCH 15/18] fix: ci --- crates/hotshot/examples/infra/mod.rs | 1 + justfile | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/hotshot/examples/infra/mod.rs b/crates/hotshot/examples/infra/mod.rs index e48f1d3984..32ae61f3e0 100644 --- a/crates/hotshot/examples/infra/mod.rs +++ b/crates/hotshot/examples/infra/mod.rs @@ -267,6 +267,7 @@ async fn libp2p_network_from_config( // NOTE: this introduces an invariant that the keys are assigned using this indexed // function all_keys, + None, da_keys.clone(), da_keys.contains(&pub_key), ) diff --git a/justfile b/justfile index d47709d61d..6d650b31e4 100644 --- a/justfile +++ b/justfile @@ -27,10 +27,10 @@ async := "async_std" export RUST_MIN_STACK=4194304 RUSTDOCFLAGS='-D warnings --cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std" {{original_rustdocflags}}' RUSTFLAGS='--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std" {{original_rustflags}}' && just {{target}} {{ARGS}} build: - cargo build --workspace --bins --tests --lib --benches --release + cargo build --workspace --examples --bins --tests --lib --benches --release build_release: - cargo build --workspace --exclude hotshot-testing --bins --examples --no-default-features --features="docs, doc-images" + cargo build --package hotshot --no-default-features --features="docs, doc-images" example *ARGS: cargo run --profile=release-lto --example {{ARGS}} @@ -116,11 +116,11 @@ check: lint: fmt echo linting - cargo clippy --workspace --bins --tests -- -D warnings + cargo clippy --workspace --examples --bins --tests -- -D warnings lint_release: fmt echo linting - cargo clippy --workspace --exclude hotshot-testing --bins --examples --no-default-features --features="demo, docs, doc-images" -- -D warnings + cargo clippy --package hotshot --no-default-features --features="docs, doc-images" -- -D warnings fmt: echo Running cargo fmt From 30d5ceb684228cbbc3a80ae174724b2519f868f5 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sun, 21 Jan 2024 13:03:46 -0500 Subject: [PATCH 16/18] fix: revert CI to use build just command --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 17ceb54c6d..7380b67db0 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -46,7 +46,7 @@ jobs: sudo cp just /usr/bin/just - name: Build all crates in workspace - run: just ${{ matrix.just_variants }} build_release + run: just ${{ matrix.just_variants }} build - name: Unit and integration tests for all crates in workspace run: | @@ -91,7 +91,7 @@ jobs: prefix-key: arm-${{ matrix.just_variants }} - name: Build all crates in workspace - run: just ${{ matrix.just_variants }} build_release + run: just ${{ matrix.just_variants }} build - name: Upload Binaries uses: actions/upload-artifact@v3 From 0dc6155ce41b9622523bf3b94c08030c25764de1 Mon Sep 17 00:00:00 2001 From: Justin Restivo Date: Sun, 21 Jan 2024 19:03:55 -0500 Subject: [PATCH 17/18] fix: increase timeout --- crates/testing/tests/unreliable_network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/testing/tests/unreliable_network.rs b/crates/testing/tests/unreliable_network.rs index b959185abc..c2e4a56249 100644 --- a/crates/testing/tests/unreliable_network.rs +++ b/crates/testing/tests/unreliable_network.rs @@ -30,7 +30,7 @@ async fn libp2p_network_sync() { }, completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( TimeBasedCompletionTaskDescription { - duration: Duration::new(240, 0), + duration: Duration::new(360, 0), }, ), unreliable_network: Some(Box::new(SynchronousNetwork { From 3f5fd167aadfa1abe10d1a3c2251ecfe048379ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 03:05:23 +0000 Subject: [PATCH 18/18] Bump derive_builder from 0.12.0 to 0.13.0 Bumps [derive_builder](https://github.com/colin-kiegel/rust-derive-builder) from 0.12.0 to 0.13.0. - [Release notes](https://github.com/colin-kiegel/rust-derive-builder/releases) - [Commits](https://github.com/colin-kiegel/rust-derive-builder/compare/v0.12.0...v0.13.0) --- updated-dependencies: - dependency-name: derive_builder dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 12 ++++++------ crates/libp2p-networking/Cargo.toml | 2 +- crates/testing-macros/Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 533811eb10..cb704fc783 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1843,18 +1843,18 @@ dependencies = [ [[package]] name = "derive_builder" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +checksum = "660047478bc508c0fde22c868991eec0c40a63e48d610befef466d48e2bee574" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +checksum = "9b217e6dd1011a54d12f3b920a411b5abd44b1716ecfe94f5f2f2f7b52e08ab7" dependencies = [ "darling 0.14.4", "proc-macro2", @@ -1864,9 +1864,9 @@ dependencies = [ [[package]] name = "derive_builder_macro" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +checksum = "7a5f77d7e20ac9153428f7ca14a88aba652adfc7a0ef0a06d654386310ef663b" dependencies = [ "derive_builder_core", "syn 1.0.109", diff --git a/crates/libp2p-networking/Cargo.toml b/crates/libp2p-networking/Cargo.toml index a0e56ed4da..6e6eb49461 100644 --- a/crates/libp2p-networking/Cargo.toml +++ b/crates/libp2p-networking/Cargo.toml @@ -18,7 +18,7 @@ async-trait = { workspace = true } bincode = { workspace = true } blake3 = { workspace = true } custom_debug = { workspace = true } -derive_builder = "0.12.0" +derive_builder = "0.13.0" either = { workspace = true } futures = { workspace = true } hotshot-constants = { path = "../constants" } diff --git a/crates/testing-macros/Cargo.toml b/crates/testing-macros/Cargo.toml index 398a03e720..7445ecda3b 100644 --- a/crates/testing-macros/Cargo.toml +++ b/crates/testing-macros/Cargo.toml @@ -26,7 +26,7 @@ serde = { workspace = true } quote = "1.0.33" syn = { version = "2.0.43", features = ["full", "extra-traits"] } proc-macro2 = "1.0.78" -derive_builder = "0.12.0" +derive_builder = "0.13.0" [dev-dependencies] async-lock = { workspace = true }