From dbc98e31a1f9ea9d4b1ae70fa31fc236b73c426e Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Wed, 28 Aug 2024 14:36:39 -0600 Subject: [PATCH 01/10] first pass --- Cargo.lock | 22 +-- crates/orchestrator/run-config.toml | 293 +++++++++++++++++++++++++++- crates/orchestrator/src/config.rs | 7 + crates/orchestrator/src/lib.rs | 66 ++++++- 4 files changed, 365 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e64fd4b7b..4ce747f931 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3076,7 +3076,7 @@ dependencies = [ [[package]] name = "hotshot" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-broadcast", @@ -3142,7 +3142,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-broadcast", @@ -3174,7 +3174,7 @@ dependencies = [ [[package]] name = "hotshot-examples" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-broadcast", @@ -3224,7 +3224,7 @@ dependencies = [ [[package]] name = "hotshot-fakeapi" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-lock 2.8.0", @@ -3242,7 +3242,7 @@ dependencies = [ [[package]] name = "hotshot-macros" -version = "0.5.71" +version = "0.5.72" dependencies = [ "derive_builder", "proc-macro2", @@ -3252,7 +3252,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-compatibility-layer", @@ -3281,7 +3281,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" -version = "0.5.71" +version = "0.5.72" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -3302,7 +3302,7 @@ dependencies = [ [[package]] name = "hotshot-task" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-broadcast", @@ -3316,7 +3316,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-broadcast", @@ -3352,7 +3352,7 @@ dependencies = [ [[package]] name = "hotshot-testing" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-broadcast", @@ -4513,7 +4513,7 @@ dependencies = [ [[package]] name = "libp2p-networking" -version = "0.5.71" +version = "0.5.72" dependencies = [ "anyhow", "async-compatibility-layer", diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml index 7fe8bb7e2a..f4a7b2edd6 100644 --- a/crates/orchestrator/run-config.toml +++ b/crates/orchestrator/run-config.toml @@ -39,22 +39,299 @@ seed = [ ] start_delay_seconds = 0 cdn_marshal_address = "127.0.0.1:9000" +public_keys = [ + [ + 156, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 66, + 76, + 83, + 95, + 86, + 69, + 82, + 95, + 75, + 69, + 89, + 128, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 109, + 11, + 51, + 75, + 228, + 10, + 98, + 245, + 34, + 143, + 104, + 54, + 209, + 90, + 146, + 241, + 171, + 45, + 180, + 100, + 155, + 247, + 147, + 107, + 78, + 237, + 143, + 82, + 61, + 46, + 50, + 29, + 66, + 5, + 76, + 77, + 203, + 81, + 106, + 200, + 240, + 227, + 102, + 160, + 118, + 244, + 206, + 217, + 110, + 53, + 170, + 143, + 191, + 80, + 144, + 67, + 150, + 146, + 152, + 26, + 243, + 128, + 38, + 13, + 133, + 30, + 78, + 229, + 166, + 203, + 237, + 15, + 212, + 7, + 74, + 28, + 61, + 222, + 133, + 207, + 157, + 42, + 247, + 162, + 239, + 192, + 67, + 143, + 149, + 18, + 193, + 115, + 201, + 136, + 53, + 37, + 11, + 27, + 216, + 179, + 255, + 107, + 89, + 136, + 90, + 129, + 170, + 190, + 231, + 242, + 215, + 16, + 218, + 251, + 243, + 23, + 165, + 215, + 187, + 41, + 126, + 211, + 93, + 228, + 192, + 53, + 51, + 14, + 84, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 48, + 120, + 49, + 96, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 83, + 67, + 72, + 78, + 79, + 82, + 82, + 95, + 86, + 69, + 82, + 95, + 75, + 69, + 89, + 64, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 148, + 154, + 131, + 105, + 86, + 114, + 51, + 72, + 86, + 63, + 96, + 107, + 231, + 98, + 23, + 228, + 87, + 132, + 249, + 208, + 128, + 32, + 47, + 157, + 61, + 126, + 219, + 47, + 159, + 170, + 83, + 31, + 175, + 141, + 187, + 167, + 193, + 239, + 132, + 52, + 231, + 153, + 198, + 62, + 133, + 184, + 236, + 175, + 12, + 33, + 115, + 160, + 103, + 143, + 239, + 38, + 116, + 124, + 207, + 125, + 230, + 161, + 126, + 9, + 17, + ], +] [config] num_nodes_with_stake = 10 num_nodes_without_stake = 0 -start_threshold = [ - 8, - 10, -] +start_threshold = [8, 10] staked_da_nodes = 10 non_staked_da_nodes = 0 fixed_leader_for_gpuvid = 1 next_view_timeout = 30000 -timeout_ratio = [ - 11, - 10, -] +timeout_ratio = [11, 10] round_start_delay = 1 start_delay = 1 num_bootstrap = 5 diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index 3e8c1747d8..2cbe560690 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -208,6 +208,8 @@ pub struct NetworkConfig { pub builder: BuilderType, /// random builder config pub random_builder: Option, + /// The list of public keys that are allowed to connect to the orchestrator + pub public_keys: Vec>, } /// the source of the network config @@ -439,6 +441,7 @@ impl Default for NetworkConfig { commit_sha: String::new(), builder: BuilderType::default(), random_builder: None, + public_keys: vec![], } } } @@ -491,6 +494,9 @@ pub struct NetworkConfigFile { /// random builder configuration #[serde(default)] pub random_builder: Option, + /// The list of public keys that are allowed to connect to the orchestrator + #[serde(default)] + pub public_keys: Vec>, } impl From> for NetworkConfig { @@ -536,6 +542,7 @@ impl From> for NetworkConfig { commit_sha: String::new(), builder: val.builder, random_builder: val.random_builder, + public_keys: val.public_keys, } } } diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index 72c48a4117..1edca40422 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -83,6 +83,8 @@ struct OrchestratorState { peer_pub_ready: bool, /// A map from public keys to `(node_index, is_da)`. pub_posted: HashMap, (u64, bool)>, + /// A collection for storing `ready` public keys. + ready_posted: Vec>, /// Whether nodes should start their HotShot instances /// Will be set to true once all nodes post they are ready to start start: bool, @@ -114,6 +116,7 @@ impl OrchestratorState { config: network_config, peer_pub_ready: false, pub_posted: HashMap::new(), + ready_posted: Vec::new(), nodes_connected: 0, start: false, bench_results: BenchResults::default(), @@ -228,7 +231,7 @@ pub trait OrchestratorApi { /// post endpoint for whether or not all nodes are ready /// # Errors /// if unable to serve - fn post_ready(&mut self) -> Result<(), ServerError>; + fn post_ready(&mut self, pubkey: &mut Vec) -> Result<(), ServerError>; /// post endpoint for manually starting the orchestrator /// # Errors /// if unable to serve @@ -329,6 +332,14 @@ where let node_index = self.pub_posted.len() as u64; let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); + + if !self.config.public_keys.contains(&pubkey) { + return Err(ServerError { + status: tide_disco::StatusCode::FORBIDDEN, + message: "You are unauthorized to register with the orchestrator".to_string(), + }); + } + self.config .config .known_nodes_with_stake @@ -416,8 +427,39 @@ where } // Assumes nodes do not post 'ready' twice - // TODO ED Add a map to verify which nodes have posted they're ready - fn post_ready(&mut self) -> Result<(), ServerError> { + fn post_ready(&mut self, pubkey: &mut Vec) -> Result<(), ServerError> { + // Deserialize the payload + match PeerConfig::::from_bytes(pubkey) { + Some(staked_pubkey) => { + // Is this node allowed to connect? + if !self.config.public_keys.contains(&pubkey) { + return Err(ServerError { + status: tide_disco::StatusCode::FORBIDDEN, + message: "You are unauthorized to register with the orchestrator" + .to_string(), + }); + } + + // Have they already connected? + if self.ready_posted.contains(&pubkey) { + return Err(ServerError { + status: tide_disco::StatusCode::BAD_REQUEST, + message: "You have already reported your ready status".to_string(), + }); + } + + // Otherwise, register that we've seen their ready status + self.ready_posted.push(pubkey.clone()) + } + None => { + // Something went wrong while deserializing. + return Err(ServerError { + status: tide_disco::StatusCode::BAD_REQUEST, + message: "You supplied an invalid pubkey".to_string(), + }); + } + } + self.nodes_connected += 1; println!("Nodes connected: {}", self.nodes_connected); @@ -636,7 +678,23 @@ where })? .post( "post_ready", - |_req, state: &mut ::State| async move { state.post_ready() }.boxed(), + |req, state: &mut ::State| { + async move { + let mut body_bytes = req.body_bytes(); + body_bytes.drain(..12); + // Decode the payload-supplied pubkey + let Ok(mut pubkey) = + vbs::Serializer::::deserialize(&body_bytes) + else { + return Err(ServerError { + status: tide_disco::StatusCode::BAD_REQUEST, + message: "Malformed body".to_string(), + }); + }; + state.post_ready(&mut pubkey) + } + .boxed() + }, )? .post( "post_manual_start", From ba0cd854ff1a0c2abb21647ef0fe244d6a5eb3c7 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Wed, 28 Aug 2024 14:41:24 -0600 Subject: [PATCH 02/10] lints --- crates/orchestrator/src/lib.rs | 47 ++++++++++++---------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index 1edca40422..92e93d465a 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -333,7 +333,7 @@ where let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - if !self.config.public_keys.contains(&pubkey) { + if !self.config.public_keys.contains(pubkey) { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: "You are unauthorized to register with the orchestrator".to_string(), @@ -428,38 +428,25 @@ where // Assumes nodes do not post 'ready' twice fn post_ready(&mut self, pubkey: &mut Vec) -> Result<(), ServerError> { - // Deserialize the payload - match PeerConfig::::from_bytes(pubkey) { - Some(staked_pubkey) => { - // Is this node allowed to connect? - if !self.config.public_keys.contains(&pubkey) { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "You are unauthorized to register with the orchestrator" - .to_string(), - }); - } - - // Have they already connected? - if self.ready_posted.contains(&pubkey) { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "You have already reported your ready status".to_string(), - }); - } + // Is this node allowed to connect? + if !self.config.public_keys.contains(pubkey) { + return Err(ServerError { + status: tide_disco::StatusCode::FORBIDDEN, + message: "You are unauthorized to register with the orchestrator".to_string(), + }); + } - // Otherwise, register that we've seen their ready status - self.ready_posted.push(pubkey.clone()) - } - None => { - // Something went wrong while deserializing. - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "You supplied an invalid pubkey".to_string(), - }); - } + // Have they already connected? + if self.ready_posted.contains(pubkey) { + return Err(ServerError { + status: tide_disco::StatusCode::BAD_REQUEST, + message: "You have already reported your ready status".to_string(), + }); } + // Otherwise, register that we've seen their ready status + self.ready_posted.push(pubkey.clone()); + self.nodes_connected += 1; println!("Nodes connected: {}", self.nodes_connected); From 2728a6e6fc1bc897c8c0edea9fbdbdbc72569350 Mon Sep 17 00:00:00 2001 From: rob-maron <132852777+rob-maron@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:36:42 -0400 Subject: [PATCH 03/10] try to pull stake table key (#3622) --- crates/orchestrator/run-config.toml | 282 +---------------------- crates/orchestrator/src/config.rs | 4 +- crates/orchestrator/src/lib.rs | 22 +- crates/types/src/stake_table.rs | 7 +- crates/types/src/traits/signature_key.rs | 6 +- 5 files changed, 32 insertions(+), 289 deletions(-) diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml index f4a7b2edd6..26984db2d5 100644 --- a/crates/orchestrator/run-config.toml +++ b/crates/orchestrator/run-config.toml @@ -40,287 +40,7 @@ seed = [ start_delay_seconds = 0 cdn_marshal_address = "127.0.0.1:9000" public_keys = [ - [ - 156, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 11, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 66, - 76, - 83, - 95, - 86, - 69, - 82, - 95, - 75, - 69, - 89, - 128, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 109, - 11, - 51, - 75, - 228, - 10, - 98, - 245, - 34, - 143, - 104, - 54, - 209, - 90, - 146, - 241, - 171, - 45, - 180, - 100, - 155, - 247, - 147, - 107, - 78, - 237, - 143, - 82, - 61, - 46, - 50, - 29, - 66, - 5, - 76, - 77, - 203, - 81, - 106, - 200, - 240, - 227, - 102, - 160, - 118, - 244, - 206, - 217, - 110, - 53, - 170, - 143, - 191, - 80, - 144, - 67, - 150, - 146, - 152, - 26, - 243, - 128, - 38, - 13, - 133, - 30, - 78, - 229, - 166, - 203, - 237, - 15, - 212, - 7, - 74, - 28, - 61, - 222, - 133, - 207, - 157, - 42, - 247, - 162, - 239, - 192, - 67, - 143, - 149, - 18, - 193, - 115, - 201, - 136, - 53, - 37, - 11, - 27, - 216, - 179, - 255, - 107, - 89, - 136, - 90, - 129, - 170, - 190, - 231, - 242, - 215, - 16, - 218, - 251, - 243, - 23, - 165, - 215, - 187, - 41, - 126, - 211, - 93, - 228, - 192, - 53, - 51, - 14, - 84, - 3, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 48, - 120, - 49, - 96, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 15, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 83, - 67, - 72, - 78, - 79, - 82, - 82, - 95, - 86, - 69, - 82, - 95, - 75, - 69, - 89, - 64, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 148, - 154, - 131, - 105, - 86, - 114, - 51, - 72, - 86, - 63, - 96, - 107, - 231, - 98, - 23, - 228, - 87, - 132, - 249, - 208, - 128, - 32, - 47, - 157, - 61, - 126, - 219, - 47, - 159, - 170, - 83, - 31, - 175, - 141, - 187, - 167, - 193, - 239, - 132, - 52, - 231, - 153, - 198, - 62, - 133, - 184, - 236, - 175, - 12, - 33, - 115, - 160, - 103, - 143, - 239, - 38, - 116, - 124, - 207, - 125, - 230, - 161, - 126, - 9, - 17, - ], + "BLS_VER_KEY~p-JKk1VvO1RoMrDrqyjz0P1VGwtOaEjF5jLjpOZbJi5O747fvYEOg0OvCl_CLe4shh7vsqeG9uMF9RssM12sLSuaiVJkCClxEI5mRLV4qff1UjZAZJIBgeL1_hRhRUkpqC0Trm1qtvXtZ8FwOCIzYXv8c300Au824k7FxjjcWLBL", ] [config] diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index 2cbe560690..4a746a1830 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -209,7 +209,7 @@ pub struct NetworkConfig { /// random builder config pub random_builder: Option, /// The list of public keys that are allowed to connect to the orchestrator - pub public_keys: Vec>, + pub public_keys: Vec, } /// the source of the network config @@ -496,7 +496,7 @@ pub struct NetworkConfigFile { pub random_builder: Option, /// The list of public keys that are allowed to connect to the orchestrator #[serde(default)] - pub public_keys: Vec>, + pub public_keys: Vec, } impl From> for NetworkConfig { diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index 92e93d465a..ef94be6be0 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -23,7 +23,10 @@ use client::{BenchResults, BenchResultsDownloadConfig}; use config::BuilderType; use csv::Writer; use futures::{stream::FuturesUnordered, FutureExt, StreamExt}; -use hotshot_types::{traits::signature_key::SignatureKey, PeerConfig}; +use hotshot_types::{ + traits::signature_key::{SignatureKey, StakeTableEntryType}, + PeerConfig, +}; use libp2p::{ identity::{ ed25519::{Keypair as EdKeypair, SecretKey}, @@ -331,9 +334,15 @@ where let node_index = self.pub_posted.len() as u64; + // Deserialize the public key let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - if !self.config.public_keys.contains(pubkey) { + // Check if the node is allowed to connect + if !self + .config + .public_keys + .contains(&staked_pubkey.stake_table_entry.public_key()) + { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: "You are unauthorized to register with the orchestrator".to_string(), @@ -428,8 +437,15 @@ where // Assumes nodes do not post 'ready' twice fn post_ready(&mut self, pubkey: &mut Vec) -> Result<(), ServerError> { + // Deserialize the public key + let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); + // Is this node allowed to connect? - if !self.config.public_keys.contains(pubkey) { + if !self + .config + .public_keys + .contains(&staked_pubkey.stake_table_entry.public_key()) + { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: "You are unauthorized to register with the orchestrator".to_string(), diff --git a/crates/types/src/stake_table.rs b/crates/types/src/stake_table.rs index 9d41931f0c..eddfd1caef 100644 --- a/crates/types/src/stake_table.rs +++ b/crates/types/src/stake_table.rs @@ -21,11 +21,16 @@ pub struct StakeTableEntry { pub stake_amount: U256, } -impl StakeTableEntryType for StakeTableEntry { +impl StakeTableEntryType for StakeTableEntry { /// Get the stake amount fn stake(&self) -> U256 { self.stake_amount } + + /// Get the public key + fn public_key(&self) -> K { + self.stake_key.clone() + } } impl StakeTableEntry { diff --git a/crates/types/src/traits/signature_key.rs b/crates/types/src/traits/signature_key.rs index d764b08163..ebee7c6b75 100644 --- a/crates/types/src/traits/signature_key.rs +++ b/crates/types/src/traits/signature_key.rs @@ -25,9 +25,11 @@ use super::EncodeBytes; use crate::{utils::BuilderCommitment, vid::VidSchemeType}; /// Type representing stake table entries in a `StakeTable` -pub trait StakeTableEntryType { +pub trait StakeTableEntryType { /// Get the stake value fn stake(&self) -> U256; + /// Get the public key + fn public_key(&self) -> K; } /// Trait for abstracting public key signatures @@ -60,7 +62,7 @@ pub trait SignatureKey: + for<'a> Deserialize<'a> + Hash; /// The type of the entry that contain both public key and stake value - type StakeTableEntry: StakeTableEntryType + type StakeTableEntry: StakeTableEntryType + Send + Sync + Sized From dcb6b7c64399bd78e60749226105f2a671097182 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 29 Aug 2024 09:16:53 -0600 Subject: [PATCH 04/10] misc cleanup --- crates/orchestrator/src/config.rs | 14 ++++++++++++++ crates/orchestrator/src/lib.rs | 14 +++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index 4a746a1830..df450e03c4 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -210,6 +210,8 @@ pub struct NetworkConfig { pub random_builder: Option, /// The list of public keys that are allowed to connect to the orchestrator pub public_keys: Vec, + /// Whether or not to disable registration verification. + pub disable_registration_verification: bool, } /// the source of the network config @@ -442,6 +444,7 @@ impl Default for NetworkConfig { builder: BuilderType::default(), random_builder: None, public_keys: vec![], + disable_registration_verification: false, } } } @@ -497,10 +500,20 @@ pub struct NetworkConfigFile { /// The list of public keys that are allowed to connect to the orchestrator #[serde(default)] pub public_keys: Vec, + /// Whether or not to disable registration verification. + #[serde(default)] + pub disable_registration_verification: bool, } impl From> for NetworkConfig { fn from(val: NetworkConfigFile) -> Self { + #[cfg(not(debug_assertions))] + { + if val.disable_registration_verification { + panic!("Registration verification cannot be turned off in production builds"); + } + } + NetworkConfig { rounds: val.rounds, indexed_da: val.indexed_da, @@ -543,6 +556,7 @@ impl From> for NetworkConfig { builder: val.builder, random_builder: val.random_builder, public_keys: val.public_keys, + disable_registration_verification: val.disable_registration_verification, } } } diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index ef94be6be0..ddf5152502 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -12,7 +12,7 @@ pub mod client; pub mod config; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fs::OpenOptions, io::{self, ErrorKind}, time::Duration, @@ -87,7 +87,7 @@ struct OrchestratorState { /// A map from public keys to `(node_index, is_da)`. pub_posted: HashMap, (u64, bool)>, /// A collection for storing `ready` public keys. - ready_posted: Vec>, + ready_posted: HashSet, /// Whether nodes should start their HotShot instances /// Will be set to true once all nodes post they are ready to start start: bool, @@ -119,7 +119,7 @@ impl OrchestratorState { config: network_config, peer_pub_ready: false, pub_posted: HashMap::new(), - ready_posted: Vec::new(), + ready_posted: HashSet::new(), nodes_connected: 0, start: false, bench_results: BenchResults::default(), @@ -453,16 +453,16 @@ where } // Have they already connected? - if self.ready_posted.contains(pubkey) { + if !self + .ready_posted + .insert(staked_pubkey.stake_table_entry.public_key().clone()) + { return Err(ServerError { status: tide_disco::StatusCode::BAD_REQUEST, message: "You have already reported your ready status".to_string(), }); } - // Otherwise, register that we've seen their ready status - self.ready_posted.push(pubkey.clone()); - self.nodes_connected += 1; println!("Nodes connected: {}", self.nodes_connected); From 5ebb2de26c3179c730186bb82f6fe969eb8860d4 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 29 Aug 2024 09:35:44 -0600 Subject: [PATCH 05/10] allow disable verification --- crates/orchestrator/run-config.toml | 1 + crates/orchestrator/src/config.rs | 7 ++-- crates/orchestrator/src/lib.rs | 54 ++++++++++++++++------------- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml index 26984db2d5..ffdb82f055 100644 --- a/crates/orchestrator/run-config.toml +++ b/crates/orchestrator/run-config.toml @@ -42,6 +42,7 @@ cdn_marshal_address = "127.0.0.1:9000" public_keys = [ "BLS_VER_KEY~p-JKk1VvO1RoMrDrqyjz0P1VGwtOaEjF5jLjpOZbJi5O747fvYEOg0OvCl_CLe4shh7vsqeG9uMF9RssM12sLSuaiVJkCClxEI5mRLV4qff1UjZAZJIBgeL1_hRhRUkpqC0Trm1qtvXtZ8FwOCIzYXv8c300Au824k7FxjjcWLBL", ] +disable_registration_verification = false [config] num_nodes_with_stake = 10 diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index df450e03c4..ebec6addc0 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -507,11 +507,8 @@ pub struct NetworkConfigFile { impl From> for NetworkConfig { fn from(val: NetworkConfigFile) -> Self { - #[cfg(not(debug_assertions))] - { - if val.disable_registration_verification { - panic!("Registration verification cannot be turned off in production builds"); - } + if val.disable_registration_verification { + tracing::error!("REGISTRATION VERIFICATION IS TURNED OFF"); } NetworkConfig { diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index ddf5152502..0f4b2ceab9 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -103,6 +103,8 @@ struct OrchestratorState { accepting_new_keys: bool, /// Builder address pool builders: Vec, + /// Whether or not registration verification is disabled for this instance + disable_registraton_verification: bool, } impl OrchestratorState { @@ -127,6 +129,7 @@ impl OrchestratorState { manual_start_allowed: true, accepting_new_keys: true, builders, + disable_registraton_verification: network_config.disable_registration_verification, } } @@ -323,7 +326,7 @@ where return Ok((*node_index, *is_da)); } - if !self.accepting_new_keys { + if !self.disable_registraton_verification && !self.accepting_new_keys { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: @@ -437,30 +440,33 @@ where // Assumes nodes do not post 'ready' twice fn post_ready(&mut self, pubkey: &mut Vec) -> Result<(), ServerError> { - // Deserialize the public key - let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - - // Is this node allowed to connect? - if !self - .config - .public_keys - .contains(&staked_pubkey.stake_table_entry.public_key()) - { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "You are unauthorized to register with the orchestrator".to_string(), - }); - } + // If we have not disabled registration verification. + if !self.disable_registraton_verification { + // Deserialize the public key + let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); + + // Is this node allowed to connect? + if !self + .config + .public_keys + .contains(&staked_pubkey.stake_table_entry.public_key()) + { + return Err(ServerError { + status: tide_disco::StatusCode::FORBIDDEN, + message: "You are unauthorized to register with the orchestrator".to_string(), + }); + } - // Have they already connected? - if !self - .ready_posted - .insert(staked_pubkey.stake_table_entry.public_key().clone()) - { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "You have already reported your ready status".to_string(), - }); + // Have they already connected? + if !self + .ready_posted + .insert(staked_pubkey.stake_table_entry.public_key().clone()) + { + return Err(ServerError { + status: tide_disco::StatusCode::BAD_REQUEST, + message: "You have already reported your ready status".to_string(), + }); + } } self.nodes_connected += 1; From f2c7504a0c63ca86c71475ae7492bc081e890ffa Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 29 Aug 2024 10:11:12 -0600 Subject: [PATCH 06/10] finish all validation and bug fixes --- crates/examples/infra/mod.rs | 5 +++- crates/orchestrator/run-config.toml | 10 ++++++++ crates/orchestrator/src/client.rs | 6 +++-- crates/orchestrator/src/lib.rs | 37 ++++++++++------------------- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/crates/examples/infra/mod.rs b/crates/examples/infra/mod.rs index 8fe91fa855..5de178c57d 100755 --- a/crates/examples/infra/mod.rs +++ b/crates/examples/infra/mod.rs @@ -886,6 +886,9 @@ pub async fn main_entry_point< derive_libp2p_peer_id::(&my_own_validator_config.private_key) .expect("failed to derive Libp2p keypair"); + // For use below to register this node. + let node_public_key = my_own_validator_config.public_key.clone(); + // conditionally save/load config from file or orchestrator // This is a function that will return correct complete config from orchestrator. // It takes in a valid args.network_config_file when loading from file, or valid validator_config when loading from orchestrator, the invalid one will be ignored. @@ -956,7 +959,7 @@ pub async fn main_entry_point< if let NetworkConfigSource::Orchestrator = source { info!("Waiting for the start command from orchestrator"); orchestrator_client - .wait_for_all_nodes_ready(run_config.clone().node_index) + .wait_for_all_nodes_ready(node_public_key) .await; } diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml index ffdb82f055..d3f3110498 100644 --- a/crates/orchestrator/run-config.toml +++ b/crates/orchestrator/run-config.toml @@ -41,6 +41,16 @@ start_delay_seconds = 0 cdn_marshal_address = "127.0.0.1:9000" public_keys = [ "BLS_VER_KEY~p-JKk1VvO1RoMrDrqyjz0P1VGwtOaEjF5jLjpOZbJi5O747fvYEOg0OvCl_CLe4shh7vsqeG9uMF9RssM12sLSuaiVJkCClxEI5mRLV4qff1UjZAZJIBgeL1_hRhRUkpqC0Trm1qtvXtZ8FwOCIzYXv8c300Au824k7FxjjcWLBL", + "BLS_VER_KEY~bQszS-QKYvUij2g20VqS8asttGSb95NrTu2PUj0uMh1CBUxNy1FqyPDjZqB29M7ZbjWqj79QkEOWkpga84AmDYUeTuWmy-0P1AdKHD3ehc-dKvei78BDj5USwXPJiDUlCxvYs_9rWYhagaq-5_LXENr78xel17spftNd5MA1Mw5U", + "BLS_VER_KEY~4zQnaCOFJ7m95OjxeNls0QOOwWbz4rfxaL3NwmN2zSdnf8t5Nw_dfmMHq05ee8jCegw6Bn5T8inmrnGGAsQJMMWLv77nd7FJziz2ViAbXg-XGGF7o4HyzELCmypDOIYF3X2UWferFE_n72ZX0iQkUhOvYZZ7cfXToXxRTtb_mwRR", + "BLS_VER_KEY~rO2PIjyY30HGfapFcloFe3mNDKMIFi6JlOLkH5ZWBSYoRm5fE2-Rm6Lp3EvmAcB5r7KFJ0c1Uor308x78r04EY_sfjcsDCWt7RSJdL4cJoD_4fSTCv_bisO8k98hs_8BtqQt8BHlPeJohpUXvcfnK8suXJETiJ6Er97pfxRbzgAL", + "BLS_VER_KEY~r6b-Cwzp-b3czlt0MHmYPJIow5kMsXbrNmZsLSYg9RV49oCCO4WEeCRFR02x9bqLCa_sgNFMrIeNdEa11qNiBAohApYFIvrSa-zP5QGj3xbZaMOCrshxYit6E2TR-XsWvv6gjOrypmugjyTAth-iqQzTboSfmO9DD1-gjJIdCaD7", + "BLS_VER_KEY~IBRoz_Q1EXvcm1pNZcmVlyYZU8hZ7qmy337ePAjEMhz8Hl2q8vWPFOd3BaLwgRS1UzAPW3z4E-XIgRDGcRBTAMZX9b_0lKYjlyTlNF2EZfNnKmvv-xJ0yurkfjiveeYEsD2l5d8q_rJJbH1iZdXy-yPEbwI0SIvQfwdlcaKw9po4", + "BLS_VER_KEY~kEUEUJFBtCXl68fM_2roQw856wQlu1ZoDmPn8uu4bQgeZwyb5oz5_kMl-oAJ_OtbYV1serjWE--eXB_qYIpQLZka42-cML6WjCQjNl1hGSejtoBDkExNeUNcweFQBbEsaDiIy3-sgHTrfYpFd1icKeAVihLRn5_RtSU_RUu1TQqR", + "BLS_VER_KEY~PAAQNgOYfj3GiVX7LxSlkXfOCDSnNKZDqPVYQ_jBMxKzOCn0PXbqQ62kKPenWOmCxiCE7X158s-VenBna6MjHJgf61eBAO-3-OyTP5NWVx49RTgHhQf2iMTKk2iqK2gjnjZimBU135YU4lQFtrG-ZgRezwqkC5vy8V-q46fschIG", + "BLS_VER_KEY~96hAcdFZxQT8CEHcyV8j2ILJRsXagquENPkc9AwLSx3u6AE_uMupIKGbNJRiM99oFneK2vI5g1u61HidWeuTLRPM2537xAXeaO8e-wJYx4FaPKw_xTcLPrIm0OZT7SsLAMwFuqfMbDdKM71-RyrLwhff5517xXBKEk5Tg9iT9Qrr", + "BLS_VER_KEY~-pVi7j6TEBeG7ABata4uWWDRM2SrY8wWotWsGnTpIhnOVYJI_lNWyig6VJUuFmBsMS8rLMU7nDxDm8SbObxyA-SLFcr_jCkZqsbx8GcVQrnBAfjNRWuPZP0xcTDMu2IkQqtc3L0OpzbMEgGRGE8Wj09pNqouzl-xhPoYjTmD06Bw", + "BLS_VER_KEY~IUPSdnsNUHgNx_74ZhBPrICcDZ9Bp_DAt-6kFz8vSwJES2Vy1Ws8NJ1mxb9XGE1u13sw0FRe8kn5Ib3p2stbEtR_1Qgbuif6aoLrGaSUzy0MvwrO58u9kHZk3rXIuSAN7n4ok3-KKk2CmnBfx7fchFoqT56FXCd1EJ7XRrYj8wTh", ] disable_registration_verification = false diff --git a/crates/orchestrator/src/client.rs b/crates/orchestrator/src/client.rs index 0843844da6..c201e72ef2 100644 --- a/crates/orchestrator/src/client.rs +++ b/crates/orchestrator/src/client.rs @@ -442,12 +442,14 @@ impl OrchestratorClient { /// # Panics /// Panics if unable to post. #[instrument(skip(self), name = "orchestrator ready signal")] - pub async fn wait_for_all_nodes_ready(&self, node_index: u64) -> bool { + pub async fn wait_for_all_nodes_ready(&self, public_key: KEY) -> bool { let send_ready_f = |client: Client| { + let request_body = vbs::Serializer::::serialize(&public_key) + .expect("Failed to serialize public key"); async move { let result: Result<_, ClientError> = client .post("api/ready") - .body_json(&node_index) + .body_binary(&request_body) .unwrap() .send() .await diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index 0f4b2ceab9..f0b253c0bc 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -103,8 +103,6 @@ struct OrchestratorState { accepting_new_keys: bool, /// Builder address pool builders: Vec, - /// Whether or not registration verification is disabled for this instance - disable_registraton_verification: bool, } impl OrchestratorState { @@ -129,7 +127,6 @@ impl OrchestratorState { manual_start_allowed: true, accepting_new_keys: true, builders, - disable_registraton_verification: network_config.disable_registration_verification, } } @@ -234,10 +231,10 @@ pub trait OrchestratorApi { /// # Errors /// if unable to serve fn post_run_results(&mut self, metrics: BenchResults) -> Result<(), ServerError>; - /// post endpoint for whether or not all nodes are ready + /// A node POSTs its public key to let the orchestrator know that it is ready /// # Errors /// if unable to serve - fn post_ready(&mut self, pubkey: &mut Vec) -> Result<(), ServerError>; + fn post_ready(&mut self, bls_public_key: &KEY) -> Result<(), ServerError>; /// post endpoint for manually starting the orchestrator /// # Errors /// if unable to serve @@ -326,7 +323,7 @@ where return Ok((*node_index, *is_da)); } - if !self.disable_registraton_verification && !self.accepting_new_keys { + if !self.config.disable_registration_verification && !self.accepting_new_keys { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: @@ -346,6 +343,7 @@ where .public_keys .contains(&staked_pubkey.stake_table_entry.public_key()) { + println!("{}", staked_pubkey.stake_table_entry.public_key()); return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: "You are unauthorized to register with the orchestrator".to_string(), @@ -394,7 +392,7 @@ where } } - println!("Posted public key for node_index {node_index}"); + tracing::info!("Posted public key for node_index {node_index}"); // node_index starts at 0, so once it matches `num_nodes_with_stake` // we will have registered one node too many. hence, we want `node_index + 1`. @@ -439,18 +437,11 @@ where } // Assumes nodes do not post 'ready' twice - fn post_ready(&mut self, pubkey: &mut Vec) -> Result<(), ServerError> { + fn post_ready(&mut self, pubkey: &KEY) -> Result<(), ServerError> { // If we have not disabled registration verification. - if !self.disable_registraton_verification { - // Deserialize the public key - let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); - + if !self.config.disable_registration_verification { // Is this node allowed to connect? - if !self - .config - .public_keys - .contains(&staked_pubkey.stake_table_entry.public_key()) - { + if !self.config.public_keys.contains(pubkey) { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: "You are unauthorized to register with the orchestrator".to_string(), @@ -458,10 +449,7 @@ where } // Have they already connected? - if !self - .ready_posted - .insert(staked_pubkey.stake_table_entry.public_key().clone()) - { + if !self.ready_posted.insert(pubkey.clone()) { return Err(ServerError { status: tide_disco::StatusCode::BAD_REQUEST, message: "You have already reported your ready status".to_string(), @@ -471,7 +459,7 @@ where self.nodes_connected += 1; - println!("Nodes connected: {}", self.nodes_connected); + tracing::info!("Nodes connected: {}", self.nodes_connected); // i.e. nodes_connected >= num_nodes_with_stake * (start_threshold.0 / start_threshold.1) if self.nodes_connected * self.config.config.start_threshold.1 @@ -692,15 +680,14 @@ where let mut body_bytes = req.body_bytes(); body_bytes.drain(..12); // Decode the payload-supplied pubkey - let Ok(mut pubkey) = - vbs::Serializer::::deserialize(&body_bytes) + let Ok(pubkey) = vbs::Serializer::::deserialize(&body_bytes) else { return Err(ServerError { status: tide_disco::StatusCode::BAD_REQUEST, message: "Malformed body".to_string(), }); }; - state.post_ready(&mut pubkey) + state.post_ready(&pubkey) } .boxed() }, From c0c3ec10f64be6bc4336dfb18c2ce7102c81245f Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 29 Aug 2024 10:13:39 -0600 Subject: [PATCH 07/10] remove old log --- crates/orchestrator/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index f0b253c0bc..6e7caf31c5 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -343,7 +343,6 @@ where .public_keys .contains(&staked_pubkey.stake_table_entry.public_key()) { - println!("{}", staked_pubkey.stake_table_entry.public_key()); return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: "You are unauthorized to register with the orchestrator".to_string(), From b105bb14b3b9f4e302d5511fddc9858818c2d9b4 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 29 Aug 2024 10:18:04 -0600 Subject: [PATCH 08/10] vec->hashset --- crates/orchestrator/src/config.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index ebec6addc0..f1c3348c91 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -5,6 +5,7 @@ // along with the HotShot repository. If not, see . use std::{ + collections::HashSet, env, fs, net::SocketAddr, num::NonZeroUsize, @@ -209,7 +210,7 @@ pub struct NetworkConfig { /// random builder config pub random_builder: Option, /// The list of public keys that are allowed to connect to the orchestrator - pub public_keys: Vec, + pub public_keys: HashSet, /// Whether or not to disable registration verification. pub disable_registration_verification: bool, } @@ -443,7 +444,7 @@ impl Default for NetworkConfig { commit_sha: String::new(), builder: BuilderType::default(), random_builder: None, - public_keys: vec![], + public_keys: HashSet::new(), disable_registration_verification: false, } } @@ -499,7 +500,7 @@ pub struct NetworkConfigFile { pub random_builder: Option, /// The list of public keys that are allowed to connect to the orchestrator #[serde(default)] - pub public_keys: Vec, + pub public_keys: HashSet, /// Whether or not to disable registration verification. #[serde(default)] pub disable_registration_verification: bool, From 7668a2fa2b01db63d0655257df9747fa56cbb287 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 29 Aug 2024 11:22:41 -0600 Subject: [PATCH 09/10] misc feedback fixes --- crates/examples/infra/mod.rs | 8 +++-- crates/orchestrator/run-config.toml | 2 +- crates/orchestrator/src/client.rs | 7 ++--- crates/orchestrator/src/config.rs | 12 +++----- crates/orchestrator/src/lib.rs | 48 ++++++++++++++--------------- 5 files changed, 36 insertions(+), 41 deletions(-) diff --git a/crates/examples/infra/mod.rs b/crates/examples/infra/mod.rs index 5de178c57d..f1d8d83474 100755 --- a/crates/examples/infra/mod.rs +++ b/crates/examples/infra/mod.rs @@ -886,8 +886,10 @@ pub async fn main_entry_point< derive_libp2p_peer_id::(&my_own_validator_config.private_key) .expect("failed to derive Libp2p keypair"); - // For use below to register this node. - let node_public_key = my_own_validator_config.public_key.clone(); + // We need this to be able to register our node + let peer_config = + PeerConfig::::to_bytes(&my_own_validator_config.public_config()) + .clone(); // conditionally save/load config from file or orchestrator // This is a function that will return correct complete config from orchestrator. @@ -959,7 +961,7 @@ pub async fn main_entry_point< if let NetworkConfigSource::Orchestrator = source { info!("Waiting for the start command from orchestrator"); orchestrator_client - .wait_for_all_nodes_ready(node_public_key) + .wait_for_all_nodes_ready(peer_config) .await; } diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml index d3f3110498..e5eecaa9c5 100644 --- a/crates/orchestrator/run-config.toml +++ b/crates/orchestrator/run-config.toml @@ -52,7 +52,7 @@ public_keys = [ "BLS_VER_KEY~-pVi7j6TEBeG7ABata4uWWDRM2SrY8wWotWsGnTpIhnOVYJI_lNWyig6VJUuFmBsMS8rLMU7nDxDm8SbObxyA-SLFcr_jCkZqsbx8GcVQrnBAfjNRWuPZP0xcTDMu2IkQqtc3L0OpzbMEgGRGE8Wj09pNqouzl-xhPoYjTmD06Bw", "BLS_VER_KEY~IUPSdnsNUHgNx_74ZhBPrICcDZ9Bp_DAt-6kFz8vSwJES2Vy1Ws8NJ1mxb9XGE1u13sw0FRe8kn5Ib3p2stbEtR_1Qgbuif6aoLrGaSUzy0MvwrO58u9kHZk3rXIuSAN7n4ok3-KKk2CmnBfx7fchFoqT56FXCd1EJ7XRrYj8wTh", ] -disable_registration_verification = false +enable_registration_verification = true [config] num_nodes_with_stake = 10 diff --git a/crates/orchestrator/src/client.rs b/crates/orchestrator/src/client.rs index c201e72ef2..b4a2e6f898 100644 --- a/crates/orchestrator/src/client.rs +++ b/crates/orchestrator/src/client.rs @@ -442,14 +442,13 @@ impl OrchestratorClient { /// # Panics /// Panics if unable to post. #[instrument(skip(self), name = "orchestrator ready signal")] - pub async fn wait_for_all_nodes_ready(&self, public_key: KEY) -> bool { + pub async fn wait_for_all_nodes_ready(&self, peer_config: Vec) -> bool { let send_ready_f = |client: Client| { - let request_body = vbs::Serializer::::serialize(&public_key) - .expect("Failed to serialize public key"); + let pk = peer_config.clone(); async move { let result: Result<_, ClientError> = client .post("api/ready") - .body_binary(&request_body) + .body_binary(&pk) .unwrap() .send() .await diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index f1c3348c91..96e26e6b08 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -212,7 +212,7 @@ pub struct NetworkConfig { /// The list of public keys that are allowed to connect to the orchestrator pub public_keys: HashSet, /// Whether or not to disable registration verification. - pub disable_registration_verification: bool, + pub enable_registration_verification: bool, } /// the source of the network config @@ -445,7 +445,7 @@ impl Default for NetworkConfig { builder: BuilderType::default(), random_builder: None, public_keys: HashSet::new(), - disable_registration_verification: false, + enable_registration_verification: true, } } } @@ -503,15 +503,11 @@ pub struct NetworkConfigFile { pub public_keys: HashSet, /// Whether or not to disable registration verification. #[serde(default)] - pub disable_registration_verification: bool, + pub enable_registration_verification: bool, } impl From> for NetworkConfig { fn from(val: NetworkConfigFile) -> Self { - if val.disable_registration_verification { - tracing::error!("REGISTRATION VERIFICATION IS TURNED OFF"); - } - NetworkConfig { rounds: val.rounds, indexed_da: val.indexed_da, @@ -554,7 +550,7 @@ impl From> for NetworkConfig { builder: val.builder, random_builder: val.random_builder, public_keys: val.public_keys, - disable_registration_verification: val.disable_registration_verification, + enable_registration_verification: val.enable_registration_verification, } } } diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index 6e7caf31c5..76992ce59c 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -12,7 +12,7 @@ pub mod client; pub mod config; use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, fs::OpenOptions, io::{self, ErrorKind}, time::Duration, @@ -86,8 +86,6 @@ struct OrchestratorState { peer_pub_ready: bool, /// A map from public keys to `(node_index, is_da)`. pub_posted: HashMap, (u64, bool)>, - /// A collection for storing `ready` public keys. - ready_posted: HashSet, /// Whether nodes should start their HotShot instances /// Will be set to true once all nodes post they are ready to start start: bool, @@ -119,7 +117,6 @@ impl OrchestratorState { config: network_config, peer_pub_ready: false, pub_posted: HashMap::new(), - ready_posted: HashSet::new(), nodes_connected: 0, start: false, bench_results: BenchResults::default(), @@ -234,7 +231,7 @@ pub trait OrchestratorApi { /// A node POSTs its public key to let the orchestrator know that it is ready /// # Errors /// if unable to serve - fn post_ready(&mut self, bls_public_key: &KEY) -> Result<(), ServerError>; + fn post_ready(&mut self, peer_config: &PeerConfig) -> Result<(), ServerError>; /// post endpoint for manually starting the orchestrator /// # Errors /// if unable to serve @@ -323,7 +320,7 @@ where return Ok((*node_index, *is_da)); } - if !self.config.disable_registration_verification && !self.accepting_new_keys { + if !self.accepting_new_keys { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: @@ -338,10 +335,11 @@ where let staked_pubkey = PeerConfig::::from_bytes(pubkey).unwrap(); // Check if the node is allowed to connect - if !self - .config - .public_keys - .contains(&staked_pubkey.stake_table_entry.public_key()) + if self.config.enable_registration_verification + && !self + .config + .public_keys + .contains(&staked_pubkey.stake_table_entry.public_key()) { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, @@ -391,7 +389,7 @@ where } } - tracing::info!("Posted public key for node_index {node_index}"); + tracing::error!("Posted public key for node_index {node_index}"); // node_index starts at 0, so once it matches `num_nodes_with_stake` // we will have registered one node too many. hence, we want `node_index + 1`. @@ -436,29 +434,26 @@ where } // Assumes nodes do not post 'ready' twice - fn post_ready(&mut self, pubkey: &KEY) -> Result<(), ServerError> { + fn post_ready(&mut self, peer_config: &PeerConfig) -> Result<(), ServerError> { // If we have not disabled registration verification. - if !self.config.disable_registration_verification { + if self.config.enable_registration_verification { // Is this node allowed to connect? - if !self.config.public_keys.contains(pubkey) { + if !self + .config + .config + .known_nodes_with_stake + .contains(peer_config) + { return Err(ServerError { status: tide_disco::StatusCode::FORBIDDEN, message: "You are unauthorized to register with the orchestrator".to_string(), }); } - - // Have they already connected? - if !self.ready_posted.insert(pubkey.clone()) { - return Err(ServerError { - status: tide_disco::StatusCode::BAD_REQUEST, - message: "You have already reported your ready status".to_string(), - }); - } } self.nodes_connected += 1; - tracing::info!("Nodes connected: {}", self.nodes_connected); + tracing::error!("Nodes connected: {}", self.nodes_connected); // i.e. nodes_connected >= num_nodes_with_stake * (start_threshold.0 / start_threshold.1) if self.nodes_connected * self.config.config.start_threshold.1 @@ -679,8 +674,7 @@ where let mut body_bytes = req.body_bytes(); body_bytes.drain(..12); // Decode the payload-supplied pubkey - let Ok(pubkey) = vbs::Serializer::::deserialize(&body_bytes) - else { + let Some(pubkey) = PeerConfig::::from_bytes(&body_bytes) else { return Err(ServerError { status: tide_disco::StatusCode::BAD_REQUEST, message: "Malformed body".to_string(), @@ -779,6 +773,10 @@ where network_config.config.known_nodes_with_stake = vec![]; network_config.config.known_da_nodes = vec![]; + if network_config.enable_registration_verification { + tracing::error!("REGISTRATION VERIFICATION IS TURNED OFF"); + } + let web_api = define_api().map_err(|_e| io::Error::new(ErrorKind::Other, "Failed to define api")); From 7f2e9e4093d25161b3277a5f0bec888c05a6dad1 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 29 Aug 2024 12:48:02 -0600 Subject: [PATCH 10/10] remove check --- crates/orchestrator/src/lib.rs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index 76992ce59c..311d36c2cc 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -436,19 +436,17 @@ where // Assumes nodes do not post 'ready' twice fn post_ready(&mut self, peer_config: &PeerConfig) -> Result<(), ServerError> { // If we have not disabled registration verification. - if self.config.enable_registration_verification { - // Is this node allowed to connect? - if !self - .config - .config - .known_nodes_with_stake - .contains(peer_config) - { - return Err(ServerError { - status: tide_disco::StatusCode::FORBIDDEN, - message: "You are unauthorized to register with the orchestrator".to_string(), - }); - } + // Is this node allowed to connect? + if !self + .config + .config + .known_nodes_with_stake + .contains(peer_config) + { + return Err(ServerError { + status: tide_disco::StatusCode::FORBIDDEN, + message: "You are unauthorized to register with the orchestrator".to_string(), + }); } self.nodes_connected += 1;