Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Tech Debt] Allow nodes to rejoin by saving index and config #2168

Merged
merged 6 commits into from
Dec 9, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 12 additions & 32 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 1 addition & 15 deletions crates/hotshot/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,74 +21,60 @@ hotshot-testing = []
# libp2p
[[example]]
name = "validator-libp2p"
required-features = ["libp2p/rsa"]
path = "examples/libp2p/validator.rs"

[[example]]
name = "multi-validator-libp2p"
required-features = ["libp2p/rsa"]
path = "examples/libp2p/multi-validator.rs"

[[example]]
name = "orchestrator-libp2p"
required-features = ["libp2p/rsa"]
path = "examples/libp2p/orchestrator.rs"

[[example]]
name = "all-libp2p"
required-features = ["libp2p/rsa"]
path = "examples/libp2p/all.rs"

# webserver
[[example]]
name = "webserver"
required-features = ["libp2p/rsa"]
path = "examples/webserver/webserver.rs"

[[example]]
name = "orchestrator-webserver"
required-features = ["libp2p/rsa"]
path = "examples/webserver/orchestrator.rs"

[[example]]
name = "validator-webserver"
required-features = ["libp2p/rsa"]
path = "examples/webserver/validator.rs"

[[example]]
name = "multi-validator-webserver"
required-features = ["libp2p/rsa"]
path = "examples/webserver/multi-validator.rs"

[[example]]
name = "multi-webserver"
required-features = ["libp2p/rsa"]
path = "examples/webserver/multi-webserver.rs"

[[example]]
name = "all-webserver"
required-features = ["libp2p/rsa"]
path = "examples/webserver/all.rs"

# combined
[[example]]
name = "all-combined"
required-features = ["libp2p/rsa"]
path = "examples/combined/all.rs"

[[example]]
name = "multi-validator-combined"
required-features = ["libp2p/rsa"]
path = "examples/combined/multi-validator.rs"

[[example]]
name = "validator-combined"
required-features = ["libp2p/rsa"]
path = "examples/combined/validator.rs"

[[example]]
name = "orchestrator-combined"
required-features = ["libp2p/rsa"]
path = "examples/combined/orchestrator.rs"

[dependencies]
Expand Down Expand Up @@ -141,4 +127,4 @@ blake3 = { workspace = true }
clap = { version = "4.4", features = ["derive", "env"] }
serde_json = "1.0.108"
toml = { workspace = true }
hotshot-testing = { path = "../testing" }
hotshot-testing = { path = "../testing" }
2 changes: 1 addition & 1 deletion crates/hotshot/examples/combined/all.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ async fn main() {
NodeImpl,
>(OrchestratorArgs {
url: Url::parse("http://localhost:4444").unwrap(),

config_file: args.config_file.clone(),
}));

Expand All @@ -102,6 +101,7 @@ async fn main() {
url: "http://localhost".to_string(),
port: 4444,
public_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)),
network_config_file: None,
})
.await
});
Expand Down
14 changes: 11 additions & 3 deletions crates/hotshot/examples/combined/multi-validator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ struct MultiValidatorArgs {
/// This node's public IP address, for libp2p
/// If no IP address is passed in, it will default to 127.0.0.1
pub public_ip: Option<IpAddr>,
/// An optional network config file to save to/load from
/// Allows for rejoining the network on a complete state loss
#[arg(short, long)]
pub network_config_file: Option<String>,
}

#[cfg_attr(
Expand All @@ -45,8 +49,9 @@ async fn main() {
args.port
);
let mut nodes = Vec::new();
for _ in 0..args.num_nodes {
let url: String = args.url.clone();

for node_index in 0..args.num_nodes {
let args = args.clone();

let node = async_spawn(async move {
infra::main_entry_point::<
Expand All @@ -58,9 +63,12 @@ async fn main() {
NodeImpl,
ThisRun,
>(ValidatorArgs {
url,
url: args.url,
port: args.port,
public_ip: args.public_ip,
network_config_file: args
.network_config_file
.map(|s| format!("{}-{}", s, node_index)),
})
rob-maron marked this conversation as resolved.
Show resolved Hide resolved
.await
});
Expand Down
46 changes: 21 additions & 25 deletions crates/hotshot/examples/infra/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use hotshot::{
types::{SignatureKey, SystemContextHandle},
HotShotType, Memberships, Networks, SystemContext,
};
use hotshot_orchestrator::config::NetworkConfigSource;
use hotshot_orchestrator::{
self,
client::{OrchestratorClient, ValidatorArgs},
Expand Down Expand Up @@ -825,34 +826,27 @@ pub async fn main_entry_point<

error!("Starting validator");

let orchestrator_client: OrchestratorClient =
OrchestratorClient::connect_to_orchestrator(args.clone()).await;

// Identify with the orchestrator
// see what our public identity will be
let public_ip = match args.public_ip {
Some(ip) => ip,
None => local_ip_address::local_ip().unwrap(),
};
error!(
"Identifying with orchestrator using IP address {}",
public_ip.to_string()
);
let node_index: u16 = orchestrator_client
.identify_with_orchestrator(public_ip.to_string())
.await;
error!("Finished identifying; our node index is {node_index}");
error!("Getting config from orchestrator");

let mut run_config = orchestrator_client
.get_config_from_orchestrator::<TYPES>(node_index)
.await;

run_config.node_index = node_index.into();

let orchestrator_client: OrchestratorClient =
OrchestratorClient::new(args.clone(), public_ip.to_string()).await;

// conditionally save/load config from file or orchestrator
let (mut run_config, source) =
NetworkConfig::from_file_or_orchestrator(&orchestrator_client, args.network_config_file)
.await;

let node_index = run_config.node_index;
error!("Retrieved config; our node index is {node_index}");

run_config.config.my_own_validator_config =
ValidatorConfig::<<TYPES as NodeType>::SignatureKey>::generated_from_seed_indexed(
run_config.seed,
node_index.into(),
node_index,
1,
);
//run_config.libp2p_config.as_mut().unwrap().public_ip = args.public_ip.unwrap();
Expand Down Expand Up @@ -892,12 +886,14 @@ pub async fn main_entry_point<
}
}

error!("Waiting for start command from orchestrator");
orchestrator_client
.wait_for_all_nodes_ready(run_config.clone().node_index)
.await;
if let NetworkConfigSource::Orchestrator = source {
error!("Waiting for the start command from orchestrator");
orchestrator_client
.wait_for_all_nodes_ready(run_config.clone().node_index)
.await;
}

error!("All nodes are ready! Starting HotShot");
error!("Starting HotShot");
run.run_hotshot(
hotshot,
&mut transactions,
Expand Down
2 changes: 1 addition & 1 deletion crates/hotshot/examples/libp2p/all.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ async fn main() {
NodeImpl,
>(OrchestratorArgs {
url: Url::parse("http://localhost:4444").unwrap(),

config_file: args.config_file.clone(),
}));

Expand All @@ -69,6 +68,7 @@ async fn main() {
url: "http://localhost".to_string(),
port: 4444,
public_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)),
network_config_file: None,
})
.await
});
Expand Down
13 changes: 10 additions & 3 deletions crates/hotshot/examples/libp2p/multi-validator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ struct MultiValidatorArgs {
/// This node's public IP address, for libp2p
/// If no IP address is passed in, it will default to 127.0.0.1
pub public_ip: Option<IpAddr>,
/// An optional network config file to save to/load from
/// Allows for rejoining the network on a complete state loss
#[arg(short, long)]
pub network_config_file: Option<String>,
}

#[cfg_attr(
Expand All @@ -45,8 +49,8 @@ async fn main() {
args.port
);
let mut nodes = Vec::new();
for _ in 0..args.num_nodes {
let url: String = args.url.clone();
for node_index in 0..args.num_nodes {
let args = args.clone();

let node = async_spawn(async move {
infra::main_entry_point::<
Expand All @@ -58,9 +62,12 @@ async fn main() {
NodeImpl,
ThisRun,
>(ValidatorArgs {
url,
url: args.url,
port: args.port,
public_ip: args.public_ip,
network_config_file: args
.network_config_file
.map(|s| format!("{}-{}", s, node_index)),
})
rob-maron marked this conversation as resolved.
Show resolved Hide resolved
.await
});
Expand Down
Loading
Loading