diff --git a/m1/.cargo/config.toml b/m1/.cargo/config.toml index f93b5556..769caad3 100644 --- a/m1/.cargo/config.toml +++ b/m1/.cargo/config.toml @@ -30,4 +30,4 @@ rustflags = [ "force-unwind-tables=yes", "-C", "link-arg=/STACK:8000000" # Set stack to 8 MB -] +] \ No newline at end of file diff --git a/m1/Cargo.lock b/m1/Cargo.lock index eb9d3fc7..74a996ee 100644 --- a/m1/Cargo.lock +++ b/m1/Cargo.lock @@ -2565,8 +2565,7 @@ dependencies = [ [[package]] name = "avalanche-network-runner-sdk" version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355b909145de091c1c55b9aab6c507d77da853104c32ede00d3968998dfc4c18" +source = "git+https://github.com/0xmovses/avalanche-network-runner-sdk-rs?branch=main#c7bd446cb58931b33648810e6f0618231b27ae89" dependencies = [ "log", "prost", @@ -3177,6 +3176,23 @@ dependencies = [ "vec_map", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive 3.2.25", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap 0.16.0", +] + [[package]] name = "clap" version = "4.4.11" @@ -3184,7 +3200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", - "clap_derive", + "clap_derive 4.4.7", ] [[package]] @@ -3195,10 +3211,23 @@ checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", - "clap_lex", + "clap_lex 0.6.0", "strsim 0.10.0", ] +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "clap_derive" version = "4.4.7" @@ -3211,6 +3240,15 @@ dependencies = [ "syn 2.0.41", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "0.6.0" @@ -3404,14 +3442,25 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" +dependencies = [ + "percent-encoding 2.3.1", + "time", + "version_check", +] + [[package]] name = "cookie_store" -version = "0.16.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d606d0fba62e13cf04db20536c05cb7f13673c161cb47a47a82b9b9e7d3f1daa" +checksum = "387461abbc748185c3a6e1673d826918b450b87ff22639429c694619a83b6cf6" dependencies = [ - "cookie", - "idna 0.2.3", + "cookie 0.17.0", + "idna 0.3.0", "log", "publicsuffix", "serde 1.0.193", @@ -3961,6 +4010,7 @@ dependencies = [ "log", "random-manager", "serde_json", + "simulator", "subnet", "tempfile", "tokio", @@ -5075,17 +5125,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.3.0" @@ -6820,6 +6859,12 @@ dependencies = [ "num-traits 0.2.17", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "ouroboros" version = "0.9.5" @@ -7264,7 +7309,7 @@ dependencies = [ "async-trait", "bytes", "chrono", - "cookie", + "cookie 0.16.2", "futures-util", "headers", "http", @@ -8020,13 +8065,13 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64 0.21.5", "bytes", - "cookie", + "cookie 0.17.0", "cookie_store", "encoding_rs", "futures-core", @@ -8051,6 +8096,7 @@ dependencies = [ "serde 1.0.193", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -8941,6 +8987,29 @@ dependencies = [ "termcolor", ] +[[package]] +name = "simulator" +version = "0.1.0" +dependencies = [ + "anyhow", + "aptos-sdk", + "avalanche-installer", + "avalanche-network-runner-sdk", + "avalanche-types", + "clap 3.2.25", + "env_logger", + "log", + "once_cell", + "rand 0.7.3", + "random-manager", + "reqwest", + "serde 1.0.193", + "serde_json", + "tokio", + "tonic 0.9.2", + "url 2.5.0", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -9384,6 +9453,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.50" @@ -10450,9 +10525,9 @@ checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", diff --git a/m1/Cargo.toml b/m1/Cargo.toml index 13858016..4d65489b 100644 --- a/m1/Cargo.toml +++ b/m1/Cargo.toml @@ -3,7 +3,8 @@ resolver = "2" members = [ "subnet", "tests/e2e", - "e2e-benchmark" + "e2e-benchmark", + "simulator", ] [workspace.package] @@ -35,6 +36,7 @@ codespan-reporting = "0.11.1" criterion = "0.3.5" criterion-cpu-time = "0.1.0" dirs = "4.0.0" +env_logger = "0.10.1" hex = "0.4.3" hkdf = "0.10.0" hostname = "0.3.1" diff --git a/m1/justfile b/m1/justfile new file mode 100644 index 00000000..979ce5b9 --- /dev/null +++ b/m1/justfile @@ -0,0 +1,2 @@ +build: + ./scripts/build.debug.sh && cargo build -p simuatlor --bin simulator \ No newline at end of file diff --git a/m1/rustfmt.toml b/m1/rustfmt.toml new file mode 100644 index 00000000..055778f5 --- /dev/null +++ b/m1/rustfmt.toml @@ -0,0 +1,5 @@ +comment_width = 100 +format_code_in_doc_comments = true +imports_granularity = "Crate" +imports_layout = "Vertical" +wrap_comments = true \ No newline at end of file diff --git a/m1/scripts/build.debug.sh b/m1/scripts/build.debug.sh index d4b63c7c..c9289a61 100755 --- a/m1/scripts/build.debug.sh +++ b/m1/scripts/build.debug.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +#!/usr/bin/env bash set -xue if ! [[ "$0" =~ scripts/build.debug.sh ]]; then @@ -7,8 +8,10 @@ if ! [[ "$0" =~ scripts/build.debug.sh ]]; then fi PROTOC_VERSION=$(protoc --version | cut -f2 -d' ') -if [[ "${PROTOC_VERSION}" == "" ]] || [[ "${PROTOC_VERSION}" < 3.15.0 ]]; then - echo "protoc must be installed and the version must be greater than 3.15.0" +MIN_VERSION="3.15" + +if ! printf "%s\n%s" "$PROTOC_VERSION" "$MIN_VERSION" | sort -V | tail -n 1 | grep -q "$PROTOC_VERSION"; then + echo "protoc must be installed and the version must be greater than 3.15" exit 255 fi @@ -18,4 +21,4 @@ cargo build -p subnet --bin subnet ./target/debug/subnet --help ./target/debug/subnet genesis "hello world" -./target/debug/subnet vm-id subnet \ No newline at end of file +./target/debug/subnet vm-id subnet diff --git a/m1/scripts/subnet-cli-setup.sh b/m1/scripts/subnet-cli-setup.sh new file mode 100755 index 00000000..1c0cc2ad --- /dev/null +++ b/m1/scripts/subnet-cli-setup.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +# Install subnet-cli +VERSION=0.0.4 # Populate latest here + +GOARCH=$(go env GOARCH) +GOOS=$(go env GOOS) +DOWNLOAD_PATH=/tmp/subnet-cli.tar.gz +DOWNLOAD_URL=https://github.com/ava-labs/subnet-cli/releases/download/v${VERSION}/subnet-cli_${VERSION}_linux_${GOARCH}.tar.gz +if [[ ${GOOS} == "darwin" ]]; then + DOWNLOAD_URL=https://github.com/ava-labs/subnet-cli/releases/download/v${VERSION}/subnet-cli_${VERSION}_darwin_${GOARCH}.tar.gz +fi + +rm -f ${DOWNLOAD_PATH} +rm -f /tmp/subnet-cli + +echo "downloading subnet-cli ${VERSION} at ${DOWNLOAD_URL}" +curl -L ${DOWNLOAD_URL} -o ${DOWNLOAD_PATH} + +echo "extracting downloaded subnet-cli" +tar xzvf ${DOWNLOAD_PATH} -C /tmp + +/tmp/subnet-cli -h + +cp /tmp/subnet-cli $HOME/bin/subnet-cli \ No newline at end of file diff --git a/m1/simulator/Cargo.toml b/m1/simulator/Cargo.toml new file mode 100644 index 00000000..1b4c68d1 --- /dev/null +++ b/m1/simulator/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "simulator" +version = "0.1.0" +edition = "2021" + +[dependencies] +avalanche-installer = "0.0.77" +avalanche-network-runner-sdk = { git = "https://github.com/0xmovses/avalanche-network-runner-sdk-rs", branch = "main" } +log = "0.4.19" +random-manager = "0.0.5" +serde = { workspace = true } +serde_json = "1.0.108" # https://github.com/serde-rs/json/releases +avalanche-types = { workspace = true } # https://crates.io/crates/avalanche-types +aptos-sdk = {workspace = true } +anyhow = { workspace = true } +env_logger = { workspace = true } +url = { workspace = true } +tokio = { workspace = true } +tonic = "0.9.2" +once_cell = { workspace = true } +rand = { workspace = true } +reqwest = "0.11.24" +clap = { workspace = true } \ No newline at end of file diff --git a/m1/simulator/src/commands.rs b/m1/simulator/src/commands.rs new file mode 100644 index 00000000..d81f7333 --- /dev/null +++ b/m1/simulator/src/commands.rs @@ -0,0 +1,150 @@ +use clap::{Parser, Subcommand}; + +#[derive(Debug, Parser, Clone)] +#[clap(name = "forc index", about = "M1 network simulator", version = "0.1")] +pub struct Cli { + /// The command to run + #[clap(subcommand)] + pub command: SubCommands, + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, +} + +/// Start the simulator +#[derive(Debug, Parser, Clone)] +pub struct StartCommand { + /// The number of validators for the network + #[clap( + long, + default_value = "5", + help = "The number of validators for the network." + )] + pub nodes: u64, + + /// Sets if the validators join the network at once, or in a staggered way + #[clap( + long, + default_value = "false", + help = "Sets if the validators join the network at once, or in a staggered way." + )] + pub staggered: bool, + + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, + + /// The GRPC endpoint of the network runner to connect to + #[clap(long, help = "The GRPC endpoint of the network runner to connect to.")] + pub grpc_endpoint: Option, +} + +/// Partition the network +#[derive(Debug, Parser, Clone)] +pub struct PartitionCommand { + /// The percentage of validators that will be partitioned + #[clap( + long, + default_value = "5", + help = "The percentage of validators that will be in a partitioned state" + )] + pub amount: u8, + + /// Sets if the validators become paritioned at once or in a staggered way + #[clap( + long, + default_value = "false", + help = "Sets if the validators become partitioned at once or in a staggered way." + )] + pub staggered: bool, + + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, +} + +#[derive(Debug, Parser, Clone)] +pub struct ReconnectCommand { + /// The nodes to reconnect by `NodeId` + pub nodes: Vec, + + /// Sets if the validators rejoin the network together or in a staggered way + #[clap(long, default_value = "false")] + pub staggered: bool, + + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, +} + +/// Add a node to the network +#[derive(Debug, Parser, Clone)] +pub struct AddNodeCommand { + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, + + /// The name of the node to add + #[clap(long, help = "The name of the node to add.")] + pub name: Option, +} + +#[derive(Debug, Parser, Clone)] +pub struct RemoveNodeCommand { + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, + + /// The name of the node to remove + #[clap(long, help = "The name of the node to remove.")] + pub name: String, +} + +#[derive(Debug, Parser, Clone)] +pub struct AddValidatorCommand { + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, + + /// The name of the validator to add + #[clap(long, help = "The name of the validator to add.")] + pub name: String, +} + +#[derive(Debug, Parser, Clone)] +pub struct RemoveValidatorCommand { + /// Verbose output + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, + + /// The name of the validator to remove + #[clap(long, help = "The name of the validator to remove.")] + pub name: String, +} + +#[derive(Debug, Parser, Clone)] +pub struct HealthCommand { + /// Verbose ouput + #[clap(short, long, help = "Verbose output.")] + pub verbose: bool, +} + +#[derive(Debug, Subcommand, Clone)] +pub enum SubCommands { + /// Starts the network with a number of validators + Start(StartCommand), + /// Adds a node to the network + AddNode(AddNodeCommand), + /// Removes a node from the network + RemoveNode(RemoveNodeCommand), + /// Adds a validator to the network + AddValidator(AddValidatorCommand), + /// Removes a validator from the network + RemoveValidator(RemoveValidatorCommand), + /// Simulates a network partition. + Partition(PartitionCommand), + /// Reconnects the validators after they have become partitioned + Reconnect(ReconnectCommand), + /// Output the overall network and consensus health + Health(HealthCommand), +} diff --git a/m1/simulator/src/lib.rs b/m1/simulator/src/lib.rs new file mode 100644 index 00000000..0d631eae --- /dev/null +++ b/m1/simulator/src/lib.rs @@ -0,0 +1,465 @@ +use anyhow::{anyhow, Context, Error, Result}; +use std::{ + collections::HashMap, + env, + fs::{self, File}, + io::{self, Write}, + path::Path, + str::FromStr, + sync::Arc, + thread, + time::{Duration, Instant}, +}; + +use avalanche_network_runner_sdk::{ + rpcpb::{ + AddSubnetValidatorsRequest, CustomChainInfo, RemoveSubnetValidatorsRequest, + RemoveSubnetValidatorsSpec, SubnetValidatorsSpec, + }, + AddNodeRequest, BlockchainSpec, Client, GlobalConfig, RemoveNodeRequest, StartRequest, +}; +use avalanche_types::{ + ids, + jsonrpc::client::info as avalanche_sdk_info, + subnet::{self, rpc::snowman::block}, +}; +use commands::*; +use tonic::transport::Channel; + +pub mod commands; + +const AVALANCHEGO_VERSION: &str = "v1.10.12"; +pub const LOCAL_GRPC_ENDPOINT: &str = "http://127.0.0.1:12342"; +const VM_NAME: &str = "subnet"; + +/// The Simulator is used to run commands on the avalanche-go-network-runner. +/// It can be used across multiple threads and is thread safe. +pub struct Simulator { + /// The network runner client + pub cli: Arc>, + /// The command to run + pub command: SubCommands, + /// The path to the avalanchego binary, must be version no higher than 1.10.12 + /// higher versions use a VM plugin version higher that `28`, which is used by `subnet` + pub avalanchego_path: String, + /// The path to the VM plugin + pub vm_plugin_path: String, + /// The subnet ID created by the network runner, + /// this is not the same value as the VM ID. + pub subnet_id: Option, + /// The network ID + pub network_id: Option, +} + +impl Simulator { + pub async fn new(command: SubCommands) -> Result { + let cli = Client::new(LOCAL_GRPC_ENDPOINT).await; + Ok(Self { + cli: Arc::new(cli), + command, + avalanchego_path: get_avalanchego_path()?, + vm_plugin_path: get_vm_plugin_path()?, + subnet_id: None, + network_id: None, + }) + } + + pub async fn exec(&mut self, verbose: bool) -> Result<()> { + self.init_logger(verbose); + match &self.command { + SubCommands::Start(cmd) => self.start_network(cmd.clone()).await?, + SubCommands::AddNode(cmd) => self.add_node(cmd.clone()).await?, + SubCommands::RemoveNode(cmd) => self.remove_node(cmd.clone()).await?, + SubCommands::AddValidator(cmd) => self.add_validator(cmd.clone()).await?, + SubCommands::RemoveValidator(cmd) => self.remove_validator(cmd.clone()).await?, + SubCommands::Partition(cmd) => self.partition_network(cmd.clone()).await?, + SubCommands::Reconnect(cmd) => self.reconnect_validators(cmd.clone()).await?, + SubCommands::Health(cmd) => self.network_health(cmd.clone()).await?, + } + Ok(()) + } + + async fn start_network(&mut self, cmd: StartCommand) -> Result<()> { + log::debug!("Running command: {:?}", cmd); + + let vm_id = subnet::vm_name_to_id("subnet").unwrap(); + + let plugins_dir = if !&self.avalanchego_path.is_empty() { + let manifest_dir = env::var("CARGO_MANIFEST_DIR").context("No manifest dir found")?; + let workspace_dir = Path::new(&manifest_dir) + .parent() + .context("No parent dir found")?; + workspace_dir + .join("plugins") + .as_os_str() + .to_str() + .unwrap() + .to_string() + } else { + // Don't think this block will ever get hit in the current state + let exec_path = avalanche_installer::avalanchego::github::download( + None, + None, + Some(AVALANCHEGO_VERSION.to_string()), + ) + .await + .unwrap(); + self.avalanchego_path = exec_path.clone(); + avalanche_installer::avalanchego::get_plugin_dir(&self.avalanchego_path) + }; + + log::info!( + "copying vm plugin {} to {}/{}", + self.vm_plugin_path, + plugins_dir, + vm_id + ); + + fs::copy( + &self.vm_plugin_path, + Path::new(&plugins_dir).join(vm_id.to_string()), + )?; + + // write some random genesis file + let genesis = random_manager::secure_string(10); + + let genesis_file_path = random_manager::tmp_path(10, None).unwrap(); + sync_genesis(genesis.as_ref(), &genesis_file_path).unwrap(); + + log::info!( + "starting {} with avalanchego {}, genesis file path {}", + vm_id, + &self.avalanchego_path, + genesis_file_path, + ); + log::debug!( + "plugins dir: {}, global node config: {:?}", + plugins_dir, + serde_json::to_string(&GlobalConfig { + log_level: String::from("info"), + }) + ); + let resp = self + .cli + .start(StartRequest { + exec_path: self.avalanchego_path.clone(), + num_nodes: Some(cmd.nodes as u32), + plugin_dir: plugins_dir, + global_node_config: Some( + serde_json::to_string(&GlobalConfig { + log_level: String::from("info"), + }) + .unwrap(), + ), + blockchain_specs: vec![BlockchainSpec { + vm_name: String::from(VM_NAME), + genesis: genesis_file_path.to_string(), + //blockchain_alias : String::from("subnet"), // todo: this doesn't always work oddly enough, need to debug + ..Default::default() + }], + ..Default::default() + }) + .await?; + log::info!( + "started avalanchego cluster with network-runner: {:?}", + resp + ); + + // enough time for network-runner to get ready + thread::sleep(Duration::from_secs(20)); + + log::info!("checking cluster healthiness..."); + let mut ready = false; + + let timeout = Duration::from_secs(300); + let interval = Duration::from_secs(15); + let start = Instant::now(); + let mut cnt: u128 = 0; + loop { + let elapsed = start.elapsed(); + if elapsed.gt(&timeout) { + break; + } + + let itv = { + if cnt == 0 { + // first poll with no wait + Duration::from_secs(1) + } else { + interval + } + }; + thread::sleep(itv); + + ready = { + match self.cli.health().await { + Ok(_) => { + log::info!("healthy now!"); + true + } + Err(e) => { + log::warn!("not healthy yet {}", e); + false + } + } + }; + if ready { + break; + } + + cnt += 1; + } + assert!(ready); + + log::info!("checking status..."); + let mut status = self.cli.status().await.expect("failed status"); + loop { + let elapsed = start.elapsed(); + if elapsed.gt(&timeout) { + break; + } + + if let Some(ci) = &status.cluster_info { + if !ci.custom_chains.is_empty() { + break; + } + } + + log::info!("retrying checking status..."); + thread::sleep(interval); + status = self.cli.status().await.expect("failed status"); + } + + assert!(status.cluster_info.is_some()); + let cluster_info = status.cluster_info.unwrap(); + let mut rpc_eps: Vec = Vec::new(); + for (node_name, iv) in cluster_info.node_infos.into_iter() { + log::info!("{}: {}", node_name, iv.uri); + rpc_eps.push(iv.uri.clone()); + } + for (k, v) in cluster_info.custom_chains.iter() { + log::info!("custom chain info: {}={:?}", k, v); + } + log::info!("avalanchego RPC endpoints: {:?}", rpc_eps); + + let resp = avalanche_sdk_info::get_network_id(&rpc_eps[0]) + .await + .unwrap(); + let network_id = resp.result.unwrap().network_id; + log::info!("network Id: {}", network_id); + self.network_id = Some(network_id); + + Ok(()) + } + + async fn partition_network(&self, _: PartitionCommand) -> Result<()> { + Ok(()) + } + + async fn reconnect_validators(&self, _: ReconnectCommand) -> Result<()> { + Ok(()) + } + + async fn network_health(&self, cmd: HealthCommand) -> Result<()> { + log::debug!("Running command: {:?}", cmd); + let resp = self.cli.health().await?; + log::info!("network health: {:?}", resp); + Ok(()) + } + + async fn add_node(&self, cmd: AddNodeCommand) -> Result<()> { + log::debug!("Running command: {:?}", cmd.clone()); + let name = match cmd.name { + Some(n) => format!("node-{}", n), + None => format!("node-{}", random_manager::secure_string(5)), + }; + let resp = self + .cli + .add_node(AddNodeRequest { + name, + exec_path: self.avalanchego_path.clone(), + node_config: None, + ..Default::default() + }) + .await?; + log::info!("added node: {:?}", resp); + Ok(()) + } + + async fn remove_node(&self, cmd: RemoveNodeCommand) -> Result<()> { + log::debug!("Running command: {:?}", cmd); + let resp = self + .cli + .remove_node(RemoveNodeRequest { + name: cmd.name, + ..Default::default() + }) + .await?; + log::info!("removed node: {:?}", resp); + Ok(()) + } + + async fn add_validator(&self, cmd: AddValidatorCommand) -> Result<()> { + let resp = self + .cli + .add_validator(AddSubnetValidatorsRequest { + validator_spec: vec![{ + SubnetValidatorsSpec { + subnet_id: self.subnet_id().await?, + node_names: vec![cmd.name], + } + }], + }) + .await?; + log::info!("added validator: {:?}", resp); + Ok(()) + } + + async fn remove_validator(&self, cmd: RemoveValidatorCommand) -> Result<()> { + log::debug!("Running command: {:?}", cmd); + let resp = self + .cli + .remove_validator(RemoveSubnetValidatorsRequest { + validator_spec: vec![{ + RemoveSubnetValidatorsSpec { + subnet_id: self.subnet_id().await?, + node_names: vec![cmd.name], + } + }], + }) + .await?; + log::info!("removed validator: {:?}", resp); + Ok(()) + } + + async fn subnet_id(&self) -> Result { + let blockchain_id = self.blockchain_id().await?; + let resp = self.cli.health().await?; + let cluster_info = resp.cluster_info.context("no cluster info found")?; + let subnet_id = &cluster_info + .custom_chains + .get(&blockchain_id.to_string()) + .context("no custom chains found")? + .subnet_id; + return Ok(subnet_id.to_owned()); + } + + async fn blockchain_id(&self) -> Result { + log::info!("checking status..."); + let status = self.cli.status().await?; + + let mut blockchain_id = ids::Id::empty(); + + let cluster_info = status.cluster_info.context("no cluster info found")?; + for (k, v) in cluster_info.custom_chains.iter() { + log::info!("custom chain info: {}={:?}", k, v); + if v.chain_name == VM_NAME { + blockchain_id = ids::Id::from_str(&v.chain_id)?; + break; + } + } + Ok(blockchain_id) + } + + fn init_logger(&self, verbose: bool) { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(if verbose { + "debug" + } else { + "info" + })) + .init(); + } +} + +#[must_use] +pub fn get_network_runner_grpc_endpoint() -> (String, bool) { + match std::env::var("NETWORK_RUNNER_GRPC_ENDPOINT") { + Ok(s) => (s, true), + _ => (String::new(), false), + } +} + +#[must_use] +pub fn get_network_runner_enable_shutdown() -> bool { + matches!(std::env::var("NETWORK_RUNNER_ENABLE_SHUTDOWN"), Ok(_)) +} + +#[must_use] +pub fn get_avalanchego_path() -> Result { + let manifest_dir = env::var("CARGO_MANIFEST_DIR").context("No manifest dir found")?; + let manifest_path = Path::new(&manifest_dir); + + //Navigate two levels up from the Cargo manifest directory ../../ + let avalanchego_path = manifest_path + .parent() + .context("No parent dirctory found")? + .parent() + .context("No parent directory found")? + .parent() + .context("No parent directory found")? + .join("avalanchego") + .join("build") + .join("avalanchego"); + + if !avalanchego_path.exists() { + log::debug!("avalanchego path: {:?}", avalanchego_path); + return Err(anyhow!( + " + avalanchego binary not in expected path. + Install the binary at the expected path {:?}", + avalanchego_path + )); + } + + let path_buf = avalanchego_path + .to_str() + .context("Failed to convert path to string")?; + log::debug!("avalanchego path: {}", path_buf); + Ok(path_buf.to_string()) +} + +#[must_use] +pub fn get_vm_plugin_path() -> Result { + let manifest_dir = env::var("CARGO_MANIFEST_DIR").context("No manifest dir found")?; + let manifest_path = Path::new(&manifest_dir); + + // Construct the path to the binary with ./target/debug/subnet + let subnet_path = manifest_path + .parent() + .context("Could not find the parent dir")? + .join("target") + .join("debug") + .join("subnet"); + if !subnet_path.exists() { + log::debug!("vm plugin path: {:?}", subnet_path); + return Err(anyhow!( + " + vm plugin not in expected path. + Install the plugin at the expected path {:?}", + subnet_path + )); + } + + let path_buf = subnet_path + .to_str() + .context("Failed to convert path to string")?; + log::debug!("vm plugin path: {}", path_buf); + Ok(path_buf.to_string()) +} + +// todo: extracted from genesis method +// todo: really we should use a genesis once more +pub fn sync_genesis(byte_string: &str, file_path: &str) -> io::Result<()> { + log::info!("syncing genesis to '{}'", file_path); + + let path = Path::new(file_path); + let parent_dir = path.parent().expect("Invalid path"); + fs::create_dir_all(parent_dir)?; + + let d = byte_string.as_bytes(); + + let mut f = File::create(file_path)?; + f.write_all(&d)?; + + Ok(()) +} diff --git a/m1/simulator/src/main.rs b/m1/simulator/src/main.rs new file mode 100644 index 00000000..4c989bd9 --- /dev/null +++ b/m1/simulator/src/main.rs @@ -0,0 +1,10 @@ +use clap::Parser; +use simulator::{commands::Cli, Simulator}; + +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + let cli = Cli::parse(); + let mut simulator = Simulator::new(cli.command).await?; + simulator.exec(cli.verbose).await?; + Ok(()) +} diff --git a/m1/tests/e2e/Cargo.toml b/m1/tests/e2e/Cargo.toml index 69c36c12..6b7acf3a 100644 --- a/m1/tests/e2e/Cargo.toml +++ b/m1/tests/e2e/Cargo.toml @@ -9,10 +9,12 @@ license = "BSD-3-Clause" homepage = "https://avax.network" [dependencies] +tokio = { workspace = true } +simulator = { path = "../../simulator" } [dev-dependencies] avalanche-installer = "0.0.77" -avalanche-network-runner-sdk = "0.3.3" # https://crates.io/crates/avalanche-network-runner-sdk +avalanche-network-runner-sdk = { git = "https://github.com/0xmovses/avalanche-network-runner-sdk-rs", branch = "main" } avalanche-types = { workspace = true } # https://crates.io/crates/avalanche-types env_logger = "0.10.1" log = "0.4.19" diff --git a/m1/tests/e2e/src/lib.rs b/m1/tests/e2e/src/lib.rs index 6e11c1a5..318e31bc 100644 --- a/m1/tests/e2e/src/lib.rs +++ b/m1/tests/e2e/src/lib.rs @@ -1,31 +1,4 @@ #[cfg(test)] mod tests; -#[must_use] -pub fn get_network_runner_grpc_endpoint() -> (String, bool) { - match std::env::var("NETWORK_RUNNER_GRPC_ENDPOINT") { - Ok(s) => (s, true), - _ => (String::new(), false), - } -} -#[must_use] -pub fn get_network_runner_enable_shutdown() -> bool { - matches!(std::env::var("NETWORK_RUNNER_ENABLE_SHUTDOWN"), Ok(_)) -} - -#[must_use] -pub fn get_avalanchego_path() -> (String, bool) { - match std::env::var("AVALANCHEGO_PATH") { - Ok(s) => (s, true), - _ => (String::new(), false), - } -} - -#[must_use] -pub fn get_vm_plugin_path() -> (String, bool) { - match std::env::var("VM_PLUGIN_PATH") { - Ok(s) => (s, true), - _ => (String::new(), false), - } -} diff --git a/m1/tests/e2e/src/tests/mod.rs b/m1/tests/e2e/src/tests/mod.rs index 6382b619..1cfaefe4 100644 --- a/m1/tests/e2e/src/tests/mod.rs +++ b/m1/tests/e2e/src/tests/mod.rs @@ -1,245 +1,21 @@ -use core::time; -use std::{ - io, - fs::{self, File}, - path::Path, - str::FromStr, - thread, - time::{Duration, Instant}, io::Write, +use simulator::{ + commands::{StartCommand, SubCommands}, + Simulator, }; -use avalanche_network_runner_sdk::{BlockchainSpec, Client, GlobalConfig, StartRequest}; -use avalanche_types::{ids, jsonrpc::client::info as avalanche_sdk_info, subnet}; - -const AVALANCHEGO_VERSION: &str = "v1.10.9"; - -// todo: extracted from genesis method -// todo: really we should use a genesis once more -pub fn sync_genesis(byte_string : &str, file_path: &str) -> io::Result<()> { - log::info!("syncing genesis to '{}'", file_path); - - let path = Path::new(file_path); - let parent_dir = path.parent().expect("Invalid path"); - fs::create_dir_all(parent_dir)?; - - let d = byte_string.as_bytes(); - - let mut f = File::create(file_path)?; - f.write_all(&d)?; - - Ok(()) -} - #[tokio::test] async fn e2e() { - let _ = env_logger::builder() - .filter_level(log::LevelFilter::Info) - .is_test(true) - .try_init(); - - let (ep, is_set) = crate::get_network_runner_grpc_endpoint(); - assert!(is_set); - - let cli = Client::new(&ep).await; - - log::info!("ping..."); - let resp = cli.ping().await.expect("failed ping"); - log::info!("network-runner is running (ping response {:?})", resp); - - let (vm_plugin_path, exists) = crate::get_vm_plugin_path(); - log::info!("Vm Plugin path: {vm_plugin_path}"); - assert!(exists); - assert!(Path::new(&vm_plugin_path).exists()); - - let vm_id = Path::new(&vm_plugin_path) - .file_stem() - .unwrap() - .to_str() - .unwrap() - .to_string(); - // ! for now, we hardcode the id to be subnet for orchestration - let vm_id = subnet::vm_name_to_id("subnet").unwrap(); - - let (mut avalanchego_exec_path, _) = crate::get_avalanchego_path(); - let plugins_dir = if !avalanchego_exec_path.is_empty() { - let parent_dir = Path::new(&avalanchego_exec_path) - .parent() - .expect("unexpected None parent"); - parent_dir - .join("plugins") - .as_os_str() - .to_str() - .unwrap() - .to_string() - } else { - let exec_path = avalanche_installer::avalanchego::github::download( - None, - None, - Some(AVALANCHEGO_VERSION.to_string()), - ) - .await - .unwrap(); - avalanchego_exec_path = exec_path; - avalanche_installer::avalanchego::get_plugin_dir(&avalanchego_exec_path) + let cmd = StartCommand { + nodes: 5, + staggered: false, + verbose: false, + grpc_endpoint: None, }; - - log::info!( - "copying vm plugin {} to {}/{}", - vm_plugin_path, - plugins_dir, - vm_id - ); - - fs::create_dir(&plugins_dir).unwrap(); - fs::copy( - &vm_plugin_path, - Path::new(&plugins_dir).join(vm_id.to_string()), - ) - .unwrap(); - - // write some random genesis file - let genesis = random_manager::secure_string(10); - - let genesis_file_path = random_manager::tmp_path(10, None).unwrap(); - sync_genesis(genesis.as_ref(), &genesis_file_path).unwrap(); - - log::info!( - "starting {} with avalanchego {}, genesis file path {}", - vm_id, - &avalanchego_exec_path, - genesis_file_path, - ); - let resp = cli - .start(StartRequest { - exec_path: avalanchego_exec_path, - num_nodes: Some(5), - plugin_dir: plugins_dir, - global_node_config: Some( - serde_json::to_string(&GlobalConfig { - log_level: String::from("info"), - }) - .unwrap(), - ), - blockchain_specs: vec![BlockchainSpec { - vm_name: String::from("subnet"), - genesis: genesis_file_path.to_string(), - // blockchain_alias : String::from("subnet"), // todo: this doesn't always work oddly enough, need to debug - ..Default::default() - }], - ..Default::default() - }) + let mut simulator = Simulator::new(SubCommands::Start(cmd)) .await - .expect("failed start"); - log::info!( - "started avalanchego cluster with network-runner: {:?}", - resp - ); - - // enough time for network-runner to get ready - thread::sleep(Duration::from_secs(20)); - - log::info!("checking cluster healthiness..."); - let mut ready = false; - - let timeout = Duration::from_secs(300); - let interval = Duration::from_secs(15); - let start = Instant::now(); - let mut cnt: u128 = 0; - loop { - let elapsed = start.elapsed(); - if elapsed.gt(&timeout) { - break; - } - - let itv = { - if cnt == 0 { - // first poll with no wait - Duration::from_secs(1) - } else { - interval - } - }; - thread::sleep(itv); - - ready = { - match cli.health().await { - Ok(_) => { - log::info!("healthy now!"); - true - } - Err(e) => { - log::warn!("not healthy yet {}", e); - false - } - } - }; - if ready { - break; - } - - cnt += 1; - } - assert!(ready); - - log::info!("checking status..."); - let mut status = cli.status().await.expect("failed status"); - loop { - let elapsed = start.elapsed(); - if elapsed.gt(&timeout) { - break; - } - - if let Some(ci) = &status.cluster_info { - if !ci.custom_chains.is_empty() { - break; - } - } - - log::info!("retrying checking status..."); - thread::sleep(interval); - status = cli.status().await.expect("failed status"); - } - - assert!(status.cluster_info.is_some()); - let cluster_info = status.cluster_info.unwrap(); - let mut rpc_eps: Vec = Vec::new(); - for (node_name, iv) in cluster_info.node_infos.into_iter() { - log::info!("{}: {}", node_name, iv.uri); - rpc_eps.push(iv.uri.clone()); - } - let mut blockchain_id = ids::Id::empty(); - for (k, v) in cluster_info.custom_chains.iter() { - log::info!("custom chain info: {}={:?}", k, v); - if v.chain_name == "subnet" { - blockchain_id = ids::Id::from_str(&v.chain_id).unwrap(); - break; - } - } - log::info!("avalanchego RPC endpoints: {:?}", rpc_eps); - - let resp = avalanche_sdk_info::get_network_id(&rpc_eps[0]) + .expect("Failed to create simulator"); + simulator + .exec(cmd.verbose) .await - .unwrap(); - let network_id = resp.result.unwrap().network_id; - log::info!("network Id: {}", network_id); - - // keep alive by sleeping for duration provided by SUBNET_TIMEOUT environment variable - // use sensible default - - let val = std::env::var("SUBNET_TIMEOUT") - .unwrap_or_else(|_| "0".to_string()) - .parse::() - .unwrap(); - - log::info!("sleeping for {} seconds", timeout.as_secs()); - if val < 0 { - // run forever - loop { - thread::sleep(Duration::from_secs(1000)); - } - } else { - let timeout = Duration::from_secs(val as u64); - thread::sleep(timeout); - } - + .expect("Failed to execute simulator"); }