diff --git a/crates/configuration/Cargo.toml b/crates/configuration/Cargo.toml index ad20f731e..56d70d80a 100644 --- a/crates/configuration/Cargo.toml +++ b/crates/configuration/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" regex = { workspace = true } lazy_static = { workspace = true } multiaddr = { workspace = true } -url = { workspace = true } +url = { workspace = true, features = ["serde"] } thiserror = { workspace = true } anyhow = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/crates/configuration/src/global_settings.rs b/crates/configuration/src/global_settings.rs index 8c6f5b85c..607951e89 100644 --- a/crates/configuration/src/global_settings.rs +++ b/crates/configuration/src/global_settings.rs @@ -1,7 +1,7 @@ use std::{error::Error, fmt::Display, net::IpAddr, str::FromStr}; use multiaddr::Multiaddr; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::shared::{ errors::{ConfigError, FieldError}, @@ -10,9 +10,9 @@ use crate::shared::{ }; /// Global settings applied to an entire network. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct GlobalSettings { - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] bootnodes_addresses: Vec, // TODO: parse both case in zombienet node version to avoid renamed ? #[serde(rename = "timeout")] diff --git a/crates/configuration/src/hrmp_channel.rs b/crates/configuration/src/hrmp_channel.rs index ad2a0b53f..228ff2ef4 100644 --- a/crates/configuration/src/hrmp_channel.rs +++ b/crates/configuration/src/hrmp_channel.rs @@ -1,11 +1,11 @@ use std::marker::PhantomData; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::shared::{macros::states, types::ParaId}; /// HRMP channel configuration, with fine-grained configuration options. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct HrmpChannelConfig { sender: ParaId, recipient: ParaId, diff --git a/crates/configuration/src/lib.rs b/crates/configuration/src/lib.rs index 624ac6909..dc7158f44 100644 --- a/crates/configuration/src/lib.rs +++ b/crates/configuration/src/lib.rs @@ -1,3 +1,4 @@ +#![allow(clippy::expect_fun_call)] mod global_settings; mod hrmp_channel; mod network; diff --git a/crates/configuration/src/network.rs b/crates/configuration/src/network.rs index 239b74f6c..f5a6160c3 100644 --- a/crates/configuration/src/network.rs +++ b/crates/configuration/src/network.rs @@ -1,25 +1,35 @@ -use std::{cell::RefCell, marker::PhantomData, rc::Rc}; +use std::{cell::RefCell, fs, marker::PhantomData, rc::Rc}; +use anyhow::anyhow; use regex::Regex; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{ global_settings::{GlobalSettings, GlobalSettingsBuilder}, hrmp_channel::{self, HrmpChannelConfig, HrmpChannelConfigBuilder}, parachain::{self, ParachainConfig, ParachainConfigBuilder}, relaychain::{self, RelaychainConfig, RelaychainConfigBuilder}, - shared::{helpers::merge_errors_vecs, macros::states, types::ValidationContext}, + shared::{ + constants::{ + NO_ERR_DEF_BUILDER, RELAY_NOT_NONE, RW_FAILED, THIS_IS_A_BUG, VALIDATION_CHECK, + VALID_REGEX, + }, + helpers::merge_errors_vecs, + macros::states, + node::NodeConfig, + types::{Arg, AssetLocation, Chain, Command, Image, ValidationContext}, + }, }; /// A network configuration, composed of a relaychain, parachains and HRMP channels. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct NetworkConfig { #[serde(rename = "settings")] global_settings: GlobalSettings, relaychain: Option, - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] parachains: Vec, - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] hrmp_channels: Vec, } @@ -33,7 +43,7 @@ impl NetworkConfig { pub fn relaychain(&self) -> &RelaychainConfig { self.relaychain .as_ref() - .expect("typestate should ensure the relaychain isn't None at this point, this is a bug please report it: https://github.com/paritytech/zombienet-sdk/issues") + .expect(&format!("{}, {}", RELAY_NOT_NONE, THIS_IS_A_BUG)) } /// The parachains of the network. @@ -48,11 +58,110 @@ impl NetworkConfig { pub fn dump_to_toml(&self) -> Result { // This regex is used to replace the "" enclosed u128 value to a raw u128 because u128 is not supported for TOML serialization/deserialization. - let re = Regex::new(r#""U128%(?\d+)""#).expect("regex should be valid, this is a bug please report it: https://github.com/paritytech/zombienet-sdk/issues"); + let re = Regex::new(r#""U128%(?\d+)""#) + .expect(&format!("{} {}", VALID_REGEX, THIS_IS_A_BUG)); let toml_string = toml::to_string_pretty(&self)?; Ok(re.replace_all(&toml_string, "$u128_value").to_string()) } + + pub fn load_from_toml(path: &str) -> Result { + let file_str = fs::read_to_string(path).expect(&format!("{} {}", RW_FAILED, THIS_IS_A_BUG)); + let re: Regex = Regex::new(r"(?(initial_)?balance)\s+=\s+(?\d+)") + .expect(&format!("{} {}", VALID_REGEX, THIS_IS_A_BUG)); + + let mut network_config: NetworkConfig = toml::from_str( + re.replace_all(&file_str, "$field_name = \"$u128_value\"") + .as_ref(), + )?; + + // All unwraps below are safe, because we ensure that the relaychain is not None at this point + if network_config.relaychain.is_none() { + Err(anyhow!("Relay chain does not exist."))? + } + + // retrieve the defaults relaychain for assigning to nodes if needed + let relaychain_default_command: Option = + network_config.relaychain().default_command().cloned(); + + let relaychain_default_image: Option = + network_config.relaychain().default_image().cloned(); + + let relaychain_default_db_snapshot: Option = + network_config.relaychain().default_db_snapshot().cloned(); + + let default_args: Vec = network_config + .relaychain() + .default_args() + .into_iter() + .cloned() + .collect(); + + let mut nodes: Vec = network_config + .relaychain() + .nodes() + .into_iter() + .cloned() + .collect(); + + // Validation checks for relay + TryInto::::try_into(network_config.relaychain().chain().as_str())?; + if relaychain_default_image.is_some() { + TryInto::::try_into(relaychain_default_image.clone().expect(VALIDATION_CHECK))?; + } + if relaychain_default_command.is_some() { + TryInto::::try_into( + relaychain_default_command.clone().expect(VALIDATION_CHECK), + )?; + } + + for node in nodes.iter_mut() { + if relaychain_default_command.is_some() { + // we modify only nodes which don't already have a command + if node.command.is_none() { + node.command = relaychain_default_command.clone(); + } + } + + if relaychain_default_image.is_some() && node.image.is_none() { + node.image = relaychain_default_image.clone(); + } + + if relaychain_default_db_snapshot.is_some() && node.db_snapshot.is_none() { + node.db_snapshot = relaychain_default_db_snapshot.clone(); + } + + if !default_args.is_empty() && node.args().is_empty() { + node.set_args(default_args.clone()); + } + } + + network_config + .relaychain + .as_mut() + .expect(&format!("{}, {}", NO_ERR_DEF_BUILDER, THIS_IS_A_BUG)) + .set_nodes(nodes); + + // Validation checks for parachains + network_config.parachains().iter().for_each(|parachain| { + let _ = TryInto::::try_into( + parachain + .chain() + .ok_or("chain name must exist") + .unwrap() + .as_str(), + ); + + if parachain.default_image().is_some() { + let _ = TryInto::::try_into(parachain.default_image().unwrap().as_str()); + } + if parachain.default_command().is_some() { + let _ = TryInto::::try_into(parachain.default_command().unwrap().as_str()); + } + }); + + Ok(network_config) + } } states! { @@ -143,9 +252,9 @@ impl Default for NetworkConfigBuilder { fn default() -> Self { Self { config: NetworkConfig { - global_settings: GlobalSettingsBuilder::new().build().expect( - "should have no errors for default builder. this is a bug, please report it", - ), + global_settings: GlobalSettingsBuilder::new() + .build() + .expect(&format!("{}, {}", NO_ERR_DEF_BUILDER, THIS_IS_A_BUG)), relaychain: None, parachains: vec![], hrmp_channels: vec![], @@ -851,4 +960,436 @@ mod tests { fs::read_to_string("./testing/snapshots/0002-overridden-defaults.toml").unwrap(); assert_eq!(got, expected); } + + #[test] + fn the_toml_config_should_be_imported_and_match_a_network() { + let load_from_toml = + NetworkConfig::load_from_toml("./testing/snapshots/0000-small-network.toml").unwrap(); + + let expected = NetworkConfigBuilder::new() + .with_relaychain(|relaychain| { + relaychain + .with_chain("rococo-local") + .with_default_command("polkadot") + .with_default_image("docker.io/parity/polkadot:latest") + .with_default_args(vec![("-lparachain", "debug").into()]) + .with_node(|node| { + node.with_name("alice") + .validator(true) + .invulnerable(true) + .validator(true) + .bootnode(false) + .with_initial_balance(2000000000000) + }) + .with_node(|node| { + node.with_name("bob") + .with_args(vec![("--database", "paritydb-experimental").into()]) + .validator(true) + .invulnerable(false) + .bootnode(true) + .with_initial_balance(2000000000000) + }) + }) + .build() + .unwrap(); + + // We need to assert parts of the network config separately because the expected one contains the chain default context which + // is used for dumbing to tomp while the + // while loaded + assert_eq!( + expected.relaychain().chain(), + load_from_toml.relaychain().chain() + ); + assert_eq!( + expected.relaychain().default_args(), + load_from_toml.relaychain().default_args() + ); + assert_eq!( + expected.relaychain().default_command(), + load_from_toml.relaychain().default_command() + ); + assert_eq!( + expected.relaychain().default_image(), + load_from_toml.relaychain().default_image() + ); + + // Check the nodes without the Chain Default Context + expected + .relaychain() + .nodes() + .iter() + .zip(load_from_toml.relaychain().nodes().iter()) + .for_each(|(expected_node, loaded_node)| { + assert_eq!(expected_node.name(), loaded_node.name()); + assert_eq!(expected_node.command(), loaded_node.command()); + assert_eq!(expected_node.args(), loaded_node.args()); + assert_eq!( + expected_node.is_invulnerable(), + loaded_node.is_invulnerable() + ); + assert_eq!(expected_node.is_validator(), loaded_node.is_validator()); + assert_eq!(expected_node.is_bootnode(), loaded_node.is_bootnode()); + assert_eq!( + expected_node.initial_balance(), + loaded_node.initial_balance() + ); + }); + } + + #[test] + fn the_toml_config_should_be_imported_and_match_a_network_with_parachains() { + let load_from_toml = + NetworkConfig::load_from_toml("./testing/snapshots/0001-big-network.toml").unwrap(); + + let expected = NetworkConfigBuilder::new() + .with_relaychain(|relaychain| { + relaychain + .with_chain("polkadot") + .with_default_command("polkadot") + .with_default_image("docker.io/parity/polkadot:latest") + .with_default_resources(|resources| { + resources + .with_request_cpu(100000) + .with_request_memory("500M") + .with_limit_cpu("10Gi") + .with_limit_memory("4000M") + }) + .with_node(|node| { + node.with_name("alice") + .with_initial_balance(1_000_000_000) + .validator(true) + .bootnode(true) + .invulnerable(true) + }) + .with_node(|node| { + node.with_name("bob") + .validator(true) + .invulnerable(true) + .bootnode(true) + }) + }) + .with_parachain(|parachain| { + parachain + .with_id(1000) + .with_chain("myparachain") + .with_chain_spec_path("/path/to/my/chain/spec.json") + .with_registration_strategy(RegistrationStrategy::UsingExtrinsic) + .onboard_as_parachain(false) + .with_default_db_snapshot("https://storage.com/path/to/db_snapshot.tgz") + .with_collator(|collator| { + collator + .with_name("john") + .bootnode(true) + .validator(true) + .invulnerable(true) + .with_initial_balance(5_000_000_000) + }) + .with_collator(|collator| { + collator + .with_name("charles") + .bootnode(true) + .invulnerable(true) + .with_initial_balance(0) + }) + .with_collator(|collator| { + collator + .with_name("frank") + .validator(true) + .bootnode(true) + .with_initial_balance(1_000_000_000) + }) + }) + .with_parachain(|parachain| { + parachain + .with_id(2000) + .with_chain("myotherparachain") + .with_chain_spec_path("/path/to/my/other/chain/spec.json") + .with_collator(|collator| { + collator + .with_name("mike") + .bootnode(true) + .validator(true) + .invulnerable(true) + .with_initial_balance(5_000_000_000) + }) + .with_collator(|collator| { + collator + .with_name("georges") + .bootnode(true) + .invulnerable(true) + .with_initial_balance(0) + }) + .with_collator(|collator| { + collator + .with_name("victor") + .validator(true) + .bootnode(true) + .with_initial_balance(1_000_000_000) + }) + }) + .with_hrmp_channel(|hrmp_channel| { + hrmp_channel + .with_sender(1000) + .with_recipient(2000) + .with_max_capacity(150) + .with_max_message_size(5000) + }) + .with_hrmp_channel(|hrmp_channel| { + hrmp_channel + .with_sender(2000) + .with_recipient(1000) + .with_max_capacity(200) + .with_max_message_size(8000) + }) + .build() + .unwrap(); + + // Check the relay chain + assert_eq!( + expected.relaychain().default_resources(), + load_from_toml.relaychain().default_resources() + ); + + // Check the nodes without the Chain Default Context + expected + .relaychain() + .nodes() + .iter() + .zip(load_from_toml.relaychain().nodes().iter()) + .for_each(|(expected_node, loaded_node)| { + assert_eq!(expected_node.name(), loaded_node.name()); + assert_eq!(expected_node.command(), loaded_node.command()); + assert_eq!(expected_node.args(), loaded_node.args()); + assert_eq!(expected_node.is_validator(), loaded_node.is_validator()); + assert_eq!(expected_node.is_bootnode(), loaded_node.is_bootnode()); + assert_eq!( + expected_node.initial_balance(), + loaded_node.initial_balance() + ); + assert_eq!( + expected_node.is_invulnerable(), + loaded_node.is_invulnerable() + ); + }); + + expected + .parachains() + .iter() + .zip(load_from_toml.parachains().iter()) + .for_each(|(expected_parachain, loaded_parachain)| { + assert_eq!(expected_parachain.id(), loaded_parachain.id()); + assert_eq!(expected_parachain.chain(), loaded_parachain.chain()); + assert_eq!( + expected_parachain.chain_spec_path(), + loaded_parachain.chain_spec_path() + ); + assert_eq!( + expected_parachain.registration_strategy(), + loaded_parachain.registration_strategy() + ); + assert_eq!( + expected_parachain.onboard_as_parachain(), + loaded_parachain.onboard_as_parachain() + ); + assert_eq!( + expected_parachain.default_db_snapshot(), + loaded_parachain.default_db_snapshot() + ); + assert_eq!( + expected_parachain.default_command(), + loaded_parachain.default_command() + ); + assert_eq!( + expected_parachain.default_image(), + loaded_parachain.default_image() + ); + assert_eq!( + expected_parachain.collators().len(), + loaded_parachain.collators().len() + ); + expected_parachain + .collators() + .iter() + .zip(loaded_parachain.collators().iter()) + .for_each(|(expected_collator, loaded_collator)| { + assert_eq!(expected_collator.name(), loaded_collator.name()); + assert_eq!(expected_collator.command(), loaded_collator.command()); + assert_eq!(expected_collator.image(), loaded_collator.image()); + assert_eq!( + expected_collator.is_validator(), + loaded_collator.is_validator() + ); + assert_eq!( + expected_collator.is_bootnode(), + loaded_collator.is_bootnode() + ); + assert_eq!( + expected_collator.is_invulnerable(), + loaded_collator.is_invulnerable() + ); + assert_eq!( + expected_collator.initial_balance(), + loaded_collator.initial_balance() + ); + }); + }); + + expected + .hrmp_channels() + .iter() + .zip(load_from_toml.hrmp_channels().iter()) + .for_each(|(expected_hrmp_channel, loaded_hrmp_channel)| { + assert_eq!(expected_hrmp_channel.sender(), loaded_hrmp_channel.sender()); + assert_eq!( + expected_hrmp_channel.recipient(), + loaded_hrmp_channel.recipient() + ); + assert_eq!( + expected_hrmp_channel.max_capacity(), + loaded_hrmp_channel.max_capacity() + ); + assert_eq!( + expected_hrmp_channel.max_message_size(), + loaded_hrmp_channel.max_message_size() + ); + }); + } + + #[test] + fn the_toml_config_should_be_imported_and_match_a_network_with_overriden_defaults() { + let load_from_toml = + NetworkConfig::load_from_toml("./testing/snapshots/0002-overridden-defaults.toml") + .unwrap(); + + let expected = NetworkConfigBuilder::new() + .with_relaychain(|relaychain| { + relaychain + .with_chain("polkadot") + .with_default_command("polkadot") + .with_default_image("docker.io/parity/polkadot:latest") + .with_default_args(vec![("-name", "value").into(), "--flag".into()]) + .with_default_db_snapshot("https://storage.com/path/to/db_snapshot.tgz") + .with_default_resources(|resources| { + resources + .with_request_cpu(100000) + .with_request_memory("500M") + .with_limit_cpu("10Gi") + .with_limit_memory("4000M") + }) + .with_node(|node| { + node.with_name("alice") + .with_initial_balance(1_000_000_000) + .validator(true) + .bootnode(true) + .invulnerable(true) + }) + .with_node(|node| { + node.with_name("bob") + .validator(true) + .invulnerable(true) + .bootnode(true) + .with_image("mycustomimage:latest") + .with_command("my-custom-command") + .with_db_snapshot("https://storage.com/path/to/other/db_snapshot.tgz") + .with_resources(|resources| { + resources + .with_request_cpu(1000) + .with_request_memory("250Mi") + .with_limit_cpu("5Gi") + .with_limit_memory("2Gi") + }) + .with_args(vec![("-myothername", "value").into()]) + }) + }) + .with_parachain(|parachain| { + parachain + .with_id(1000) + .with_chain("myparachain") + .with_chain_spec_path("/path/to/my/chain/spec.json") + .with_default_db_snapshot("https://storage.com/path/to/other_snapshot.tgz") + .with_default_command("my-default-command") + .with_default_image("mydefaultimage:latest") + .with_collator(|collator| { + collator + .with_name("john") + .bootnode(true) + .validator(true) + .invulnerable(true) + .with_initial_balance(5_000_000_000) + .with_command("my-non-default-command") + .with_image("anotherimage:latest") + }) + .with_collator(|collator| { + collator + .with_name("charles") + .bootnode(true) + .invulnerable(true) + .with_initial_balance(0) + }) + }) + .build() + .unwrap(); + + expected + .parachains() + .iter() + .zip(load_from_toml.parachains().iter()) + .for_each(|(expected_parachain, loaded_parachain)| { + assert_eq!(expected_parachain.id(), loaded_parachain.id()); + assert_eq!(expected_parachain.chain(), loaded_parachain.chain()); + assert_eq!( + expected_parachain.chain_spec_path(), + loaded_parachain.chain_spec_path() + ); + assert_eq!( + expected_parachain.registration_strategy(), + loaded_parachain.registration_strategy() + ); + assert_eq!( + expected_parachain.onboard_as_parachain(), + loaded_parachain.onboard_as_parachain() + ); + assert_eq!( + expected_parachain.default_db_snapshot(), + loaded_parachain.default_db_snapshot() + ); + assert_eq!( + expected_parachain.default_command(), + loaded_parachain.default_command() + ); + assert_eq!( + expected_parachain.default_image(), + loaded_parachain.default_image() + ); + assert_eq!( + expected_parachain.collators().len(), + loaded_parachain.collators().len() + ); + expected_parachain + .collators() + .iter() + .zip(loaded_parachain.collators().iter()) + .for_each(|(expected_collator, loaded_collator)| { + assert_eq!(expected_collator.name(), loaded_collator.name()); + assert_eq!(expected_collator.command(), loaded_collator.command()); + assert_eq!(expected_collator.image(), loaded_collator.image()); + assert_eq!( + expected_collator.is_validator(), + loaded_collator.is_validator() + ); + assert_eq!( + expected_collator.is_bootnode(), + loaded_collator.is_bootnode() + ); + assert_eq!( + expected_collator.is_invulnerable(), + loaded_collator.is_invulnerable() + ); + assert_eq!( + expected_collator.initial_balance(), + loaded_collator.initial_balance() + ); + }); + }); + } } diff --git a/crates/configuration/src/parachain.rs b/crates/configuration/src/parachain.rs index 74c8197f3..543b778a1 100644 --- a/crates/configuration/src/parachain.rs +++ b/crates/configuration/src/parachain.rs @@ -1,17 +1,24 @@ use std::{cell::RefCell, error::Error, fmt::Display, marker::PhantomData, rc::Rc}; use multiaddr::Multiaddr; -use serde::{ser::SerializeStruct, Serialize}; - -use crate::shared::{ - errors::{ConfigError, FieldError}, - helpers::{merge_errors, merge_errors_vecs}, - macros::states, - node::{self, NodeConfig, NodeConfigBuilder}, - resources::{Resources, ResourcesBuilder}, - types::{ - Arg, AssetLocation, Chain, ChainDefaultContext, Command, Image, ValidationContext, U128, +use serde::{ + de::{self, Visitor}, + ser::SerializeStruct, + Deserialize, Serialize, +}; + +use crate::{ + shared::{ + errors::{ConfigError, FieldError}, + helpers::{merge_errors, merge_errors_vecs}, + macros::states, + node::{self, NodeConfig, NodeConfigBuilder}, + resources::{Resources, ResourcesBuilder}, + types::{ + Arg, AssetLocation, Chain, ChainDefaultContext, Command, Image, ValidationContext, U128, + }, }, + utils::default_as_true, }; #[derive(Debug, Clone, PartialEq)] @@ -36,14 +43,67 @@ impl Serialize for RegistrationStrategy { } } +struct RegistrationStrategyVisitor; + +impl<'de> Visitor<'de> for RegistrationStrategyVisitor { + type Value = RegistrationStrategy; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct RegistrationStrategy") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut add_to_genesis = false; + let mut register_para = false; + + while let Some(key) = map.next_key::()? { + match key.as_str() { + "add_to_genesis" => add_to_genesis = map.next_value()?, + "register_para" => register_para = map.next_value()?, + _ => { + return Err(de::Error::unknown_field( + &key, + &["add_to_genesis", "register_para"], + )) + }, + } + } + + match (add_to_genesis, register_para) { + (true, false) => Ok(RegistrationStrategy::InGenesis), + (false, true) => Ok(RegistrationStrategy::UsingExtrinsic), + _ => Err(de::Error::missing_field("add_to_genesis or register_para")), + } + } +} + +impl<'de> Deserialize<'de> for RegistrationStrategy { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_struct( + "RegistrationStrategy", + &["add_to_genesis", "register_para"], + RegistrationStrategyVisitor, + ) + } +} + /// A parachain configuration, composed of collators and fine-grained configuration options. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ParachainConfig { id: u32, chain: Option, #[serde(flatten)] registration_strategy: Option, - #[serde(skip_serializing_if = "super::utils::is_true")] + #[serde( + skip_serializing_if = "super::utils::is_true", + default = "default_as_true" + )] onboard_as_parachain: bool, #[serde(rename = "balance")] initial_balance: U128, @@ -51,7 +111,7 @@ pub struct ParachainConfig { default_image: Option, default_resources: Option, default_db_snapshot: Option, - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] default_args: Vec, genesis_wasm_path: Option, genesis_wasm_generator: Option, @@ -60,9 +120,9 @@ pub struct ParachainConfig { chain_spec_path: Option, #[serde(rename = "cumulus_based")] is_cumulus_based: bool, - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] bootnodes_addresses: Vec, - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] collators: Vec, } @@ -617,6 +677,7 @@ impl ParachainConfigBuilder { #[cfg(test)] mod tests { use super::*; + use crate::NetworkConfig; #[test] fn parachain_config_builder_should_succeeds_and_returns_a_new_parachain_config() { @@ -672,6 +733,7 @@ mod tests { assert_eq!(collator2.command().unwrap().as_str(), "command2"); assert!(collator2.is_validator()); assert_eq!(parachain_config.chain().unwrap().as_str(), "mychainname"); + assert_eq!( parachain_config.registration_strategy().unwrap(), &RegistrationStrategy::UsingExtrinsic @@ -1000,6 +1062,36 @@ mod tests { ); } + #[test] + fn import_toml_registration_strategy_should_deserialize() { + let load_from_toml = + NetworkConfig::load_from_toml("./testing/snapshots/0001-big-network.toml").unwrap(); + + for parachain in load_from_toml.parachains().iter() { + if parachain.id() == 1000 { + assert_eq!( + parachain.registration_strategy(), + Some(&RegistrationStrategy::UsingExtrinsic) + ); + } + if parachain.id() == 2000 { + assert_eq!( + parachain.registration_strategy(), + Some(&RegistrationStrategy::InGenesis) + ); + } + } + + let load_from_toml_small = NetworkConfig::load_from_toml( + "./testing/snapshots/0003-small-network_w_parachain.toml", + ) + .unwrap(); + + let parachain = load_from_toml_small.parachains()[0]; + + assert_eq!(parachain.registration_strategy(), None); + } + #[test] fn onboard_as_parachain_should_default_to_true() { let config = ParachainConfigBuilder::new(Default::default()) diff --git a/crates/configuration/src/relaychain.rs b/crates/configuration/src/relaychain.rs index 359be3aa2..b087ddcf7 100644 --- a/crates/configuration/src/relaychain.rs +++ b/crates/configuration/src/relaychain.rs @@ -1,8 +1,9 @@ use std::{cell::RefCell, error::Error, fmt::Debug, marker::PhantomData, rc::Rc}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::shared::{ + constants::{DEFAULT_TYPESTATE, THIS_IS_A_BUG}, errors::{ConfigError, FieldError}, helpers::{merge_errors, merge_errors_vecs}, macros::states, @@ -12,19 +13,19 @@ use crate::shared::{ }; /// A relay chain configuration, composed of nodes and fine-grained configuration options. -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RelaychainConfig { chain: Chain, default_command: Option, default_image: Option, default_resources: Option, default_db_snapshot: Option, - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] default_args: Vec, chain_spec_path: Option, random_nominators_count: Option, max_nominations: Option, - #[serde(skip_serializing_if = "std::vec::Vec::is_empty")] + #[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)] nodes: Vec, } @@ -78,6 +79,10 @@ impl RelaychainConfig { pub fn nodes(&self) -> Vec<&NodeConfig> { self.nodes.iter().collect::>() } + + pub(crate) fn set_nodes(&mut self, nodes: Vec) { + self.nodes = nodes; + } } states! { @@ -100,7 +105,7 @@ impl Default for RelaychainConfigBuilder { config: RelaychainConfig { chain: "default" .try_into() - .expect("'default' overriding should be ensured by typestate. this is a bug, please report it: https://github.com/paritytech/zombienet-sdk/issues"), + .expect(&format!("{} {}", DEFAULT_TYPESTATE, THIS_IS_A_BUG)), default_command: None, default_image: None, default_resources: None, diff --git a/crates/configuration/src/shared.rs b/crates/configuration/src/shared.rs index bb1d7bf19..36b71c43e 100644 --- a/crates/configuration/src/shared.rs +++ b/crates/configuration/src/shared.rs @@ -1,3 +1,4 @@ +pub mod constants; pub mod errors; pub mod helpers; pub mod macros; diff --git a/crates/configuration/src/shared/constants.rs b/crates/configuration/src/shared/constants.rs new file mode 100644 index 000000000..9d5f302be --- /dev/null +++ b/crates/configuration/src/shared/constants.rs @@ -0,0 +1,14 @@ +pub const VALID_REGEX: &str = "regex should be valid "; +pub const BORROWABLE: &str = "must be borrowable as mutable "; +pub const RELAY_NOT_NONE: &str = "typestate should ensure the relaychain isn't None at this point "; +pub const SHOULD_COMPILE: &str = "should compile with success "; +pub const INFAILABLE: &str = "infaillible "; +pub const NO_ERR_DEF_BUILDER: &str = "should have no errors for default builder "; +pub const RW_FAILED: &str = "should be able to read/write - failed "; +pub const DEFAULT_TYPESTATE: &str = "'default' overriding should be ensured by typestate "; +pub const VALIDATION_CHECK: &str = "validation failed "; + +pub const PREFIX_CANT_BE_NONE: &str = "name prefix can't be None if a value exists "; + +pub const THIS_IS_A_BUG: &str = + "- this is a bug please report it: https://github.com/paritytech/zombienet-sdk/issues"; diff --git a/crates/configuration/src/shared/helpers.rs b/crates/configuration/src/shared/helpers.rs index ae9eb6328..ee0a65e2f 100644 --- a/crates/configuration/src/shared/helpers.rs +++ b/crates/configuration/src/shared/helpers.rs @@ -1,6 +1,7 @@ use std::{cell::RefCell, rc::Rc}; use super::{ + constants::{BORROWABLE, THIS_IS_A_BUG}, errors::ValidationError, types::{Port, ValidationContext}, }; @@ -31,7 +32,7 @@ pub fn ensure_node_name_unique( ) -> Result<(), anyhow::Error> { let mut context = validation_context .try_borrow_mut() - .expect("must be borrowable as mutable, this is a bug please report it: https://github.com/paritytech/zombienet-sdk/issues"); + .expect(&format!("{}, {}", BORROWABLE, THIS_IS_A_BUG)); if !context.used_nodes_names.contains(&node_name) { context.used_nodes_names.push(node_name); @@ -47,7 +48,7 @@ pub fn ensure_port_unique( ) -> Result<(), anyhow::Error> { let mut context = validation_context .try_borrow_mut() - .expect("must be borrowable as mutable, this is a bug please report it: https://github.com/paritytech/zombienet-sdk/issues"); + .expect(&format!("{}, {}", BORROWABLE, THIS_IS_A_BUG)); if !context.used_ports.contains(&port) { context.used_ports.push(port); diff --git a/crates/configuration/src/shared/node.rs b/crates/configuration/src/shared/node.rs index bcde9fd1f..d1e0bb747 100644 --- a/crates/configuration/src/shared/node.rs +++ b/crates/configuration/src/shared/node.rs @@ -1,7 +1,7 @@ use std::{cell::RefCell, error::Error, fmt::Display, marker::PhantomData, rc::Rc}; use multiaddr::Multiaddr; -use serde::{ser::SerializeStruct, Serialize}; +use serde::{ser::SerializeStruct, Deserialize, Serialize}; use super::{ errors::FieldError, @@ -33,7 +33,7 @@ use crate::shared::{ /// } /// ) /// ``` -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct EnvVar { /// The name of the environment variable. pub name: String, @@ -52,27 +52,36 @@ impl From<(&str, &str)> for EnvVar { } /// A node configuration, with fine-grained configuration options. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Deserialize)] pub struct NodeConfig { name: String, - image: Option, - command: Option, + pub(crate) image: Option, + pub(crate) command: Option, + #[serde(default)] args: Vec, - is_validator: bool, - is_invulnerable: bool, - is_bootnode: bool, + #[serde(alias = "validator")] + pub(crate) is_validator: bool, + #[serde(alias = "invulnerable")] + pub(crate) is_invulnerable: bool, + #[serde(alias = "bootnode")] + pub(crate) is_bootnode: bool, + #[serde(alias = "balance")] + #[serde(default)] initial_balance: U128, + #[serde(default)] env: Vec, + #[serde(default)] bootnodes_addresses: Vec, - resources: Option, + pub(crate) resources: Option, ws_port: Option, rpc_port: Option, prometheus_port: Option, p2p_port: Option, p2p_cert_hash: Option, - db_snapshot: Option, + pub(crate) db_snapshot: Option, + #[serde(default)] // used to skip serialization of fields with defaults to avoid duplication - chain_context: ChainDefaultContext, + pub(crate) chain_context: ChainDefaultContext, } impl Serialize for NodeConfig { @@ -162,6 +171,11 @@ impl NodeConfig { self.args.iter().collect() } + /// Arguments to use for node. + pub(crate) fn set_args(&mut self, args: Vec) { + self.args = args; + } + /// Whether the node is a validator. pub fn is_validator(&self) -> bool { self.is_validator diff --git a/crates/configuration/src/shared/resources.rs b/crates/configuration/src/shared/resources.rs index 46f3a4aab..8537b188d 100644 --- a/crates/configuration/src/shared/resources.rs +++ b/crates/configuration/src/shared/resources.rs @@ -2,12 +2,17 @@ use std::error::Error; use lazy_static::lazy_static; use regex::Regex; -use serde::{ser::SerializeStruct, Serialize}; +use serde::{ + de::{self}, + ser::SerializeStruct, + Deserialize, Serialize, +}; use super::{ errors::{ConversionError, FieldError}, helpers::merge_errors, }; +use crate::shared::constants::{SHOULD_COMPILE, THIS_IS_A_BUG}; /// A resource quantity used to define limits (k8s/podman only). /// It can be constructed from a `&str` or u64, if it fails, it returns a [`ConversionError`]. @@ -28,7 +33,7 @@ use super::{ /// assert_eq!(quantity3.as_str(), "1Gi"); /// assert_eq!(quantity4.as_str(), "10000"); /// ``` -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ResourceQuantity(String); impl ResourceQuantity { @@ -43,7 +48,7 @@ impl TryFrom<&str> for ResourceQuantity { fn try_from(value: &str) -> Result { lazy_static! { static ref RE: Regex = Regex::new(r"^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$") - .expect("should compile with success. this is a bug, please report it: https://github.com/paritytech/zombienet-sdk/issues"); + .expect(&format!("{}, {}", SHOULD_COMPILE, THIS_IS_A_BUG)); } if !RE.is_match(value) { @@ -72,6 +77,12 @@ pub struct Resources { limit_cpu: Option, } +#[derive(Serialize, Deserialize)] +struct ResourcesField { + memory: Option, + cpu: Option, +} + impl Serialize for Resources { fn serialize(&self, serializer: S) -> Result where @@ -79,12 +90,6 @@ impl Serialize for Resources { { let mut state = serializer.serialize_struct("Resources", 2)?; - #[derive(Serialize)] - struct ResourcesField { - memory: Option, - cpu: Option, - } - if self.request_memory.is_some() || self.request_memory.is_some() { state.serialize_field( "requests", @@ -113,6 +118,52 @@ impl Serialize for Resources { } } +struct ResourcesVisitor; + +impl<'de> de::Visitor<'de> for ResourcesVisitor { + type Value = Resources; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a resources object") + } + + fn visit_map(self, mut map: A) -> Result + where + A: de::MapAccess<'de>, + { + let mut resources: Resources = Resources::default(); + + while let Some((key, value)) = map.next_entry::()? { + match key.as_str() { + "requests" => { + resources.request_memory = value.memory; + resources.request_cpu = value.cpu; + }, + "limits" => { + resources.limit_memory = value.memory; + resources.limit_cpu = value.cpu; + }, + _ => { + return Err(de::Error::unknown_field( + &key, + &["requests", "limits", "cpu", "memory"], + )) + }, + } + } + Ok(resources) + } +} + +impl<'de> Deserialize<'de> for Resources { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(ResourcesVisitor) + } +} + impl Resources { /// Memory limit applied to requests. pub fn request_memory(&self) -> Option<&ResourceQuantity> { @@ -249,6 +300,7 @@ impl ResourcesBuilder { #[allow(non_snake_case)] mod tests { use super::*; + use crate::NetworkConfig; macro_rules! impl_resources_quantity_unit_test { ($val:literal) => {{ @@ -350,6 +402,18 @@ mod tests { assert_eq!(resources.limit_memory().unwrap().as_str(), "2G"); } + #[test] + fn resources_config_toml_import_should_succeeds_and_returns_a_resources_config() { + let load_from_toml = + NetworkConfig::load_from_toml("./testing/snapshots/0001-big-network.toml").unwrap(); + + let resources = load_from_toml.relaychain().default_resources().unwrap(); + assert_eq!(resources.request_memory().unwrap().as_str(), "500M"); + assert_eq!(resources.request_cpu().unwrap().as_str(), "100000"); + assert_eq!(resources.limit_cpu().unwrap().as_str(), "10Gi"); + assert_eq!(resources.limit_memory().unwrap().as_str(), "4000M"); + } + #[test] fn resources_config_builder_should_fails_and_returns_an_error_if_couldnt_parse_request_memory() { diff --git a/crates/configuration/src/shared/types.rs b/crates/configuration/src/shared/types.rs index 04ae8a442..590fa6070 100644 --- a/crates/configuration/src/shared/types.rs +++ b/crates/configuration/src/shared/types.rs @@ -1,11 +1,17 @@ -use std::{fmt::Display, path::PathBuf, str::FromStr}; +use std::{ + error::Error, + fmt::{self, Display}, + path::PathBuf, + str::FromStr, +}; use lazy_static::lazy_static; use regex::Regex; -use serde::Serialize; +use serde::{de, Deserialize, Deserializer, Serialize}; use url::Url; use super::{errors::ConversionError, resources::Resources}; +use crate::shared::constants::{INFAILABLE, PREFIX_CANT_BE_NONE, SHOULD_COMPILE, THIS_IS_A_BUG}; /// An alias for a duration in seconds. pub type Duration = u32; @@ -18,7 +24,7 @@ pub type ParaId = u32; /// Custom type wrapping u128 to add custom Serialization/Deserialization logic because it's not supported /// issue tracking the problem: -#[derive(Debug, Clone, PartialEq)] +#[derive(Default, Debug, Clone, PartialEq)] pub struct U128(pub(crate) u128); impl From for U128 { @@ -27,6 +33,14 @@ impl From for U128 { } } +impl TryFrom<&str> for U128 { + type Error = Box; + + fn try_from(value: &str) -> Result { + Ok(Self(value.to_string().parse::()?)) + } +} + impl Serialize for U128 { fn serialize(&self, serializer: S) -> Result where @@ -38,6 +52,32 @@ impl Serialize for U128 { } } +struct U128Visitor; + +impl<'de> de::Visitor<'de> for U128Visitor { + type Value = U128; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("an integer between 0 and 2^128 − 1.") + } + + fn visit_str(self, v: &str) -> Result + where + E: de::Error, + { + v.try_into().map_err(de::Error::custom) + } +} + +impl<'de> Deserialize<'de> for U128 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(U128Visitor) + } +} + /// A chain name. /// It can be constructed for an `&str`, if it fails, it will returns a [`ConversionError`]. /// @@ -53,7 +93,7 @@ impl Serialize for U128 { /// assert_eq!(kusama.as_str(), "kusama"); /// assert_eq!(myparachain.as_str(), "myparachain"); /// ``` -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Chain(String); impl TryFrom<&str> for Chain { @@ -95,7 +135,7 @@ impl Chain { /// assert_eq!(image3.as_str(), "myrepo.com/name:version"); /// assert_eq!(image4.as_str(), "10.15.43.155/name:version"); /// ``` -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Image(String); impl TryFrom<&str> for Image { @@ -110,7 +150,7 @@ impl TryFrom<&str> for Image { static ref RE: Regex = Regex::new(&format!( "^({IP_PART}|{HOSTNAME_PART}/)?{TAG_NAME_PART}(:{TAG_VERSION_PART})?$", )) - .expect("should compile with success. this is a bug, please report it: https://github.com/paritytech/zombienet-sdk/issues"); + .expect(&format!("{}, {}", SHOULD_COMPILE, THIS_IS_A_BUG)); }; if !RE.is_match(value) { @@ -143,7 +183,7 @@ impl Image { /// assert_eq!(command1.as_str(), "mycommand"); /// assert_eq!(command2.as_str(), "myothercommand"); /// ``` -#[derive(Debug, Clone, PartialEq, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Command(String); impl TryFrom<&str> for Command { @@ -208,7 +248,7 @@ impl From<&str> for AssetLocation { } Self::FilePath( - PathBuf::from_str(value).expect("infaillible. this is a bug, please report it"), + PathBuf::from_str(value).expect(&format!("{}, {}", INFAILABLE, THIS_IS_A_BUG)), ) } } @@ -231,6 +271,32 @@ impl Serialize for AssetLocation { } } +struct AssetLocationVisitor; + +impl<'de> de::Visitor<'de> for AssetLocationVisitor { + type Value = AssetLocation; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, v: &str) -> Result + where + E: de::Error, + { + Ok(AssetLocation::from(v)) + } +} + +impl<'de> Deserialize<'de> for AssetLocation { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(AssetLocationVisitor) + } +} + /// A CLI argument passed to an executed command, can be an option with an assigned value or a simple flag to enable/disable a feature. /// A flag arg can be constructed from a `&str` and a option arg can be constructed from a `(&str, &str)`. /// @@ -276,18 +342,67 @@ impl Serialize for Arg { } } +struct ArgVisitor; + +impl<'de> de::Visitor<'de> for ArgVisitor { + type Value = Arg; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, v: &str) -> Result + where + E: de::Error, + { + let re = Regex::new("^(?(?-{1,2})(?[a-zA-Z]+(-[a-zA-Z]+)*))((?=| )(?.+))?$").unwrap(); + let captures = re.captures(v); + + if let Some(captures) = captures { + if let Some(value) = captures.name("value") { + return Ok(Arg::Option( + captures + .name("name_prefix") + .expect(&format!("{} {}", PREFIX_CANT_BE_NONE, THIS_IS_A_BUG)) + .as_str() + .to_string(), + value.as_str().to_string(), + )); + } + + if let Some(name_prefix) = captures.name("name_prefix") { + return Ok(Arg::Flag(name_prefix.as_str().to_string())); + } + } + + Err(de::Error::custom( + "the provided argument is invalid and doesn't match Arg::Option or Arg::Flag", + )) + } +} + +impl<'de> Deserialize<'de> for Arg { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(ArgVisitor) + } +} + #[derive(Debug, Default, Clone)] pub struct ValidationContext { pub used_ports: Vec, pub used_nodes_names: Vec, } -#[derive(Default, Debug, Clone, PartialEq)] +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] pub struct ChainDefaultContext { pub(crate) default_command: Option, pub(crate) default_image: Option, pub(crate) default_resources: Option, pub(crate) default_db_snapshot: Option, + #[serde(default)] pub(crate) default_args: Vec, } diff --git a/crates/configuration/src/utils.rs b/crates/configuration/src/utils.rs index 5c5a184d3..ced19103e 100644 --- a/crates/configuration/src/utils.rs +++ b/crates/configuration/src/utils.rs @@ -1,3 +1,7 @@ -pub fn is_true(value: &bool) -> bool { +pub(crate) fn is_true(value: &bool) -> bool { *value } + +pub(crate) fn default_as_true() -> bool { + true +} diff --git a/crates/configuration/testing/snapshots/0003-small-network_w_parachain.toml b/crates/configuration/testing/snapshots/0003-small-network_w_parachain.toml new file mode 100644 index 000000000..b998a14c0 --- /dev/null +++ b/crates/configuration/testing/snapshots/0003-small-network_w_parachain.toml @@ -0,0 +1,40 @@ +[settings] +timeout = 1000 +node_spawn_timeout = 300 + +[relaychain] +chain = "rococo-local" +default_command = "polkadot" +default_image = "docker.io/parity/polkadot:latest" +default_args = ["-lparachain=debug"] + +[[relaychain.nodes]] +name = "alice" +validator = true +invulnerable = true +bootnode = false +balance = 2000000000000 + +[[relaychain.nodes]] +name = "bob" +args = ["--database=paritydb-experimental"] +validator = true +invulnerable = false +bootnode = true +balance = 2000000000000 + +[[parachains]] +id = 1000 +chain = "myparachain" +onboard_as_parachain = false +balance = 2000000000000 +default_db_snapshot = "https://storage.com/path/to/db_snapshot.tgz" +chain_spec_path = "/path/to/my/chain/spec.json" +cumulus_based = true + +[[parachains.collators]] +name = "john" +validator = true +invulnerable = true +bootnode = true +balance = 5000000000