diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bbc5583ac..0c616dc06 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -152,3 +152,12 @@ jobs: run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard - name: Run integration test run: bash scripts/integration_tests/all-up-test.sh ETH_PAYMENTS + integration-test-multi-exit: + needs: integration-test-five-nodes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Wireguard + run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard + - name: Run integration test + run: bash scripts/integration_tests/all-up-test.sh MULTI_EXIT \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index a3e6d010e..bbd6fb562 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1629,6 +1629,7 @@ dependencies = [ "env_logger", "futures 0.3.28", "ipnetwork", + "lazy_static", "log", "nix 0.26.2", "num-traits", diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 683c53d84..9c4f5a648 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -31,3 +31,4 @@ futures = { version = "0.3", features = ["compat"] } num256 = "0.5" num-traits="0.2" web30 = "1.0" +lazy_static = "1.4" diff --git a/integration_tests/src/debts.rs b/integration_tests/src/debts.rs index 22ca3bcdc..bb3253100 100644 --- a/integration_tests/src/debts.rs +++ b/integration_tests/src/debts.rs @@ -7,8 +7,9 @@ use crate::setup_utils::database::start_postgres; use crate::setup_utils::namespaces::*; use crate::setup_utils::rita::thread_spawner; use crate::utils::{ - generate_traffic, get_default_settings, get_ip_from_namespace, query_debts, register_to_exit, - test_all_internet_connectivity, test_reach_all, test_routes, DEBT_ACCURACY_THRES, + generate_traffic, get_default_settings, get_ip_from_namespace, query_debts, + register_all_namespaces_to_exit, test_all_internet_connectivity, test_reach_all, test_routes, + DEBT_ACCURACY_THRES, }; use althea_types::Identity; use log::info; @@ -44,7 +45,8 @@ pub async fn run_debts_test() { let namespaces = node_config.0; let expected_routes = node_config.1; - let (rita_settings, rita_exit_settings) = get_default_settings(); + let (client_settings, exit_settings) = + get_default_settings("test".to_string(), namespaces.clone()); // The exit price is set to ns.cost during thread_spawner let exit_price = namespaces.get_namespace(4).unwrap().cost; @@ -55,7 +57,7 @@ pub async fn run_debts_test() { let res = setup_ns(namespaces.clone()); info!("Namespaces setup: {res:?}"); - let rita_identities = thread_spawner(namespaces.clone(), rita_settings, rita_exit_settings) + let rita_identities = thread_spawner(namespaces.clone(), client_settings, exit_settings) .expect("Could not spawn Rita threads"); // There should be only 1 exit identity let exit_identity = rita_identities @@ -71,16 +73,7 @@ pub async fn run_debts_test() { test_routes(namespaces.clone(), expected_routes); info!("Registering routers to the exit"); - for r in namespaces.names.clone() { - if let NodeType::Client = r.node_type { - let res = register_to_exit(r.get_name()).await; - if !res.is_success() { - panic!("Failed to register {} to exit with {:?}", r.get_name(), res); - } else { - info!("{} registered to exit", r.get_name()); - } - } - } + register_all_namespaces_to_exit(namespaces.clone()).await; // Let network stabalize thread::sleep(Duration::from_secs(20)); @@ -132,8 +125,9 @@ pub async fn run_debts_test() { // if to_node is none, we are querying the internet, so last node has to be an exit node if to_node.is_none() { exit_node = query_nodes.pop(); - if exit_node.clone().unwrap().node_type != NodeType::Exit { - panic!("Why is last element not an exit?"); + match exit_node.clone().unwrap().node_type { + NodeType::Exit { .. } => {} + _ => panic!("Why is last element not an exit?"), } } diff --git a/integration_tests/src/five_nodes.rs b/integration_tests/src/five_nodes.rs index 937169b0f..98af5e60a 100644 --- a/integration_tests/src/five_nodes.rs +++ b/integration_tests/src/five_nodes.rs @@ -1,8 +1,10 @@ use crate::setup_utils::database::start_postgres; use crate::setup_utils::namespaces::*; use crate::setup_utils::rita::thread_spawner; -use crate::utils::{get_default_settings, register_to_exit, test_reach_all, test_routes}; -use log::{error, info}; +use crate::utils::{ + get_default_settings, register_all_namespaces_to_exit, test_reach_all, test_routes, +}; +use log::info; use std::collections::HashMap; /// Runs a five node fixed network map test scenario, this does basic network setup and tests reachability to @@ -13,7 +15,8 @@ pub async fn run_five_node_test_scenario() { let namespaces = node_config.0; let expected_routes = node_config.1; - let (rita_settings, rita_exit_settings) = get_default_settings(); + let (client_settings, exit_settings) = + get_default_settings("test".to_string(), namespaces.clone()); namespaces.validate(); @@ -21,7 +24,7 @@ pub async fn run_five_node_test_scenario() { let res = setup_ns(namespaces.clone()); info!("Namespaces setup: {res:?}"); - let _ = thread_spawner(namespaces.clone(), rita_settings, rita_exit_settings) + let _ = thread_spawner(namespaces.clone(), client_settings, exit_settings) .expect("Could not spawn Rita threads"); info!("Thread Spawner: {res:?}"); @@ -33,14 +36,7 @@ pub async fn run_five_node_test_scenario() { test_routes(namespaces.clone(), expected_routes); info!("Registering routers to the exit"); - for r in namespaces.names { - if let NodeType::Client = r.node_type { - let res = register_to_exit(r.get_name()).await; - if !res.is_success() { - error!("Failed to register {} to exit with {:?}", r.get_name(), res); - } - } - } + register_all_namespaces_to_exit(namespaces.clone()).await; } /// This defines the network map for a five node scenario @@ -64,37 +60,51 @@ pub fn five_node_config() -> (NamespaceInfo, HashMap) { let testa = Namespace { id: 1, cost: 25, - node_type: NodeType::Client, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, }; let testb = Namespace { id: 2, cost: 500, - node_type: NodeType::Client, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, }; let testc = Namespace { id: 3, cost: 15, - node_type: NodeType::Client, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, }; let testd = Namespace { id: 4, cost: 10, - node_type: NodeType::Exit, + node_type: NodeType::Exit { + instance_name: "test_4".to_string(), + }, }; let teste = Namespace { id: 5, cost: 40, - node_type: NodeType::Client, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, }; let testf = Namespace { id: 6, cost: 20, - node_type: NodeType::Client, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, }; let testg = Namespace { id: 7, cost: 15, - node_type: NodeType::Client, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, }; let nsinfo = NamespaceInfo { diff --git a/integration_tests/src/lib.rs b/integration_tests/src/lib.rs index 9e93147de..fb181d9b0 100644 --- a/integration_tests/src/lib.rs +++ b/integration_tests/src/lib.rs @@ -1,7 +1,11 @@ +#[macro_use] +extern crate lazy_static; + use std::time::Duration; pub mod debts; pub mod five_nodes; +pub mod mutli_exit; pub mod payments_althea; pub mod payments_eth; pub mod setup_utils; diff --git a/integration_tests/src/mutli_exit.rs b/integration_tests/src/mutli_exit.rs new file mode 100644 index 000000000..bb5b40e9a --- /dev/null +++ b/integration_tests/src/mutli_exit.rs @@ -0,0 +1,364 @@ +use std::{collections::HashMap, str::from_utf8, thread, time::Duration}; + +use crate::{ + setup_utils::{ + database::start_postgres, + namespaces::{setup_ns, Namespace, NamespaceInfo, NodeType, PriceId, RouteHop}, + rita::thread_spawner, + }, + utils::{ + get_default_settings, get_node_id_from_ip, register_all_namespaces_to_exit, + test_all_internet_connectivity, test_reach_all, test_routes, + }, +}; +use althea_kernel_interface::KI; +use log::info; + +/* +Nodes are connected as such, 4 and 5 are exit: +1---------2 +| | +| | +| | +| 4 ---------7 +| +| +3---------5 +| | +| | +| | +8---------6 +*/ + +pub async fn run_multi_exit_test() { + info!("Starting mutli exit test"); + + let node_configs = multi_exit_config(); + let namespaces = node_configs.0; + let expected_routes = node_configs.1; + + let (rita_client_settings, rita_exit_settings) = + get_default_settings("test".to_string(), namespaces.clone()); + + namespaces.validate(); + start_postgres(); + + let res = setup_ns(namespaces.clone()); + + let _rita_identities = + thread_spawner(namespaces.clone(), rita_client_settings, rita_exit_settings) + .expect("Could not spawn Rita threads"); + info!("Thread Spawner: {res:?}"); + + // Test for network convergence + test_reach_all(namespaces.clone()); + + test_routes(namespaces.clone(), expected_routes); + + info!("Registering routers to the exit"); + register_all_namespaces_to_exit(namespaces.clone()).await; + + thread::sleep(Duration::from_secs(5)); + + test_all_internet_connectivity(namespaces.clone()); + + let current_exit = get_current_exit(namespaces.names[0].clone(), namespaces.clone()); + info!( + "All nodes are set up correctly and are connected to exit {}", + current_exit.get_name() + ); + + // Kill one of the exits + kill_exit(current_exit.clone()); + info!( + "Exit {} killed, verifing client migration", + current_exit.get_name() + ); + + thread::sleep(Duration::from_secs(10)); + + // Check that we migrated + let new_exit = get_current_exit(namespaces.names[0].clone(), namespaces.clone()); + assert!(new_exit != current_exit); + info!("Clients have migrated to exit {}", new_exit.get_name()); + + // verfiy that we have internet connectivity. Special case where clients migrate from n-4 to n-5, n-7 + // wont have internet connection since it routes through n-4. So we remove it from namespace list before checking + // for internet connectivity + let namespaces_without_7 = NamespaceInfo { + names: { + let mut ret = namespaces.names.clone(); + let ns_7 = namespaces.get_namespace(7).clone().unwrap(); + let ind_to_remove = ret.iter().position(|x| *x == ns_7).unwrap(); + ret.remove(ind_to_remove); + ret + }, + linked: namespaces.linked, + }; + test_all_internet_connectivity(namespaces_without_7); +} + +fn kill_exit(exit: Namespace) { + let out = KI + .run_command("ip", &["netns", "pids", &exit.get_name()]) + .unwrap(); + let out = from_utf8(&out.stdout) + .unwrap() + .split('\n') + .collect::>(); + for s in out { + KI.run_command("kill", &[s.trim()]).unwrap(); + } +} + +fn get_current_exit(ns: Namespace, namespaces: NamespaceInfo) -> Namespace { + let out = KI + .run_command( + "ip", + &["netns", "exec", &ns.get_name(), "wg", "show", "wg_exit"], + ) + .unwrap(); + let out = from_utf8(&out.stdout).unwrap(); + let out = out.split('\n').collect::>(); + + let mut out_str = None; + for o in out { + if o.contains("endpoint") { + out_str = Some(o); + break; + } + } + let out = out_str.unwrap(); + let out = out.split(']').collect::>()[0]; + let out = out.split('[').last().unwrap(); + let ns = get_node_id_from_ip(out.parse().unwrap()); + namespaces.get_namespace(ns).expect("This should be valid") +} + +/// This defines the network map for a multi exit scenario +pub fn multi_exit_config() -> (NamespaceInfo, HashMap) { + /* + Nodes are connected as such, 4 and 5 are exit: + 1---------2 + | | + | | + | | + | 4(Exit) ---------7 + | + | + 3---------5(Exit) + | | + | | + | | + 8---------6 + */ + + let testa = Namespace { + id: 1, + cost: 25, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, + }; + let testb = Namespace { + id: 2, + cost: 50, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, + }; + let testc = Namespace { + id: 3, + cost: 15, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, + }; + let testd = Namespace { + id: 4, + cost: 10, + node_type: NodeType::Exit { + instance_name: "test_4".to_string(), + }, + }; + let teste = Namespace { + id: 5, + cost: 20, + node_type: NodeType::Exit { + instance_name: "test_5".to_string(), + }, + }; + let testf = Namespace { + id: 6, + cost: 60, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, + }; + let testg = Namespace { + id: 7, + cost: 15, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, + }; + let testh = Namespace { + id: 8, + cost: 10, + node_type: NodeType::Client { + cluster_name: "test".to_string(), + }, + }; + + let nsinfo_exit = NamespaceInfo { + names: vec![ + testa.clone(), + testb.clone(), + testc.clone(), + testd.clone(), + teste.clone(), + testf.clone(), + testg.clone(), + testh.clone(), + ], + linked: vec![ + // arbitrary connections + (1, 2), + (1, 3), + (2, 4), + (3, 8), + (3, 5), + (4, 7), + (5, 6), + (6, 8), + ], + }; + + // For each namespace in the outer hashmap(A), we have an inner hashmap holding the other namespace nodes(B), how + // much the expected price from A -> B is, and what the next hop would be from A -> B. + let mut expected_routes = HashMap::new(); + + let testa_routes = RouteHop { + destination: [ + (2, PriceId { price: 0, id: 2 }), + (3, PriceId { price: 0, id: 3 }), + (4, PriceId { price: 50, id: 2 }), + (5, PriceId { price: 15, id: 3 }), + (6, PriceId { price: 25, id: 3 }), + (7, PriceId { price: 60, id: 2 }), + (8, PriceId { price: 15, id: 3 }), + ] + .iter() + .cloned() + .collect(), + }; + let testb_routes = RouteHop { + destination: [ + (1, PriceId { price: 0, id: 1 }), + (3, PriceId { price: 25, id: 1 }), + (4, PriceId { price: 0, id: 4 }), + (5, PriceId { price: 40, id: 1 }), + (6, PriceId { price: 50, id: 1 }), + (7, PriceId { price: 10, id: 4 }), + (8, PriceId { price: 40, id: 1 }), + ] + .iter() + .cloned() + .collect(), + }; + let testc_routes = RouteHop { + destination: [ + (1, PriceId { price: 0, id: 1 }), + (2, PriceId { price: 25, id: 1 }), + (4, PriceId { price: 75, id: 1 }), + (5, PriceId { price: 0, id: 5 }), + (6, PriceId { price: 10, id: 8 }), + (7, PriceId { price: 85, id: 1 }), + (8, PriceId { price: 0, id: 8 }), + ] + .iter() + .cloned() + .collect(), + }; + let testd_routes = RouteHop { + destination: [ + (1, PriceId { price: 50, id: 2 }), + (2, PriceId { price: 0, id: 2 }), + (3, PriceId { price: 75, id: 2 }), + (5, PriceId { price: 90, id: 2 }), + (6, PriceId { price: 100, id: 2 }), + (7, PriceId { price: 0, id: 7 }), + (8, PriceId { price: 90, id: 2 }), + ] + .iter() + .cloned() + .collect(), + }; + let teste_routes = RouteHop { + destination: [ + (1, PriceId { price: 15, id: 3 }), + (2, PriceId { price: 40, id: 3 }), + (3, PriceId { price: 0, id: 3 }), + (4, PriceId { price: 90, id: 3 }), + (6, PriceId { price: 0, id: 6 }), + (7, PriceId { price: 100, id: 3 }), + (8, PriceId { price: 15, id: 3 }), + ] + .iter() + .cloned() + .collect(), + }; + let testf_routes = RouteHop { + destination: [ + (1, PriceId { price: 25, id: 8 }), + (2, PriceId { price: 50, id: 8 }), + (3, PriceId { price: 10, id: 8 }), + (4, PriceId { price: 100, id: 8 }), + (5, PriceId { price: 0, id: 5 }), + (7, PriceId { price: 110, id: 8 }), + (8, PriceId { price: 0, id: 8 }), + ] + .iter() + .cloned() + .collect(), + }; + let testg_routes = RouteHop { + destination: [ + (1, PriceId { price: 60, id: 4 }), + (2, PriceId { price: 10, id: 4 }), + (3, PriceId { price: 85, id: 4 }), + (4, PriceId { price: 0, id: 4 }), + (5, PriceId { price: 100, id: 4 }), + (6, PriceId { price: 110, id: 4 }), + (8, PriceId { price: 100, id: 4 }), + ] + .iter() + .cloned() + .collect(), + }; + let testh_routes = RouteHop { + destination: [ + (1, PriceId { price: 15, id: 3 }), + (2, PriceId { price: 40, id: 3 }), + (3, PriceId { price: 0, id: 3 }), + (4, PriceId { price: 90, id: 3 }), + (5, PriceId { price: 15, id: 3 }), + (6, PriceId { price: 0, id: 6 }), + (7, PriceId { price: 100, id: 3 }), + ] + .iter() + .cloned() + .collect(), + }; + + expected_routes.insert(testa, testa_routes); + expected_routes.insert(testb, testb_routes); + expected_routes.insert(testc, testc_routes); + expected_routes.insert(testd, testd_routes); + expected_routes.insert(teste, teste_routes); + expected_routes.insert(testf, testf_routes); + expected_routes.insert(testg, testg_routes); + expected_routes.insert(testh, testh_routes); + + (nsinfo_exit, expected_routes) +} diff --git a/integration_tests/src/payments_althea.rs b/integration_tests/src/payments_althea.rs index 10ead2e4c..70b0dd9b9 100644 --- a/integration_tests/src/payments_althea.rs +++ b/integration_tests/src/payments_althea.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::thread; use std::time::Duration; @@ -6,15 +7,17 @@ use crate::setup_utils::database::start_postgres; use crate::setup_utils::namespaces::*; use crate::setup_utils::rita::thread_spawner; use crate::utils::{ - althea_system_chain_client, althea_system_chain_exit, generate_traffic, get_default_settings, - print_althea_balances, register_erc20_usdc_token, register_to_exit, send_althea_tokens, - test_reach_all, test_routes, validate_debt_entry, + generate_traffic, get_default_settings, print_althea_balances, register_all_namespaces_to_exit, + register_erc20_usdc_token, send_althea_tokens, test_reach_all, test_routes, + validate_debt_entry, TEST_PAY_THRESH, }; -use althea_types::ALTHEA_PREFIX; +use althea_types::{Denom, SystemChain, ALTHEA_PREFIX}; use deep_space::Address as AltheaAddress; use deep_space::{EthermintPrivateKey, PrivateKey}; use log::info; use rita_common::debt_keeper::GetDebtsResult; +use settings::client::RitaClientSettings; +use settings::exit::RitaExitSettingsStruct; const USDC_TO_WEI_DECIMAL: u64 = 1_000_000_000_000u64; @@ -40,7 +43,8 @@ pub async fn run_althea_payments_test_scenario() { let namespaces = node_config.0; let expected_routes = node_config.1; - let (rita_settings, rita_exit_settings) = get_default_settings(); + let (mut client_settings, mut exit_settings) = + get_default_settings("test".to_string(), namespaces.clone()); namespaces.validate(); start_postgres(); @@ -49,10 +53,10 @@ pub async fn run_althea_payments_test_scenario() { info!("Namespaces setup: {res:?}"); // Modify configs to use Althea chain - let rita_exit_settings = althea_system_chain_exit(rita_exit_settings); - let rita_settings = althea_system_chain_client(rita_settings); + let (client_settings, exit_settings) = + althea_payments_map(&mut client_settings, &mut exit_settings); - let rita_identities = thread_spawner(namespaces.clone(), rita_settings, rita_exit_settings) + let rita_identities = thread_spawner(namespaces.clone(), client_settings, exit_settings) .expect("Could not spawn Rita threads"); info!("Thread Spawner: {res:?}"); @@ -62,16 +66,7 @@ pub async fn run_althea_payments_test_scenario() { test_routes(namespaces.clone(), expected_routes); info!("Registering routers to the exit"); - for r in namespaces.names.clone() { - if let NodeType::Client = r.node_type { - let res = register_to_exit(r.get_name()).await; - if !res.is_success() { - panic!("Failed to register {} to exit with {:?}", r.get_name(), res); - } else { - info!("{} registered to exit", r.get_name()); - } - } - } + register_all_namespaces_to_exit(namespaces.clone()).await; thread::sleep(Duration::from_secs(10)); @@ -123,3 +118,27 @@ fn althea_payment_conditions(debts: GetDebtsResult) -> bool { (true, true) ) } + +fn althea_payments_map( + c_set: &mut RitaClientSettings, + exit_set: &mut RitaExitSettingsStruct, +) -> (RitaClientSettings, RitaExitSettingsStruct) { + let mut accept_de = HashMap::new(); + accept_de.insert( + "usdc".to_string(), + Denom { + denom: "uUSDC".to_string(), + decimal: 1_000_000u64, + }, + ); + + c_set.payment.system_chain = SystemChain::Althea; + exit_set.payment.system_chain = SystemChain::Althea; + // set pay thres to a smaller value + c_set.payment.payment_threshold = TEST_PAY_THRESH.into(); + exit_set.payment.payment_threshold = TEST_PAY_THRESH.into(); + c_set.payment.accepted_denoms = Some(accept_de.clone()); + exit_set.payment.accepted_denoms = Some(accept_de.clone()); + + (c_set.clone(), exit_set.clone()) +} diff --git a/integration_tests/src/payments_eth.rs b/integration_tests/src/payments_eth.rs index ccb3eb065..dde886b2f 100644 --- a/integration_tests/src/payments_eth.rs +++ b/integration_tests/src/payments_eth.rs @@ -1,17 +1,18 @@ use crate::five_nodes::five_node_config; use crate::setup_utils::database::start_postgres; +use crate::setup_utils::namespaces::setup_ns; use crate::setup_utils::namespaces::Namespace; -use crate::setup_utils::namespaces::{setup_ns, NodeType}; use crate::setup_utils::rita::thread_spawner; -use crate::utils::{generate_traffic, validate_debt_entry}; +use crate::utils::{generate_traffic, register_all_namespaces_to_exit, validate_debt_entry}; use crate::utils::{ - get_default_settings, register_to_exit, send_eth_bulk, test_reach_all, test_routes, - TEST_PAY_THRESH, + get_default_settings, send_eth_bulk, test_reach_all, test_routes, TEST_PAY_THRESH, }; use clarity::Address as EthAddress; use clarity::{PrivateKey as EthPrivateKey, Uint256}; use log::info; use rita_common::debt_keeper::GetDebtsResult; +use settings::client::RitaClientSettings; +use settings::exit::RitaExitSettingsStruct; use std::thread; use std::time::Duration; use web30::client::Web3; @@ -43,11 +44,12 @@ pub async fn run_eth_payments_test_scenario() { let namespaces = node_config.0; let expected_routes = node_config.1; - let (mut rita_settings, mut rita_exit_settings) = get_default_settings(); + let (mut client_settings, mut exit_settings) = + get_default_settings("test".to_string(), namespaces.clone()); // Set payment thresholds low enough so that they get triggered after an iperf - rita_settings.payment.payment_threshold = TEST_PAY_THRESH.into(); - rita_exit_settings.payment.payment_threshold = TEST_PAY_THRESH.into(); + let (client_settings, exit_settings) = + eth_payments_map(&mut client_settings, &mut exit_settings); namespaces.validate(); start_postgres(); @@ -55,7 +57,7 @@ pub async fn run_eth_payments_test_scenario() { let res = setup_ns(namespaces.clone()); info!("Namespaces setup: {res:?}"); - let rita_identities = thread_spawner(namespaces.clone(), rita_settings, rita_exit_settings) + let rita_identities = thread_spawner(namespaces.clone(), client_settings, exit_settings) .expect("Could not spawn Rita threads"); info!("Thread Spawner: {res:?}"); @@ -63,16 +65,7 @@ pub async fn run_eth_payments_test_scenario() { test_routes(namespaces.clone(), expected_routes); info!("Registering routers to the exit"); - for r in namespaces.names.clone() { - if let NodeType::Client = r.node_type { - let res = register_to_exit(r.get_name()).await; - if !res.is_success() { - panic!("Failed to register {} to exit with {:?}", r.get_name(), res); - } else { - info!("{} registered to exit", r.get_name()); - } - } - } + register_all_namespaces_to_exit(namespaces.clone()).await; thread::sleep(Duration::from_secs(10)); @@ -117,3 +110,12 @@ fn eth_payment_conditions(debts: GetDebtsResult) -> bool { (true, true) ) } + +fn eth_payments_map( + c_set: &mut RitaClientSettings, + exit_set: &mut RitaExitSettingsStruct, +) -> (RitaClientSettings, RitaExitSettingsStruct) { + c_set.payment.payment_threshold = TEST_PAY_THRESH.into(); + exit_set.payment.payment_threshold = TEST_PAY_THRESH.into(); + (c_set.clone(), exit_set.clone()) +} diff --git a/integration_tests/src/setup_utils/namespaces.rs b/integration_tests/src/setup_utils/namespaces.rs index 83ce77d58..d7030bb9e 100644 --- a/integration_tests/src/setup_utils/namespaces.rs +++ b/integration_tests/src/setup_utils/namespaces.rs @@ -20,8 +20,13 @@ pub struct Namespace { #[derive(Clone, Eq, PartialEq, Hash, Debug)] pub enum NodeType { - Client, - Exit, + Client { + // The exit this client should be connected to at network init + cluster_name: String, + }, + Exit { + instance_name: String, + }, } impl Namespace { @@ -55,7 +60,7 @@ impl NamespaceInfo { panic!("Duplicate id in namespace definition {}", space.id) } - if let NodeType::Exit = space.node_type { + if let NodeType::Exit { .. } = space.node_type { if space.id == 1 { // this would conflict with ip requirements for the internal // bridge @@ -234,7 +239,7 @@ pub fn setup_ns(spaces: NamespaceInfo) -> Result<(), KernelInterfaceError> { // and the ability to reach the postgresql database in the native namespace let mut links_to_native_namespace = Vec::new(); for name in spaces.names.iter() { - if let NodeType::Exit = name.node_type { + if let NodeType::Exit { .. } = name.node_type { let veth_native_to_exit = format!("vout-o-{}", name.get_name()); let veth_exit_to_native = format!("vout-{}-o", name.get_name()); let exit_ip = format!( diff --git a/integration_tests/src/setup_utils/rita.rs b/integration_tests/src/setup_utils/rita.rs index 2d6435872..88e331001 100644 --- a/integration_tests/src/setup_utils/rita.rs +++ b/integration_tests/src/setup_utils/rita.rs @@ -1,9 +1,13 @@ +use crate::utils::TEST_EXIT_DETAILS; + use super::babel::spawn_babel; use super::namespaces::get_nsfd; use super::namespaces::NamespaceInfo; use super::namespaces::NodeType; use althea_kernel_interface::KernelInterfaceError; use althea_types::Identity; +use ipnetwork::IpNetwork; +use ipnetwork::Ipv6Network; use log::info; use nix::sched::{setns, CloneFlags}; use rita_client::{ @@ -44,8 +48,8 @@ pub struct InstanceData { /// returns data about the spanwed instances that is used for coordination pub fn thread_spawner( namespaces: NamespaceInfo, - rita_settings: RitaClientSettings, - rita_exit_settings: RitaExitSettingsStruct, + client_settings: RitaClientSettings, + exit_settings: RitaExitSettingsStruct, ) -> Result { let mut instance_data = InstanceData::default(); let babeld_path = "/var/babeld/babeld/babeld".to_string(); @@ -65,21 +69,22 @@ pub fn thread_spawner( // todo spawn exits first in order to pass data to the clients? Or configure via endpoints later? - match ns.node_type { - NodeType::Client => { + match ns.node_type.clone() { + NodeType::Client { cluster_name: _ } => { let instance_info = spawn_rita( ns.get_name(), veth_interfaces, - rita_settings.clone(), + client_settings.clone(), ns.cost, ); instance_data.client_identities.push(instance_info); } - NodeType::Exit => { + NodeType::Exit { instance_name } => { let instance_info = spawn_rita_exit( ns.get_name(), + instance_name, veth_interfaces, - rita_exit_settings.clone(), + exit_settings.clone(), ns.cost as u64, ns.cost, ); @@ -183,6 +188,7 @@ pub fn spawn_rita( /// Spawn a thread for rita given a NamespaceInfo which will be assigned to the namespace given pub fn spawn_rita_exit( ns: String, + instance_name: String, veth_interfaces: HashSet, mut resettings: RitaExitSettingsStruct, exit_fee: u64, @@ -217,6 +223,17 @@ pub fn spawn_rita_exit( 0, id.try_into().unwrap(), ))); + let instance = TEST_EXIT_DETAILS + .get("test") + .unwrap() + .instances + .get(&instance_name) + .expect("Why is there no instance?"); + resettings.exit_network.subnet = Some(IpNetwork::V6( + Ipv6Network::new(instance.subnet, 40).unwrap(), + )); + resettings.network.wg_private_key = Some(instance.wg_priv_key); + resettings.network.wg_public_key = Some(instance.wg_pub_key); resettings.network.wg_private_key_path = wg_keypath; resettings.network.peer_interfaces = veth_interfaces; resettings.payment.local_fee = local_fee; @@ -227,10 +244,7 @@ pub fn spawn_rita_exit( resettings.db_uri = "postgresql://postgres@10.0.0.1/test".to_string(); // mirrored from rita_bin/src/exit.rs - let mut resettings = clu::exit_init("linux", resettings); - - // the exit must be added to the cluster after generating appropriate details - resettings.exit_network.cluster_exits = vec![resettings.get_identity().unwrap()]; + let resettings = clu::exit_init("linux", resettings); set_flag_config(config_path.into()); settings::set_rita_exit(resettings.clone()); diff --git a/integration_tests/src/utils.rs b/integration_tests/src/utils.rs index 83105a2fc..724b5adc6 100644 --- a/integration_tests/src/utils.rs +++ b/integration_tests/src/utils.rs @@ -13,13 +13,14 @@ use althea_proto::{ query_client::QueryClient, Metadata, QueryDenomMetadataRequest, }, }; -use althea_types::{ContactType, Denom, SystemChain}; +use althea_types::{ContactType, Denom, Identity, SystemChain, WgKey}; use awc::http::StatusCode; use babel_monitor::{open_babel_stream, parse_routes, structs::Route}; use clarity::{Transaction, Uint256}; use deep_space::{Address as AltheaAddress, Coin, Contact, CosmosPrivateKey, PrivateKey}; use futures::future::join_all; -use ipnetwork::{IpNetwork, Ipv6Network}; +use ipnetwork::IpNetwork; +use lazy_static; use log::{error, info, trace, warn}; use nix::{ fcntl::{open, OFlag}, @@ -62,6 +63,65 @@ pub const MIN_GLOBAL_FEE_AMOUNT: u128 = 10; pub const TOTAL_TIMEOUT: Duration = Duration::from_secs(300); pub const DEBT_ACCURACY_THRES: u8 = 15; +lazy_static! { + pub static ref TEST_EXIT_DETAILS: HashMap = { + let mut details = HashMap::new(); + let instance_4 = ExitInstances { + instance_name: "test_4".to_string(), + mesh_ip_v2: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 4)), + subnet: Ipv6Addr::new(0xfbad, 200, 0, 0, 0, 0, 0, 0), + wg_priv_key: "OGzbcm6czrjOEAViK7ZzlWM8mtjCxp7UPbuLS/dATV4=" + .parse() + .unwrap(), + wg_pub_key: "bvM10HW73yePrxdtCQQ4U20W5ogogdiZtUihrPc/oGY=" + .parse() + .unwrap(), + }; + + let instance_5 = ExitInstances { + instance_name: "test_5".to_string(), + mesh_ip_v2: IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 5)), + subnet: Ipv6Addr::new(0xfbad, 400, 0, 0, 0, 0, 0, 0), + wg_priv_key: "SEBve3ERCYCriBEfNFnWbED5OwWo/Ylppg1KEt0HZnA=" + .parse() + .unwrap(), + wg_pub_key: "R8F6IhDvy6PwcENEFQEBZXEY2fi6jEvmPTVvleR1IUw=" + .parse() + .unwrap(), + }; + + let exit = ExitInfo { + exit_name: "test".to_string(), + root_ip: IpAddr::V6(Ipv6Addr::new(0xfd00, 200, 199, 198, 197, 196, 195, 194)), + instances: { + let mut ret = HashMap::new(); + ret.insert(instance_4.instance_name.clone(), instance_4); + ret.insert(instance_5.instance_name.clone(), instance_5); + ret + }, + }; + details.insert(exit.exit_name.clone(), exit); + details + }; +} + +/// Struct used to store info of each exit instance in tests +#[derive(Clone, Debug)] +pub struct ExitInfo { + pub exit_name: String, + pub root_ip: IpAddr, + pub instances: HashMap, +} + +#[derive(Clone, Debug)] +pub struct ExitInstances { + pub instance_name: String, + pub mesh_ip_v2: IpAddr, + pub subnet: Ipv6Addr, + pub wg_pub_key: WgKey, + pub wg_priv_key: WgKey, +} + pub fn get_althea_grpc() -> String { format!("http://{}:9091", NODE_IP) } @@ -241,31 +301,45 @@ pub const CONFIG_FILE_PATH: &str = "/config.toml"; /// Gets the default client and exit settings handling the pre-launch exchange of exit into and its insertion into /// the -pub fn get_default_settings() -> (RitaClientSettings, RitaExitSettingsStruct) { - let mut exit = RitaExitSettingsStruct::new("/althea_rs/settings/test_exit.toml").unwrap(); +pub fn get_default_settings( + cluster_name: String, + namespaces: NamespaceInfo, +) -> (RitaClientSettings, RitaExitSettingsStruct) { + let exit = RitaExitSettingsStruct::new("/althea_rs/settings/test_exit.toml").unwrap(); + let client = RitaClientSettings::new("/althea_rs/settings/test.toml").unwrap(); - // exit should allow instant registration by any requester - exit.verif_settings = None; - exit.network.mesh_ip = Some(EXIT_ROOT_IP); - exit.exit_network.subnet = Some(IpNetwork::V6(Ipv6Network::new(EXIT_SUBNET, 40).unwrap())); + let cluster = TEST_EXIT_DETAILS + .get(&cluster_name) + .expect("Please provide a valid cluster name"); - let mut client = RitaClientSettings::new(CONFIG_FILE_PATH).unwrap(); + let mut cluster_exits = Vec::new(); + let mut client_exit_servers = HashMap::new(); - client.exit_client.contact_info = Some( - ContactType::Both { - number: "+11111111".parse().unwrap(), - email: "fake@fake.com".parse().unwrap(), - sequence_number: Some(0), + let mut exit_mesh_ips = HashSet::new(); + for ns in namespaces.names { + if let NodeType::Exit { instance_name: _ } = ns.node_type.clone() { + exit_mesh_ips.insert(get_ip_from_namespace(ns)); } - .into(), - ); - client.exit_client.current_exit = Some(TEST_EXIT_NAME.to_string()); - client.exit_client.exits.insert( - TEST_EXIT_NAME.to_string(), + } + + for instance in cluster.instances.values() { + if exit_mesh_ips.contains(&instance.mesh_ip_v2.to_string()) { + cluster_exits.push(Identity { + mesh_ip: instance.mesh_ip_v2, + eth_address: exit.payment.eth_address.unwrap(), + wg_public_key: instance.wg_pub_key, + nickname: None, + }); + } + } + + client_exit_servers.insert( + cluster_name.clone(), settings::client::ExitServer { - root_ip: EXIT_ROOT_IP, + root_ip: cluster.root_ip, subnet: None, eth_address: exit.payment.eth_address.unwrap(), + // This is the wg key that is common among all instances wg_public_key: exit.exit_network.wg_public_key, registration_port: exit.exit_network.exit_hello_port, description: exit.description.clone(), @@ -273,13 +347,28 @@ pub fn get_default_settings() -> (RitaClientSettings, RitaExitSettingsStruct) { }, ); + let mut exit = exit.clone(); + let mut client = client.clone(); + // exit should allow instant registration by any requester + exit.verif_settings = None; + exit.network.mesh_ip = Some(cluster.root_ip); + exit.exit_network.cluster_exits = cluster_exits.clone(); + client.exit_client.contact_info = Some( + ContactType::Both { + number: "+11111111".parse().unwrap(), + email: "fake@fake.com".parse().unwrap(), + sequence_number: Some(0), + } + .into(), + ); + client.exit_client.current_exit = Some(cluster_name); + client.exit_client.exits = client_exit_servers.clone(); // first node is passed through to the host machine for testing second node is used // for testnet queries exit.payment.althea_grpc_list = vec![get_althea_grpc()]; exit.payment.eth_node_list = vec![get_eth_node()]; client.payment.althea_grpc_list = vec![get_althea_grpc()]; client.payment.eth_node_list = vec![get_eth_node()]; - (client, exit) } @@ -318,9 +407,12 @@ pub fn althea_system_chain_exit(settings: RitaExitSettingsStruct) -> RitaExitSet } // Calls the register to exit rpc function within the provided namespace -pub async fn register_to_exit(namespace_name: String) -> StatusCode { +pub async fn register_to_exit(namespace_name: String, exit_name: String) -> StatusCode { // thread safe lock that allows us to pass data between the router thread and this thread // one copy of the reference is sent into the closure and the other is kept in this scope. + let exit_network = TEST_EXIT_DETAILS + .get(&exit_name) + .expect("Please provide a valid exit"); let response: Arc>> = Arc::new(RwLock::new(None)); let response_local = response.clone(); let namespace_local = namespace_name.clone(); @@ -335,7 +427,7 @@ pub async fn register_to_exit(namespace_name: String) -> StatusCode { let req = client .post(format!( "http://localhost:4877/exits/{}/register", - TEST_EXIT_NAME + exit_network.exit_name )) .send() .await @@ -432,7 +524,14 @@ pub fn generate_traffic(from: Namespace, to: Option, data: String) { pub fn get_ip_from_namespace(node: Namespace) -> String { match node.node_type { - NodeType::Exit => EXIT_ROOT_IP.to_string(), + NodeType::Exit { instance_name } => TEST_EXIT_DETAILS + .get("test") + .unwrap() + .instances + .get(&instance_name) + .unwrap() + .mesh_ip_v2 + .to_string(), _ => format!("fd00::{}", node.id), } } @@ -460,7 +559,7 @@ pub async fn query_debts( for node in nodes { let response: Arc>>> = Arc::new(RwLock::new(None)); let response_local = response.clone(); - let node_name = node.get_name(); + let node_name: String = node.get_name(); let _ = thread::spawn(move || { info!("Starting inner thraed"); @@ -816,13 +915,17 @@ pub async fn print_althea_balances(addresses: Vec, denom: String) // Relies on nodes to be named using fd00::5, fd00::25 etc pub fn get_node_id_from_ip(ip: IpAddr) -> u16 { - match ip { - EXIT_ROOT_IP => 4, - _ => { - let addr = ip.to_string(); - addr.split("::").last().unwrap().parse().unwrap() - } + let exit_4 = IpAddr::V6(Ipv6Addr::new(0xfd00, 200, 199, 198, 197, 196, 195, 194)); + let exit_5 = IpAddr::V6(Ipv6Addr::new(0xfd00, 400, 399, 398, 397, 396, 395, 394)); + + if ip == exit_4 { + return 4; + } + if ip == exit_5 { + return 5; } + let addr = ip.to_string(); + addr.split("::").last().unwrap().parse().unwrap() } pub const HIGH_GAS_PRICE: u64 = 1_000_000_000u64; @@ -967,3 +1070,24 @@ pub fn generate_config_file(path: String) -> Result<(), KernelInterfaceError> { write_out(&path, lines) } + +pub async fn register_all_namespaces_to_exit(namespaces: NamespaceInfo) { + let register_timeout = Duration::from_secs(20); + for r in namespaces.names.clone() { + if let NodeType::Client { cluster_name } = r.node_type.clone() { + let start: Instant = Instant::now(); + loop { + let res = register_to_exit(r.get_name(), cluster_name.clone()).await; + if res.is_success() { + break; + } + if Instant::now() - start > register_timeout { + panic!("Failed to register {} to exit", r.get_name()); + } + warn!("Failed {} registration to exit, trying again", r.get_name()); + thread::sleep(Duration::from_secs(1)); + } + info!("{} registered to exit {}", r.get_name(), cluster_name); + } + } +} diff --git a/settings/test_exit.toml b/settings/test_exit.toml index 8cb5ed9ed..92f32eaaf 100644 --- a/settings/test_exit.toml +++ b/settings/test_exit.toml @@ -47,7 +47,6 @@ pass = "Some pass here" mesh_ip = "fd00::5" eth_address = "0xbe398dc24de37c73cec974d688018e58f94d6e0a" wg_public_key = "bvM10HW73yePrxdtCQQ4U20W5ogogdiZtUihrPc/oGY=" -althea_address = "althea11lrsu892mqx2mndyvjufrh2ux56tyfxl2e3eht3" [dao] dao_enforcement = false diff --git a/test_runner/src/main.rs b/test_runner/src/main.rs index 4d51ac3d4..bba203d74 100644 --- a/test_runner/src/main.rs +++ b/test_runner/src/main.rs @@ -1,6 +1,7 @@ use integration_tests::debts::run_debts_test; /// Binary crate for actually running the integration tests use integration_tests::five_nodes::run_five_node_test_scenario; +use integration_tests::mutli_exit::run_multi_exit_test; use integration_tests::utils::{generate_config_file, CONFIG_FILE_PATH}; use integration_tests::{ payments_althea::run_althea_payments_test_scenario, @@ -40,6 +41,8 @@ async fn main() { run_eth_payments_test_scenario().await; } else if test_type == "PAYMENTS_ALTHEA" || test_type == "ALTHEA_PAYMENTS" { run_althea_payments_test_scenario().await + } else if test_type == "MULTI_EXIT" { + run_multi_exit_test().await } else { panic!("Error unknown test type {}!", test_type); }