Add a multi exit test to test runner

This commit edits test runner setup to spawn additional exits in the environment to
tests multi exit scenarios. Currently this test spawns two exits in a network, kills the first and
verfies that the exits switch over to the second one

.github/workflows/rust.yml

@@ -152,3 +152,12 @@ jobs:
         run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard
       - name: Run integration test
         run: bash scripts/integration_tests/all-up-test.sh ETH_PAYMENTS
+  integration-test-multi-exit:
+    needs: integration-test-five-nodes
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Wireguard
+        run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard
+      - name: Run integration test
+        run: bash scripts/integration_tests/all-up-test.sh MULTI_EXIT

integration_tests/Cargo.toml

@@ -31,3 +31,4 @@ futures = { version = "0.3", features = ["compat"] }
 num256 = "0.5"
 num-traits="0.2"
 web30 = "1.0"
+lazy_static = "1.4"

integration_tests/src/debts.rs

@@ -7,8 +7,9 @@ use crate::setup_utils::database::start_postgres;
 use crate::setup_utils::namespaces::*;
 use crate::setup_utils::rita::thread_spawner;
 use crate::utils::{
-    generate_traffic, get_default_settings, get_ip_from_namespace, query_debts, register_to_exit,
-    test_all_internet_connectivity, test_reach_all, test_routes, DEBT_ACCURACY_THRES,
+    generate_traffic, get_default_settings, get_ip_from_namespace, query_debts,
+    register_all_namespaces_to_exit, test_all_internet_connectivity, test_reach_all, test_routes,
+    DEBT_ACCURACY_THRES,
 };
 use althea_types::Identity;
 use log::info;
@@ -44,7 +45,8 @@ pub async fn run_debts_test() {
     let namespaces = node_config.0;
     let expected_routes = node_config.1;
 
-    let (rita_settings, rita_exit_settings) = get_default_settings();
+    let (client_settings, exit_settings) =
+        get_default_settings("test".to_string(), namespaces.clone());
 
     // The exit price is set to ns.cost during thread_spawner
     let exit_price = namespaces.get_namespace(4).unwrap().cost;
@@ -55,7 +57,7 @@ pub async fn run_debts_test() {
     let res = setup_ns(namespaces.clone());
     info!("Namespaces setup: {res:?}");
 
-    let rita_identities = thread_spawner(namespaces.clone(), rita_settings, rita_exit_settings)
+    let rita_identities = thread_spawner(namespaces.clone(), client_settings, exit_settings)
         .expect("Could not spawn Rita threads");
     // There should be only 1 exit identity
     let exit_identity = rita_identities
@@ -71,16 +73,7 @@ pub async fn run_debts_test() {
     test_routes(namespaces.clone(), expected_routes);
 
     info!("Registering routers to the exit");
-    for r in namespaces.names.clone() {
-        if let NodeType::Client = r.node_type {
-            let res = register_to_exit(r.get_name()).await;
-            if !res.is_success() {
-                panic!("Failed to register {} to exit with {:?}", r.get_name(), res);
-            } else {
-                info!("{} registered to exit", r.get_name());
-            }
-        }
-    }
+    register_all_namespaces_to_exit(namespaces.clone()).await;
 
     // Let network stabalize
     thread::sleep(Duration::from_secs(20));
@@ -132,8 +125,9 @@ pub async fn run_debts_test() {
         // if to_node is none, we are querying the internet, so last node has to be an exit node
         if to_node.is_none() {
             exit_node = query_nodes.pop();
-            if exit_node.clone().unwrap().node_type != NodeType::Exit {
-                panic!("Why is last element not an exit?");
+            match exit_node.clone().unwrap().node_type {
+                NodeType::Exit { .. } => {}
+                _ => panic!("Why is last element not an exit?"),
             }
         }
 

integration_tests/src/five_nodes.rs

@@ -1,8 +1,10 @@
 use crate::setup_utils::database::start_postgres;
 use crate::setup_utils::namespaces::*;
 use crate::setup_utils::rita::thread_spawner;
-use crate::utils::{get_default_settings, register_to_exit, test_reach_all, test_routes};
-use log::{error, info};
+use crate::utils::{
+    get_default_settings, register_all_namespaces_to_exit, test_reach_all, test_routes,
+};
+use log::info;
 use std::collections::HashMap;
 
 /// Runs a five node fixed network map test scenario, this does basic network setup and tests reachability to
@@ -13,15 +15,16 @@ pub async fn run_five_node_test_scenario() {
     let namespaces = node_config.0;
     let expected_routes = node_config.1;
 
-    let (rita_settings, rita_exit_settings) = get_default_settings();
+    let (client_settings, exit_settings) =
+        get_default_settings("test".to_string(), namespaces.clone());
 
     namespaces.validate();
 
     start_postgres();
     let res = setup_ns(namespaces.clone());
     info!("Namespaces setup: {res:?}");
 
-    let _ = thread_spawner(namespaces.clone(), rita_settings, rita_exit_settings)
+    let _ = thread_spawner(namespaces.clone(), client_settings, exit_settings)
         .expect("Could not spawn Rita threads");
     info!("Thread Spawner: {res:?}");
 
@@ -33,14 +36,7 @@ pub async fn run_five_node_test_scenario() {
     test_routes(namespaces.clone(), expected_routes);
 
     info!("Registering routers to the exit");
-    for r in namespaces.names {
-        if let NodeType::Client = r.node_type {
-            let res = register_to_exit(r.get_name()).await;
-            if !res.is_success() {
-                error!("Failed to register {} to exit with {:?}", r.get_name(), res);
-            }
-        }
-    }
+    register_all_namespaces_to_exit(namespaces.clone()).await;
 }
 
 /// This defines the network map for a five node scenario
@@ -64,37 +60,51 @@ pub fn five_node_config() -> (NamespaceInfo, HashMap<Namespace, RouteHop>) {
     let testa = Namespace {
         id: 1,
         cost: 25,
-        node_type: NodeType::Client,
+        node_type: NodeType::Client {
+            cluster_name: "test".to_string(),
+        },
     };
     let testb = Namespace {
         id: 2,
         cost: 500,
-        node_type: NodeType::Client,
+        node_type: NodeType::Client {
+            cluster_name: "test".to_string(),
+        },
     };
     let testc = Namespace {
         id: 3,
         cost: 15,
-        node_type: NodeType::Client,
+        node_type: NodeType::Client {
+            cluster_name: "test".to_string(),
+        },
     };
     let testd = Namespace {
         id: 4,
         cost: 10,
-        node_type: NodeType::Exit,
+        node_type: NodeType::Exit {
+            instance_name: "test_4".to_string(),
+        },
     };
     let teste = Namespace {
         id: 5,
         cost: 40,
-        node_type: NodeType::Client,
+        node_type: NodeType::Client {
+            cluster_name: "test".to_string(),
+        },
     };
     let testf = Namespace {
         id: 6,
         cost: 20,
-        node_type: NodeType::Client,
+        node_type: NodeType::Client {
+            cluster_name: "test".to_string(),
+        },
     };
     let testg = Namespace {
         id: 7,
         cost: 15,
-        node_type: NodeType::Client,
+        node_type: NodeType::Client {
+            cluster_name: "test".to_string(),
+        },
     };
 
     let nsinfo = NamespaceInfo {

integration_tests/src/lib.rs

@@ -1,8 +1,12 @@
+#[macro_use]
+extern crate lazy_static;
+
 use std::time::Duration;
 
 pub mod config;
 pub mod debts;
 pub mod five_nodes;
+pub mod mutli_exit;
 pub mod payments_althea;
 pub mod payments_eth;
 pub mod setup_utils;