Merge pull request #35 from EspressoSystems/jr/remove_json_config

Replace identity_mappings.json with cli variables

libp2p-networking/README → libp2p-networking/README.md

@@ -58,3 +58,18 @@ In the direct message case, the conductor will increment the state of a randomly
 In both cases, the test terminates as successful when the conductor receives the incremented state from all other nodes. Then, the conductor sends a special "kill" message to all known nodes and waits for them to disconnect.
 
 Metadata about the toplogy is currently read from an `identity_mapping.json` file that manually labels the type of node (bootstrap, regular, conductor). The conductor uses this to figure out information about all nodes in the network. The regular nodes use this to learn about their ip address and the addresses necessary to bootstrap onto the network. The boostrap nodes only use this to learn about their ip addresses.
+
+### Running counter multi-machine tests
+
+A sample invocation locally:
+
+```bash
+# run each line in a separate terminal
+nix develop -c cargo run --features webui --release --example counter -- --bound_addr 127.0.0.1:9000 --node_type Bootstrap --num_nodes 5 --bootstrap 127.0.0.1:9000 --webui 127.0.0.1:8000
+nix develop -c cargo run --features webui --release --example counter -- --bound_addr 127.0.0.1:9001 --node_type Regular --num_nodes 5 --bootstrap 127.0.0.1:9000 --webui 127.0.0.1:8001
+nix develop -c cargo run --features webui --release --example counter -- --bound_addr 127.0.0.1:9002 --node_type Regular --num_nodes 5 --bootstrap 127.0.0.1:9000 --webui 127.0.0.1:8002
+nix develop -c cargo run --features webui --release --example counter -- --bound_addr 127.0.0.1:9003 --node_type Regular --num_nodes 5 --bootstrap 127.0.0.1:9000 --webui 127.0.0.1:8003
+nix develop -c cargo run --features webui --release --example counter -- --bound_addr 127.0.0.1:9004 --node_type Conductor --num_nodes 5 --bootstrap 127.0.0.1:9000 --webui 127.0.0.1:8004
+```
+
+To run on the AWS cluster, see [here](https://github.com/EspressoSystems/cloud-infrastructure/blob/c86873a5c647772836907fc206fce5702a5878bb/ansible/networking-demo/README.md).

libp2p-networking/examples/common/web.rs

@@ -17,9 +17,12 @@ where
     let mut tide = tide::with_state(state);
     // Unwrap this in the calling thread so that if it fails we fail completely
     // instead of not knowing why the web UI does not work
-    tide.at("/")
-        .serve_file("web/index.html")
-        .expect("Could not register web/index.html");
+    tide.at("/").get(|_| async move {
+        Ok(tide::Response::builder(200)
+            .content_type(tide::http::mime::HTML)
+            .body(include_str!("../../web/index.html"))
+            .build())
+    });
     tide.at("/sse").get(tide::sse::endpoint(
         |req: tide::Request<Arc<NetworkNodeHandle<S>>>, sender| async move {
             let peer_addr = req.peer_addr();

libp2p-networking/examples/counter.rs

@@ -9,13 +9,7 @@ pub mod common;
 #[instrument]
 async fn main() -> Result<()> {
     let args = CliOpt::from_args();
-    start_main(
-        args.ip.unwrap(),
-        args.path,
-        #[cfg(feature = "webui")]
-        args.webui,
-    )
-    .await?;
+    start_main(args).await?;
 
     // optional UI perhaps? for monitoring purposes
     Ok(())

libp2p-networking/flake.nix

@@ -121,7 +121,7 @@
 
         defaultPackage = self.packages.${system}.${crateName};
 
-        devShell = pkgs.mkShell {
+        devShells.staticShell = pkgs.mkShell {
           shellHook = ''
             ulimit -n 1024
             export RUSTFLAGS='-C target-feature=+crt-static'
@@ -131,6 +131,11 @@
             with pkgs; [ fenix.packages.${system}.rust-analyzer fenixStable ] ++ buildDeps;
         };
 
+        devShell = pkgs.mkShell {
+          buildInputs =
+            with pkgs; [ fenix.packages.${system}.rust-analyzer fenixStable ] ++ buildDeps;
+        };
+
         devShells.perfShell = pkgs.mkShell {
           buildInputs = with pkgs; [ grcov recent_flamegraph fd fenixNightly fenix.packages.${system}.rust-analyzer ] ++ buildDeps;
           shellHook = ''

libp2p-networking/src/network_node.rs

@@ -7,6 +7,8 @@ use rand::{seq::IteratorRandom, thread_rng};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::Debug;
+use std::str::FromStr;
+
 use std::{
     collections::HashSet,
     io::Error,
@@ -126,8 +128,10 @@ pub struct ConnectionData {
     pub connected_peers: HashSet<PeerId>,
     /// set of peers that were at one point connected
     pub connecting_peers: HashSet<PeerId>,
-    /// set of events to send to client
+    /// set of known peers
     pub known_peers: HashSet<PeerId>,
+    /// set of peers that are immune to pruning
+    pub ignored_peers: HashSet<PeerId>,
 }
 
 impl NetworkDef {
@@ -275,6 +279,22 @@ pub enum NetworkNodeType {
     Conductor,
 }
 
+impl FromStr for NetworkNodeType {
+    type Err = String;
+
+    fn from_str(input: &str) -> Result<NetworkNodeType, Self::Err> {
+        match input {
+            "Conductor" => Ok(NetworkNodeType::Conductor),
+            "Regular" => Ok(NetworkNodeType::Regular),
+            "Bootstrap" => Ok(NetworkNodeType::Bootstrap),
+            _ => Err(
+                "Couldn't parse node type. Must be one of Conductor, Bootstrap, Regular"
+                    .to_string(),
+            ),
+        }
+    }
+}
+
 /// serialize an arbitrary message
 /// # Errors
 /// when unable to serialize a message
@@ -361,6 +381,9 @@ pub enum ClientRequest {
     Pruning(bool),
     /// add vec of known peers or addresses
     AddKnownPeers(Vec<(Option<PeerId>, Multiaddr)>),
+    /// Ignore peers. Only here for debugging purposes.
+    /// Allows us to have nodes that are never pruned
+    IgnorePeers(Vec<PeerId>),
 }
 
 /// events generated by the swarm that we wish
@@ -705,6 +728,12 @@ impl NetworkNode {
                 #[allow(clippy::enum_glob_use)]
                 use ClientRequest::*;
                 match msg {
+                    IgnorePeers(peers) => {
+                        self.swarm
+                            .behaviour_mut()
+                            .ignored_peers
+                            .extend(peers.iter());
+                    }
                     Shutdown => {
                         warn!("Libp2p listener shutting down");
                         return Ok(true);

libp2p-networking/tests/common/mod.rs

@@ -1,5 +1,5 @@
 use std::{
-    collections::{HashMap, HashSet},
+    collections::HashMap,
     sync::{Arc, Once},
     time::Duration,
 };
@@ -111,6 +111,7 @@ pub async fn print_connections<S>(handles: &[Arc<NetworkNodeHandle<S>>]) {
     }
 }
 
+#[allow(dead_code)]
 pub async fn check_connection_state<S>(handles: &[Arc<NetworkNodeHandle<S>>]) {
     let mut err_msg = "".to_string();
     for (i, handle) in handles.iter().enumerate() {

libp2p-networking/tests/counter.rs

@@ -1,7 +1,7 @@
 use std::{sync::Arc, time::Duration};
 mod common;
 use async_std::future::timeout;
-use common::{check_connection_state, test_bed, HandleSnafu, TestError};
+use common::{test_bed, HandleSnafu, TestError};
 
 use bincode::Options;