Add mulitnode test for two nodes rotating at 1 tick per slot

src/fullnode.rs

@@ -326,7 +326,8 @@ impl Fullnode {
                 break;
             }
             trace!(
-                "Waiting for bank tick_height to catch up from {} to {}",
+                "{:?}: Waiting for bank tick_height to catch up from {} to {}",
+                self.id,
                 bank_tick_height,
                 tick_height
             );

tests/multinode.rs

@@ -1655,31 +1655,62 @@ fn retry_send_tx_and_retry_get_balance(
     None
 }
 
-fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64) {
+fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_validator: bool) {
     solana_logger::setup();
     info!(
         "fullnode_rotate_fast: ticks_per_slot={} slots_per_epoch={}",
         ticks_per_slot, slots_per_epoch
     );
 
+    // Create the leader node information
     let leader_keypair = Arc::new(Keypair::new());
-    let leader_pubkey = leader_keypair.pubkey().clone();
     let leader = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
     let leader_info = leader.info.clone();
-    let (_mint, leader_ledger_path, _last_entry_height, _last_id, last_entry_id) =
+
+    // Create the validator node information
+    let validator_keypair = Arc::new(Keypair::new());
+    let validator = Node::new_localhost_with_pubkey(validator_keypair.pubkey());
+
+    // Make a common mint and a genesis entry for both leader + validator ledgers
+    let (mint_keypair, leader_ledger_path, genesis_entry_height, last_id, mut last_entry_id) =
         create_tmp_sample_ledger(
-            "fullnode_transact_while_rotating_fast",
+            "test_fullnode_rotate",
             1_000_000_000_000_000_000,
             0,
-            leader_pubkey,
+            leader_keypair.pubkey(),
             123,
         );
+
+    let mut ledger_paths = Vec::new();
+    ledger_paths.push(leader_ledger_path.clone());
     info!("ledger is {}", leader_ledger_path);
 
-    // Setup the cluster with a single node
+    if include_validator {
+        // Add validator vote on tick height 1
+        let (active_set_entries, _) = make_active_set_entries(
+            &validator_keypair,
+            &mint_keypair,
+            100,
+            1,
+            &last_entry_id,
+            &last_id,
+            0,
+        );
+
+        let blocktree = Blocktree::open(&leader_ledger_path).unwrap();
+        blocktree
+            .write_entries(0, genesis_entry_height, &active_set_entries)
+            .unwrap();
+        last_entry_id = active_set_entries.last().unwrap().id;
+    }
+
+    // Setup leader scheduler config
     let mut fullnode_config = FullnodeConfig::default();
     fullnode_config.leader_scheduler_config.ticks_per_slot = ticks_per_slot;
     fullnode_config.leader_scheduler_config.slots_per_epoch = slots_per_epoch;
+    fullnode_config
+        .leader_scheduler_config
+        .active_window_tick_length = std::u64::MAX;
 
     let mut tick_height_of_next_rotation = ticks_per_slot;
     if fullnode_config.leader_scheduler_config.ticks_per_slot == 1 {
@@ -1698,6 +1729,56 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64) {
         tick_height_of_next_rotation += 1;
     }
 
+    // Start up the node(s)
+    let mut node_exits = vec![];
+
+    if include_validator {
+        info!("validator id: {}", validator_keypair.pubkey());
+
+        let validator_ledger_path = tmp_copy_ledger(&leader_ledger_path, "test_fullnode_rotate");
+        ledger_paths.push(validator_ledger_path.clone());
+        let validator_fullnode = Fullnode::new(
+            validator,
+            &validator_keypair,
+            &validator_ledger_path,
+            VotingKeypair::new_local(&validator_keypair),
+            Some(&leader_info),
+            &fullnode_config,
+        );
+
+        let (validator_rotation_sender, validator_rotation_receiver) = channel();
+        node_exits.push(validator_fullnode.run(Some(validator_rotation_sender)));
+
+        let mut validator_tick_height_of_next_rotation = tick_height_of_next_rotation;
+        let mut validator_should_be_leader = false;
+        Builder::new()
+            .name("validator_rotation_receiver".to_string())
+            .spawn(move || loop {
+                match validator_rotation_receiver.recv() {
+                    Ok((rotation_type, tick_height)) => {
+                        info!(
+                            "validator rotation event {:?} at tick_height={}",
+                            rotation_type, tick_height
+                        );
+                        assert_eq!(validator_tick_height_of_next_rotation, tick_height);
+                        assert_eq!(
+                            rotation_type,
+                            if validator_should_be_leader {
+                                FullnodeReturnType::LeaderToValidatorRotation
+                            } else {
+                                FullnodeReturnType::ValidatorToLeaderRotation
+                            }
+                        );
+                        validator_should_be_leader = !validator_should_be_leader;
+                        validator_tick_height_of_next_rotation += ticks_per_slot;
+                    }
+                    Err(_) => break,
+                };
+            })
+            .unwrap();
+    }
+    let (leader_rotation_sender, leader_rotation_receiver) = channel();
+    info!("leader id: {}", leader_keypair.pubkey());
     let leader_fullnode = Fullnode::new(
         leader,
         &leader_keypair,
@@ -1706,13 +1787,16 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64) {
         None,
         &fullnode_config,
     );
-    let (leader_rotation_sender, leader_rotation_receiver) = channel();
-    let leader_fullnode_exit = leader_fullnode.run(Some(leader_rotation_sender));
+
+    node_exits.push(leader_fullnode.run(Some(leader_rotation_sender)));
+
+    converge(&leader_info, node_exits.len());
     info!(
         "found leader: {:?}",
         poll_gossip_for_leader(leader_info.gossip, Some(5)).unwrap()
     );
 
+    let mut should_be_leader = true;
     const N: usize = 10;
     for i in 1..=N {
         info!("waiting for rotation #{}/{}", i, N);
@@ -1723,22 +1807,53 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64) {
                     rotation_type, tick_height
                 );
                 assert_eq!(tick_height_of_next_rotation, tick_height);
+                if include_validator {
+                    assert_eq!(
+                        rotation_type,
+                        if should_be_leader {
+                            FullnodeReturnType::LeaderToValidatorRotation
+                        } else {
+                            FullnodeReturnType::ValidatorToLeaderRotation
+                        }
+                    );
+                    should_be_leader = !should_be_leader;
+                } else {
+                    assert_eq!(rotation_type, FullnodeReturnType::LeaderToLeaderRotation);
+                }
                 tick_height_of_next_rotation += ticks_per_slot;
             }
             err => panic!("Unexpected response: {:?}", err),
         };
     }
 
     info!("Shutting down");
-    leader_fullnode_exit();
+    for node_exit in node_exits {
+        node_exit();
+    }
+
+    for path in ledger_paths {
+        Blocktree::destroy(&path).expect("Expected successful database destruction");
+        remove_dir_all(path).unwrap();
+    }
+}
+
+#[test]
+fn test_one_fullnode_rotate_every_tick() {
+    test_fullnode_rotate(1, 1, false);
+}
+
+#[test]
+fn test_one_fullnode_rotate_every_second_tick() {
+    test_fullnode_rotate(2, 1, false);
 }
 
 #[test]
-fn test_fullnode_rotate_every_tick() {
-    test_fullnode_rotate(1, 1);
+#[ignore] // Currently fails!
+fn test_two_fullnodes_rotate_every_tick() {
+    test_fullnode_rotate(1, 1, true);
 }
 
 #[test]
-fn test_fullnode_rotate_every_second_tick() {
-    test_fullnode_rotate(2, 1);
+fn test_two_fullnodes_rotate_every_second_tick() {
+    test_fullnode_rotate(2, 1, true);
 }