diff --git a/tests/multinode.rs b/tests/multinode.rs index 49dfd7329ef567..e9c108a56c4e5d 100644 --- a/tests/multinode.rs +++ b/tests/multinode.rs @@ -631,94 +631,112 @@ fn test_multi_node_dynamic_network() { let mut validators: Vec<_> = t2.into_iter().map(|t| t.join().unwrap()).collect(); + let mut client = mk_client(&leader_data); + let mut last_finality = Duration::from_millis(client.get_finality() as u64); + info!( + "Last finality {}.{:03}", + last_finality.as_secs(), + last_finality.subsec_millis() + ); let now = Instant::now(); let mut consecutive_success = 0; - let mut failures = 0; let mut max_distance_increase = 0i64; + let mut expected_balance = leader_balance; for i in 0..std::cmp::max(20, num_nodes) { - //verify leader can do transfer - let expected = ((i + 3) * 500) as i64; - let leader_balance = retry_send_tx_and_retry_get_balance( - &leader_data, - &alice_arc.read().unwrap(), - &bob_pubkey, - Some(expected), - ).unwrap(); - if leader_balance != expected { - info!( - "leader dropped transaction {} {:?} {:?}", - i, leader_balance, expected - ); + trace!("getting leader last_id"); + let last_id = client.get_last_id(); + trace!("executing leader transfer"); + let sig = client + .transfer( + 500, + &alice_arc.read().unwrap().keypair(), + bob_pubkey, + &last_id, + ) + .unwrap(); + + expected_balance += 500; + + assert!(client.poll_for_signature(&sig).is_ok()); + + let now = Instant::now(); + let mut finality = Duration::from_millis(client.get_finality() as u64); + while finality == last_finality { + // Check if new finality is less than last finality (+5 seconds of grace period) + assert!(now.elapsed() < last_finality.checked_add(Duration::new(5, 0)).unwrap()); + sleep(Duration::from_millis(100)); + finality = Duration::from_millis(client.get_finality() as u64); } - //verify all validators have the same balance - { - let mut success = 0usize; - let mut max_distance = 0i64; - let mut total_distance = 0i64; - let mut num_nodes_behind = 0i64; - validators.retain(|server| { - let mut retain_me = true; - let mut client = mk_client(&server.0); - trace!("{:x} {} get_balance start", server.0.debug_id(), i); - let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(leader_balance)); - trace!( - "{:x} {} get_balance: {:?} leader_balance: {}", - server.0.debug_id(), - i, - getbal, - leader_balance - ); - let bal = getbal.unwrap_or(0); - let distance = (leader_balance - bal) / 500; - max_distance = max(distance, max_distance); - total_distance += distance; - if distance > max_distance_increase { - info!("Node {:x} is behind by {}", server.0.debug_id(), distance); - max_distance_increase = distance; - if max_distance_increase as u64 > purge_lag as u64 { - server.1.exit(); - info!("Node {:x} is exiting", server.0.debug_id()); - retain_me = false; - } - } - if distance > 0 { - num_nodes_behind += 1; - } - if let Some(bal) = getbal { - if bal == leader_balance { - success += 1; - } - } - retain_me - }); - if num_nodes_behind != 0 { - info!("{} nodes are lagging behind leader", num_nodes_behind); - } - info!( - "SUCCESS[{}] {} out of {} distance: {} max_distance: {} finality: {}", - i, - success, - validators.len(), - total_distance, - max_distance, - get_finality(&leader_data) - ); - if success == validators.len() && total_distance == 0 { - consecutive_success += 1; - } else { - consecutive_success = 0; - failures += 1; - } - if consecutive_success == 10 { - break; + + last_finality = finality; + + let balance = retry_get_balance(&mut client, &bob_pubkey, Some(expected_balance)); + info!( + "Leader expected balance {}, current balance {:?}", + expected_balance, balance + ); + assert_eq!(balance, Some(expected_balance)); + consecutive_success += 1; + + info!( + "SUCCESS[{}] balance: {} finality: {}.{:03} seconds", + i, + expected_balance, + last_finality.as_secs(), + last_finality.subsec_millis(), + ); + + if consecutive_success == 10 { + break; + } + } + + //verify all validators have the same balance + let mut max_distance = 0i64; + let mut total_distance = 0i64; + let mut num_nodes_behind = 0i64; + validators.retain(|server| { + let mut retain_me = true; + let mut client = mk_client(&server.0); + trace!("{:x} get_balance start", server.0.debug_id()); + let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(expected_balance)); + trace!( + "{:x} get_balance: {:?} expected_balance: {}", + server.0.debug_id(), + getbal, + expected_balance + ); + let bal = getbal.unwrap_or(0); + let distance = (expected_balance - bal) / 500; + max_distance = max(distance, max_distance); + total_distance += distance; + if distance > max_distance_increase { + info!("Node {:x} is behind by {}", server.0.debug_id(), distance); + max_distance_increase = distance; + if max_distance_increase as u64 > purge_lag as u64 { + server.1.exit(); + info!("Node {:x} is exiting", server.0.debug_id()); + retain_me = false; } } + if distance > 0 { + num_nodes_behind += 1; + } + retain_me + }); + + if num_nodes_behind != 0 { + info!("{} nodes are lagging behind leader", num_nodes_behind); } info!( - "Took {} s to converge total failures: {}", - duration_as_s(&now.elapsed()), - failures + "Validators lagging: {}/{}, distance: {}, max_distance: {}", + num_nodes_behind, + validators.len(), + total_distance, + max_distance, ); + + info!("Took {} s to converge", duration_as_s(&now.elapsed()),); assert_eq!(consecutive_success, 10); for (_, node) in &validators { node.exit(); @@ -821,9 +839,3 @@ fn retry_send_tx_and_retry_get_balance( } None } - -fn get_finality(leader: &NodeInfo) -> usize { - let mut client = mk_client(leader); - trace!("getting leader finality"); - client.get_finality() -}