Skip to content

Commit

Permalink
Refactored large network test to use finality to assert success
Browse files Browse the repository at this point in the history
  • Loading branch information
@pgarg66
pgarg66 committed Aug 15, 2018
1 parent db35f22 commit cc1ac20
Showing 1 changed file with 96 additions and 84 deletions.
180 changes: 96 additions & 84 deletions tests/multinode.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,94 +631,112 @@ fn test_multi_node_dynamic_network() {

let mut validators: Vec<_> = t2.into_iter().map(|t| t.join().unwrap()).collect();

let mut client = mk_client(&leader_data);
let mut last_finality = Duration::from_millis(client.get_finality() as u64);
info!(
"Last finality {}.{:03}",
last_finality.as_secs(),
last_finality.subsec_millis()
);
let now = Instant::now();
let mut consecutive_success = 0;
let mut failures = 0;
let mut max_distance_increase = 0i64;
let mut expected_balance = leader_balance;
for i in 0..std::cmp::max(20, num_nodes) {
//verify leader can do transfer
let expected = ((i + 3) * 500) as i64;
let leader_balance = retry_send_tx_and_retry_get_balance(
&leader_data,
&alice_arc.read().unwrap(),
&bob_pubkey,
Some(expected),
).unwrap();
if leader_balance != expected {
info!(
"leader dropped transaction {} {:?} {:?}",
i, leader_balance, expected
);
trace!("getting leader last_id");
let last_id = client.get_last_id();
trace!("executing leader transfer");
let sig = client
.transfer(
500,
&alice_arc.read().unwrap().keypair(),
bob_pubkey,
&last_id,
)
.unwrap();

expected_balance += 500;

assert!(client.poll_for_signature(&sig).is_ok());

let now = Instant::now();
let mut finality = Duration::from_millis(client.get_finality() as u64);
while finality == last_finality {
// Check if new finality is less than last finality (+5 seconds of grace period)
assert!(now.elapsed() < last_finality.checked_add(Duration::new(5, 0)).unwrap());
sleep(Duration::from_millis(100));
finality = Duration::from_millis(client.get_finality() as u64);
}
//verify all validators have the same balance
{
let mut success = 0usize;
let mut max_distance = 0i64;
let mut total_distance = 0i64;
let mut num_nodes_behind = 0i64;
validators.retain(|server| {
let mut retain_me = true;
let mut client = mk_client(&server.0);
trace!("{:x} {} get_balance start", server.0.debug_id(), i);
let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(leader_balance));
trace!(
"{:x} {} get_balance: {:?} leader_balance: {}",
server.0.debug_id(),
i,
getbal,
leader_balance
);
let bal = getbal.unwrap_or(0);
let distance = (leader_balance - bal) / 500;
max_distance = max(distance, max_distance);
total_distance += distance;
if distance > max_distance_increase {
info!("Node {:x} is behind by {}", server.0.debug_id(), distance);
max_distance_increase = distance;
if max_distance_increase as u64 > purge_lag as u64 {
server.1.exit();
info!("Node {:x} is exiting", server.0.debug_id());
retain_me = false;
}
}
if distance > 0 {
num_nodes_behind += 1;
}
if let Some(bal) = getbal {
if bal == leader_balance {
success += 1;
}
}
retain_me
});
if num_nodes_behind != 0 {
info!("{} nodes are lagging behind leader", num_nodes_behind);
}
info!(
"SUCCESS[{}] {} out of {} distance: {} max_distance: {} finality: {}",
i,
success,
validators.len(),
total_distance,
max_distance,
get_finality(&leader_data)
);
if success == validators.len() && total_distance == 0 {
consecutive_success += 1;
} else {
consecutive_success = 0;
failures += 1;
}
if consecutive_success == 10 {
break;

last_finality = finality;

let balance = retry_get_balance(&mut client, &bob_pubkey, Some(expected_balance));
info!(
"Leader expected balance {}, current balance {:?}",
expected_balance, balance
);
assert_eq!(balance, Some(expected_balance));
consecutive_success += 1;

info!(
"SUCCESS[{}] balance: {} finality: {}.{:03} seconds",
i,
expected_balance,
last_finality.as_secs(),
last_finality.subsec_millis(),
);

if consecutive_success == 10 {
break;
}
}

//verify all validators have the same balance
let mut max_distance = 0i64;
let mut total_distance = 0i64;
let mut num_nodes_behind = 0i64;
validators.retain(|server| {
let mut retain_me = true;
let mut client = mk_client(&server.0);
trace!("{:x} get_balance start", server.0.debug_id());
let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(expected_balance));
trace!(
"{:x} get_balance: {:?} expected_balance: {}",
server.0.debug_id(),
getbal,
expected_balance
);
let bal = getbal.unwrap_or(0);
let distance = (expected_balance - bal) / 500;
max_distance = max(distance, max_distance);
total_distance += distance;
if distance > max_distance_increase {
info!("Node {:x} is behind by {}", server.0.debug_id(), distance);
max_distance_increase = distance;
if max_distance_increase as u64 > purge_lag as u64 {
server.1.exit();
info!("Node {:x} is exiting", server.0.debug_id());
retain_me = false;
}
}
if distance > 0 {
num_nodes_behind += 1;
}
retain_me
});

if num_nodes_behind != 0 {
info!("{} nodes are lagging behind leader", num_nodes_behind);
}
info!(
"Took {} s to converge total failures: {}",
duration_as_s(&now.elapsed()),
failures
"Validators lagging: {}/{}, distance: {}, max_distance: {}",
num_nodes_behind,
validators.len(),
total_distance,
max_distance,
);

info!("Took {} s to converge", duration_as_s(&now.elapsed()),);
assert_eq!(consecutive_success, 10);
for (_, node) in &validators {
node.exit();
Expand Down Expand Up @@ -821,9 +839,3 @@ fn retry_send_tx_and_retry_get_balance(
}
None
}

fn get_finality(leader: &NodeInfo) -> usize {
let mut client = mk_client(leader);
trace!("getting leader finality");
client.get_finality()
}

0 comments on commit cc1ac20

Please sign in to comment.