Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored large network test to use finality to assert success #978

Merged
merged 1 commit into from
Aug 16, 2018
Merged

Refactored large network test to use finality to assert success #978

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 62 additions & 93 deletions tests/multinode.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ use solana::result;
use solana::service::Service;
use solana::signature::{Keypair, KeypairUtil, Pubkey};
use solana::thin_client::ThinClient;
use solana::timing::duration_as_s;
use solana::timing::{duration_as_ms, duration_as_s};
use solana::window::{default_window, WINDOW_SIZE};
use std::cmp::max;
use std::env;
use std::fs::{copy, create_dir_all, remove_dir_all};
use std::net::UdpSocket;
Expand Down Expand Up @@ -524,15 +523,7 @@ fn test_multi_node_dynamic_network() {
Ok(val) => val
.parse()
.expect(&format!("env var {} is not parse-able as usize", key)),
Err(_) => 170,
};

let purge_key = "SOLANA_DYNAMIC_NODES_PURGE_LAG";
let purge_lag: usize = match env::var(purge_key) {
Ok(val) => val
.parse()
.expect(&format!("env var {} is not parse-able as usize", purge_key)),
Err(_) => std::usize::MAX,
Err(_) => 200,
};

let leader_keypair = Keypair::new();
Expand Down Expand Up @@ -631,94 +622,78 @@ fn test_multi_node_dynamic_network() {

let mut validators: Vec<_> = t2.into_iter().map(|t| t.join().unwrap()).collect();

let now = Instant::now();
let mut client = mk_client(&leader_data);
let mut last_finality = client.get_finality();
info!("Last finality {}", last_finality);
let start = Instant::now();
let mut consecutive_success = 0;
let mut failures = 0;
let mut max_distance_increase = 0i64;
let mut expected_balance = leader_balance;
for i in 0..std::cmp::max(20, num_nodes) {
//verify leader can do transfer
let expected = ((i + 3) * 500) as i64;
let leader_balance = retry_send_tx_and_retry_get_balance(
&leader_data,
&alice_arc.read().unwrap(),
&bob_pubkey,
Some(expected),
).unwrap();
if leader_balance != expected {
info!(
"leader dropped transaction {} {:?} {:?}",
i, leader_balance, expected
);
trace!("getting leader last_id");
let last_id = client.get_last_id();
trace!("executing leader transfer");
let sig = client
.transfer(
500,
&alice_arc.read().unwrap().keypair(),
bob_pubkey,
&last_id,
)
.unwrap();

expected_balance += 500;

assert!(client.poll_for_signature(&sig).is_ok());

let now = Instant::now();
let mut finality = client.get_finality();

// Need this to make sure the finality is updated
// (i.e. the node is not returning stale value)
while last_finality == finality {
finality = client.get_finality();
}

while duration_as_ms(&now.elapsed()) < finality as u64 {
sleep(Duration::from_millis(100));
finality = client.get_finality()
}
//verify all validators have the same balance
{
let mut success = 0usize;
let mut max_distance = 0i64;
let mut total_distance = 0i64;

last_finality = finality;

let balance = retry_get_balance(&mut client, &bob_pubkey, Some(expected_balance));
assert_eq!(balance, Some(expected_balance));
consecutive_success += 1;

info!(
"SUCCESS[{}] balance: {}, finality: {} ms",
i, expected_balance, last_finality,
);

if consecutive_success == 10 {
info!("Took {} s to converge", duration_as_s(&start.elapsed()),);
info!("Verifying signature of the last transaction in the validators");

let mut num_nodes_behind = 0i64;
validators.retain(|server| {
let mut retain_me = true;
let mut client = mk_client(&server.0);
trace!("{:x} {} get_balance start", server.0.debug_id(), i);
let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(leader_balance));
trace!(
"{:x} {} get_balance: {:?} leader_balance: {}",
server.0.debug_id(),
i,
getbal,
leader_balance
);
let bal = getbal.unwrap_or(0);
let distance = (leader_balance - bal) / 500;
max_distance = max(distance, max_distance);
total_distance += distance;
if distance > max_distance_increase {
info!("Node {:x} is behind by {}", server.0.debug_id(), distance);
max_distance_increase = distance;
if max_distance_increase as u64 > purge_lag as u64 {
server.1.exit();
info!("Node {:x} is exiting", server.0.debug_id());
retain_me = false;
}
}
if distance > 0 {
num_nodes_behind += 1;
}
if let Some(bal) = getbal {
if bal == leader_balance {
success += 1;
}
}
retain_me
trace!("{:x} polling for signature", server.0.debug_id());
num_nodes_behind += match client.poll_for_signature(&sig) {
Ok(_) => 0,
Err(_) => 1,
};
true
});
if num_nodes_behind != 0 {
info!("{} nodes are lagging behind leader", num_nodes_behind);
}

info!(
"SUCCESS[{}] {} out of {} distance: {} max_distance: {} finality: {}",
i,
success,
"Validators lagging: {}/{}",
num_nodes_behind,
validators.len(),
total_distance,
max_distance,
get_finality(&leader_data)
);
if success == validators.len() && total_distance == 0 {
consecutive_success += 1;
} else {
consecutive_success = 0;
failures += 1;
}
if consecutive_success == 10 {
break;
}
break;
}
}
info!(
"Took {} s to converge total failures: {}",
duration_as_s(&now.elapsed()),
failures
);

assert_eq!(consecutive_success, 10);
for (_, node) in &validators {
node.exit();
Expand Down Expand Up @@ -821,9 +796,3 @@ fn retry_send_tx_and_retry_get_balance(
}
None
}

fn get_finality(leader: &NodeInfo) -> usize {
let mut client = mk_client(leader);
trace!("getting leader finality");
client.get_finality()
}