Skip to content

Commit

Permalink
Tune the number of test runs and test timing
Browse files Browse the repository at this point in the history
  • Loading branch information
teor2345 committed May 30, 2023
1 parent 04c525a commit ae836a6
Showing 1 changed file with 13 additions and 8 deletions.
21 changes: 13 additions & 8 deletions zebra-network/src/peer/connection/tests/vectors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -784,18 +784,20 @@ async fn connection_is_randomly_disconnected_on_overload() {
let _init_guard = zebra_test::init();

// The number of times we repeat the test
const TEST_RUNS: usize = 21_000;

// The expected number of tests before failing due to random all-successes or all-failures
const TESTS_BEFORE_FAILURE: f32 = 1000.0;

let test_runs = TEST_RUNS as f32;
const TEST_RUNS: usize = 220;
// The expected number of tests before a test failure due to random chance.
// Based on 10 tests per PR, 100 PR pushes per week, 50 weeks per year.
const TESTS_BEFORE_FAILURE: f32 = 50_000.0;

let test_runs = TEST_RUNS.try_into().expect("constant fits in i32");
// The probability of random test failure is:
// MIN_OVERLOAD_DROP_PROBABILITY^TEST_RUNS + MAX_OVERLOAD_DROP_PROBABILITY^TEST_RUNS
assert!(
test_runs / TESTS_BEFORE_FAILURE > 1.0 / MIN_OVERLOAD_DROP_PROBABILITY,
1.0 / MIN_OVERLOAD_DROP_PROBABILITY.powi(test_runs) > TESTS_BEFORE_FAILURE,
"not enough test runs: failures must be frequent enough to happen in almost all tests"
);
assert!(
test_runs / TESTS_BEFORE_FAILURE > 1.0 / (1.0 - MAX_OVERLOAD_DROP_PROBABILITY),
1.0 / MAX_OVERLOAD_DROP_PROBABILITY.powi(test_runs) > TESTS_BEFORE_FAILURE,
"not enough test runs: successes must be frequent enough to happen in almost all tests"
);

Expand Down Expand Up @@ -824,6 +826,7 @@ async fn connection_is_randomly_disconnected_on_overload() {

// Start the connection run loop future in a spawned task
let connection_handle = tokio::spawn(connection.run(peer_rx));
tokio::time::sleep(Duration::from_millis(1)).await;

// The connection hasn't received any messages, so it must not have errors
let error = shared_error_slot.try_get_error();
Expand All @@ -838,6 +841,7 @@ async fn connection_is_randomly_disconnected_on_overload() {
.send(Ok(inbound_req))
.await
.expect("send to channel always succeeds");
tokio::time::sleep(Duration::from_millis(1)).await;

// The connection hasn't got a response, so it must not have errors
let error = shared_error_slot.try_get_error();
Expand All @@ -850,6 +854,7 @@ async fn connection_is_randomly_disconnected_on_overload() {
.expect_request(Request::Peers)
.await
.respond_error(Overloaded::new().into());
tokio::time::sleep(Duration::from_millis(1)).await;

let outbound_result = peer_outbound_messages.try_next();
assert!(
Expand Down

0 comments on commit ae836a6

Please sign in to comment.