A0-1499: E2e ban test threshold

.github/workflows/e2e-tests-main-devnet.yml

@@ -457,6 +457,21 @@ jobs:
           follow-up-finalization-check: true
         timeout-minutes: 15
 
+  run-e2e-ban-manual:
+    needs: [build-test-docker, build-test-client]
+    name: Run ban manual test
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: ban_manual
+          follow-up-finalization-check: true
+        timeout-minutes: 15
+
   run-e2e-ban-counter-clearing:
     needs: [build-test-docker, build-test-client]
     name: Run ban counter clearing test
@@ -472,6 +487,21 @@ jobs:
           follow-up-finalization-check: true
         timeout-minutes: 15
 
+  run-e2e-ban-threshold:
+    needs: [build-test-docker, build-test-client]
+    name: Run ban threshold test
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: ban_threshold
+          follow-up-finalization-check: true
+        timeout-minutes: 15
+
   run-e2e-version-upgrade:
     needs: [build-test-docker, build-test-client]
     name: Run basic (positive) version-upgrade test
@@ -510,21 +540,6 @@ jobs:
           ONLY_LEGACY: true
         timeout-minutes: 10
 
-  run-e2e-ban-manual:
-    needs: [build-test-docker, build-test-client]
-    name: Run ban manual test
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v2
-
-      - name: Run e2e test
-        uses: ./.github/actions/run-e2e-test
-        with:
-          test-case: ban_manual
-          follow-up-finalization-check: true
-        timeout-minutes: 15
-
   run-e2e-version-upgrade-catchup:
     needs: [build-test-docker, build-cliain-image]
     name: Run series of tests where some of the nodes need to do version-upgrade during catch-up
@@ -606,8 +621,9 @@ jobs:
       run-e2e-rewards-points-basic,
       run-e2e-authorities-are-staking,
       run-e2e-ban-automatic,
-      run-e2e-ban-counter-clearing,
       run-e2e-ban-manual,
+      run-e2e-ban-counter-clearing,
+      run-e2e-ban-threshold,
       run-e2e-version-upgrade,
       run-e2e-failing-version-upgrade,
       run-e2e-version-upgrade-catchup,

aleph-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aleph_client"
-version = "1.12.0"
+version = "1.13.0"
 edition = "2021"
 license = "Apache 2.0"
 

aleph-client/src/elections.rs

@@ -1,3 +1,4 @@
+use log::info;
 use primitives::{
     BanConfig, BanInfo, CommitteeSeats, EraIndex, EraValidators, SessionCount, SessionIndex,
 };
@@ -76,18 +77,19 @@ pub fn get_ban_config<C: ReadStorage>(connection: &C) -> BanConfig {
 pub fn get_underperformed_validator_session_count<C: ReadStorage>(
     connection: &C,
     account_id: &AccountId,
+    block_hash: Option<H256>,
 ) -> SessionCount {
     connection
         .read_storage_map(
             PALLET,
             "UnderperformedValidatorSessionCount",
             account_id,
-            None,
+            block_hash,
         )
         .unwrap_or(0)
 }
 
-pub fn get_ban_reason_for_validator<C: ReadStorage>(
+pub fn get_ban_info_for_validator<C: ReadStorage>(
     connection: &C,
     account_id: &AccountId,
 ) -> Option<BanInfo> {
@@ -129,21 +131,26 @@ pub fn change_ban_config(
     ban_period: Option<EraIndex>,
     status: XtStatus,
 ) {
+    info!(target: "aleph-client", "Changing ban config");
+    let call_name = "set_ban_config";
+
     let call = compose_call!(
         sudo_connection.as_connection().metadata,
         PALLET,
-        "set_ban_config",
+        call_name,
         minimal_expected_performance,
         underperformed_session_count_threshold,
         clean_session_counter_delay,
         ban_period
     );
+
     let xt = compose_extrinsic!(
         sudo_connection.as_connection(),
         "Sudo",
         "sudo_unchecked_weight",
         call,
         0_u64
     );
-    send_xt(sudo_connection, xt, Some("set_ban_config"), status);
+
+    send_xt(sudo_connection, xt, Some(call_name), status);
 }

aleph-client/src/lib.rs

@@ -6,7 +6,7 @@ pub use balances::total_issuance;
 use codec::{Decode, Encode};
 pub use debug::print_storages;
 pub use elections::{
-    ban_from_committee, change_ban_config, get_ban_config, get_ban_reason_for_validator,
+    ban_from_committee, change_ban_config, get_ban_config, get_ban_info_for_validator,
     get_committee_seats, get_current_era_non_reserved_validators,
     get_current_era_reserved_validators, get_current_era_validators, get_era_validators,
     get_next_era_committee_seats, get_next_era_non_reserved_validators,

e2e-tests/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aleph-e2e-client"
-version = "0.7.0"
+version = "0.8.0"
 edition = "2021"
 license = "Apache 2.0"
 

e2e-tests/src/ban.rs

@@ -1,27 +1,39 @@
 use aleph_client::{
-    change_validators, get_ban_config, get_ban_reason_for_validator,
-    get_underperformed_validator_session_count, wait_for_event, wait_for_full_era_completion,
-    AccountId, AnyConnection, RootConnection, XtStatus,
+    change_validators, get_ban_config, get_ban_info_for_validator, get_block_hash,
+    get_session_period, get_underperformed_validator_session_count, wait_for_event,
+    wait_for_full_era_completion, AccountId, AnyConnection, RootConnection, XtStatus,
 };
 use codec::Decode;
 use log::info;
-use primitives::{BanConfig, BanInfo, CommitteeSeats, EraValidators, SessionCount};
+use primitives::{BanConfig, BanInfo, CommitteeSeats, EraValidators, SessionCount, SessionIndex};
+use sp_core::H256;
 use sp_runtime::Perbill;
 
-use crate::{accounts::account_ids_from_keys, validators::get_test_validators, Config};
+use crate::{
+    accounts::account_ids_from_keys, elections::get_members_subset_for_session,
+    validators::get_test_validators, Config,
+};
+
+const RESERVED_SEATS: u32 = 2;
+const NON_RESERVED_SEATS: u32 = 2;
 
-pub fn setup_test(
-    config: &Config,
-) -> anyhow::Result<(RootConnection, Vec<AccountId>, Vec<AccountId>)> {
+type BanTestSetup = (
+    RootConnection,
+    Vec<AccountId>,
+    Vec<AccountId>,
+    CommitteeSeats,
+);
+
+pub fn setup_test(config: &Config) -> anyhow::Result<BanTestSetup> {
     let root_connection = config.create_root_connection();
 
     let validator_keys = get_test_validators(config);
     let reserved_validators = account_ids_from_keys(&validator_keys.reserved);
     let non_reserved_validators = account_ids_from_keys(&validator_keys.non_reserved);
 
     let seats = CommitteeSeats {
-        reserved_seats: 2,
-        non_reserved_seats: 2,
+        reserved_seats: RESERVED_SEATS,
+        non_reserved_seats: NON_RESERVED_SEATS,
     };
 
     change_validators(
@@ -38,6 +50,7 @@ pub fn setup_test(
         root_connection,
         reserved_validators,
         non_reserved_validators,
+        seats,
     ))
 }
 
@@ -83,9 +96,10 @@ pub fn check_underperformed_validator_session_count<C: AnyConnection>(
     connection: &C,
     validator: &AccountId,
     expected_session_count: &SessionCount,
+    block_hash: Option<H256>,
 ) -> SessionCount {
     let underperformed_validator_session_count =
-        get_underperformed_validator_session_count(connection, validator);
+        get_underperformed_validator_session_count(connection, validator, block_hash);
 
     assert_eq!(
         &underperformed_validator_session_count,
@@ -100,7 +114,7 @@ pub fn check_ban_info_for_validator<C: AnyConnection>(
     validator: &AccountId,
     expected_info: Option<&BanInfo>,
 ) -> Option<BanInfo> {
-    let validator_ban_info = get_ban_reason_for_validator(connection, validator);
+    let validator_ban_info = get_ban_info_for_validator(connection, validator);
 
     assert_eq!(validator_ban_info.as_ref(), expected_info);
 
@@ -124,3 +138,90 @@ pub fn check_ban_event<C: AnyConnection>(
 
     Ok(event)
 }
+
+pub fn get_members_for_session(
+    reserved_validators: &[AccountId],
+    non_reserved_validators: &[AccountId],
+    seats: &CommitteeSeats,
+    session: SessionIndex,
+) -> Vec<AccountId> {
+    let reserved_members =
+        get_members_subset_for_session(seats.reserved_seats, reserved_validators, session);
+    let non_reserved_members =
+        get_members_subset_for_session(seats.non_reserved_seats, non_reserved_validators, session);
+    reserved_members
+        .into_iter()
+        .chain(non_reserved_members.into_iter())
+        .collect()
+}
+
+/// Checks whether underperformed counts for validators change predictably. Assumes: (a) that the
+/// sessions checked are in the past, (b) that all the checked validators are underperforming in
+/// those sessions (e.g. due to a prohibitively high performance threshold).
+pub fn check_underperformed_count_for_sessions<C: AnyConnection>(
+    connection: &C,
+    reserved_validators: &[AccountId],
+    non_reserved_validators: &[AccountId],
+    seats: &CommitteeSeats,
+    start_session: SessionIndex,
+    end_session: SessionIndex,
+    ban_session_threshold: SessionCount,
+) -> anyhow::Result<()> {
+    let session_period = get_session_period(connection);
+
+    let validators: Vec<_> = reserved_validators
+        .iter()
+        .chain(non_reserved_validators.iter())
+        .collect();
+
+    for session in start_session..end_session {
+        let session_end_block = (session + 1) * session_period;
+        let session_end_block_hash = get_block_hash(connection, session_end_block);
+
+        let previous_session_end_block = session_end_block - session_period;
+        let previous_session_end_block_hash =
+            get_block_hash(connection, previous_session_end_block);
+
+        let members =
+            get_members_for_session(reserved_validators, non_reserved_validators, seats, session);
+
+        validators.iter().for_each(|&val| {
+            info!(
+                "Checking session count | session {} | validator {}",
+                session, val
+            );
+            let session_underperformed_count = get_underperformed_validator_session_count(
+                connection,
+                val,
+                Some(session_end_block_hash),
+            );
+            let previous_session_underperformed_count = get_underperformed_validator_session_count(
+                connection,
+                val,
+                Some(previous_session_end_block_hash),
+            );
+
+            let underperformed_diff =
+                session_underperformed_count.abs_diff(previous_session_underperformed_count);
+
+            if members.contains(val) {
+                // Counter for committee members legally incremented by 1 or reset to 0 (decremented
+                // by ban_session_threshold - 1).
+                if underperformed_diff != 1 && underperformed_diff != (ban_session_threshold - 1) {
+                    panic!(
+                        "Underperformed session count for committee validator {} for session {} changed from {} to {}.",
+                        val, session, previous_session_underperformed_count, session_underperformed_count
+                    );
+                }
+            } else if underperformed_diff != 0 {
+                // Counter for validators on the bench should stay the same.
+                panic!(
+                    "Underperformed session count for non-committee validator {} for session {} changed from {} to {}.",
+                    val, session, previous_session_underperformed_count, session_underperformed_count
+                );
+            }
+        });
+    }
+
+    Ok(())
+}

e2e-tests/src/cases.rs

@@ -3,7 +3,7 @@ use crate::{
     test::{
         authorities_are_staking as test_authorities_are_staking,
         ban_automatic as test_ban_automatic, ban_manual as test_ban_manual,
-        batch_transactions as test_batch_transactions,
+        ban_threshold as test_ban_threshold, batch_transactions as test_batch_transactions,
         change_stake_and_force_new_era as test_change_stake_and_force_new_era,
         change_validators as test_change_validators,
         channeling_fee_and_tip as test_channeling_fee_and_tip,
@@ -70,5 +70,6 @@ pub fn possible_test_cases() -> PossibleTestCases {
             "clearing_session_count",
             test_clearing_session_count as TestCase,
         ),
+        ("ban_threshold", test_ban_threshold as TestCase),
     ]
 }