feat: implement chunk validator assignment (#9983)

Implements chunk validator assignment as described in

- the docs linked from the [Stateless Validation
Organizer](https://docs.google.com/document/d/1gFv3GzPHR5CAX7_X2l5MpuMg7GIjW4Ne8e_3kZLnZQQ/edit#heading=h.d4bbvnvyxo9f)
- this [zulip
conversation](https://near.zulipchat.com/#narrow/stream/407237-pagoda.2Fcore.2Fstateless-validation/topic/validator.20seat.20assignment)

### The term _seat_

In `nearcore` it is already used:

-
[`seat_price`](https://github.com/near/nearcore/blob/6f324e84a7a7162956f0b9985094ee146919f5ae/core/primitives/src/epoch_manager.rs#L710-L716)
is the minimum stake required to become a validator.
-
[`NumSeats`](https://github.com/near/nearcore/blob/02f06d1c844296d9b7ea01289ea9e1842f404dd1/core/primitives-core/src/types.rs#L39C1-L40)
is the number of seats available for block producers.

This might lead to conflicts and confusion with the concept of _seat_ to
be introduced for chunk validator assignment. To avoid that, this PR
uses the term _mandate_ for now. Accordingly the stake of chunk
validators is splitt into _mandates_ which are randomly shuffled and
assigned to shards.

I’m happy to rename _mandate_ to anything else, though if possible I
would try to avoid conflicts with the existing usage of _seat_.

### Overview

- Introduces `EpochInfoV4` which contains the epoch’s
`ValidatorMandates`.
- Adds `EpochInfo::sample_chunk_validators` which allows sampling for a
given height.
- Module `core::primitives::validator_mandates` encapsulates splitting
validator stake into mandates, shuffling them, and assigning them to
shards.

### Follow-up concerns

As discussed offline
([ref](https://near.zulipchat.com/#narrow/stream/407237-pagoda.2Fcore.2Fstateless-validation/topic/validator.20seat.20assignment/near/396265731)),
the following are separate concerns:

- Properly integrating stateless validator assignment with the rest of
the system.
- Dynamically calculating and updating `stake_per_mandate` and
`min_mandates_per_shard`.

Hence there are some open `TODO`s in the diff, they are marked with
#10014.

chain/chain/Cargo.toml

@@ -57,9 +57,13 @@ protocol_feature_reject_blocks_with_outdated_protocol_version = [
 protocol_feature_simple_nightshade_v2 = [
   "near-primitives/protocol_feature_simple_nightshade_v2",
 ]
+protocol_feature_chunk_validation = [
+  "near-primitives/protocol_feature_chunk_validation",
+]
 
 nightly = [
   "nightly_protocol",
+  "protocol_feature_chunk_validation",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",
   "protocol_feature_simple_nightshade_v2",
   "near-chain-configs/nightly",

chain/chain/src/test_utils/kv_runtime.rs

@@ -524,6 +524,8 @@ impl EpochManagerAdapter for MockEpochManager {
             1,
             1,
             RngSeed::default(),
+            #[cfg(feature = "protocol_feature_chunk_validation")]
+            Default::default(),
         )))
     }
 

chain/epoch-manager/Cargo.toml

@@ -33,8 +33,12 @@ expensive_tests = []
 protocol_feature_fix_staking_threshold = [
   "near-primitives/protocol_feature_fix_staking_threshold",
 ]
+protocol_feature_chunk_validation = [
+  "near-primitives/protocol_feature_chunk_validation",
+]
 nightly = [
   "nightly_protocol",
+  "protocol_feature_chunk_validation",
   "protocol_feature_fix_staking_threshold",
   "near-chain-configs/nightly",
   "near-primitives/nightly",

chain/epoch-manager/src/proposals.rs

@@ -85,6 +85,8 @@ mod old_validator_selection {
     use near_primitives::types::{
         AccountId, Balance, NumSeats, ValidatorId, ValidatorKickoutReason,
     };
+    #[cfg(feature = "protocol_feature_chunk_validation")]
+    use near_primitives::validator_mandates::ValidatorMandates;
     use near_primitives::version::ProtocolVersion;
     use rand::{RngCore, SeedableRng};
     use rand_hc::Hc128Rng;
@@ -248,6 +250,10 @@ mod old_validator_selection {
             .map(|(index, s)| (s.account_id().clone(), index as ValidatorId))
             .collect::<HashMap<_, _>>();
 
+        // Old validator selection is not aware of chunk validator mandates.
+        #[cfg(feature = "protocol_feature_chunk_validation")]
+        let validator_mandates: ValidatorMandates = Default::default();
+
         Ok(EpochInfo::new(
             prev_epoch_info.epoch_height() + 1,
             final_proposals,
@@ -264,6 +270,8 @@ mod old_validator_selection {
             threshold,
             next_version,
             rng_seed,
+            #[cfg(feature = "protocol_feature_chunk_validation")]
+            validator_mandates,
         ))
     }
 

chain/epoch-manager/src/test_utils.rs

@@ -20,6 +20,8 @@ use near_primitives::types::{
     ValidatorId, ValidatorKickoutReason,
 };
 use near_primitives::utils::get_num_seats_per_shard;
+#[cfg(feature = "protocol_feature_chunk_validation")]
+use near_primitives::validator_mandates::{ValidatorMandates, ValidatorMandatesConfig};
 use near_primitives::version::PROTOCOL_VERSION;
 use near_store::test_utils::create_test_store;
 
@@ -104,9 +106,19 @@ pub fn epoch_info_with_num_seats(
             })
             .collect()
     };
+    let all_validators = account_to_validators(accounts);
+    #[cfg(feature = "protocol_feature_chunk_validation")]
+    let validator_mandates = {
+        // TODO(#10014) determine required stake per mandate instead of reusing seat price.
+        // TODO(#10014) determine `min_mandates_per_shard`
+        let num_shards = chunk_producers_settlement.len();
+        let min_mandates_per_shard = 0;
+        let config = ValidatorMandatesConfig::new(seat_price, min_mandates_per_shard, num_shards);
+        ValidatorMandates::new(config, &all_validators)
+    };
     EpochInfo::new(
         epoch_height,
-        account_to_validators(accounts),
+        all_validators,
         validator_to_index,
         block_producers_settlement,
         chunk_producers_settlement,
@@ -120,6 +132,8 @@ pub fn epoch_info_with_num_seats(
         seat_price,
         PROTOCOL_VERSION,
         TEST_SEED,
+        #[cfg(feature = "protocol_feature_chunk_validation")]
+        validator_mandates,
     )
 }
 

chain/epoch-manager/src/validator_selection.rs

@@ -7,6 +7,8 @@ use near_primitives::types::validator_stake::ValidatorStake;
 use near_primitives::types::{
     AccountId, Balance, ProtocolVersion, ValidatorId, ValidatorKickoutReason,
 };
+#[cfg(feature = "protocol_feature_chunk_validation")]
+use near_primitives::validator_mandates::{ValidatorMandates, ValidatorMandatesConfig};
 use num_rational::Ratio;
 use std::cmp::{self, Ordering};
 use std::collections::hash_map;
@@ -96,6 +98,7 @@ pub fn proposals_to_epoch_info(
     }
 
     let num_chunk_producers = chunk_producers.len();
+    // Constructing `all_validators` such that a validators position corresponds to its `ValidatorId`.
     let mut all_validators: Vec<ValidatorStake> = Vec::with_capacity(num_chunk_producers);
     let mut validator_to_index = HashMap::new();
     let mut block_producers_settlement = Vec::with_capacity(block_producers.len());
@@ -170,6 +173,18 @@ pub fn proposals_to_epoch_info(
             .collect()
     };
 
+    #[cfg(feature = "protocol_feature_chunk_validation")]
+    let validator_mandates = {
+        // TODO(#10014) determine required stake per mandate instead of reusing seat price.
+        // TODO(#10014) determine `min_mandates_per_shard`
+        let min_mandates_per_shard = 0;
+        let validator_mandates_config =
+            ValidatorMandatesConfig::new(threshold, min_mandates_per_shard, num_shards as usize);
+        // We can use `all_validators` to construct mandates Since a validator's position in
+        // `all_validators` corresponds to its `ValidatorId`
+        ValidatorMandates::new(validator_mandates_config, &all_validators)
+    };
+
     let fishermen_to_index = fishermen
         .iter()
         .enumerate()
@@ -192,6 +207,8 @@ pub fn proposals_to_epoch_info(
         threshold,
         next_version,
         rng_seed,
+        #[cfg(feature = "protocol_feature_chunk_validation")]
+        validator_mandates,
     ))
 }
 
@@ -620,6 +637,52 @@ mod tests {
         }
     }
 
+    /// This test only verifies that chunk validator mandates are correctly wired up with
+    /// `EpochInfo`. The internals of mandate assignment are tested in the module containing
+    /// [`ValidatorMandates`].
+    #[cfg(feature = "protocol_feature_chunk_validation")]
+    #[test]
+    fn test_chunk_validators_sampling() {
+        let num_shards = 4;
+        let epoch_config = create_epoch_config(
+            num_shards,
+            2 * num_shards,
+            0,
+            ValidatorSelectionConfig {
+                num_chunk_only_producer_seats: 0,
+                minimum_validators_per_shard: 1,
+                minimum_stake_ratio: Ratio::new(160, 1_000_000),
+            },
+        );
+        let prev_epoch_height = 7;
+        let prev_epoch_info = create_prev_epoch_info(prev_epoch_height, &["test1", "test2"], &[]);
+        let proposals =
+            create_proposals(&[("test1", 15), ("test2", 9), ("test3", 5), ("test4", 3)]);
+
+        let epoch_info = proposals_to_epoch_info(
+            &epoch_config,
+            [0; 32],
+            &prev_epoch_info,
+            proposals,
+            Default::default(),
+            Default::default(),
+            0,
+            PROTOCOL_VERSION,
+            PROTOCOL_VERSION,
+        )
+        .unwrap();
+
+        // Given `epoch_info` and `proposals` above, the sample at a given height is deterministic.
+        let height = 42;
+        let expected_assignments: Vec<HashMap<ValidatorId, u16>> = vec![
+            HashMap::from([(0, 1), (1, 5), (2, 1), (3, 1)]),
+            HashMap::from([(0, 6), (1, 1), (2, 1)]),
+            HashMap::from([(0, 5), (1, 2), (2, 1)]),
+            HashMap::from([(0, 3), (1, 1), (2, 2), (3, 2)]),
+        ];
+        assert_eq!(epoch_info.sample_chunk_validators(height), expected_assignments);
+    }
+
     #[test]
     fn test_validator_assignment_ratio_condition() {
         // There are more seats than proposals, however the

core/primitives-core/Cargo.toml

@@ -36,9 +36,11 @@ protocol_feature_fix_staking_threshold = []
 protocol_feature_fix_contract_loading_cost = []
 protocol_feature_reject_blocks_with_outdated_protocol_version = []
 protocol_feature_simple_nightshade_v2 = []
+protocol_feature_chunk_validation = []
 
 nightly = [
   "nightly_protocol",
+  "protocol_feature_chunk_validation",
   "protocol_feature_fix_contract_loading_cost",
   "protocol_feature_fix_staking_threshold",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",

core/primitives-core/src/version.rs

@@ -128,6 +128,10 @@ pub enum ProtocolFeature {
     PostStateRoot,
     /// Increases the number of chunk producers.
     TestnetFewerBlockProducers,
+    /// Enables chunk validation which is introduced with stateless validation.
+    /// NEP: https://github.com/near/NEPs/pull/509
+    #[cfg(feature = "protocol_feature_chunk_validation")]
+    ChunkValidation,
 }
 
 impl ProtocolFeature {
@@ -183,6 +187,8 @@ impl ProtocolFeature {
             #[cfg(feature = "protocol_feature_simple_nightshade_v2")]
             ProtocolFeature::SimpleNightshadeV2 => 135,
             ProtocolFeature::PostStateRoot => 136,
+            #[cfg(feature = "protocol_feature_chunk_validation")]
+            ProtocolFeature::ChunkValidation => 137,
             ProtocolFeature::TestnetFewerBlockProducers => 140,
         }
     }

core/primitives/Cargo.toml

@@ -24,6 +24,7 @@ num-rational.workspace = true
 once_cell.workspace = true
 primitive-types.workspace = true
 rand.workspace = true
+rand_chacha.workspace = true
 reed-solomon-erasure.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -49,8 +50,10 @@ protocol_feature_fix_staking_threshold = ["near-primitives-core/protocol_feature
 protocol_feature_fix_contract_loading_cost = ["near-primitives-core/protocol_feature_fix_contract_loading_cost"]
 protocol_feature_reject_blocks_with_outdated_protocol_version = ["near-primitives-core/protocol_feature_reject_blocks_with_outdated_protocol_version"]
 protocol_feature_simple_nightshade_v2 = ["near-primitives-core/protocol_feature_simple_nightshade_v2"]
+protocol_feature_chunk_validation = []
 nightly = [
   "nightly_protocol",
+  "protocol_feature_chunk_validation",
   "protocol_feature_fix_contract_loading_cost",
   "protocol_feature_fix_staking_threshold",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",