feat: Introduce cutoff threshold for endorsement ratio

chain/epoch-manager/src/lib.rs

@@ -28,6 +28,7 @@ use near_primitives::views::{
 };
 use near_store::{DBCol, Store, StoreUpdate, HEADER_HEAD_KEY};
 use primitive_types::U256;
+use reward_calculator::ValidatorOnlineThresholds;
 use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
@@ -507,7 +508,15 @@ impl EpochManager {
             {
                 let mut sorted_validators = validator_block_chunk_stats
                     .iter()
-                    .map(|(account, stats)| (get_sortable_validator_online_ratio(stats), account))
+                    .map(|(account, stats)| {
+                        (
+                            get_sortable_validator_online_ratio(
+                                stats,
+                                Some(chunk_validator_only_kickout_threshold),
+                            ),
+                            account,
+                        )
+                    })
                     .collect::<Vec<_>>();
                 sorted_validators.sort();
                 sorted_validators
@@ -752,13 +761,28 @@ impl EpochManager {
                     validator_block_chunk_stats.remove(account_id);
                 }
             }
+            let epoch_config = self.get_epoch_config(block_info.epoch_id())?;
+            // If ChunkEndorsementsInBlockHeader feature is enabled, we use the chunk validator kickout threshold
+            // as the cutoff threshold for the endorsement ratio to remap the ratio to 0 or 1.
+            let online_thresholds = ValidatorOnlineThresholds {
+                online_min_threshold: epoch_config.online_min_threshold,
+                online_max_threshold: epoch_config.online_max_threshold,
+                endorsement_cutoff_threshold: if ProtocolFeature::ChunkEndorsementsInBlockHeader
+                    .enabled(epoch_protocol_version)
+                {
+                    Some(epoch_config.chunk_validator_only_kickout_threshold)
+                } else {
+                    None
+                },
+            };
             self.reward_calculator.calculate_reward(
                 validator_block_chunk_stats,
                 &validator_stake,
                 *block_info.total_supply(),
                 epoch_protocol_version,
                 self.genesis_protocol_version,
                 epoch_duration,
+                online_thresholds,
             )
         };
         let next_next_epoch_config = self.config.for_protocol_version(next_next_epoch_version);

chain/epoch-manager/src/reward_calculator.rs

@@ -13,15 +13,27 @@ use crate::validator_stats::get_validator_online_ratio;
 pub(crate) const NUM_NS_IN_SECOND: u64 = 1_000_000_000;
 pub const NUM_SECONDS_IN_A_YEAR: u64 = 24 * 60 * 60 * 365;
 
+/// Contains online thresholds for validators.
+#[derive(Clone, Debug)]
+pub struct ValidatorOnlineThresholds {
+    /// Online minimum threshold below which validator doesn't receive reward.
+    pub online_min_threshold: Rational32,
+    /// Online maximum threshold above which validator gets full reward.
+    pub online_max_threshold: Rational32,
+    /// If set, contains a number between 0 and 100 (percentage), and endorsement ratio
+    /// below this threshold will be treated 0, and otherwise be treated 1,
+    /// before calculating the average uptime ratio of the validator.
+    /// If not set, endorsement ratio will be used as is.
+    pub endorsement_cutoff_threshold: Option<u8>,
+}
+
 #[derive(Clone, Debug)]
 pub struct RewardCalculator {
     pub max_inflation_rate: Rational32,
     pub num_blocks_per_year: u64,
     pub epoch_length: u64,
     pub protocol_reward_rate: Rational32,
     pub protocol_treasury_account: AccountId,
-    pub online_min_threshold: Rational32,
-    pub online_max_threshold: Rational32,
     pub num_seconds_per_year: u64,
 }
 
@@ -33,8 +45,6 @@ impl RewardCalculator {
             epoch_length: config.epoch_length,
             protocol_reward_rate: config.protocol_reward_rate,
             protocol_treasury_account: config.protocol_treasury_account.clone(),
-            online_max_threshold: config.online_max_threshold,
-            online_min_threshold: config.online_min_threshold,
             num_seconds_per_year: NUM_SECONDS_IN_A_YEAR,
         }
     }
@@ -49,6 +59,7 @@ impl RewardCalculator {
         protocol_version: ProtocolVersion,
         genesis_protocol_version: ProtocolVersion,
         epoch_duration: u64,
+        online_thresholds: ValidatorOnlineThresholds,
     ) -> (HashMap<AccountId, Balance>, Balance) {
         let mut res = HashMap::new();
         let num_validators = validator_block_chunk_stats.len();
@@ -90,16 +101,19 @@ impl RewardCalculator {
         let mut epoch_actual_reward = epoch_protocol_treasury;
         let total_stake: Balance = validator_stake.values().sum();
         for (account_id, stats) in validator_block_chunk_stats {
-            let production_ratio = get_validator_online_ratio(&stats);
+            let production_ratio =
+                get_validator_online_ratio(&stats, online_thresholds.endorsement_cutoff_threshold);
             let average_produced_numer = production_ratio.numer();
             let average_produced_denom = production_ratio.denom();
 
             let expected_blocks = stats.block_stats.expected;
             let expected_chunks = stats.chunk_stats.expected();
             let expected_endorsements = stats.chunk_stats.endorsement_stats().expected;
 
-            let online_min_numer = U256::from(*self.online_min_threshold.numer() as u64);
-            let online_min_denom = U256::from(*self.online_min_threshold.denom() as u64);
+            let online_min_numer =
+                U256::from(*online_thresholds.online_min_threshold.numer() as u64);
+            let online_min_denom =
+                U256::from(*online_thresholds.online_min_threshold.denom() as u64);
             // If average of produced blocks below online min threshold, validator gets 0 reward.
             let chunk_only_producers_enabled =
                 checked_feature!("stable", ChunkOnlyProducers, protocol_version);
@@ -121,8 +135,10 @@ impl RewardCalculator {
                     .get(&account_id)
                     .unwrap_or_else(|| panic!("{} is not a validator", account_id));
                 // Online reward multiplier is min(1., (uptime - online_threshold_min) / (online_threshold_max - online_threshold_min).
-                let online_max_numer = U256::from(*self.online_max_threshold.numer() as u64);
-                let online_max_denom = U256::from(*self.online_max_threshold.denom() as u64);
+                let online_max_numer =
+                    U256::from(*online_thresholds.online_max_threshold.numer() as u64);
+                let online_max_denom =
+                    U256::from(*online_thresholds.online_max_threshold.denom() as u64);
                 let online_numer =
                     online_max_numer * online_min_denom - online_min_numer * online_max_denom;
                 let mut uptime_numer = (average_produced_numer * online_min_denom
@@ -161,8 +177,6 @@ mod tests {
             epoch_length,
             protocol_reward_rate: Ratio::new(0, 1),
             protocol_treasury_account: "near".parse().unwrap(),
-            online_min_threshold: Ratio::new(9, 10),
-            online_max_threshold: Ratio::new(1, 1),
             num_seconds_per_year: 1000000,
         };
         let validator_block_chunk_stats = HashMap::from([
@@ -191,6 +205,11 @@ mod tests {
             PROTOCOL_VERSION,
             PROTOCOL_VERSION,
             epoch_length * NUM_NS_IN_SECOND,
+            ValidatorOnlineThresholds {
+                online_min_threshold: Ratio::new(9, 10),
+                online_max_threshold: Ratio::new(1, 1),
+                endorsement_cutoff_threshold: None,
+            },
         );
         assert_eq!(
             result.0,
@@ -212,8 +231,6 @@ mod tests {
             epoch_length,
             protocol_reward_rate: Ratio::new(0, 10),
             protocol_treasury_account: "near".parse().unwrap(),
-            online_min_threshold: Ratio::new(9, 10),
-            online_max_threshold: Ratio::new(99, 100),
             num_seconds_per_year: 1000,
         };
         let validator_block_chunk_stats = HashMap::from([
@@ -252,6 +269,11 @@ mod tests {
             PROTOCOL_VERSION,
             PROTOCOL_VERSION,
             epoch_length * NUM_NS_IN_SECOND,
+            ValidatorOnlineThresholds {
+                online_min_threshold: Ratio::new(9, 10),
+                online_max_threshold: Ratio::new(99, 100),
+                endorsement_cutoff_threshold: None,
+            },
         );
         // Total reward is 10_000_000. Divided by 3 equal stake validators - each gets 3_333_333.
         // test1 with 94.5% online gets 50% because of linear between (0.99-0.9) online.
@@ -277,8 +299,6 @@ mod tests {
             epoch_length,
             protocol_reward_rate: Ratio::new(0, 10),
             protocol_treasury_account: "near".parse().unwrap(),
-            online_min_threshold: Ratio::new(9, 10),
-            online_max_threshold: Ratio::new(99, 100),
             num_seconds_per_year: 1000,
         };
         let validator_block_chunk_stats = HashMap::from([
@@ -329,6 +349,11 @@ mod tests {
             PROTOCOL_VERSION,
             PROTOCOL_VERSION,
             epoch_length * NUM_NS_IN_SECOND,
+            ValidatorOnlineThresholds {
+                online_min_threshold: Ratio::new(9, 10),
+                online_max_threshold: Ratio::new(99, 100),
+                endorsement_cutoff_threshold: None,
+            },
         );
         // Total reward is 10_000_000. Divided by 4 equal stake validators - each gets 2_500_000.
         // test1 with 94.5% online gets 50% because of linear between (0.99-0.9) online.
@@ -347,7 +372,6 @@ mod tests {
         }
     }
 
-    // Test rewards when some validators are only responsible for endorsements
     #[test]
     fn test_reward_stateless_validation() {
         let epoch_length = 1000;
@@ -357,8 +381,6 @@ mod tests {
             epoch_length,
             protocol_reward_rate: Ratio::new(0, 10),
             protocol_treasury_account: "near".parse().unwrap(),
-            online_min_threshold: Ratio::new(9, 10),
-            online_max_threshold: Ratio::new(99, 100),
             num_seconds_per_year: 1000,
         };
         let validator_block_chunk_stats = HashMap::from([
@@ -415,6 +437,11 @@ mod tests {
             PROTOCOL_VERSION,
             PROTOCOL_VERSION,
             epoch_length * NUM_NS_IN_SECOND,
+            ValidatorOnlineThresholds {
+                online_min_threshold: Ratio::new(9, 10),
+                online_max_threshold: Ratio::new(99, 100),
+                endorsement_cutoff_threshold: None,
+            },
         );
         // Total reward is 10_000_000. Divided by 4 equal stake validators - each gets 2_500_000.
         // test1 with 94.5% online gets 50% because of linear between (0.99-0.9) online.
@@ -433,6 +460,95 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_reward_stateless_validation_with_endorsement_cutoff() {
+        let epoch_length = 1000;
+        let reward_calculator = RewardCalculator {
+            max_inflation_rate: Ratio::new(1, 100),
+            num_blocks_per_year: 1000,
+            epoch_length,
+            protocol_reward_rate: Ratio::new(0, 10),
+            protocol_treasury_account: "near".parse().unwrap(),
+            num_seconds_per_year: 1000,
+        };
+        let validator_block_chunk_stats = HashMap::from([
+            // Blocks, chunks, endorsements - endorsement ratio cutoff is exceeded
+            (
+                "test1".parse().unwrap(),
+                BlockChunkValidatorStats {
+                    block_stats: ValidatorStats { produced: 945, expected: 1000 },
+                    chunk_stats: ChunkStats {
+                        production: ValidatorStats { produced: 944, expected: 1000 },
+                        endorsement: ValidatorStats { produced: 946, expected: 1000 },
+                    },
+                },
+            ),
+            // Blocks, chunks, endorsements - endorsement ratio cutoff is not exceeded
+            (
+                "test2".parse().unwrap(),
+                BlockChunkValidatorStats {
+                    block_stats: ValidatorStats { produced: 945, expected: 1000 },
+                    chunk_stats: ChunkStats {
+                        production: ValidatorStats { produced: 944, expected: 1000 },
+                        endorsement: ValidatorStats { produced: 446, expected: 1000 },
+                    },
+                },
+            ),
+            // Endorsements only - endorsement ratio cutoff is exceeded
+            (
+                "test3".parse().unwrap(),
+                BlockChunkValidatorStats {
+                    block_stats: ValidatorStats { produced: 0, expected: 0 },
+                    chunk_stats: ChunkStats::new_with_endorsement(946, 1000),
+                },
+            ),
+            // Endorsements only - endorsement ratio cutoff is not exceeded
+            (
+                "test4".parse().unwrap(),
+                BlockChunkValidatorStats {
+                    block_stats: ValidatorStats { produced: 0, expected: 0 },
+                    chunk_stats: ChunkStats::new_with_endorsement(446, 1000),
+                },
+            ),
+        ]);
+        let validator_stake = HashMap::from([
+            ("test1".parse().unwrap(), 500_000),
+            ("test2".parse().unwrap(), 500_000),
+            ("test3".parse().unwrap(), 500_000),
+            ("test4".parse().unwrap(), 500_000),
+        ]);
+        let total_supply = 1_000_000_000;
+        let result = reward_calculator.calculate_reward(
+            validator_block_chunk_stats,
+            &validator_stake,
+            total_supply,
+            PROTOCOL_VERSION,
+            PROTOCOL_VERSION,
+            epoch_length * NUM_NS_IN_SECOND,
+            ValidatorOnlineThresholds {
+                online_min_threshold: Ratio::new(9, 10),
+                online_max_threshold: Ratio::new(99, 100),
+                endorsement_cutoff_threshold: Some(50),
+            },
+        );
+        // test2 does not get reward since its uptime ration goes below online_min_threshold,
+        // because its endorsement ratio is below the cutoff threshold.
+        // test3 does not get reward since its endorsement ratio is below the cutoff threshold.
+        {
+            assert_eq!(
+                result.0,
+                HashMap::from([
+                    ("near".parse().unwrap(), 0),
+                    ("test1".parse().unwrap(), 1_750_000u128),
+                    ("test2".parse().unwrap(), 0),
+                    ("test3".parse().unwrap(), 2_500_000u128),
+                    ("test4".parse().unwrap(), 0)
+                ])
+            );
+            assert_eq!(result.1, 4_250_000u128);
+        }
+    }
+
     /// Test that under an extreme setting (total supply 100b, epoch length half a day),
     /// reward calculation will not overflow.
     #[test]
@@ -445,8 +561,6 @@ mod tests {
             epoch_length,
             protocol_reward_rate: Ratio::new(1, 10),
             protocol_treasury_account: "near".parse().unwrap(),
-            online_min_threshold: Ratio::new(9, 10),
-            online_max_threshold: Ratio::new(1, 1),
             num_seconds_per_year: 60 * 60 * 24 * 365,
         };
         let validator_block_chunk_stats = HashMap::from([(
@@ -469,6 +583,11 @@ mod tests {
             PROTOCOL_VERSION,
             PROTOCOL_VERSION,
             epoch_length * NUM_NS_IN_SECOND,
+            ValidatorOnlineThresholds {
+                online_min_threshold: Ratio::new(9, 10),
+                online_max_threshold: Ratio::new(1, 1),
+                endorsement_cutoff_threshold: None,
+            },
         );
     }
 }

chain/epoch-manager/src/shard_tracker.rs

@@ -255,8 +255,6 @@ mod tests {
             epoch_length: 1,
             protocol_reward_rate: Ratio::from_integer(0),
             protocol_treasury_account: "test".parse().unwrap(),
-            online_max_threshold: initial_epoch_config.online_max_threshold,
-            online_min_threshold: initial_epoch_config.online_min_threshold,
             num_seconds_per_year: 1000000,
         };
         EpochManager::new(

chain/epoch-manager/src/test_utils.rs

@@ -190,8 +190,6 @@ pub fn default_reward_calculator() -> RewardCalculator {
         epoch_length: 1,
         protocol_reward_rate: Ratio::from_integer(0),
         protocol_treasury_account: "near".parse().unwrap(),
-        online_min_threshold: Ratio::new(90, 100),
-        online_max_threshold: Ratio::new(99, 100),
         num_seconds_per_year: NUM_SECONDS_IN_A_YEAR,
     }
 }