Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Introduce cutoff threshold for endorsement ratio #12047

Merged
merged 11 commits into from
Sep 9, 2024
26 changes: 25 additions & 1 deletion chain/epoch-manager/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use near_primitives::views::{
};
use near_store::{DBCol, Store, StoreUpdate, HEADER_HEAD_KEY};
use primitive_types::U256;
use reward_calculator::ValidatorOnlineThresholds;
use std::cmp::Ordering;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
Expand Down Expand Up @@ -507,7 +508,15 @@ impl EpochManager {
{
let mut sorted_validators = validator_block_chunk_stats
.iter()
.map(|(account, stats)| (get_sortable_validator_online_ratio(stats), account))
.map(|(account, stats)| {
(
get_sortable_validator_online_ratio(
stats,
Some(chunk_validator_only_kickout_threshold),
),
account,
)
})
.collect::<Vec<_>>();
sorted_validators.sort();
sorted_validators
Expand Down Expand Up @@ -752,13 +761,28 @@ impl EpochManager {
validator_block_chunk_stats.remove(account_id);
}
}
let epoch_config = self.get_epoch_config(block_info.epoch_id())?;
// If ChunkEndorsementsInBlockHeader feature is enabled, we use the chunk validator kickout threshold
// as the cutoff threshold for the endorsement ratio to remap the ratio to 0 or 1.
let online_thresholds = ValidatorOnlineThresholds {
online_min_threshold: epoch_config.online_min_threshold,
online_max_threshold: epoch_config.online_max_threshold,
endorsement_cutoff_threshold: if ProtocolFeature::ChunkEndorsementsInBlockHeader
.enabled(epoch_protocol_version)
{
Some(epoch_config.chunk_validator_only_kickout_threshold)
} else {
None
},
};
self.reward_calculator.calculate_reward(
validator_block_chunk_stats,
&validator_stake,
*block_info.total_supply(),
epoch_protocol_version,
self.genesis_protocol_version,
epoch_duration,
online_thresholds,
)
};
let next_next_epoch_config = self.config.for_protocol_version(next_next_epoch_version);
Expand Down
159 changes: 139 additions & 20 deletions chain/epoch-manager/src/reward_calculator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,27 @@ use crate::validator_stats::get_validator_online_ratio;
pub(crate) const NUM_NS_IN_SECOND: u64 = 1_000_000_000;
pub const NUM_SECONDS_IN_A_YEAR: u64 = 24 * 60 * 60 * 365;

/// Contains online thresholds for validators.
#[derive(Clone, Debug)]
pub struct ValidatorOnlineThresholds {
/// Online minimum threshold below which validator doesn't receive reward.
pub online_min_threshold: Rational32,
/// Online maximum threshold above which validator gets full reward.
pub online_max_threshold: Rational32,
/// If set, contains a number between 0 and 100 (percentage), and endorsement ratio
/// below this threshold will be treated 0, and otherwise be treated 1,
/// before calculating the average uptime ratio of the validator.
/// If not set, endorsement ratio will be used as is.
pub endorsement_cutoff_threshold: Option<u8>,
}

#[derive(Clone, Debug)]
pub struct RewardCalculator {
pub max_inflation_rate: Rational32,
pub num_blocks_per_year: u64,
pub epoch_length: u64,
pub protocol_reward_rate: Rational32,
pub protocol_treasury_account: AccountId,
pub online_min_threshold: Rational32,
Copy link
Contributor Author

@tayfunelmas tayfunelmas Sep 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved these to the ValidatorOnlineThresholds struct passed to calculate_reward.
In fact we do not re-initialize the RewardCalculator when there is a new epoch or protocol change, so this way it is safer as we will need to change the online thresholds (endorsement cutoff in this case) via protocol upgrades.

pub online_max_threshold: Rational32,
pub num_seconds_per_year: u64,
}

Expand All @@ -33,8 +45,6 @@ impl RewardCalculator {
epoch_length: config.epoch_length,
protocol_reward_rate: config.protocol_reward_rate,
protocol_treasury_account: config.protocol_treasury_account.clone(),
online_max_threshold: config.online_max_threshold,
online_min_threshold: config.online_min_threshold,
num_seconds_per_year: NUM_SECONDS_IN_A_YEAR,
}
}
Expand All @@ -49,6 +59,7 @@ impl RewardCalculator {
protocol_version: ProtocolVersion,
genesis_protocol_version: ProtocolVersion,
epoch_duration: u64,
online_thresholds: ValidatorOnlineThresholds,
) -> (HashMap<AccountId, Balance>, Balance) {
let mut res = HashMap::new();
let num_validators = validator_block_chunk_stats.len();
Expand Down Expand Up @@ -90,16 +101,19 @@ impl RewardCalculator {
let mut epoch_actual_reward = epoch_protocol_treasury;
let total_stake: Balance = validator_stake.values().sum();
for (account_id, stats) in validator_block_chunk_stats {
let production_ratio = get_validator_online_ratio(&stats);
let production_ratio =
get_validator_online_ratio(&stats, online_thresholds.endorsement_cutoff_threshold);
let average_produced_numer = production_ratio.numer();
let average_produced_denom = production_ratio.denom();

let expected_blocks = stats.block_stats.expected;
let expected_chunks = stats.chunk_stats.expected();
let expected_endorsements = stats.chunk_stats.endorsement_stats().expected;

let online_min_numer = U256::from(*self.online_min_threshold.numer() as u64);
let online_min_denom = U256::from(*self.online_min_threshold.denom() as u64);
let online_min_numer =
U256::from(*online_thresholds.online_min_threshold.numer() as u64);
let online_min_denom =
U256::from(*online_thresholds.online_min_threshold.denom() as u64);
// If average of produced blocks below online min threshold, validator gets 0 reward.
let chunk_only_producers_enabled =
checked_feature!("stable", ChunkOnlyProducers, protocol_version);
Expand All @@ -121,8 +135,10 @@ impl RewardCalculator {
.get(&account_id)
.unwrap_or_else(|| panic!("{} is not a validator", account_id));
// Online reward multiplier is min(1., (uptime - online_threshold_min) / (online_threshold_max - online_threshold_min).
let online_max_numer = U256::from(*self.online_max_threshold.numer() as u64);
let online_max_denom = U256::from(*self.online_max_threshold.denom() as u64);
let online_max_numer =
U256::from(*online_thresholds.online_max_threshold.numer() as u64);
let online_max_denom =
U256::from(*online_thresholds.online_max_threshold.denom() as u64);
let online_numer =
online_max_numer * online_min_denom - online_min_numer * online_max_denom;
let mut uptime_numer = (average_produced_numer * online_min_denom
Expand Down Expand Up @@ -161,8 +177,6 @@ mod tests {
epoch_length,
protocol_reward_rate: Ratio::new(0, 1),
protocol_treasury_account: "near".parse().unwrap(),
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(1, 1),
num_seconds_per_year: 1000000,
};
let validator_block_chunk_stats = HashMap::from([
Expand Down Expand Up @@ -191,6 +205,11 @@ mod tests {
PROTOCOL_VERSION,
PROTOCOL_VERSION,
epoch_length * NUM_NS_IN_SECOND,
ValidatorOnlineThresholds {
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(1, 1),
endorsement_cutoff_threshold: None,
},
);
assert_eq!(
result.0,
Expand All @@ -212,8 +231,6 @@ mod tests {
epoch_length,
protocol_reward_rate: Ratio::new(0, 10),
protocol_treasury_account: "near".parse().unwrap(),
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(99, 100),
num_seconds_per_year: 1000,
};
let validator_block_chunk_stats = HashMap::from([
Expand Down Expand Up @@ -252,6 +269,11 @@ mod tests {
PROTOCOL_VERSION,
PROTOCOL_VERSION,
epoch_length * NUM_NS_IN_SECOND,
ValidatorOnlineThresholds {
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(99, 100),
endorsement_cutoff_threshold: None,
},
);
// Total reward is 10_000_000. Divided by 3 equal stake validators - each gets 3_333_333.
// test1 with 94.5% online gets 50% because of linear between (0.99-0.9) online.
Expand All @@ -277,8 +299,6 @@ mod tests {
epoch_length,
protocol_reward_rate: Ratio::new(0, 10),
protocol_treasury_account: "near".parse().unwrap(),
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(99, 100),
num_seconds_per_year: 1000,
};
let validator_block_chunk_stats = HashMap::from([
Expand Down Expand Up @@ -329,6 +349,11 @@ mod tests {
PROTOCOL_VERSION,
PROTOCOL_VERSION,
epoch_length * NUM_NS_IN_SECOND,
ValidatorOnlineThresholds {
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(99, 100),
endorsement_cutoff_threshold: None,
},
);
// Total reward is 10_000_000. Divided by 4 equal stake validators - each gets 2_500_000.
// test1 with 94.5% online gets 50% because of linear between (0.99-0.9) online.
Expand All @@ -347,7 +372,6 @@ mod tests {
}
}

// Test rewards when some validators are only responsible for endorsements
#[test]
fn test_reward_stateless_validation() {
let epoch_length = 1000;
Expand All @@ -357,8 +381,6 @@ mod tests {
epoch_length,
protocol_reward_rate: Ratio::new(0, 10),
protocol_treasury_account: "near".parse().unwrap(),
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(99, 100),
num_seconds_per_year: 1000,
};
let validator_block_chunk_stats = HashMap::from([
Expand Down Expand Up @@ -415,6 +437,11 @@ mod tests {
PROTOCOL_VERSION,
PROTOCOL_VERSION,
epoch_length * NUM_NS_IN_SECOND,
ValidatorOnlineThresholds {
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(99, 100),
endorsement_cutoff_threshold: None,
},
);
// Total reward is 10_000_000. Divided by 4 equal stake validators - each gets 2_500_000.
// test1 with 94.5% online gets 50% because of linear between (0.99-0.9) online.
Expand All @@ -433,6 +460,95 @@ mod tests {
}
}

#[test]
fn test_reward_stateless_validation_with_endorsement_cutoff() {
let epoch_length = 1000;
let reward_calculator = RewardCalculator {
max_inflation_rate: Ratio::new(1, 100),
num_blocks_per_year: 1000,
epoch_length,
protocol_reward_rate: Ratio::new(0, 10),
protocol_treasury_account: "near".parse().unwrap(),
num_seconds_per_year: 1000,
};
let validator_block_chunk_stats = HashMap::from([
// Blocks, chunks, endorsements - endorsement ratio cutoff is exceeded
(
"test1".parse().unwrap(),
BlockChunkValidatorStats {
block_stats: ValidatorStats { produced: 945, expected: 1000 },
chunk_stats: ChunkStats {
production: ValidatorStats { produced: 944, expected: 1000 },
endorsement: ValidatorStats { produced: 946, expected: 1000 },
},
},
),
// Blocks, chunks, endorsements - endorsement ratio cutoff is not exceeded
(
"test2".parse().unwrap(),
BlockChunkValidatorStats {
block_stats: ValidatorStats { produced: 945, expected: 1000 },
chunk_stats: ChunkStats {
production: ValidatorStats { produced: 944, expected: 1000 },
endorsement: ValidatorStats { produced: 446, expected: 1000 },
},
},
),
// Endorsements only - endorsement ratio cutoff is exceeded
(
"test3".parse().unwrap(),
BlockChunkValidatorStats {
block_stats: ValidatorStats { produced: 0, expected: 0 },
chunk_stats: ChunkStats::new_with_endorsement(946, 1000),
},
),
// Endorsements only - endorsement ratio cutoff is not exceeded
(
"test4".parse().unwrap(),
BlockChunkValidatorStats {
block_stats: ValidatorStats { produced: 0, expected: 0 },
chunk_stats: ChunkStats::new_with_endorsement(446, 1000),
},
),
]);
let validator_stake = HashMap::from([
("test1".parse().unwrap(), 500_000),
("test2".parse().unwrap(), 500_000),
("test3".parse().unwrap(), 500_000),
("test4".parse().unwrap(), 500_000),
]);
let total_supply = 1_000_000_000;
let result = reward_calculator.calculate_reward(
validator_block_chunk_stats,
&validator_stake,
total_supply,
PROTOCOL_VERSION,
PROTOCOL_VERSION,
epoch_length * NUM_NS_IN_SECOND,
ValidatorOnlineThresholds {
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(99, 100),
endorsement_cutoff_threshold: Some(50),
},
);
// test2 does not get reward since its uptime ration goes below online_min_threshold,
// because its endorsement ratio is below the cutoff threshold.
// test3 does not get reward since its endorsement ratio is below the cutoff threshold.
{
assert_eq!(
result.0,
HashMap::from([
("near".parse().unwrap(), 0),
("test1".parse().unwrap(), 1_750_000u128),
("test2".parse().unwrap(), 0),
("test3".parse().unwrap(), 2_500_000u128),
("test4".parse().unwrap(), 0)
])
);
assert_eq!(result.1, 4_250_000u128);
}
}

/// Test that under an extreme setting (total supply 100b, epoch length half a day),
/// reward calculation will not overflow.
#[test]
Expand All @@ -445,8 +561,6 @@ mod tests {
epoch_length,
protocol_reward_rate: Ratio::new(1, 10),
protocol_treasury_account: "near".parse().unwrap(),
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(1, 1),
num_seconds_per_year: 60 * 60 * 24 * 365,
};
let validator_block_chunk_stats = HashMap::from([(
Expand All @@ -469,6 +583,11 @@ mod tests {
PROTOCOL_VERSION,
PROTOCOL_VERSION,
epoch_length * NUM_NS_IN_SECOND,
ValidatorOnlineThresholds {
online_min_threshold: Ratio::new(9, 10),
online_max_threshold: Ratio::new(1, 1),
endorsement_cutoff_threshold: None,
},
);
}
}
2 changes: 0 additions & 2 deletions chain/epoch-manager/src/shard_tracker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,6 @@ mod tests {
epoch_length: 1,
protocol_reward_rate: Ratio::from_integer(0),
protocol_treasury_account: "test".parse().unwrap(),
online_max_threshold: initial_epoch_config.online_max_threshold,
online_min_threshold: initial_epoch_config.online_min_threshold,
num_seconds_per_year: 1000000,
};
EpochManager::new(
Expand Down
2 changes: 0 additions & 2 deletions chain/epoch-manager/src/test_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,6 @@ pub fn default_reward_calculator() -> RewardCalculator {
epoch_length: 1,
protocol_reward_rate: Ratio::from_integer(0),
protocol_treasury_account: "near".parse().unwrap(),
online_min_threshold: Ratio::new(90, 100),
online_max_threshold: Ratio::new(99, 100),
num_seconds_per_year: NUM_SECONDS_IN_A_YEAR,
}
}
Expand Down
Loading
Loading