feat: Export number of expected chunks/blocks in epoch to prometheus (#…

…8759)

The producer to block/chunk assignment is known ahead of time so we can
estimate what we expect to see at the end of the epoch.

We are computing these numbers only once per epoch.

Tested on mainnet and localnet. 
On localnet:
- started 4 nodes
- stopped one of them for a delta of 200 blocks
- started it again

CHANGELOG.md

@@ -15,6 +15,7 @@
 * The contract runtime switched to using our fork of wasmer, with various improvements.
 * undo-block tool to reset the chain head from current head to its prev block. Use the tool by running: `./target/release/neard --home {path_to_config_directory} undo-block`. [#8681](https://github.com/near/nearcore/pull/8681)
 * Add per shard granularity for chunks in validator info metric. [#8934](https://github.com/near/nearcore/pull/8934)
+* Add prometheus metrics for expected number of blocks/chunks at the end of the epoch. [#8759](https://github.com/near/nearcore/pull/8759)
 
 ## 1.33.0
 

chain/client/src/info.rs

@@ -11,7 +11,7 @@ use near_primitives::telemetry::{
     TelemetryAgentInfo, TelemetryChainInfo, TelemetryInfo, TelemetrySystemInfo,
 };
 use near_primitives::types::{
-    AccountId, Balance, BlockHeight, EpochHeight, EpochId, Gas, NumBlocks, ShardId,
+    AccountId, Balance, BlockHeight, EpochHeight, EpochId, Gas, NumBlocks, ShardId, ValidatorId,
     ValidatorInfoIdentifier,
 };
 use near_primitives::unwrap_or_return;
@@ -23,6 +23,7 @@ use near_primitives::views::{
 };
 use near_telemetry::{telemetry, TelemetryActor};
 use std::cmp::min;
+use std::collections::HashMap;
 use std::fmt::Write;
 use std::sync::Arc;
 use std::time::Instant;
@@ -59,6 +60,8 @@ pub struct InfoHelper {
     telemetry_actor: Option<Addr<TelemetryActor>>,
     /// Log coloring enabled
     log_summary_style: LogSummaryStyle,
+    /// Epoch height
+    epoch_id: Option<EpochId>,
     /// Timestamp of starting the client.
     pub boot_time_seconds: i64,
 }
@@ -83,6 +86,7 @@ impl InfoHelper {
             validator_signer,
             log_summary_style: client_config.log_summary_style,
             boot_time_seconds: StaticClock::utc().timestamp(),
+            epoch_id: None,
         }
     }
 
@@ -181,6 +185,60 @@ impl InfoHelper {
         }
     }
 
+    /// The value obtained by multiplying the stake fraction with the expected number of blocks in an epoch
+    /// is an estimation, and not an exact value. To obtain a more precise result, it is necessary to examine
+    /// all the blocks in the epoch. However, even this method may not be completely accurate because additional
+    /// blocks could potentially be added at the end of the epoch.
+    fn record_epoch_settlement_info(head: &Tip, client: &crate::client::Client) {
+        let epoch_info = client.runtime_adapter.get_epoch_info(&head.epoch_id);
+        let blocks_in_epoch = client.config.epoch_length;
+        let number_of_shards =
+            client.runtime_adapter.num_shards(&head.epoch_id).unwrap_or_default();
+        if let Ok(epoch_info) = epoch_info {
+            let mut stake_per_bp = HashMap::<ValidatorId, Balance>::new();
+
+            let stake_to_blocks = |stake: Balance, stake_sum: Balance| -> i64 {
+                if stake == 0 {
+                    0
+                } else {
+                    (((stake as f64) / (stake_sum as f64)) * (blocks_in_epoch as f64)) as i64
+                }
+            };
+
+            let mut stake_sum = 0;
+            for &id in epoch_info.block_producers_settlement() {
+                let stake = epoch_info.validator_stake(id);
+                stake_per_bp.insert(id, stake);
+                stake_sum += stake;
+            }
+
+            stake_per_bp.iter().for_each(|(&id, &stake)| {
+                metrics::VALIDATORS_BLOCKS_EXPECTED_IN_EPOCH
+                    .with_label_values(&[epoch_info.get_validator(id).account_id().as_str()])
+                    .set(stake_to_blocks(stake, stake_sum))
+            });
+
+            for shard_id in 0..number_of_shards {
+                let mut stake_per_cp = HashMap::<ValidatorId, Balance>::new();
+                stake_sum = 0;
+                for &id in &epoch_info.chunk_producers_settlement()[shard_id as usize] {
+                    let stake = epoch_info.validator_stake(id);
+                    stake_per_cp.insert(id, stake);
+                    stake_sum += stake;
+                }
+
+                stake_per_cp.iter().for_each(|(&id, &stake)| {
+                    metrics::VALIDATORS_CHUNKS_EXPECTED_IN_EPOCH
+                        .with_label_values(&[
+                            epoch_info.get_validator(id).account_id().as_str(),
+                            &shard_id.to_string(),
+                        ])
+                        .set(stake_to_blocks(stake, stake_sum))
+                });
+            }
+        }
+    }
+
     /// Print current summary.
     pub fn log_summary(
         &mut self,
@@ -234,9 +292,15 @@ impl InfoHelper {
                 .unwrap_or_default()
         };
 
-        InfoHelper::record_tracked_shards(&head, client);
-        InfoHelper::record_block_producers(&head, client);
-        InfoHelper::record_chunk_producers(&head, client);
+        InfoHelper::record_tracked_shards(&head, &client);
+        InfoHelper::record_block_producers(&head, &client);
+        InfoHelper::record_chunk_producers(&head, &client);
+        let next_epoch_id = Some(head.epoch_id.clone());
+        if self.epoch_id.ne(&next_epoch_id) {
+            // We only want to compute this once per epoch.
+            InfoHelper::record_epoch_settlement_info(&head, &client);
+            self.epoch_id = next_epoch_id;
+        }
 
         self.info(
             &head,

chain/client/src/metrics.rs

@@ -123,6 +123,15 @@ pub(crate) static VALIDATORS_CHUNKS_EXPECTED_BY_SHARD: Lazy<IntGaugeVec> = Lazy:
     .unwrap()
 });
 
+pub(crate) static VALIDATORS_CHUNKS_EXPECTED_IN_EPOCH: Lazy<IntGaugeVec> = Lazy::new(|| {
+    try_create_int_gauge_vec(
+        "near_validators_chunks_expected_in_epoch",
+        "Number of chunks expected to be produced by a validator within current epoch",
+        &["account_id", "shard_id"],
+    )
+    .unwrap()
+});
+
 pub(crate) static VALIDATORS_BLOCKS_PRODUCED: Lazy<IntGaugeVec> = Lazy::new(|| {
     try_create_int_gauge_vec(
         "near_validators_blocks_produced",
@@ -141,6 +150,15 @@ pub(crate) static VALIDATORS_BLOCKS_EXPECTED: Lazy<IntGaugeVec> = Lazy::new(|| {
     .unwrap()
 });
 
+pub(crate) static VALIDATORS_BLOCKS_EXPECTED_IN_EPOCH: Lazy<IntGaugeVec> = Lazy::new(|| {
+    try_create_int_gauge_vec(
+        "near_validators_blocks_expected_in_epoch",
+        "Number of blocks expected to be produced by a validator within current epoch",
+        &["account_id"],
+    )
+    .unwrap()
+});
+
 pub(crate) static TRACKED_SHARDS: Lazy<IntGaugeVec> = Lazy::new(|| {
     try_create_int_gauge_vec("near_client_tracked_shards", "Tracked shards", &["shard_id"]).unwrap()
 });