Added metrics counts to monitor anomalies

runtime/src/prioritization_fee.rs

@@ -19,6 +19,9 @@ struct PrioritizationFeeMetrics {
     // Count of transactions that have zero prioritization fee.
     non_prioritized_transactions_count: u64,
 
+    // Count of attempted update on finalized PrioritizationFee
+    attempted_update_on_finalized_fee_count: u64,
+
     // Total prioritization fees included in this slot.
     total_prioritization_fee: u64,
 
@@ -41,6 +44,10 @@ impl PrioritizationFeeMetrics {
         saturating_add_assign!(self.total_update_elapsed_us, val);
     }
 
+    fn increment_attempted_update_on_finalized_fee_count(&mut self, val: u64) {
+        saturating_add_assign!(self.attempted_update_on_finalized_fee_count, val);
+    }
+
     fn update_prioritization_fee(&mut self, fee: u64) {
         if fee == 0 {
             saturating_add_assign!(self.non_prioritized_transactions_count, 1);
@@ -82,6 +89,11 @@ impl PrioritizationFeeMetrics {
                 self.non_prioritized_transactions_count as i64,
                 i64
             ),
+            (
+                "attempted_update_on_finalized_fee_count",
+                self.attempted_update_on_finalized_fee_count as i64,
+                i64
+            ),
             (
                 "total_prioritization_fee",
                 self.total_prioritization_fee as i64,
@@ -159,22 +171,27 @@ impl PrioritizationFee {
     ) -> Result<(), PrioritizationFeeError> {
         let (_, update_time) = measure!(
             {
-                if transaction_fee < self.min_transaction_fee {
-                    self.min_transaction_fee = transaction_fee;
-                }
-
-                for write_account in writable_accounts.iter() {
-                    self.min_writable_account_fees
-                        .entry(*write_account)
-                        .and_modify(|write_lock_fee| {
-                            *write_lock_fee = std::cmp::min(*write_lock_fee, transaction_fee)
-                        })
-                        .or_insert(transaction_fee);
+                if !self.is_finalized {
+                    if transaction_fee < self.min_transaction_fee {
+                        self.min_transaction_fee = transaction_fee;
+                    }
+
+                    for write_account in writable_accounts.iter() {
+                        self.min_writable_account_fees
+                            .entry(*write_account)
+                            .and_modify(|write_lock_fee| {
+                                *write_lock_fee = std::cmp::min(*write_lock_fee, transaction_fee)
+                            })
+                            .or_insert(transaction_fee);
+                    }
+
+                    self.metrics
+                        .accumulate_total_prioritization_fee(transaction_fee);
+                    self.metrics.update_prioritization_fee(transaction_fee);
+                } else {
+                    self.metrics
+                        .increment_attempted_update_on_finalized_fee_count(1);
                 }
-
-                self.metrics
-                    .accumulate_total_prioritization_fee(transaction_fee);
-                self.metrics.update_prioritization_fee(transaction_fee);
             },
             "update_time",
         );

runtime/src/prioritization_fee_cache.rs

@@ -34,6 +34,9 @@ struct PrioritizationFeeCacheMetrics {
     // Count of transactions that successfully updated each slot's prioritization fee cache.
     successful_transaction_update_count: AtomicU64,
 
+    // Count of duplicated banks being purged
+    purged_duplicated_bank_count: AtomicU64,
+
     // Accumulated time spent on tracking prioritization fee for each slot.
     total_update_elapsed_us: AtomicU64,
 
@@ -53,6 +56,11 @@ impl PrioritizationFeeCacheMetrics {
             .fetch_add(val, Ordering::Relaxed);
     }
 
+    fn accumulate_total_purged_duplicated_bank_count(&self, val: u64) {
+        self.purged_duplicated_bank_count
+            .fetch_add(val, Ordering::Relaxed);
+    }
+
     fn accumulate_total_update_elapsed_us(&self, val: u64) {
         self.total_update_elapsed_us
             .fetch_add(val, Ordering::Relaxed);
@@ -83,6 +91,11 @@ impl PrioritizationFeeCacheMetrics {
                     .swap(0, Ordering::Relaxed) as i64,
                 i64
             ),
+            (
+                "purged_duplicated_bank_count",
+                self.purged_duplicated_bank_count.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
             (
                 "total_update_elapsed_us",
                 self.total_update_elapsed_us.swap(0, Ordering::Relaxed) as i64,
@@ -313,8 +326,16 @@ impl PrioritizationFeeCache {
         // block minimum fee.
         let (_, slot_finalize_time) = measure!(
             {
-                // retain prioritization fee for OC-ed bank
+                let pre_purge_bank_count = slot_prioritization_fee.len() as u64;
                 slot_prioritization_fee.retain(|id, _| id == bank_id);
+                let post_purge_bank_count = slot_prioritization_fee.len() as u64;
+                metrics.accumulate_total_purged_duplicated_bank_count(
+                    pre_purge_bank_count.saturating_sub(post_purge_bank_count),
+                );
+                if post_purge_bank_count == 0 {
+                    warn!("Prioritization fee cache unexpected finalized on non-existing bank. slot {slot} bank id {bank_id}");
+                }
+
                 let mut block_prioritization_fee = slot_prioritization_fee
                     .entry(*bank_id)
                     .or_insert(PrioritizationFee::default());
@@ -550,21 +571,21 @@ mod tests {
             &1
         )
         .entry(1)
-        .or_insert(PrioritizationFee::default())
+        .or_default()
         .mark_block_completed()
         .is_ok());
         assert!(PrioritizationFeeCache::get_prioritization_fee(
             prioritization_fee_cache.cache.clone(),
             &2
         )
         .entry(2)
-        .or_insert(PrioritizationFee::default())
+        .or_default()
         .mark_block_completed()
         .is_ok());
         // add slot 3 entry to cache, but not finalize it
         PrioritizationFeeCache::get_prioritization_fee(prioritization_fee_cache.cache.clone(), &3)
             .entry(3)
-            .or_insert(PrioritizationFee::default());
+            .or_default();
 
         // assert available block count should be 2 finalized blocks
         assert_eq!(2, prioritization_fee_cache.available_block_count());