From bb407d2b6256d997411e1958b18d2326899ba81e Mon Sep 17 00:00:00 2001
From: "Jeff Washington (jwash)" <jeff.washington@solana.com>
Date: Mon, 31 Jul 2023 07:07:13 -0700
Subject: [PATCH] add metric read_only_cache_load_us (#32559)

---
 runtime/src/accounts_db.rs              | 13 +++++--
 runtime/src/read_only_accounts_cache.rs | 45 +++++++++++++++----------
 2 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
index 9002bd9a2d90fe..11512958b0210e 100644
--- a/runtime/src/accounts_db.rs
+++ b/runtime/src/accounts_db.rs
@@ -8389,8 +8389,12 @@ impl AccountsDb {
 
     fn report_store_timings(&self) {
         if self.stats.last_store_report.should_update(1000) {
-            let (read_only_cache_hits, read_only_cache_misses, read_only_cache_evicts) =
-                self.read_only_accounts_cache.get_and_reset_stats();
+            let (
+                read_only_cache_hits,
+                read_only_cache_misses,
+                read_only_cache_evicts,
+                read_only_cache_load_us,
+            ) = self.read_only_accounts_cache.get_and_reset_stats();
             datapoint_info!(
                 "accounts_db_store_timings",
                 (
@@ -8456,6 +8460,11 @@ impl AccountsDb {
                     read_only_cache_evicts,
                     i64
                 ),
+                (
+                    "read_only_accounts_cache_load_us",
+                    read_only_cache_load_us,
+                    i64
+                ),
                 (
                     "calc_stored_meta_us",
                     self.stats.calc_stored_meta.swap(0, Ordering::Relaxed),
diff --git a/runtime/src/read_only_accounts_cache.rs b/runtime/src/read_only_accounts_cache.rs
index 0532a570596d53..23a0af60f4731e 100644
--- a/runtime/src/read_only_accounts_cache.rs
+++ b/runtime/src/read_only_accounts_cache.rs
@@ -3,6 +3,7 @@
 use {
     dashmap::{mapref::entry::Entry, DashMap},
     index_list::{Index, IndexList},
+    solana_measure::measure_us,
     solana_sdk::{
         account::{AccountSharedData, ReadableAccount},
         clock::Slot,
@@ -39,6 +40,7 @@ pub(crate) struct ReadOnlyAccountsCache {
     hits: AtomicU64,
     misses: AtomicU64,
     evicts: AtomicU64,
+    load_us: AtomicU64,
 }
 
 impl ReadOnlyAccountsCache {
@@ -51,6 +53,7 @@ impl ReadOnlyAccountsCache {
             hits: AtomicU64::default(),
             misses: AtomicU64::default(),
             evicts: AtomicU64::default(),
+            load_us: AtomicU64::default(),
         }
     }
 
@@ -63,6 +66,7 @@ impl ReadOnlyAccountsCache {
         self.hits.store(0, Ordering::Relaxed);
         self.misses.store(0, Ordering::Relaxed);
         self.evicts.store(0, Ordering::Relaxed);
+        self.load_us.store(0, Ordering::Relaxed);
     }
 
     /// true if pubkey is in cache at slot
@@ -71,21 +75,27 @@ impl ReadOnlyAccountsCache {
     }
 
     pub(crate) fn load(&self, pubkey: Pubkey, slot: Slot) -> Option<AccountSharedData> {
-        let key = (pubkey, slot);
-        let Some(mut entry) = self.cache.get_mut(&key) else {
-            self.misses.fetch_add(1, Ordering::Relaxed);
-            return None;
-        };
-        self.hits.fetch_add(1, Ordering::Relaxed);
-        // Move the entry to the end of the queue.
-        // self.queue is modified while holding a reference to the cache entry;
-        // so that another thread cannot write to the same key.
-        {
-            let mut queue = self.queue.lock().unwrap();
-            queue.remove(entry.index);
-            entry.index = queue.insert_last(key);
-        }
-        Some(entry.account.clone())
+        let (account, load_us) = measure_us!({
+            let key = (pubkey, slot);
+            let Some(mut entry) = self.cache.get_mut(&key) else {
+                self.misses.fetch_add(1, Ordering::Relaxed);
+                return None;
+            };
+            // Move the entry to the end of the queue.
+            // self.queue is modified while holding a reference to the cache entry;
+            // so that another thread cannot write to the same key.
+            {
+                let mut queue = self.queue.lock().unwrap();
+                queue.remove(entry.index);
+                entry.index = queue.insert_last(key);
+            }
+            let account = entry.account.clone();
+            drop(entry);
+            self.hits.fetch_add(1, Ordering::Relaxed);
+            Some(account)
+        });
+        self.load_us.fetch_add(load_us, Ordering::Relaxed);
+        account
     }
 
     fn account_size(&self, account: &AccountSharedData) -> usize {
@@ -147,12 +157,13 @@ impl ReadOnlyAccountsCache {
         self.data_size.load(Ordering::Relaxed)
     }
 
-    pub(crate) fn get_and_reset_stats(&self) -> (u64, u64, u64) {
+    pub(crate) fn get_and_reset_stats(&self) -> (u64, u64, u64, u64) {
         let hits = self.hits.swap(0, Ordering::Relaxed);
         let misses = self.misses.swap(0, Ordering::Relaxed);
         let evicts = self.evicts.swap(0, Ordering::Relaxed);
+        let load_us = self.load_us.swap(0, Ordering::Relaxed);
 
-        (hits, misses, evicts)
+        (hits, misses, evicts, load_us)
     }
 }