From d9fa958b26d1b12b95e3dda155eb93befb6a4db3 Mon Sep 17 00:00:00 2001 From: Koute Date: Thu, 26 Jan 2023 14:38:00 +0900 Subject: [PATCH] Rework the trie cache (#12982) * Rework the trie cache * Align `state-machine` tests * Bump `schnellru` to 0.1.1 * Fix off-by-one * Align to review comments * Bump `ahash` to 0.8.2 * Bump `schnellru` to 0.2.0 * Bump `schnellru` to 0.2.1 * Remove unnecessary bound * Remove unnecessary loop when calculating maximum memory usage * Remove unnecessary `mut`s --- Cargo.lock | 8 +- client/db/src/bench.rs | 2 +- client/db/src/lib.rs | 2 +- client/finality-grandpa/Cargo.toml | 2 +- client/network-gossip/Cargo.toml | 2 +- primitives/state-machine/src/trie_backend.rs | 20 +- primitives/trie/Cargo.toml | 6 +- primitives/trie/src/cache/mod.rs | 576 ++++++++--- primitives/trie/src/cache/shared_cache.rs | 948 +++++++++++-------- 9 files changed, 1016 insertions(+), 550 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53e5d4882b9dd..7c412030d560c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8383,7 +8383,7 @@ dependencies = [ name = "sc-finality-grandpa" version = "0.10.0-dev" dependencies = [ - "ahash 0.7.6", + "ahash 0.8.2", "array-bytes", "assert_matches", "async-trait", @@ -8583,7 +8583,7 @@ dependencies = [ name = "sc-network-gossip" version = "0.10.0-dev" dependencies = [ - "ahash 0.7.6", + "ahash 0.8.2", "futures", "futures-timer", "libp2p", @@ -10352,18 +10352,18 @@ dependencies = [ name = "sp-trie" version = "7.0.0" dependencies = [ - "ahash 0.7.6", + "ahash 0.8.2", "array-bytes", "criterion", "hash-db", "hashbrown 0.12.3", "lazy_static", - "lru", "memory-db", "nohash-hasher", "parity-scale-codec", "parking_lot 0.12.1", "scale-info", + "schnellru", "sp-core", "sp-runtime", "sp-std", diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 13d91fff0b555..e53209f27706c 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -110,7 +110,7 @@ impl BenchmarkingState { proof_recorder_root: Cell::new(root), enable_tracking, // Enable the cache, but do not sync anything to the shared state. - shared_trie_cache: SharedTrieCache::new(CacheSize::Maximum(0)), + shared_trie_cache: SharedTrieCache::new(CacheSize::new(0)), }; state.add_whitelist_to_tracker(); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 09ccfef1cc28b..f217eb6480abc 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1243,7 +1243,7 @@ impl Backend { blocks_pruning: config.blocks_pruning, genesis_state: RwLock::new(None), shared_trie_cache: config.trie_cache_maximum_size.map(|maximum_size| { - SharedTrieCache::new(sp_trie::cache::CacheSize::Maximum(maximum_size)) + SharedTrieCache::new(sp_trie::cache::CacheSize::new(maximum_size)) }), }; diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 9a31a1d4bf6a7..8a4e0449f2d1c 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -14,7 +14,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -ahash = "0.7.6" +ahash = "0.8.2" array-bytes = "4.1" async-trait = "0.1.57" dyn-clone = "1.0" diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index ef23062768d1e..7811e86c095ba 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -14,7 +14,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -ahash = "0.7.6" +ahash = "0.8.2" futures = "0.3.21" futures-timer = "3.0.1" libp2p = "0.50.0" diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index da4250b6ba3e1..ff5cce24c5124 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -412,19 +412,19 @@ pub mod tests { fn $name() { let parameters = vec![ (StateVersion::V0, None, None), - (StateVersion::V0, Some(SharedCache::new(CacheSize::Unlimited)), None), + (StateVersion::V0, Some(SharedCache::new(CacheSize::unlimited())), None), (StateVersion::V0, None, Some(Recorder::default())), ( StateVersion::V0, - Some(SharedCache::new(CacheSize::Unlimited)), + Some(SharedCache::new(CacheSize::unlimited())), Some(Recorder::default()), ), (StateVersion::V1, None, None), - (StateVersion::V1, Some(SharedCache::new(CacheSize::Unlimited)), None), + (StateVersion::V1, Some(SharedCache::new(CacheSize::unlimited())), None), (StateVersion::V1, None, Some(Recorder::default())), ( StateVersion::V1, - Some(SharedCache::new(CacheSize::Unlimited)), + Some(SharedCache::new(CacheSize::unlimited())), Some(Recorder::default()), ), ]; @@ -760,7 +760,7 @@ pub mod tests { .clone() .for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); - for cache in [Some(SharedTrieCache::new(CacheSize::Unlimited)), None] { + for cache in [Some(SharedTrieCache::new(CacheSize::unlimited())), None] { // Run multiple times to have a different cache conditions. for i in 0..5 { if let Some(cache) = &cache { @@ -793,7 +793,7 @@ pub mod tests { proof_record_works_with_iter_inner(StateVersion::V1); } fn proof_record_works_with_iter_inner(state_version: StateVersion) { - for cache in [Some(SharedTrieCache::new(CacheSize::Unlimited)), None] { + for cache in [Some(SharedTrieCache::new(CacheSize::unlimited())), None] { // Run multiple times to have a different cache conditions. for i in 0..5 { if let Some(cache) = &cache { @@ -870,7 +870,7 @@ pub mod tests { assert_eq!(in_memory.child_storage(child_info_2, &[i]).unwrap().unwrap(), vec![i]) }); - for cache in [Some(SharedTrieCache::new(CacheSize::Unlimited)), None] { + for cache in [Some(SharedTrieCache::new(CacheSize::unlimited())), None] { // Run multiple times to have a different cache conditions. for i in 0..5 { eprintln!("Running with cache {}, iteration {}", cache.is_some(), i); @@ -1002,7 +1002,7 @@ pub mod tests { nodes }; - let cache = SharedTrieCache::::new(CacheSize::Unlimited); + let cache = SharedTrieCache::::new(CacheSize::unlimited()); { let local_cache = cache.local_cache(); let mut trie_cache = local_cache.as_trie_db_cache(child_1_root); @@ -1093,7 +1093,7 @@ pub mod tests { #[test] fn new_data_is_added_to_the_cache() { - let shared_cache = SharedTrieCache::new(CacheSize::Unlimited); + let shared_cache = SharedTrieCache::new(CacheSize::unlimited()); let new_data = vec![ (&b"new_data0"[..], Some(&b"0"[..])), (&b"new_data1"[..], Some(&b"1"[..])), @@ -1159,7 +1159,7 @@ pub mod tests { assert_eq!(in_memory.child_storage(child_info_1, &key).unwrap().unwrap(), child_trie_1_val); assert_eq!(in_memory.child_storage(child_info_2, &key).unwrap().unwrap(), child_trie_2_val); - for cache in [Some(SharedTrieCache::new(CacheSize::Unlimited)), None] { + for cache in [Some(SharedTrieCache::new(CacheSize::unlimited())), None] { // Run multiple times to have a different cache conditions. for i in 0..5 { eprintln!("Running with cache {}, iteration {}", cache.is_some(), i); diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 3f045a1cb216d..78b0b5d1bbda3 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -18,12 +18,11 @@ name = "bench" harness = false [dependencies] -ahash = { version = "0.7.6", optional = true } +ahash = { version = "0.8.2", optional = true } codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false } hashbrown = { version = "0.12.3", optional = true } hash-db = { version = "0.15.2", default-features = false } lazy_static = { version = "1.4.0", optional = true } -lru = { version = "0.8.1", optional = true } memory-db = { version = "0.31.0", default-features = false } nohash-hasher = { version = "0.2.0", optional = true } parking_lot = { version = "0.12.1", optional = true } @@ -34,6 +33,7 @@ trie-db = { version = "0.24.0", default-features = false } trie-root = { version = "0.17.0", default-features = false } sp-core = { version = "7.0.0", default-features = false, path = "../core" } sp-std = { version = "5.0.0", default-features = false, path = "../std" } +schnellru = { version = "0.2.1", optional = true } [dev-dependencies] array-bytes = "4.1" @@ -50,7 +50,7 @@ std = [ "hashbrown", "hash-db/std", "lazy_static", - "lru", + "schnellru", "memory-db/std", "nohash-hasher", "parking_lot", diff --git a/primitives/trie/src/cache/mod.rs b/primitives/trie/src/cache/mod.rs index 85539cf626857..3c1e5b8d0ff0b 100644 --- a/primitives/trie/src/cache/mod.rs +++ b/primitives/trie/src/cache/mod.rs @@ -36,13 +36,17 @@ use crate::{Error, NodeCodec}; use hash_db::Hasher; -use hashbrown::HashSet; use nohash_hasher::BuildNoHashHasher; -use parking_lot::{Mutex, MutexGuard, RwLockReadGuard}; -use shared_cache::{SharedValueCache, ValueCacheKey}; +use parking_lot::{Mutex, MutexGuard}; +use schnellru::LruMap; +use shared_cache::{ValueCacheKey, ValueCacheRef}; use std::{ - collections::{hash_map::Entry as MapEntry, HashMap}, - sync::Arc, + collections::HashMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::Duration, }; use trie_db::{node::NodeOwned, CachedValue}; @@ -50,29 +54,267 @@ mod shared_cache; pub use shared_cache::SharedTrieCache; -use self::shared_cache::{SharedTrieCacheInner, ValueCacheKeyHash}; +use self::shared_cache::ValueCacheKeyHash; const LOG_TARGET: &str = "trie-cache"; -/// The size of the cache. +/// The maximum amount of time we'll wait trying to acquire the shared cache lock +/// when the local cache is dropped and synchronized with the share cache. +/// +/// This is just a failsafe; normally this should never trigger. +const SHARED_CACHE_WRITE_LOCK_TIMEOUT: Duration = Duration::from_millis(100); + +/// The maximum number of existing keys in the shared cache that a single local cache +/// can promote to the front of the LRU cache in one go. +/// +/// If we have a big shared cache and the local cache hits all of those keys we don't +/// want to spend forever bumping all of them. +const SHARED_NODE_CACHE_MAX_PROMOTED_KEYS: u32 = 1792; +/// Same as [`SHARED_NODE_CACHE_MAX_PROMOTED_KEYS`]. +const SHARED_VALUE_CACHE_MAX_PROMOTED_KEYS: u32 = 1792; + +/// The maximum portion of the shared cache (in percent) that a single local +/// cache can replace in one go. +/// +/// We don't want a single local cache instance to have the ability to replace +/// everything in the shared cache. +const SHARED_NODE_CACHE_MAX_REPLACE_PERCENT: usize = 33; +/// Same as [`SHARED_NODE_CACHE_MAX_REPLACE_PERCENT`]. +const SHARED_VALUE_CACHE_MAX_REPLACE_PERCENT: usize = 33; + +/// The maximum inline capacity of the local cache, in bytes. +/// +/// This is just an upper limit; since the maps are resized in powers of two +/// their actual size will most likely not exactly match this. +const LOCAL_NODE_CACHE_MAX_INLINE_SIZE: usize = 512 * 1024; +/// Same as [`LOCAL_NODE_CACHE_MAX_INLINE_SIZE`]. +const LOCAL_VALUE_CACHE_MAX_INLINE_SIZE: usize = 512 * 1024; + +/// The maximum size of the memory allocated on the heap by the local cache, in bytes. +const LOCAL_NODE_CACHE_MAX_HEAP_SIZE: usize = 2 * 1024 * 1024; +/// Same as [`LOCAL_NODE_CACHE_MAX_HEAP_SIZE`]. +const LOCAL_VALUE_CACHE_MAX_HEAP_SIZE: usize = 4 * 1024 * 1024; + +/// The size of the shared cache. #[derive(Debug, Clone, Copy)] -pub enum CacheSize { - /// Do not limit the cache size. - Unlimited, - /// Let the cache in maximum use the given amount of bytes. - Maximum(usize), -} +pub struct CacheSize(usize); impl CacheSize { - /// Returns `true` if the `current_size` exceeds the allowed size. - fn exceeds(&self, current_size: usize) -> bool { - match self { - Self::Unlimited => false, - Self::Maximum(max) => *max < current_size, + /// An unlimited cache size. + pub const fn unlimited() -> Self { + CacheSize(usize::MAX) + } + + /// A cache size `bytes` big. + pub const fn new(bytes: usize) -> Self { + CacheSize(bytes) + } +} + +/// A limiter for the local node cache. This makes sure the local cache doesn't grow too big. +#[derive(Default)] +pub struct LocalNodeCacheLimiter { + /// The current size (in bytes) of data allocated by this cache on the heap. + /// + /// This doesn't include the size of the map itself. + current_heap_size: usize, +} + +impl schnellru::Limiter> for LocalNodeCacheLimiter +where + H: AsRef<[u8]> + std::fmt::Debug, +{ + type KeyToInsert<'a> = H; + type LinkType = u32; + + #[inline] + fn is_over_the_limit(&self, length: usize) -> bool { + // Only enforce the limit if there's more than one element to make sure + // we can always add a new element to the cache. + if length <= 1 { + return false } + + self.current_heap_size > LOCAL_NODE_CACHE_MAX_HEAP_SIZE + } + + #[inline] + fn on_insert<'a>( + &mut self, + _length: usize, + key: H, + cached_node: NodeCached, + ) -> Option<(H, NodeCached)> { + self.current_heap_size += cached_node.heap_size(); + Some((key, cached_node)) + } + + #[inline] + fn on_replace( + &mut self, + _length: usize, + _old_key: &mut H, + _new_key: H, + old_node: &mut NodeCached, + new_node: &mut NodeCached, + ) -> bool { + debug_assert_eq!(_old_key.as_ref().len(), _new_key.as_ref().len()); + self.current_heap_size = + self.current_heap_size + new_node.heap_size() - old_node.heap_size(); + true + } + + #[inline] + fn on_removed(&mut self, _key: &mut H, cached_node: &mut NodeCached) { + self.current_heap_size -= cached_node.heap_size(); + } + + #[inline] + fn on_cleared(&mut self) { + self.current_heap_size = 0; + } + + #[inline] + fn on_grow(&mut self, new_memory_usage: usize) -> bool { + new_memory_usage <= LOCAL_NODE_CACHE_MAX_INLINE_SIZE + } +} + +/// A limiter for the local value cache. This makes sure the local cache doesn't grow too big. +#[derive(Default)] +pub struct LocalValueCacheLimiter { + /// The current size (in bytes) of data allocated by this cache on the heap. + /// + /// This doesn't include the size of the map itself. + current_heap_size: usize, +} + +impl schnellru::Limiter, CachedValue> for LocalValueCacheLimiter +where + H: AsRef<[u8]>, +{ + type KeyToInsert<'a> = ValueCacheRef<'a, H>; + type LinkType = u32; + + #[inline] + fn is_over_the_limit(&self, length: usize) -> bool { + // Only enforce the limit if there's more than one element to make sure + // we can always add a new element to the cache. + if length <= 1 { + return false + } + + self.current_heap_size > LOCAL_VALUE_CACHE_MAX_HEAP_SIZE + } + + #[inline] + fn on_insert( + &mut self, + _length: usize, + key: Self::KeyToInsert<'_>, + value: CachedValue, + ) -> Option<(ValueCacheKey, CachedValue)> { + self.current_heap_size += key.storage_key.len(); + Some((key.into(), value)) + } + + #[inline] + fn on_replace( + &mut self, + _length: usize, + _old_key: &mut ValueCacheKey, + _new_key: ValueCacheRef, + _old_value: &mut CachedValue, + _new_value: &mut CachedValue, + ) -> bool { + debug_assert_eq!(_old_key.storage_key.len(), _new_key.storage_key.len()); + true + } + + #[inline] + fn on_removed(&mut self, key: &mut ValueCacheKey, _: &mut CachedValue) { + self.current_heap_size -= key.storage_key.len(); + } + + #[inline] + fn on_cleared(&mut self) { + self.current_heap_size = 0; + } + + #[inline] + fn on_grow(&mut self, new_memory_usage: usize) -> bool { + new_memory_usage <= LOCAL_VALUE_CACHE_MAX_INLINE_SIZE + } +} + +/// A struct to gather hit/miss stats to aid in debugging the performance of the cache. +#[derive(Default)] +struct HitStats { + shared_hits: AtomicU64, + shared_fetch_attempts: AtomicU64, + local_hits: AtomicU64, + local_fetch_attempts: AtomicU64, +} + +impl std::fmt::Display for HitStats { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + let shared_hits = self.shared_hits.load(Ordering::Relaxed); + let shared_fetch_attempts = self.shared_fetch_attempts.load(Ordering::Relaxed); + let local_hits = self.local_hits.load(Ordering::Relaxed); + let local_fetch_attempts = self.local_fetch_attempts.load(Ordering::Relaxed); + if shared_fetch_attempts == 0 && local_hits == 0 { + write!(fmt, "empty") + } else { + let percent_local = (local_hits as f32 / local_fetch_attempts as f32) * 100.0; + let percent_shared = (shared_hits as f32 / shared_fetch_attempts as f32) * 100.0; + write!( + fmt, + "local hit rate = {}% [{}/{}], shared hit rate = {}% [{}/{}]", + percent_local as u32, + local_hits, + local_fetch_attempts, + percent_shared as u32, + shared_hits, + shared_fetch_attempts + ) + } + } +} + +/// A struct to gather hit/miss stats for the node cache and the value cache. +#[derive(Default)] +struct TrieHitStats { + node_cache: HitStats, + value_cache: HitStats, +} + +/// An internal struct to store the cached trie nodes. +pub(crate) struct NodeCached { + /// The cached node. + pub node: NodeOwned, + /// Whether this node was fetched from the shared cache or not. + pub is_from_shared_cache: bool, +} + +impl NodeCached { + /// Returns the number of bytes allocated on the heap by this node. + fn heap_size(&self) -> usize { + self.node.size_in_bytes() - std::mem::size_of::>() } } +type NodeCacheMap = LruMap, LocalNodeCacheLimiter, schnellru::RandomState>; + +type ValueCacheMap = LruMap< + ValueCacheKey, + CachedValue, + LocalValueCacheLimiter, + BuildNoHashHasher>, +>; + +type ValueAccessSet = + LruMap>; + /// The local trie cache. /// /// This cache should be used per state instance created by the backend. One state instance is @@ -86,21 +328,13 @@ impl CacheSize { pub struct LocalTrieCache { /// The shared trie cache that created this instance. shared: SharedTrieCache, + /// The local cache for the trie nodes. - node_cache: Mutex>>, - /// Keeps track of all the trie nodes accessed in the shared cache. - /// - /// This will be used to ensure that these nodes are brought to the front of the lru when this - /// local instance is merged back to the shared cache. - shared_node_cache_access: Mutex>, + node_cache: Mutex>, + /// The local cache for the values. - value_cache: Mutex< - HashMap< - ValueCacheKey<'static, H::Out>, - CachedValue, - BuildNoHashHasher>, - >, - >, + value_cache: Mutex>, + /// Keeps track of all values accessed in the shared cache. /// /// This will be used to ensure that these nodes are brought to the front of the lru when this @@ -109,8 +343,9 @@ pub struct LocalTrieCache { /// as we only use this set to update the lru position it is fine, even if we bring the wrong /// value to the top. The important part is that we always get the correct value from the value /// cache for a given key. - shared_value_cache_access: - Mutex>>, + shared_value_cache_access: Mutex, + + stats: TrieHitStats, } impl LocalTrieCache { @@ -118,19 +353,18 @@ impl LocalTrieCache { /// /// The given `storage_root` needs to be the storage root of the trie this cache is used for. pub fn as_trie_db_cache(&self, storage_root: H::Out) -> TrieCache<'_, H> { - let shared_inner = self.shared.read_lock_inner(); - let value_cache = ValueCache::ForStorageRoot { storage_root, local_value_cache: self.value_cache.lock(), shared_value_cache_access: self.shared_value_cache_access.lock(), + buffered_value: None, }; TrieCache { - shared_inner, + shared_cache: self.shared.clone(), local_cache: self.node_cache.lock(), value_cache, - shared_node_cache_access: self.shared_node_cache_access.lock(), + stats: &self.stats, } } @@ -143,63 +377,89 @@ impl LocalTrieCache { /// would break because of this. pub fn as_trie_db_mut_cache(&self) -> TrieCache<'_, H> { TrieCache { - shared_inner: self.shared.read_lock_inner(), + shared_cache: self.shared.clone(), local_cache: self.node_cache.lock(), value_cache: ValueCache::Fresh(Default::default()), - shared_node_cache_access: self.shared_node_cache_access.lock(), + stats: &self.stats, } } } impl Drop for LocalTrieCache { fn drop(&mut self) { - let mut shared_inner = self.shared.write_lock_inner(); + tracing::debug!( + target: LOG_TARGET, + "Local node trie cache dropped: {}", + self.stats.node_cache + ); + + tracing::debug!( + target: LOG_TARGET, + "Local value trie cache dropped: {}", + self.stats.value_cache + ); - shared_inner - .node_cache_mut() - .update(self.node_cache.lock().drain(), self.shared_node_cache_access.lock().drain()); + let mut shared_inner = match self.shared.write_lock_inner() { + Some(inner) => inner, + None => { + tracing::warn!( + target: LOG_TARGET, + "Timeout while trying to acquire a write lock for the shared trie cache" + ); + return + }, + }; - shared_inner - .value_cache_mut() - .update(self.value_cache.lock().drain(), self.shared_value_cache_access.lock().drain()); + shared_inner.node_cache_mut().update(self.node_cache.get_mut().drain()); + + shared_inner.value_cache_mut().update( + self.value_cache.get_mut().drain(), + self.shared_value_cache_access.get_mut().drain().map(|(key, ())| key), + ); } } /// The abstraction of the value cache for the [`TrieCache`]. -enum ValueCache<'a, H> { +enum ValueCache<'a, H: Hasher> { /// The value cache is fresh, aka not yet associated to any storage root. /// This is used for example when a new trie is being build, to cache new values. - Fresh(HashMap, CachedValue>), + Fresh(HashMap, CachedValue>), /// The value cache is already bound to a specific storage root. ForStorageRoot { - shared_value_cache_access: MutexGuard< - 'a, - HashSet>, - >, - local_value_cache: MutexGuard< - 'a, - HashMap< - ValueCacheKey<'static, H>, - CachedValue, - nohash_hasher::BuildNoHashHasher>, - >, - >, - storage_root: H, + shared_value_cache_access: MutexGuard<'a, ValueAccessSet>, + local_value_cache: MutexGuard<'a, ValueCacheMap>, + storage_root: H::Out, + // The shared value cache needs to be temporarily locked when reading from it + // so we need to clone the value that is returned, but we need to be able to + // return a reference to the value, so we just buffer it here. + buffered_value: Option>, }, } -impl + std::hash::Hash + Eq + Clone + Copy> ValueCache<'_, H> { +impl ValueCache<'_, H> { /// Get the value for the given `key`. fn get<'a>( &'a mut self, key: &[u8], - shared_value_cache: &'a SharedValueCache, - ) -> Option<&CachedValue> { - match self { - Self::Fresh(map) => map.get(key), - Self::ForStorageRoot { local_value_cache, shared_value_cache_access, storage_root } => { - let key = ValueCacheKey::new_ref(key, *storage_root); + shared_cache: &SharedTrieCache, + stats: &HitStats, + ) -> Option<&CachedValue> { + stats.local_fetch_attempts.fetch_add(1, Ordering::Relaxed); + match self { + Self::Fresh(map) => + if let Some(value) = map.get(key) { + stats.local_hits.fetch_add(1, Ordering::Relaxed); + Some(value) + } else { + None + }, + Self::ForStorageRoot { + local_value_cache, + shared_value_cache_access, + storage_root, + buffered_value, + } => { // We first need to look up in the local cache and then the shared cache. // It can happen that some value is cached in the shared cache, but the // weak reference of the data can not be upgraded anymore. This for example @@ -207,35 +467,39 @@ impl + std::hash::Hash + Eq + Clone + Copy> ValueCache<'_, H> { // // So, the logic of the trie would lookup the data and the node and store both // in our local caches. - local_value_cache - .get(unsafe { - // SAFETY - // - // We need to convert the lifetime to make the compiler happy. However, as - // we only use the `key` to looking up the value this lifetime conversion is - // safe. - std::mem::transmute::<&ValueCacheKey<'_, H>, &ValueCacheKey<'static, H>>( - &key, - ) - }) - .or_else(|| { - shared_value_cache.get(&key).map(|v| { - shared_value_cache_access.insert(key.get_hash()); - v - }) - }) + + let hash = ValueCacheKey::hash_data(key, storage_root); + + if let Some(value) = local_value_cache + .peek_by_hash(hash.raw(), |existing_key, _| { + existing_key.is_eq(storage_root, key) + }) { + stats.local_hits.fetch_add(1, Ordering::Relaxed); + + return Some(value) + } + + stats.shared_fetch_attempts.fetch_add(1, Ordering::Relaxed); + if let Some(value) = shared_cache.peek_value_by_hash(hash, storage_root, key) { + stats.shared_hits.fetch_add(1, Ordering::Relaxed); + shared_value_cache_access.insert(hash, ()); + *buffered_value = Some(value.clone()); + return buffered_value.as_ref() + } + + None }, } } /// Insert some new `value` under the given `key`. - fn insert(&mut self, key: &[u8], value: CachedValue) { + fn insert(&mut self, key: &[u8], value: CachedValue) { match self { Self::Fresh(map) => { map.insert(key.into(), value); }, Self::ForStorageRoot { local_value_cache, storage_root, .. } => { - local_value_cache.insert(ValueCacheKey::new_value(key, *storage_root), value); + local_value_cache.insert(ValueCacheRef::new(key, *storage_root), value); }, } } @@ -247,10 +511,10 @@ impl + std::hash::Hash + Eq + Clone + Copy> ValueCache<'_, H> { /// be merged back into the [`LocalTrieCache`] with [`Self::merge_into`] after all operations are /// done. pub struct TrieCache<'a, H: Hasher> { - shared_inner: RwLockReadGuard<'a, SharedTrieCacheInner>, - shared_node_cache_access: MutexGuard<'a, HashSet>, - local_cache: MutexGuard<'a, HashMap>>, - value_cache: ValueCache<'a, H::Out>, + shared_cache: SharedTrieCache, + local_cache: MutexGuard<'a, NodeCacheMap>, + value_cache: ValueCache<'a, H>, + stats: &'a TrieHitStats, } impl<'a, H: Hasher> TrieCache<'a, H> { @@ -267,16 +531,11 @@ impl<'a, H: Hasher> TrieCache<'a, H> { let mut value_cache = local.value_cache.lock(); let partial_hash = ValueCacheKey::hash_partial_data(&storage_root); - cache - .into_iter() - .map(|(k, v)| { - let hash = - ValueCacheKeyHash::from_hasher_and_storage_key(partial_hash.clone(), &k); - (ValueCacheKey::Value { storage_key: k, storage_root, hash }, v) - }) - .for_each(|(k, v)| { - value_cache.insert(k, v); - }); + cache.into_iter().for_each(|(k, v)| { + let hash = ValueCacheKeyHash::from_hasher_and_storage_key(partial_hash.clone(), &k); + let k = ValueCacheRef { storage_root, storage_key: &k, hash }; + value_cache.insert(k, v); + }); } } } @@ -287,53 +546,85 @@ impl<'a, H: Hasher> trie_db::TrieCache> for TrieCache<'a, H> { hash: H::Out, fetch_node: &mut dyn FnMut() -> trie_db::Result, H::Out, Error>, ) -> trie_db::Result<&NodeOwned, H::Out, Error> { - if let Some(res) = self.shared_inner.node_cache().get(&hash) { - tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from shared cache"); - self.shared_node_cache_access.insert(hash); - return Ok(res) - } + let mut is_local_cache_hit = true; + self.stats.node_cache.local_fetch_attempts.fetch_add(1, Ordering::Relaxed); - match self.local_cache.entry(hash) { - MapEntry::Occupied(res) => { - tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from local cache"); - Ok(res.into_mut()) - }, - MapEntry::Vacant(vacant) => { - let node = (*fetch_node)(); + // First try to grab the node from the local cache. + let node = self.local_cache.get_or_insert_fallible(hash, || { + is_local_cache_hit = false; - tracing::trace!( - target: LOG_TARGET, - ?hash, - fetch_successful = node.is_ok(), - "Node not found, needed to fetch it." - ); + // It was not in the local cache; try the shared cache. + self.stats.node_cache.shared_fetch_attempts.fetch_add(1, Ordering::Relaxed); + if let Some(node) = self.shared_cache.peek_node(&hash) { + self.stats.node_cache.shared_hits.fetch_add(1, Ordering::Relaxed); + tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from shared cache"); - Ok(vacant.insert(node?)) - }, + return Ok(NodeCached:: { node: node.clone(), is_from_shared_cache: true }) + } + + // It was not in the shared cache; try fetching it from the database. + match fetch_node() { + Ok(node) => { + tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from database"); + Ok(NodeCached:: { node, is_from_shared_cache: false }) + }, + Err(error) => { + tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from database failed"); + Err(error) + }, + } + }); + + if is_local_cache_hit { + tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from local cache"); + self.stats.node_cache.local_hits.fetch_add(1, Ordering::Relaxed); } + + Ok(&node? + .expect("you can always insert at least one element into the local cache; qed") + .node) } fn get_node(&mut self, hash: &H::Out) -> Option<&NodeOwned> { - if let Some(node) = self.shared_inner.node_cache().get(hash) { - tracing::trace!(target: LOG_TARGET, ?hash, "Getting node from shared cache"); - self.shared_node_cache_access.insert(*hash); - return Some(node) - } + let mut is_local_cache_hit = true; + self.stats.node_cache.local_fetch_attempts.fetch_add(1, Ordering::Relaxed); - let res = self.local_cache.get(hash); + // First try to grab the node from the local cache. + let cached_node = self.local_cache.get_or_insert_fallible(*hash, || { + is_local_cache_hit = false; - tracing::trace!( - target: LOG_TARGET, - ?hash, - found = res.is_some(), - "Getting node from local cache" - ); + // It was not in the local cache; try the shared cache. + self.stats.node_cache.shared_fetch_attempts.fetch_add(1, Ordering::Relaxed); + if let Some(node) = self.shared_cache.peek_node(&hash) { + self.stats.node_cache.shared_hits.fetch_add(1, Ordering::Relaxed); + tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from shared cache"); - res + Ok(NodeCached:: { node: node.clone(), is_from_shared_cache: true }) + } else { + tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from cache failed"); + + Err(()) + } + }); + + if is_local_cache_hit { + tracing::trace!(target: LOG_TARGET, ?hash, "Serving node from local cache"); + self.stats.node_cache.local_hits.fetch_add(1, Ordering::Relaxed); + } + + match cached_node { + Ok(Some(cached_node)) => Some(&cached_node.node), + Ok(None) => { + unreachable!( + "you can always insert at least one element into the local cache; qed" + ); + }, + Err(()) => None, + } } fn lookup_value_for_key(&mut self, key: &[u8]) -> Option<&CachedValue> { - let res = self.value_cache.get(key, self.shared_inner.value_cache()); + let res = self.value_cache.get(key, &self.shared_cache, &self.stats.value_cache); tracing::trace!( target: LOG_TARGET, @@ -352,7 +643,7 @@ impl<'a, H: Hasher> trie_db::TrieCache> for TrieCache<'a, H> { "Caching value for key", ); - self.value_cache.insert(key.into(), data); + self.value_cache.insert(key, data); } } @@ -369,7 +660,7 @@ mod tests { const TEST_DATA: &[(&[u8], &[u8])] = &[(b"key1", b"val1"), (b"key2", &[2; 64]), (b"key3", b"val3"), (b"key4", &[4; 64])]; const CACHE_SIZE_RAW: usize = 1024 * 10; - const CACHE_SIZE: CacheSize = CacheSize::Maximum(CACHE_SIZE_RAW); + const CACHE_SIZE: CacheSize = CacheSize::new(CACHE_SIZE_RAW); fn create_trie() -> (MemoryDB, TrieHash) { let mut db = MemoryDB::default(); @@ -418,7 +709,7 @@ mod tests { let fake_data = Bytes::from(&b"fake_data"[..]); let local_cache = shared_cache.local_cache(); - shared_cache.write_lock_inner().value_cache_mut().lru.put( + shared_cache.write_lock_inner().unwrap().value_cache_mut().lru.insert( ValueCacheKey::new_value(TEST_DATA[1].0, root), (fake_data.clone(), Default::default()).into(), ); @@ -591,7 +882,7 @@ mod tests { .lru .iter() .map(|d| d.0) - .all(|l| TEST_DATA.iter().any(|d| l.storage_key().unwrap() == d.0))); + .all(|l| TEST_DATA.iter().any(|d| &*l.storage_key == d.0))); // Run this in a loop. The first time we check that with the filled value cache, // the expected values are at the top of the LRU. @@ -617,7 +908,7 @@ mod tests { .iter() .take(2) .map(|d| d.0) - .all(|l| { TEST_DATA.iter().take(2).any(|d| l.storage_key().unwrap() == d.0) })); + .all(|l| { TEST_DATA.iter().take(2).any(|d| &*l.storage_key == d.0) })); // Delete the value cache, so that we access the nodes. shared_cache.reset_value_cache(); @@ -684,9 +975,6 @@ mod tests { } } - let node_cache_size = shared_cache.read_lock_inner().node_cache().size_in_bytes; - let value_cache_size = shared_cache.read_lock_inner().value_cache().size_in_bytes; - - assert!(node_cache_size + value_cache_size < CACHE_SIZE_RAW); + assert!(shared_cache.used_memory_size() < CACHE_SIZE_RAW); } } diff --git a/primitives/trie/src/cache/shared_cache.rs b/primitives/trie/src/cache/shared_cache.rs index 9d4d36b83a28a..8c60d5043d062 100644 --- a/primitives/trie/src/cache/shared_cache.rs +++ b/primitives/trie/src/cache/shared_cache.rs @@ -17,15 +17,14 @@ ///! Provides the [`SharedNodeCache`], the [`SharedValueCache`] and the [`SharedTrieCache`] ///! that combines both caches and is exported to the outside. -use super::{CacheSize, LOG_TARGET}; +use super::{CacheSize, NodeCached}; use hash_db::Hasher; use hashbrown::{hash_set::Entry as SetEntry, HashSet}; -use lru::LruCache; use nohash_hasher::BuildNoHashHasher; -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; +use parking_lot::{Mutex, RwLock, RwLockWriteGuard}; +use schnellru::LruMap; use std::{ hash::{BuildHasher, Hasher as _}, - mem, sync::Arc, }; use trie_db::{node::NodeOwned, CachedValue}; @@ -34,94 +33,300 @@ lazy_static::lazy_static! { static ref RANDOM_STATE: ahash::RandomState = ahash::RandomState::default(); } -/// No hashing [`LruCache`]. -type NoHashingLruCache = LruCache>; +pub struct SharedNodeCacheLimiter { + /// The maximum size (in bytes) the cache can hold inline. + /// + /// This space is always consumed whether there are any items in the map or not. + max_inline_size: usize, + + /// The maximum size (in bytes) the cache can hold on the heap. + max_heap_size: usize, + + /// The current size (in bytes) of data allocated by this cache on the heap. + /// + /// This doesn't include the size of the map itself. + heap_size: usize, + + /// A counter with the number of elements that got evicted from the cache. + /// + /// Reset to zero before every update. + items_evicted: usize, + + /// The maximum number of elements that we allow to be evicted. + /// + /// Reset on every update. + max_items_evicted: usize, +} + +impl schnellru::Limiter> for SharedNodeCacheLimiter +where + H: AsRef<[u8]>, +{ + type KeyToInsert<'a> = H; + type LinkType = u32; + + #[inline] + fn is_over_the_limit(&self, _length: usize) -> bool { + // Once we hit the limit of max items evicted this will return `false` and prevent + // any further evictions, but this is fine because the outer loop which inserts + // items into this cache will just detect this and stop inserting new items. + self.items_evicted <= self.max_items_evicted && self.heap_size > self.max_heap_size + } + + #[inline] + fn on_insert( + &mut self, + _length: usize, + key: Self::KeyToInsert<'_>, + node: NodeOwned, + ) -> Option<(H, NodeOwned)> { + let new_item_heap_size = node.size_in_bytes() - std::mem::size_of::>(); + if new_item_heap_size > self.max_heap_size { + // Item's too big to add even if the cache's empty; bail. + return None + } + + self.heap_size += new_item_heap_size; + Some((key, node)) + } + + #[inline] + fn on_replace( + &mut self, + _length: usize, + _old_key: &mut H, + _new_key: H, + old_node: &mut NodeOwned, + new_node: &mut NodeOwned, + ) -> bool { + debug_assert_eq!(_old_key.as_ref(), _new_key.as_ref()); + + let new_item_heap_size = new_node.size_in_bytes() - std::mem::size_of::>(); + if new_item_heap_size > self.max_heap_size { + // Item's too big to add even if the cache's empty; bail. + return false + } + + let old_item_heap_size = old_node.size_in_bytes() - std::mem::size_of::>(); + self.heap_size = self.heap_size - old_item_heap_size + new_item_heap_size; + true + } + + #[inline] + fn on_cleared(&mut self) { + self.heap_size = 0; + } + + #[inline] + fn on_removed(&mut self, _: &mut H, node: &mut NodeOwned) { + self.heap_size -= node.size_in_bytes() - std::mem::size_of::>(); + self.items_evicted += 1; + } + + #[inline] + fn on_grow(&mut self, new_memory_usage: usize) -> bool { + new_memory_usage <= self.max_inline_size + } +} + +pub struct SharedValueCacheLimiter { + /// The maximum size (in bytes) the cache can hold inline. + /// + /// This space is always consumed whether there are any items in the map or not. + max_inline_size: usize, + + /// The maximum size (in bytes) the cache can hold on the heap. + max_heap_size: usize, + + /// The current size (in bytes) of data allocated by this cache on the heap. + /// + /// This doesn't include the size of the map itself. + heap_size: usize, + + /// A set with all of the keys deduplicated to save on memory. + known_storage_keys: HashSet>, + + /// A counter with the number of elements that got evicted from the cache. + /// + /// Reset to zero before every update. + items_evicted: usize, + + /// The maximum number of elements that we allow to be evicted. + /// + /// Reset on every update. + max_items_evicted: usize, +} + +impl schnellru::Limiter, CachedValue> for SharedValueCacheLimiter +where + H: AsRef<[u8]>, +{ + type KeyToInsert<'a> = ValueCacheKey; + type LinkType = u32; + + #[inline] + fn is_over_the_limit(&self, _length: usize) -> bool { + self.items_evicted <= self.max_items_evicted && self.heap_size > self.max_heap_size + } + + #[inline] + fn on_insert( + &mut self, + _length: usize, + mut key: Self::KeyToInsert<'_>, + value: CachedValue, + ) -> Option<(ValueCacheKey, CachedValue)> { + match self.known_storage_keys.entry(key.storage_key.clone()) { + SetEntry::Vacant(entry) => { + let new_item_heap_size = key.storage_key.len(); + if new_item_heap_size > self.max_heap_size { + // Item's too big to add even if the cache's empty; bail. + return None + } + + self.heap_size += new_item_heap_size; + entry.insert(); + }, + SetEntry::Occupied(entry) => { + key.storage_key = entry.get().clone(); + }, + } + + Some((key, value)) + } + + #[inline] + fn on_replace( + &mut self, + _length: usize, + _old_key: &mut ValueCacheKey, + _new_key: ValueCacheKey, + _old_value: &mut CachedValue, + _new_value: &mut CachedValue, + ) -> bool { + debug_assert_eq!(_new_key.storage_key, _old_key.storage_key); + true + } + + #[inline] + fn on_removed(&mut self, key: &mut ValueCacheKey, _: &mut CachedValue) { + if Arc::strong_count(&key.storage_key) == 2 { + // There are only two instances of this key: + // 1) one memoized in `known_storage_keys`, + // 2) one inside the map. + // + // This means that after this remove goes through the `Arc` will be deallocated. + self.heap_size -= key.storage_key.len(); + self.known_storage_keys.remove(&key.storage_key); + } + self.items_evicted += 1; + } + + #[inline] + fn on_cleared(&mut self) { + self.heap_size = 0; + self.known_storage_keys.clear(); + } + + #[inline] + fn on_grow(&mut self, new_memory_usage: usize) -> bool { + new_memory_usage <= self.max_inline_size + } +} + +type SharedNodeCacheMap = + LruMap, SharedNodeCacheLimiter, schnellru::RandomState>; /// The shared node cache. /// -/// Internally this stores all cached nodes in a [`LruCache`]. It ensures that when updating the +/// Internally this stores all cached nodes in a [`LruMap`]. It ensures that when updating the /// cache, that the cache stays within its allowed bounds. -pub(super) struct SharedNodeCache { +pub(super) struct SharedNodeCache +where + H: AsRef<[u8]>, +{ /// The cached nodes, ordered by least recently used. - pub(super) lru: LruCache>, - /// The size of [`Self::lru`] in bytes. - pub(super) size_in_bytes: usize, - /// The maximum cache size of [`Self::lru`]. - maximum_cache_size: CacheSize, + pub(super) lru: SharedNodeCacheMap, } impl + Eq + std::hash::Hash> SharedNodeCache { /// Create a new instance. - fn new(cache_size: CacheSize) -> Self { - Self { lru: LruCache::unbounded(), size_in_bytes: 0, maximum_cache_size: cache_size } + fn new(max_inline_size: usize, max_heap_size: usize) -> Self { + Self { + lru: LruMap::new(SharedNodeCacheLimiter { + max_inline_size, + max_heap_size, + heap_size: 0, + items_evicted: 0, + max_items_evicted: 0, // Will be set during `update`. + }), + } } - /// Get the node for `key`. - /// - /// This doesn't change the least recently order in the internal [`LruCache`]. - pub fn get(&self, key: &H) -> Option<&NodeOwned> { - self.lru.peek(key) - } + /// Update the cache with the `list` of nodes which were either newly added or accessed. + pub fn update(&mut self, list: impl IntoIterator)>) { + let mut access_count = 0; + let mut add_count = 0; - /// Update the cache with the `added` nodes and the `accessed` nodes. - /// - /// The `added` nodes are the ones that have been collected by doing operations on the trie and - /// now should be stored in the shared cache. The `accessed` nodes are only referenced by hash - /// and represent the nodes that were retrieved from this shared cache through [`Self::get`]. - /// These `accessed` nodes are being put to the front of the internal [`LruCache`] like the - /// `added` ones. - /// - /// After the internal [`LruCache`] was updated, it is ensured that the internal [`LruCache`] is - /// inside its bounds ([`Self::maximum_size_in_bytes`]). - pub fn update( - &mut self, - added: impl IntoIterator)>, - accessed: impl IntoIterator, - ) { - let update_size_in_bytes = |size_in_bytes: &mut usize, key: &H, node: &NodeOwned| { - if let Some(new_size_in_bytes) = - size_in_bytes.checked_sub(key.as_ref().len() + node.size_in_bytes()) - { - *size_in_bytes = new_size_in_bytes; - } else { - *size_in_bytes = 0; - tracing::error!(target: LOG_TARGET, "`SharedNodeCache` underflow detected!",); - } - }; + self.lru.limiter_mut().items_evicted = 0; + self.lru.limiter_mut().max_items_evicted = + self.lru.len() * 100 / super::SHARED_NODE_CACHE_MAX_REPLACE_PERCENT; - accessed.into_iter().for_each(|key| { - // Access every node in the lru to put it to the front. - self.lru.get(&key); - }); - added.into_iter().for_each(|(key, node)| { - self.size_in_bytes += key.as_ref().len() + node.size_in_bytes(); + for (key, cached_node) in list { + if cached_node.is_from_shared_cache { + if self.lru.get(&key).is_some() { + access_count += 1; - if let Some((r_key, r_node)) = self.lru.push(key, node) { - update_size_in_bytes(&mut self.size_in_bytes, &r_key, &r_node); - } + if access_count >= super::SHARED_NODE_CACHE_MAX_PROMOTED_KEYS { + // Stop when we've promoted a large enough number of items. + break + } - // Directly ensure that we respect the maximum size. By doing it directly here we ensure - // that the internal map of the [`LruCache`] doesn't grow too much. - while self.maximum_cache_size.exceeds(self.size_in_bytes) { - // This should always be `Some(_)`, otherwise something is wrong! - if let Some((key, node)) = self.lru.pop_lru() { - update_size_in_bytes(&mut self.size_in_bytes, &key, &node); + continue } } - }); + + self.lru.insert(key, cached_node.node); + add_count += 1; + + if self.lru.limiter().items_evicted > self.lru.limiter().max_items_evicted { + // Stop when we've evicted a big enough chunk of the shared cache. + break + } + } + + tracing::debug!( + target: super::LOG_TARGET, + "Updated the shared node cache: {} accesses, {} new values, {}/{} evicted (length = {}, inline size={}/{}, heap size={}/{})", + access_count, + add_count, + self.lru.limiter().items_evicted, + self.lru.limiter().max_items_evicted, + self.lru.len(), + self.lru.memory_usage(), + self.lru.limiter().max_inline_size, + self.lru.limiter().heap_size, + self.lru.limiter().max_heap_size, + ); } /// Reset the cache. fn reset(&mut self) { - self.size_in_bytes = 0; self.lru.clear(); } } /// The hash of [`ValueCacheKey`]. -#[derive(Eq, Clone, Copy)] +#[derive(PartialEq, Eq, Clone, Copy, Hash)] +#[repr(transparent)] pub struct ValueCacheKeyHash(u64); +impl ValueCacheKeyHash { + pub fn raw(self) -> u64 { + self.0 + } +} + impl ValueCacheKeyHash { pub fn from_hasher_and_storage_key( mut hasher: impl std::hash::Hasher, @@ -133,88 +338,75 @@ impl ValueCacheKeyHash { } } -impl PartialEq for ValueCacheKeyHash { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } +impl nohash_hasher::IsEnabled for ValueCacheKeyHash {} + +/// The key type that is being used to address a [`CachedValue`]. +#[derive(Eq)] +pub(super) struct ValueCacheKey { + /// The storage root of the trie this key belongs to. + pub storage_root: H, + /// The key to access the value in the storage. + pub storage_key: Arc<[u8]>, + /// The hash that identifies this instance of `storage_root` and `storage_key`. + pub hash: ValueCacheKeyHash, } -impl std::hash::Hash for ValueCacheKeyHash { - fn hash(&self, state: &mut Hasher) { - state.write_u64(self.0); +/// A borrowed variant of [`ValueCacheKey`]. +pub(super) struct ValueCacheRef<'a, H> { + /// The storage root of the trie this key belongs to. + pub storage_root: H, + /// The key to access the value in the storage. + pub storage_key: &'a [u8], + /// The hash that identifies this instance of `storage_root` and `storage_key`. + pub hash: ValueCacheKeyHash, +} + +impl<'a, H> ValueCacheRef<'a, H> { + pub fn new(storage_key: &'a [u8], storage_root: H) -> Self + where + H: AsRef<[u8]>, + { + let hash = ValueCacheKey::::hash_data(&storage_key, &storage_root); + Self { storage_root, storage_key, hash } } } -impl nohash_hasher::IsEnabled for ValueCacheKeyHash {} +impl<'a, H> From> for ValueCacheKey { + fn from(value: ValueCacheRef<'a, H>) -> Self { + ValueCacheKey { + storage_root: value.storage_root, + storage_key: value.storage_key.into(), + hash: value.hash, + } + } +} -/// A type that can only be constructed inside of this file. -/// -/// It "requires" that the user has read the docs to prevent fuck ups. -#[derive(Eq, PartialEq)] -pub(super) struct IReadTheDocumentation(()); +impl<'a, H: std::hash::Hash> std::hash::Hash for ValueCacheRef<'a, H> { + fn hash(&self, state: &mut Hasher) { + self.hash.hash(state) + } +} -/// The key type that is being used to address a [`CachedValue`]. -/// -/// This type is implemented as `enum` to improve the performance when accessing the value cache. -/// The problem being that we need to calculate the `hash` of [`Self`] in worst case three times -/// when trying to find a value in the value cache. First to lookup the local cache, then the shared -/// cache and if we found it in the shared cache a third time to insert it into the list of accessed -/// values. To work around each variant stores the `hash` to identify a unique combination of -/// `storage_key` and `storage_root`. However, be aware that this `hash` can lead to collisions when -/// there are two different `storage_key` and `storage_root` pairs that map to the same `hash`. This -/// type also has the `Hash` variant. This variant should only be used for the use case of updating -/// the lru for a key. Because when using only the `Hash` variant to getting a value from a hash map -/// it could happen that a wrong value is returned when there is another key in the same hash map -/// that maps to the same `hash`. The [`PartialEq`] implementation is written in a way that when one -/// of the two compared instances is the `Hash` variant, we will only compare the hashes. This -/// ensures that we can use the `Hash` variant to bring values up in the lru. -#[derive(Eq)] -pub(super) enum ValueCacheKey<'a, H> { - /// Variant that stores the `storage_key` by value. - Value { - /// The storage root of the trie this key belongs to. - storage_root: H, - /// The key to access the value in the storage. - storage_key: Arc<[u8]>, - /// The hash that identifying this instance of `storage_root` and `storage_key`. - hash: ValueCacheKeyHash, - }, - /// Variant that only references the `storage_key`. - Ref { - /// The storage root of the trie this key belongs to. - storage_root: H, - /// The key to access the value in the storage. - storage_key: &'a [u8], - /// The hash that identifying this instance of `storage_root` and `storage_key`. - hash: ValueCacheKeyHash, - }, - /// Variant that only stores the hash that represents the `storage_root` and `storage_key`. - /// - /// This should be used by caution, because it can lead to accessing the wrong value in a - /// hash map/set when there exists two different `storage_root`s and `storage_key`s that - /// map to the same `hash`. - Hash { hash: ValueCacheKeyHash, _i_read_the_documentation: IReadTheDocumentation }, +impl<'a, H> PartialEq> for ValueCacheRef<'a, H> +where + H: AsRef<[u8]>, +{ + fn eq(&self, rhs: &ValueCacheKey) -> bool { + self.storage_root.as_ref() == rhs.storage_root.as_ref() && + self.storage_key == &*rhs.storage_key + } } -impl<'a, H> ValueCacheKey<'a, H> { +impl ValueCacheKey { /// Constructs [`Self::Value`]. + #[cfg(test)] // Only used in tests. pub fn new_value(storage_key: impl Into>, storage_root: H) -> Self where H: AsRef<[u8]>, { let storage_key = storage_key.into(); let hash = Self::hash_data(&storage_key, &storage_root); - Self::Value { storage_root, storage_key, hash } - } - - /// Constructs [`Self::Ref`]. - pub fn new_ref(storage_key: &'a [u8], storage_root: H) -> Self - where - H: AsRef<[u8]>, - { - let storage_key = storage_key.into(); - let hash = Self::hash_data(storage_key, &storage_root); - Self::Ref { storage_root, storage_key, hash } + Self { storage_root, storage_key, hash } } /// Returns a hasher prepared to build the final hash to identify [`Self`]. @@ -241,231 +433,133 @@ impl<'a, H> ValueCacheKey<'a, H> { ValueCacheKeyHash::from_hasher_and_storage_key(hasher, key) } - /// Returns the `hash` that identifies the current instance. - pub fn get_hash(&self) -> ValueCacheKeyHash { - match self { - Self::Value { hash, .. } | Self::Ref { hash, .. } | Self::Hash { hash, .. } => *hash, - } - } - - /// Returns the stored storage root. - pub fn storage_root(&self) -> Option<&H> { - match self { - Self::Value { storage_root, .. } | Self::Ref { storage_root, .. } => Some(storage_root), - Self::Hash { .. } => None, - } - } - - /// Returns the stored storage key. - pub fn storage_key(&self) -> Option<&[u8]> { - match self { - Self::Ref { storage_key, .. } => Some(&storage_key), - Self::Value { storage_key, .. } => Some(storage_key), - Self::Hash { .. } => None, - } + /// Checks whether the key is equal to the given `storage_key` and `storage_root`. + #[inline] + pub fn is_eq(&self, storage_root: &H, storage_key: &[u8]) -> bool + where + H: PartialEq, + { + self.storage_root == *storage_root && *self.storage_key == *storage_key } } -// Implement manually to ensure that the `Value` and `Hash` are treated equally. -impl std::hash::Hash for ValueCacheKey<'_, H> { +// Implement manually so that only `hash` is accessed. +impl std::hash::Hash for ValueCacheKey { fn hash(&self, state: &mut Hasher) { - self.get_hash().hash(state) + self.hash.hash(state) } } -impl nohash_hasher::IsEnabled for ValueCacheKey<'_, H> {} +impl nohash_hasher::IsEnabled for ValueCacheKey {} -// Implement manually to ensure that the `Value` and `Hash` are treated equally. -impl PartialEq for ValueCacheKey<'_, H> { +// Implement manually to not have to compare `hash`. +impl PartialEq for ValueCacheKey { + #[inline] fn eq(&self, other: &Self) -> bool { - // First check if `self` or `other` is only the `Hash`. - // Then we only compare the `hash`. So, there could actually be some collision - // if two different storage roots and keys are mapping to the same key. See the - // [`ValueCacheKey`] docs for more information. - match (self, other) { - (Self::Hash { hash, .. }, Self::Hash { hash: other_hash, .. }) => hash == other_hash, - (Self::Hash { hash, .. }, _) => *hash == other.get_hash(), - (_, Self::Hash { hash: other_hash, .. }) => self.get_hash() == *other_hash, - // If both are not the `Hash` variant, we compare all the values. - _ => - self.get_hash() == other.get_hash() && - self.storage_root() == other.storage_root() && - self.storage_key() == other.storage_key(), - } + self.is_eq(&other.storage_root, &other.storage_key) } } +type SharedValueCacheMap = schnellru::LruMap< + ValueCacheKey, + CachedValue, + SharedValueCacheLimiter, + BuildNoHashHasher>, +>; + /// The shared value cache. /// /// The cache ensures that it stays in the configured size bounds. -pub(super) struct SharedValueCache { +pub(super) struct SharedValueCache +where + H: AsRef<[u8]>, +{ /// The cached nodes, ordered by least recently used. - pub(super) lru: NoHashingLruCache, CachedValue>, - /// The size of [`Self::lru`] in bytes. - pub(super) size_in_bytes: usize, - /// The maximum cache size of [`Self::lru`]. - maximum_cache_size: CacheSize, - /// All known storage keys that are stored in [`Self::lru`]. - /// - /// This is used to de-duplicate keys in memory that use the - /// same [`SharedValueCache::storage_key`], but have a different - /// [`SharedValueCache::storage_root`]. - known_storage_keys: HashSet>, + pub(super) lru: SharedValueCacheMap, } impl> SharedValueCache { /// Create a new instance. - fn new(cache_size: CacheSize) -> Self { + fn new(max_inline_size: usize, max_heap_size: usize) -> Self { Self { - lru: NoHashingLruCache::unbounded_with_hasher(Default::default()), - size_in_bytes: 0, - maximum_cache_size: cache_size, - known_storage_keys: Default::default(), + lru: schnellru::LruMap::with_hasher( + SharedValueCacheLimiter { + max_inline_size, + max_heap_size, + heap_size: 0, + known_storage_keys: Default::default(), + items_evicted: 0, + max_items_evicted: 0, // Will be set during `update`. + }, + Default::default(), + ), } } - /// Get the [`CachedValue`] for `key`. - /// - /// This doesn't change the least recently order in the internal [`LruCache`]. - pub fn get<'a>(&'a self, key: &ValueCacheKey) -> Option<&'a CachedValue> { - debug_assert!( - !matches!(key, ValueCacheKey::Hash { .. }), - "`get` can not be called with `Hash` variant as this may returns the wrong value." - ); - - self.lru.peek(unsafe { - // SAFETY - // - // We need to convert the lifetime to make the compiler happy. However, as - // we only use the `key` to looking up the value this lifetime conversion is - // safe. - mem::transmute::<&ValueCacheKey<'_, H>, &ValueCacheKey<'static, H>>(key) - }) - } - /// Update the cache with the `added` values and the `accessed` values. /// /// The `added` values are the ones that have been collected by doing operations on the trie and /// now should be stored in the shared cache. The `accessed` values are only referenced by the - /// [`ValueCacheKeyHash`] and represent the values that were retrieved from this shared cache - /// through [`Self::get`]. These `accessed` values are being put to the front of the internal - /// [`LruCache`] like the `added` ones. - /// - /// After the internal [`LruCache`] was updated, it is ensured that the internal [`LruCache`] is - /// inside its bounds ([`Self::maximum_size_in_bytes`]). + /// [`ValueCacheKeyHash`] and represent the values that were retrieved from this shared cache. + /// These `accessed` values are being put to the front of the internal [`LruMap`] like the + /// `added` ones. pub fn update( &mut self, - added: impl IntoIterator, CachedValue)>, + added: impl IntoIterator, CachedValue)>, accessed: impl IntoIterator, ) { - // The base size in memory per ([`ValueCacheKey`], [`CachedValue`]). - let base_size = mem::size_of::>() + mem::size_of::>(); - let known_keys_entry_size = mem::size_of::>(); - - let update_size_in_bytes = - |size_in_bytes: &mut usize, r_key: Arc<[u8]>, known_keys: &mut HashSet>| { - // If the `strong_count == 2`, it means this is the last instance of the key. - // One being `r_key` and the other being stored in `known_storage_keys`. - let last_instance = Arc::strong_count(&r_key) == 2; - - let key_len = if last_instance { - known_keys.remove(&r_key); - r_key.len() + known_keys_entry_size - } else { - // The key is still in `keys`, because it is still used by another - // `ValueCacheKey`. - 0 - }; - - if let Some(new_size_in_bytes) = size_in_bytes.checked_sub(key_len + base_size) { - *size_in_bytes = new_size_in_bytes; - } else { - *size_in_bytes = 0; - tracing::error!(target: LOG_TARGET, "`SharedValueCache` underflow detected!",); - } - }; - - accessed.into_iter().for_each(|key| { - // Access every node in the lru to put it to the front. - // As we are using the `Hash` variant here, it may leads to putting the wrong value to - // the top. However, the only consequence of this is that we may prune a recently used - // value to early. - self.lru.get(&ValueCacheKey::Hash { - hash: key, - _i_read_the_documentation: IReadTheDocumentation(()), - }); - }); - - added.into_iter().for_each(|(key, value)| { - let (storage_root, storage_key, key_hash) = match key { - ValueCacheKey::Hash { .. } => { - // Ignore the hash variant and try the next. - tracing::error!( - target: LOG_TARGET, - "`SharedValueCached::update` was called with a key to add \ - that uses the `Hash` variant. This would lead to potential hash collision!", - ); - return - }, - ValueCacheKey::Ref { storage_key, storage_root, hash } => - (storage_root, storage_key.into(), hash), - ValueCacheKey::Value { storage_root, storage_key, hash } => - (storage_root, storage_key, hash), - }; - - let (size_update, storage_key) = - match self.known_storage_keys.entry(storage_key.clone()) { - SetEntry::Vacant(v) => { - let len = v.get().len(); - v.insert(); - - // If the key was unknown, we need to also take its length and the size of - // the entry of `known_keys` into account. - (len + base_size + known_keys_entry_size, storage_key) - }, - SetEntry::Occupied(o) => { - // Key is known - (base_size, o.get().clone()) - }, - }; - - self.size_in_bytes += size_update; - - if let Some((r_key, _)) = self - .lru - .push(ValueCacheKey::Value { storage_key, storage_root, hash: key_hash }, value) - { - if let ValueCacheKey::Value { storage_key, .. } = r_key { - update_size_in_bytes( - &mut self.size_in_bytes, - storage_key, - &mut self.known_storage_keys, - ); - } - } + let mut access_count = 0; + let mut add_count = 0; - // Directly ensure that we respect the maximum size. By doing it directly here we - // ensure that the internal map of the [`LruCache`] doesn't grow too much. - while self.maximum_cache_size.exceeds(self.size_in_bytes) { - // This should always be `Some(_)`, otherwise something is wrong! - if let Some((r_key, _)) = self.lru.pop_lru() { - if let ValueCacheKey::Value { storage_key, .. } = r_key { - update_size_in_bytes( - &mut self.size_in_bytes, - storage_key, - &mut self.known_storage_keys, - ); - } - } + for hash in accessed { + // Access every node in the map to put it to the front. + // + // Since we are only comparing the hashes here it may lead us to promoting the wrong + // values as the most recently accessed ones. However this is harmless as the only + // consequence is that we may accidentally prune a recently used value too early. + self.lru.get_by_hash(hash.raw(), |existing_key, _| existing_key.hash == hash); + access_count += 1; + } + + // Insert all of the new items which were *not* found in the shared cache. + // + // Limit how many items we'll replace in the shared cache in one go so that + // we don't evict the whole shared cache nor we keep spinning our wheels + // evicting items which we've added ourselves in previous iterations of this loop. + + self.lru.limiter_mut().items_evicted = 0; + self.lru.limiter_mut().max_items_evicted = + self.lru.len() * 100 / super::SHARED_VALUE_CACHE_MAX_REPLACE_PERCENT; + + for (key, value) in added { + self.lru.insert(key, value); + add_count += 1; + + if self.lru.limiter().items_evicted > self.lru.limiter().max_items_evicted { + // Stop when we've evicted a big enough chunk of the shared cache. + break } - }); + } + + tracing::debug!( + target: super::LOG_TARGET, + "Updated the shared value cache: {} accesses, {} new values, {}/{} evicted (length = {}, known_storage_keys = {}, inline size={}/{}, heap size={}/{})", + access_count, + add_count, + self.lru.limiter().items_evicted, + self.lru.limiter().max_items_evicted, + self.lru.len(), + self.lru.limiter().known_storage_keys.len(), + self.lru.memory_usage(), + self.lru.limiter().max_inline_size, + self.lru.limiter().heap_size, + self.lru.limiter().max_heap_size + ); } /// Reset the cache. fn reset(&mut self) { - self.size_in_bytes = 0; self.lru.clear(); - self.known_storage_keys.clear(); } } @@ -477,6 +571,7 @@ pub(super) struct SharedTrieCacheInner { impl SharedTrieCacheInner { /// Returns a reference to the [`SharedValueCache`]. + #[cfg(test)] pub(super) fn value_cache(&self) -> &SharedValueCache { &self.value_cache } @@ -487,6 +582,7 @@ impl SharedTrieCacheInner { } /// Returns a reference to the [`SharedNodeCache`]. + #[cfg(test)] pub(super) fn node_cache(&self) -> &SharedNodeCache { &self.node_cache } @@ -517,23 +613,50 @@ impl Clone for SharedTrieCache { impl SharedTrieCache { /// Create a new [`SharedTrieCache`]. pub fn new(cache_size: CacheSize) -> Self { - let (node_cache_size, value_cache_size) = match cache_size { - CacheSize::Maximum(max) => { - // Allocate 20% for the value cache. - let value_cache_size_in_bytes = (max as f32 * 0.20) as usize; - - ( - CacheSize::Maximum(max - value_cache_size_in_bytes), - CacheSize::Maximum(value_cache_size_in_bytes), - ) - }, - CacheSize::Unlimited => (CacheSize::Unlimited, CacheSize::Unlimited), - }; + let total_budget = cache_size.0; + + // Split our memory budget between the two types of caches. + let value_cache_budget = (total_budget as f32 * 0.20) as usize; // 20% for the value cache + let node_cache_budget = total_budget - value_cache_budget; // 80% for the node cache + + // Split our memory budget between what we'll be holding inline in the map, + // and what we'll be holding on the heap. + let value_cache_inline_budget = (value_cache_budget as f32 * 0.70) as usize; + let node_cache_inline_budget = (node_cache_budget as f32 * 0.70) as usize; + + // Calculate how much memory the maps will be allowed to hold inline given our budget. + let value_cache_max_inline_size = + SharedValueCacheMap::::memory_usage_for_memory_budget( + value_cache_inline_budget, + ); + + let node_cache_max_inline_size = + SharedNodeCacheMap::::memory_usage_for_memory_budget(node_cache_inline_budget); + + // And this is how much data we'll at most keep on the heap for each cache. + let value_cache_max_heap_size = value_cache_budget - value_cache_max_inline_size; + let node_cache_max_heap_size = node_cache_budget - node_cache_max_inline_size; + + tracing::debug!( + target: super::LOG_TARGET, + "Configured a shared trie cache with a budget of ~{} bytes (node_cache_max_inline_size = {}, node_cache_max_heap_size = {}, value_cache_max_inline_size = {}, value_cache_max_heap_size = {})", + total_budget, + node_cache_max_inline_size, + node_cache_max_heap_size, + value_cache_max_inline_size, + value_cache_max_heap_size, + ); Self { inner: Arc::new(RwLock::new(SharedTrieCacheInner { - node_cache: SharedNodeCache::new(node_cache_size), - value_cache: SharedValueCache::new(value_cache_size), + node_cache: SharedNodeCache::new( + node_cache_max_inline_size, + node_cache_max_heap_size, + ), + value_cache: SharedValueCache::new( + value_cache_max_inline_size, + value_cache_max_heap_size, + ), })), } } @@ -544,16 +667,50 @@ impl SharedTrieCache { shared: self.clone(), node_cache: Default::default(), value_cache: Default::default(), - shared_node_cache_access: Default::default(), - shared_value_cache_access: Default::default(), + shared_value_cache_access: Mutex::new(super::ValueAccessSet::with_hasher( + schnellru::ByLength::new(super::SHARED_VALUE_CACHE_MAX_PROMOTED_KEYS), + Default::default(), + )), + stats: Default::default(), } } + /// Get a copy of the node for `key`. + /// + /// This will temporarily lock the shared cache for reading. + /// + /// This doesn't change the least recently order in the internal [`LruMap`]. + #[inline] + pub fn peek_node(&self, key: &H::Out) -> Option> { + self.inner.read().node_cache.lru.peek(key).cloned() + } + + /// Get a copy of the [`CachedValue`] for `key`. + /// + /// This will temporarily lock the shared cache for reading. + /// + /// This doesn't reorder any of the elements in the internal [`LruMap`]. + pub fn peek_value_by_hash( + &self, + hash: ValueCacheKeyHash, + storage_root: &H::Out, + storage_key: &[u8], + ) -> Option> { + self.inner + .read() + .value_cache + .lru + .peek_by_hash(hash.0, |existing_key, _| existing_key.is_eq(storage_root, storage_key)) + .cloned() + } + /// Returns the used memory size of this cache in bytes. pub fn used_memory_size(&self) -> usize { let inner = self.inner.read(); - let value_cache_size = inner.value_cache.size_in_bytes; - let node_cache_size = inner.node_cache.size_in_bytes; + let value_cache_size = + inner.value_cache.lru.memory_usage() + inner.value_cache.lru.limiter().heap_size; + let node_cache_size = + inner.node_cache.lru.memory_usage() + inner.node_cache.lru.limiter().heap_size; node_cache_size + value_cache_size } @@ -575,13 +732,19 @@ impl SharedTrieCache { } /// Returns the read locked inner. - pub(super) fn read_lock_inner(&self) -> RwLockReadGuard<'_, SharedTrieCacheInner> { + #[cfg(test)] + pub(super) fn read_lock_inner( + &self, + ) -> parking_lot::RwLockReadGuard<'_, SharedTrieCacheInner> { self.inner.read() } /// Returns the write locked inner. - pub(super) fn write_lock_inner(&self) -> RwLockWriteGuard<'_, SharedTrieCacheInner> { - self.inner.write() + pub(super) fn write_lock_inner(&self) -> Option>> { + // This should never happen, but we *really* don't want to deadlock. So let's have it + // timeout, just in case. At worst it'll do nothing, and at best it'll avert a catastrophe + // and notify us that there's a problem. + self.inner.try_write_for(super::SHARED_CACHE_WRITE_LOCK_TIMEOUT) } } @@ -592,12 +755,7 @@ mod tests { #[test] fn shared_value_cache_works() { - let base_size = mem::size_of::>() + mem::size_of::>(); - let arc_size = mem::size_of::>(); - - let mut cache = SharedValueCache::::new(CacheSize::Maximum( - (base_size + arc_size + 10) * 10, - )); + let mut cache = SharedValueCache::::new(usize::MAX, 10 * 10); let key = vec![0; 10]; @@ -613,65 +771,85 @@ mod tests { ); // Ensure that the basics are working - assert_eq!(1, cache.known_storage_keys.len()); - assert_eq!(3, Arc::strong_count(cache.known_storage_keys.get(&key[..]).unwrap())); - assert_eq!(base_size * 2 + key.len() + arc_size, cache.size_in_bytes); + assert_eq!(1, cache.lru.limiter_mut().known_storage_keys.len()); + assert_eq!( + 3, // Two instances inside the cache + one extra in `known_storage_keys`. + Arc::strong_count(cache.lru.limiter_mut().known_storage_keys.get(&key[..]).unwrap()) + ); + assert_eq!(key.len(), cache.lru.limiter().heap_size); + assert_eq!(cache.lru.len(), 2); + assert_eq!(cache.lru.peek_newest().unwrap().0.storage_root, root1); + assert_eq!(cache.lru.peek_oldest().unwrap().0.storage_root, root0); + assert!(cache.lru.limiter().heap_size <= cache.lru.limiter().max_heap_size); + assert_eq!(cache.lru.limiter().heap_size, 10); // Just accessing a key should not change anything on the size and number of entries. cache.update(vec![], vec![ValueCacheKey::hash_data(&key[..], &root0)]); - assert_eq!(1, cache.known_storage_keys.len()); - assert_eq!(3, Arc::strong_count(cache.known_storage_keys.get(&key[..]).unwrap())); - assert_eq!(base_size * 2 + key.len() + arc_size, cache.size_in_bytes); - - // Add 9 other entries and this should move out the key for `root1`. + assert_eq!(1, cache.lru.limiter_mut().known_storage_keys.len()); + assert_eq!( + 3, + Arc::strong_count(cache.lru.limiter_mut().known_storage_keys.get(&key[..]).unwrap()) + ); + assert_eq!(key.len(), cache.lru.limiter().heap_size); + assert_eq!(cache.lru.len(), 2); + assert_eq!(cache.lru.peek_newest().unwrap().0.storage_root, root0); + assert_eq!(cache.lru.peek_oldest().unwrap().0.storage_root, root1); + assert!(cache.lru.limiter().heap_size <= cache.lru.limiter().max_heap_size); + assert_eq!(cache.lru.limiter().heap_size, 10); + + // Updating the cache again with exactly the same data should not change anything. cache.update( - (1..10) + vec![ + (ValueCacheKey::new_value(&key[..], root1), CachedValue::NonExisting), + (ValueCacheKey::new_value(&key[..], root0), CachedValue::NonExisting), + ], + vec![], + ); + assert_eq!(1, cache.lru.limiter_mut().known_storage_keys.len()); + assert_eq!( + 3, + Arc::strong_count(cache.lru.limiter_mut().known_storage_keys.get(&key[..]).unwrap()) + ); + assert_eq!(key.len(), cache.lru.limiter().heap_size); + assert_eq!(cache.lru.len(), 2); + assert_eq!(cache.lru.peek_newest().unwrap().0.storage_root, root0); + assert_eq!(cache.lru.peek_oldest().unwrap().0.storage_root, root1); + assert!(cache.lru.limiter().heap_size <= cache.lru.limiter().max_heap_size); + assert_eq!(cache.lru.limiter().items_evicted, 0); + assert_eq!(cache.lru.limiter().heap_size, 10); + + // Add 10 other entries and this should move out two of the initial entries. + cache.update( + (1..11) .map(|i| vec![i; 10]) .map(|key| (ValueCacheKey::new_value(&key[..], root0), CachedValue::NonExisting)), vec![], ); - assert_eq!(10, cache.known_storage_keys.len()); - assert_eq!(2, Arc::strong_count(cache.known_storage_keys.get(&key[..]).unwrap())); - assert_eq!((base_size + key.len() + arc_size) * 10, cache.size_in_bytes); + assert_eq!(cache.lru.limiter().items_evicted, 2); + assert_eq!(10, cache.lru.len()); + assert_eq!(10, cache.lru.limiter_mut().known_storage_keys.len()); + assert!(cache.lru.limiter_mut().known_storage_keys.get(&key[..]).is_none()); + assert_eq!(key.len() * 10, cache.lru.limiter().heap_size); + assert_eq!(cache.lru.len(), 10); + assert!(cache.lru.limiter().heap_size <= cache.lru.limiter().max_heap_size); + assert_eq!(cache.lru.limiter().heap_size, 100); + assert!(matches!( - cache.get(&ValueCacheKey::new_ref(&key, root0)).unwrap(), + cache.lru.peek(&ValueCacheKey::new_value(&[1; 10][..], root0)).unwrap(), CachedValue::::NonExisting )); - assert!(cache.get(&ValueCacheKey::new_ref(&key, root1)).is_none()); + + assert!(cache.lru.peek(&ValueCacheKey::new_value(&[1; 10][..], root1)).is_none(),); + + assert!(cache.lru.peek(&ValueCacheKey::new_value(&key[..], root0)).is_none()); + assert!(cache.lru.peek(&ValueCacheKey::new_value(&key[..], root1)).is_none()); cache.update( vec![(ValueCacheKey::new_value(vec![10; 10], root0), CachedValue::NonExisting)], vec![], ); - assert!(cache.known_storage_keys.get(&key[..]).is_none()); - } - - #[test] - fn value_cache_key_eq_works() { - let storage_key = &b"something"[..]; - let storage_key2 = &b"something2"[..]; - let storage_root = Hash::random(); - - let value = ValueCacheKey::new_value(storage_key, storage_root); - // Ref gets the same hash, but a different storage key - let ref_ = - ValueCacheKey::Ref { storage_root, storage_key: storage_key2, hash: value.get_hash() }; - let hash = ValueCacheKey::Hash { - hash: value.get_hash(), - _i_read_the_documentation: IReadTheDocumentation(()), - }; - - // Ensure that the hash variants is equal to `value`, `ref_` and itself. - assert!(hash == value); - assert!(value == hash); - assert!(hash == ref_); - assert!(ref_ == hash); - assert!(hash == hash); - - // But when we compare `value` and `ref_` the different storage key is detected. - assert!(value != ref_); - assert!(ref_ != value); + assert!(cache.lru.limiter_mut().known_storage_keys.get(&key[..]).is_none()); } }