diff --git a/src/Nethermind/Nethermind.Core/Caching/LruCacheNonConcurrent.cs b/src/Nethermind/Nethermind.Core/Caching/LruCacheNonConcurrent.cs new file mode 100644 index 00000000000..3b8622fd6da --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Caching/LruCacheNonConcurrent.cs @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Caching +{ + public sealed class LruCacheNonConcurrent where TKey : notnull + { + private readonly int _maxCapacity; + private readonly Dictionary> _cacheMap; + private readonly string _name; + private LinkedListNode? _leastRecentlyUsed; + + public LruCacheNonConcurrent(int maxCapacity, int startCapacity, string name) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxCapacity, 1); + + _name = name; + _maxCapacity = maxCapacity; + _cacheMap = typeof(TKey) == typeof(byte[]) + ? new Dictionary>((IEqualityComparer)Bytes.EqualityComparer) + : new Dictionary>(startCapacity); // do not initialize it at the full capacity + } + + public LruCacheNonConcurrent(int maxCapacity, string name) + : this(maxCapacity, 0, name) + { + } + + public void Clear() + { + _leastRecentlyUsed = null; + _cacheMap.Clear(); + } + + public TValue Get(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + TValue value = node.Value.Value; + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + return value; + } + +#pragma warning disable 8603 + // fixed C# 9 + return default; +#pragma warning restore 8603 + } + + public bool TryGet(TKey key, out TValue value) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + value = node.Value.Value; + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + return true; + } + +#pragma warning disable 8601 + // fixed C# 9 + value = default; +#pragma warning restore 8601 + return false; + } + + public bool Set(TKey key, TValue val) + { + if (val is null) + { + return DeleteNoLock(key); + } + + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + node.Value.Value = val; + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + return false; + } + + if (_cacheMap.Count >= _maxCapacity) + { + Replace(key, val); + } + else + { + LinkedListNode newNode = new(new(key, val)); + LinkedListNode.AddMostRecent(ref _leastRecentlyUsed, newNode); + _cacheMap.Add(key, newNode); + } + + return true; + } + + public bool DeleteNoLock(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + LinkedListNode.Remove(ref _leastRecentlyUsed, node); + _cacheMap.Remove(key); + return true; + } + + return false; + } + + public bool Contains(TKey key) + { + return _cacheMap.ContainsKey(key); + } + + public int Size + { + get + { + return _cacheMap.Count; + } + } + + private void Replace(TKey key, TValue value) + { + LinkedListNode? node = _leastRecentlyUsed; + if (node is null) + { + ThrowInvalidOperationException(); + } + + _cacheMap.Remove(node!.Value.Key); + + node.Value = new(key, value); + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + _cacheMap.Add(key, node); + + [DoesNotReturn] + static void ThrowInvalidOperationException() + { + throw new InvalidOperationException( + $"{nameof(LruCache)} called {nameof(Replace)} when empty."); + } + } + + private struct LruCacheItem(TKey k, TValue v) + { + public readonly TKey Key = k; + public TValue Value = v; + } + + public long MemorySize => CalculateMemorySize(0, _cacheMap.Count); + + public static long CalculateMemorySize(int keyPlusValueSize, int currentItemsCount) + { + // it may actually be different if the initial capacity not equal to max (depending on the dictionary growth path) + + const int preInit = 48 /* LinkedList */ + 80 /* Dictionary */ + 24; + int postInit = 52 /* lazy init of two internal dictionary arrays + dictionary size times (entry size + int) */ + MemorySizes.FindNextPrime(currentItemsCount) * 28 + currentItemsCount * 80 /* LinkedListNode and CacheItem times items count */; + return MemorySizes.Align(preInit + postInit + keyPlusValueSize * currentItemsCount); + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheNonConcurrent.cs b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheNonConcurrent.cs new file mode 100644 index 00000000000..ef5a2e49d2b --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheNonConcurrent.cs @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Caching +{ + public sealed class LruKeyCacheNonConcurrent where TKey : notnull + { + private readonly int _maxCapacity; + private readonly string _name; + private readonly Dictionary> _cacheMap; + private LinkedListNode? _leastRecentlyUsed; + + public LruKeyCacheNonConcurrent(int maxCapacity, int startCapacity, string name) + { + _maxCapacity = maxCapacity; + _name = name ?? throw new ArgumentNullException(nameof(name)); + _cacheMap = typeof(TKey) == typeof(byte[]) + ? new Dictionary>((IEqualityComparer)Bytes.EqualityComparer) + : new Dictionary>(startCapacity); // do not initialize it at the full capacity + } + + public LruKeyCacheNonConcurrent(int maxCapacity, string name) + : this(maxCapacity, 0, name) + { + } + + public void Clear() + { + _leastRecentlyUsed = null; + _cacheMap.Clear(); + } + + public bool Get(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + return true; + } + + return false; + } + + public bool Set(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + return false; + } + else + { + if (_cacheMap.Count >= _maxCapacity) + { + Replace(key); + } + else + { + LinkedListNode newNode = new(key); + LinkedListNode.AddMostRecent(ref _leastRecentlyUsed, newNode); + _cacheMap.Add(key, newNode); + } + + return true; + } + } + + public void Delete(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + LinkedListNode.Remove(ref _leastRecentlyUsed, node); + _cacheMap.Remove(key); + } + } + + private void Replace(TKey key) + { + LinkedListNode? node = _leastRecentlyUsed; + if (node is null) + { + ThrowInvalidOperation(); + } + + _cacheMap.Remove(node.Value); + node.Value = key; + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + _cacheMap.Add(key, node); + + [DoesNotReturn] + static void ThrowInvalidOperation() + { + throw new InvalidOperationException( + $"{nameof(LruKeyCache)} called {nameof(Replace)} when empty."); + } + } + + public long MemorySize => CalculateMemorySize(0, _cacheMap.Count); + + // TODO: memory size on the KeyCache will be smaller because we do not create LruCacheItems + public static long CalculateMemorySize(int keyPlusValueSize, int currentItemsCount) + { + // it may actually be different if the initial capacity not equal to max (depending on the dictionary growth path) + + const int preInit = 48 /* LinkedList */ + 80 /* Dictionary */ + 24; + int postInit = 52 /* lazy init of two internal dictionary arrays + dictionary size times (entry size + int) */ + MemorySizes.FindNextPrime(currentItemsCount) * 28 + currentItemsCount * 80 /* LinkedListNode and CacheItem times items count */; + return MemorySizes.Align(preInit + postInit + keyPlusValueSize * currentItemsCount); + } + } +} diff --git a/src/Nethermind/Nethermind.Db/Metrics.cs b/src/Nethermind/Nethermind.Db/Metrics.cs index 2d9a7c5ad0e..81da5a2b8c0 100644 --- a/src/Nethermind/Nethermind.Db/Metrics.cs +++ b/src/Nethermind/Nethermind.Db/Metrics.cs @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Collections.Concurrent; -using System.Collections.Generic; using System.ComponentModel; using Nethermind.Core.Attributes; @@ -14,10 +12,18 @@ public static class Metrics [Description("Number of Code DB cache reads.")] public static long CodeDbCache { get; set; } + [CounterMetric] + [Description("Number of State Trie cache hits.")] + public static long StateTreeCache { get; set; } + [CounterMetric] [Description("Number of State Trie reads.")] public static long StateTreeReads { get; set; } + [CounterMetric] + [Description("Number of State Reader reads.")] + public static long StateReaderReads { get; set; } + [CounterMetric] [Description("Number of Blocks Trie writes.")] public static long StateTreeWrites { get; set; } @@ -26,10 +32,18 @@ public static class Metrics [Description("Number of State DB duplicate writes during full pruning.")] public static int StateDbInPruningWrites; + [CounterMetric] + [Description("Number of storage trie cache hits.")] + public static long StorageTreeCache { get; set; } + [CounterMetric] [Description("Number of storage trie reads.")] public static long StorageTreeReads { get; set; } + [CounterMetric] + [Description("Number of storage reader reads.")] + public static long StorageReaderReads { get; set; } + [CounterMetric] [Description("Number of storage trie writes.")] public static long StorageTreeWrites { get; set; } diff --git a/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs b/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs index 22c3c95140c..3cc5a7983b1 100644 --- a/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs +++ b/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs @@ -5,6 +5,6 @@ namespace Nethermind.Evm { public static class MemoryAllowance { - public static int CodeCacheSize { get; } = 16_384; + public static int CodeCacheSize { get; } = 4_096 + 1_024; } } diff --git a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs index c6abed3a839..a79017fd640 100644 --- a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Runtime.InteropServices; using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Collections; @@ -19,7 +20,7 @@ namespace Nethermind.State /// Manages persistent storage allowing for snapshotting and restoring /// Persists data to ITrieStore /// - internal class PersistentStorageProvider : PartialStorageProviderBase + internal sealed class PersistentStorageProvider : PartialStorageProviderBase { private readonly ITrieStore _trieStore; private readonly StateProvider _stateProvider; @@ -34,6 +35,7 @@ internal class PersistentStorageProvider : PartialStorageProviderBase private readonly ResettableDictionary _originalValues = new(); private readonly ResettableHashSet _committedThisRound = new(); + private readonly Dictionary _blockCache = new(4_096); public PersistentStorageProvider(ITrieStore? trieStore, StateProvider? stateProvider, ILogManager? logManager, IStorageTreeFactory? storageTreeFactory = null) : base(logManager) @@ -52,6 +54,7 @@ public PersistentStorageProvider(ITrieStore? trieStore, StateProvider? stateProv public override void Reset() { base.Reset(); + _blockCache.Clear(); _storages.Reset(); _originalValues.Clear(); _committedThisRound.Clear(); @@ -281,6 +284,7 @@ private void SaveToTree(HashSet
toUpdateRoots, Change change) Db.Metrics.StorageTreeWrites++; toUpdateRoots.Add(change.StorageCell.Address); tree.Set(change.StorageCell.Index, change.Value); + _blockCache[change.StorageCell] = change.Value; } /// @@ -317,18 +321,24 @@ private StorageTree GetOrCreateStorage(Address address) private ReadOnlySpan LoadFromTree(in StorageCell storageCell) { - StorageTree tree = GetOrCreateStorage(storageCell.Address); + ref byte[]? value = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, storageCell, out bool exists); + if (!exists) + { + StorageTree tree = GetOrCreateStorage(storageCell.Address); - Db.Metrics.StorageTreeReads++; + Db.Metrics.StorageTreeReads++; - if (!storageCell.IsHash) + value = !storageCell.IsHash ? + tree.Get(storageCell.Index) : + tree.GetArray(storageCell.Hash.Bytes); + } + else { - byte[] value = tree.Get(storageCell.Index); - PushToRegistryOnly(storageCell, value); - return value; + Db.Metrics.StorageTreeCache++; } - return tree.Get(storageCell.Hash.Bytes); + if (!storageCell.IsHash) PushToRegistryOnly(storageCell, value); + return value; } private void PushToRegistryOnly(in StorageCell cell, byte[] value) @@ -368,6 +378,8 @@ private Hash256 RecalculateRootHash(Address address) public override void ClearStorage(Address address) { base.ClearStorage(address); + // Bit heavy-handed, but we need to clear all the cache for that address + _blockCache.Clear(); // here it is important to make sure that we will not reuse the same tree when the contract is revived // by means of CREATE 2 - notice that the cached trie may carry information about items that were not diff --git a/src/Nethermind/Nethermind.State/StateProvider.cs b/src/Nethermind/Nethermind.State/StateProvider.cs index 945374f5074..c73fd612610 100644 --- a/src/Nethermind/Nethermind.State/StateProvider.cs +++ b/src/Nethermind/Nethermind.State/StateProvider.cs @@ -21,14 +21,15 @@ namespace Nethermind.State internal class StateProvider { private const int StartCapacity = Resettable.StartCapacity; - private readonly ResettableDictionary> _intraBlockCache = new(); + private readonly ResettableDictionary> _intraTxCache = new(); private readonly ResettableHashSet _committedThisRound = new(); private readonly HashSet _nullAccountReads = new(); // Only guarding against hot duplicates so filter doesn't need to be too big // Note: // False negatives are fine as they will just result in a overwrite set // False positives would be problematic as the code _must_ be persisted - private readonly LruKeyCache _codeInsertFilter = new(2048, "Code Insert Filter"); + private readonly LruKeyCacheNonConcurrent _codeInsertFilter = new(1_024, "Code Insert Filter"); + private readonly Dictionary _blockCache = new(4_096); private readonly List _keptInCache = new(); private readonly ILogger _logger; @@ -90,7 +91,7 @@ public bool IsContract(Address address) public bool AccountExists(Address address) { - if (_intraBlockCache.TryGetValue(address, out Stack value)) + if (_intraTxCache.TryGetValue(address, out Stack value)) { return _changes[value.Peek()]!.ChangeType != ChangeType.Delete; } @@ -361,7 +362,7 @@ public void Restore(int snapshot) for (int i = 0; i < _currentPosition - snapshot; i++) { Change change = _changes[_currentPosition - i]; - Stack stack = _intraBlockCache[change!.Address]; + Stack stack = _intraTxCache[change!.Address]; if (stack.Count == 1) { if (change.ChangeType == ChangeType.JustCache) @@ -387,7 +388,7 @@ public void Restore(int snapshot) if (stack.Count == 0) { - _intraBlockCache.Remove(change.Address); + _intraTxCache.Remove(change.Address); } } @@ -396,7 +397,7 @@ public void Restore(int snapshot) { _currentPosition++; _changes[_currentPosition] = kept; - _intraBlockCache[kept.Address].Push(_currentPosition); + _intraTxCache[kept.Address].Push(_currentPosition); } _keptInCache.Clear(); @@ -502,7 +503,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool continue; } - Stack stack = _intraBlockCache[change.Address]; + Stack stack = _intraTxCache[change.Address]; int forAssertion = stack.Pop(); if (forAssertion != _currentPosition - i) { @@ -597,7 +598,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool Resettable.Reset(ref _changes, ref _capacity, ref _currentPosition, StartCapacity); _committedThisRound.Reset(); _nullAccountReads.Clear(); - _intraBlockCache.Reset(); + _intraTxCache.Reset(); if (isTracing) { @@ -665,13 +666,22 @@ private void ReportChanges(IStateTracer stateTracer, Dictionary value)) + if (_intraTxCache.TryGetValue(address, out Stack value)) { return _changes[value.Peek()]!.Account; } @@ -756,7 +766,7 @@ private void IncrementChangePosition() private Stack SetupCache(Address address) { - ref Stack? value = ref _intraBlockCache.GetValueRefOrAddDefault(address, out bool exists); + ref Stack? value = ref _intraTxCache.GetValueRefOrAddDefault(address, out bool exists); if (!exists) { value = new Stack(); @@ -791,7 +801,8 @@ public Change(ChangeType type, Address address, Account? account) public void Reset() { if (_logger.IsTrace) _logger.Trace("Clearing state provider caches"); - _intraBlockCache.Reset(); + _blockCache.Clear(); + _intraTxCache.Reset(); _committedThisRound.Reset(); _nullAccountReads.Clear(); _currentPosition = Resettable.EmptyPosition; diff --git a/src/Nethermind/Nethermind.State/StateReader.cs b/src/Nethermind/Nethermind.State/StateReader.cs index a978013ba34..aba5ce5c6de 100644 --- a/src/Nethermind/Nethermind.State/StateReader.cs +++ b/src/Nethermind/Nethermind.State/StateReader.cs @@ -34,7 +34,7 @@ public ReadOnlySpan GetStorage(Hash256 stateRoot, Address address, in UInt return Bytes.ZeroByte.Span; } - Metrics.StorageTreeReads++; + Metrics.StorageReaderReads++; StorageTree storage = new StorageTree(_trieStore.GetTrieStore(address.ToAccountPath), Keccak.EmptyTreeHash, _logManager); return storage.Get(index, new Hash256(storageRoot)); @@ -68,7 +68,7 @@ private bool TryGetState(Hash256 stateRoot, Address address, out AccountStruct a return false; } - Metrics.StateTreeReads++; + Metrics.StateReaderReads++; return _state.TryGetStruct(address, out account, stateRoot); } } diff --git a/src/Nethermind/Nethermind.State/StorageTree.cs b/src/Nethermind/Nethermind.State/StorageTree.cs index 6b9a520e5c4..c7f5014dfc5 100644 --- a/src/Nethermind/Nethermind.State/StorageTree.cs +++ b/src/Nethermind/Nethermind.State/StorageTree.cs @@ -59,16 +59,15 @@ public byte[] Get(in UInt256 index, Hash256? storageRoot = null) { if (index < LookupSize) { - return Get(Lookup[index], storageRoot).ToArray(); + return GetArray(Lookup[index], storageRoot); } Span key = stackalloc byte[32]; ComputeKey(index, ref key); - return Get(key, storageRoot).ToArray(); - + return GetArray(key, storageRoot); } - public override ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHash = null) + public byte[] GetArray(ReadOnlySpan rawKey, Hash256? rootHash = null) { ReadOnlySpan value = base.Get(rawKey, rootHash); @@ -81,6 +80,8 @@ public override ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootH return rlp.DecodeByteArray(); } + public override ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHash = null) => GetArray(rawKey, rootHash); + [SkipLocalsInit] public void Set(in UInt256 index, byte[] value) { diff --git a/src/Nethermind/Nethermind.State/TransientStorageProvider.cs b/src/Nethermind/Nethermind.State/TransientStorageProvider.cs index 3e2177fdf33..4c0ec3efaef 100644 --- a/src/Nethermind/Nethermind.State/TransientStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/TransientStorageProvider.cs @@ -11,7 +11,7 @@ namespace Nethermind.State /// EIP-1153 provides a transient store for contracts that doesn't persist /// storage across calls. Reverts will rollback any transient state changes. /// - internal class TransientStorageProvider : PartialStorageProviderBase + internal sealed class TransientStorageProvider : PartialStorageProviderBase { public TransientStorageProvider(ILogManager? logManager) : base(logManager) { }