diff --git a/src/DurableTask.Netherite/StorageProviders/Faster/CacheDebugger.cs b/src/DurableTask.Netherite/StorageProviders/Faster/CacheDebugger.cs index 75cd4db6..ae7bc0ff 100644 --- a/src/DurableTask.Netherite/StorageProviders/Faster/CacheDebugger.cs +++ b/src/DurableTask.Netherite/StorageProviders/Faster/CacheDebugger.cs @@ -29,6 +29,8 @@ public CacheDebugger(TestHooks testHooks) public bool EnableSizeChecking { get; set; } = true; + internal IEnumerable Keys => this.Objects.Keys; + public enum CacheEvent { // reads and RMWs on the main session @@ -299,6 +301,12 @@ internal bool CheckSize(TrackedObjectKey key, List<(long delta, long address, st { info.CacheEvents.Enqueue(new Entry { CacheEvent = CacheEvent.SizeCheckFail, Delta = actual, Address = reference }); + if (entries.Count == 0) + { + // for now, we just tolerate this since we cannot rely on evictions to reach us in time + return false; + } + // adjust the actual var firstActual = entries.FirstOrDefault().address; var firstReference = entries.FirstOrDefault().address; diff --git a/src/DurableTask.Netherite/StorageProviders/Faster/FasterKV.cs b/src/DurableTask.Netherite/StorageProviders/Faster/FasterKV.cs index ae45945b..b6d73130 100644 --- a/src/DurableTask.Netherite/StorageProviders/Faster/FasterKV.cs +++ b/src/DurableTask.Netherite/StorageProviders/Faster/FasterKV.cs @@ -991,6 +991,8 @@ public void ValidateMemoryTracker() var inMemoryIterator = this.fht.Log.Scan(this.fht.Log.HeadAddress, this.fht.Log.TailAddress); + // we now scan the in-memory part of the log and compute the total size, and store, for each key, the list of records found + long totalSize = 0; Dictionary> perKey = new Dictionary>(); void Add(TrackedObjectKey key, long delta, long address, string desc) @@ -1013,21 +1015,36 @@ void Add(TrackedObjectKey key, long delta, long address, string desc) } Add(key, delta, inMemoryIterator.CurrentAddress, $"{(recordInfo.Invalid ? "I" : "")}{(recordInfo.Tombstone ? "T" : "")}{delta}@{inMemoryIterator.CurrentAddress.ToString("x")}"); } + + foreach (var k in this.cacheDebugger.Keys) + { + if (!perKey.ContainsKey(k)) + { + perKey.Add(k, emptyList); // for keys that were not found in memory, the list of records is empty + } + } long trackedSizeAfter = this.cacheTracker.TrackedObjectSize; bool sizeMatches = true; + + // now we compare, for each key, the list of entries found in memory with what the cache debugger is tracking + foreach (var kvp in perKey) { sizeMatches = sizeMatches && this.cacheDebugger.CheckSize(kvp.Key, kvp.Value, this.Log.HeadAddress); } + // if the records matched for each key, then the total size should also match + if (sizeMatches && trackedSizeBefore == trackedSizeAfter && trackedSizeBefore != totalSize) { this.cacheDebugger.Fail("total size of tracked objects does not match"); } } + readonly static List<(long delta, long address, string desc)> emptyList = new List<(long delta, long address, string desc)>(); + internal (int numPages, long size) ComputeMemorySize(bool resetCacheDebugger) { var cacheDebugger = resetCacheDebugger ? this.cacheDebugger : null;