fix cachesize test (#172)

* fix cache size test to be aware of keys not found in memory * weaken the test since there are many false positives
microsoft · Jun 21, 2022 · 76fe0bf · 76fe0bf
1 parent e52e236
commit 76fe0bf
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 0 deletions.
diff --git a/src/DurableTask.Netherite/StorageProviders/Faster/CacheDebugger.cs b/src/DurableTask.Netherite/StorageProviders/Faster/CacheDebugger.cs
@@ -29,6 +29,8 @@ public CacheDebugger(TestHooks testHooks)
 
         public bool EnableSizeChecking { get; set; } = true;
 
+        internal IEnumerable<TrackedObjectKey> Keys => this.Objects.Keys;
+
         public enum CacheEvent
         {
             // reads and RMWs on the main session
@@ -299,6 +301,12 @@ internal bool CheckSize(TrackedObjectKey key, List<(long delta, long address, st
             {
                 info.CacheEvents.Enqueue(new Entry { CacheEvent = CacheEvent.SizeCheckFail, Delta = actual, Address = reference });
 
+                if (entries.Count == 0)
+                {
+                    // for now, we just tolerate this since we cannot rely on evictions to reach us in time
+                    return false;
+                }
+
                 // adjust the actual
                 var firstActual = entries.FirstOrDefault().address;
                 var firstReference = entries.FirstOrDefault().address;

diff --git a/src/DurableTask.Netherite/StorageProviders/Faster/FasterKV.cs b/src/DurableTask.Netherite/StorageProviders/Faster/FasterKV.cs
@@ -991,6 +991,8 @@ public void ValidateMemoryTracker()
 
             var inMemoryIterator = this.fht.Log.Scan(this.fht.Log.HeadAddress, this.fht.Log.TailAddress);
 
+            // we now scan the in-memory part of the log and compute the total size, and store, for each key, the list of records found
+
             long totalSize = 0;
             Dictionary<TrackedObjectKey, List<(long delta, long address, string desc)>> perKey = new Dictionary<TrackedObjectKey, List<(long delta, long address, string desc)>>();
             void Add(TrackedObjectKey key, long delta, long address, string desc)
@@ -1013,21 +1015,36 @@ void Add(TrackedObjectKey key, long delta, long address, string desc)
                 }
                 Add(key, delta, inMemoryIterator.CurrentAddress, $"{(recordInfo.Invalid ? "I" : "")}{(recordInfo.Tombstone ? "T" : "")}{delta}@{inMemoryIterator.CurrentAddress.ToString("x")}");
             }
+
+            foreach (var k in this.cacheDebugger.Keys)
+            {
+                if (!perKey.ContainsKey(k))
+                {
+                    perKey.Add(k, emptyList); // for keys that were not found in memory, the list of records is empty
+                }
+            }
 
             long trackedSizeAfter = this.cacheTracker.TrackedObjectSize;
 
             bool sizeMatches = true;
+
+            // now we compare, for each key, the list of entries found in memory with what the cache debugger is tracking
+
             foreach (var kvp in perKey)
             {
                 sizeMatches = sizeMatches && this.cacheDebugger.CheckSize(kvp.Key, kvp.Value, this.Log.HeadAddress);
             }
 
+            // if the records matched for each key, then the total size should also match
+
             if (sizeMatches && trackedSizeBefore == trackedSizeAfter && trackedSizeBefore != totalSize)
             {
                 this.cacheDebugger.Fail("total size of tracked objects does not match");
             }
         }
 
+        readonly static List<(long delta, long address, string desc)> emptyList = new List<(long delta, long address, string desc)>();
+
         internal (int numPages, long size) ComputeMemorySize(bool resetCacheDebugger)
         {
             var cacheDebugger = resetCacheDebugger ? this.cacheDebugger : null;