diff --git a/src/neo/Helper.cs b/src/neo/Helper.cs index b8a0a7f79e..be66ce2013 100644 --- a/src/neo/Helper.cs +++ b/src/neo/Helper.cs @@ -83,7 +83,7 @@ internal static void Remove(this HashSet set, ISet other) } } - internal static void Remove(this HashSet set, FIFOSet other) + internal static void Remove(this HashSet set, HashSetCache other) where T : IEquatable { if (set.Count > other.Count) diff --git a/src/neo/IO/Caching/FIFOSet.cs b/src/neo/IO/Caching/FIFOSet.cs deleted file mode 100644 index af65db1b8b..0000000000 --- a/src/neo/IO/Caching/FIFOSet.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.Linq; - -namespace Neo.IO.Caching -{ - internal class FIFOSet : IReadOnlyCollection where T : IEquatable - { - private readonly int maxCapacity; - private readonly int removeCount; - private readonly OrderedDictionary dictionary; - - public int Count => dictionary.Count; - - public FIFOSet(int maxCapacity, decimal batchSize = 0.1m) - { - if (maxCapacity <= 0) throw new ArgumentOutOfRangeException(nameof(maxCapacity)); - if (batchSize <= 0 || batchSize > 1) throw new ArgumentOutOfRangeException(nameof(batchSize)); - - this.maxCapacity = maxCapacity; - this.removeCount = Math.Max((int)(maxCapacity * batchSize), 1); - this.dictionary = new OrderedDictionary(maxCapacity); - } - - public bool Add(T item) - { - if (dictionary.Contains(item)) return false; - if (dictionary.Count >= maxCapacity) - { - if (removeCount == maxCapacity) - { - dictionary.Clear(); - } - else - { - for (int i = 0; i < removeCount; i++) - dictionary.RemoveAt(0); - } - } - dictionary.Add(item, null); - return true; - } - - public bool Contains(T item) - { - return dictionary.Contains(item); - } - - public void ExceptWith(IEnumerable entries) - { - foreach (var entry in entries) - { - dictionary.Remove(entry); - } - } - - public IEnumerator GetEnumerator() - { - var entries = dictionary.Keys.Cast().ToArray(); - foreach (var entry in entries) yield return entry; - } - - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - } -} diff --git a/src/neo/IO/Caching/HashSetCache.cs b/src/neo/IO/Caching/HashSetCache.cs new file mode 100644 index 0000000000..af34cd76fa --- /dev/null +++ b/src/neo/IO/Caching/HashSetCache.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Neo.IO.Caching +{ + public class HashSetCache : IReadOnlyCollection where T : IEquatable + { + /// + /// Sets where the Hashes are stored + /// + private readonly LinkedList> sets = new LinkedList>(); + + /// + /// Maximum capacity of each bucket inside each HashSet of . + /// + private readonly int bucketCapacity; + + /// + /// Maximum number of buckets for the LinkedList, meaning its maximum cardinality. + /// + private readonly int maxBucketCount; + + /// + /// Entry count + /// + public int Count { get; private set; } + + public HashSetCache(int bucketCapacity, int maxBucketCount = 10) + { + if (bucketCapacity <= 0) throw new ArgumentOutOfRangeException($"{nameof(bucketCapacity)} should be greater than 0"); + if (maxBucketCount <= 0) throw new ArgumentOutOfRangeException($"{nameof(maxBucketCount)} should be greater than 0"); + + this.Count = 0; + this.bucketCapacity = bucketCapacity; + this.maxBucketCount = maxBucketCount; + sets.AddFirst(new HashSet()); + } + + public bool Add(T item) + { + if (Contains(item)) return false; + Count++; + if (sets.First.Value.Count < bucketCapacity) return sets.First.Value.Add(item); + var newSet = new HashSet + { + item + }; + sets.AddFirst(newSet); + if (sets.Count > maxBucketCount) + { + Count -= sets.Last.Value.Count; + sets.RemoveLast(); + } + return true; + } + + public bool Contains(T item) + { + foreach (var set in sets) + { + if (set.Contains(item)) return true; + } + return false; + } + + public void ExceptWith(IEnumerable items) + { + List> removeList = null; + foreach (var item in items) + { + foreach (var set in sets) + { + if (set.Remove(item)) + { + Count--; + if (set.Count == 0) + { + removeList ??= new List>(); + removeList.Add(set); + } + break; + } + } + } + if (removeList == null) return; + foreach (var set in removeList) + { + sets.Remove(set); + } + } + + public IEnumerator GetEnumerator() + { + foreach (var set in sets) + { + foreach (var item in set) + { + yield return item; + } + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} diff --git a/src/neo/Network/P2P/ProtocolHandler.cs b/src/neo/Network/P2P/ProtocolHandler.cs index cecbb02796..a97cb46922 100644 --- a/src/neo/Network/P2P/ProtocolHandler.cs +++ b/src/neo/Network/P2P/ProtocolHandler.cs @@ -32,8 +32,8 @@ protected override UInt256 GetKeyForItem((UInt256, DateTime) item) private readonly NeoSystem system; private readonly PendingKnownHashesCollection pendingKnownHashes; - private readonly FIFOSet knownHashes; - private readonly FIFOSet sentHashes; + private readonly HashSetCache knownHashes; + private readonly HashSetCache sentHashes; private VersionPayload version; private bool verack = false; private BloomFilter bloom_filter; @@ -47,8 +47,8 @@ public ProtocolHandler(NeoSystem system) { this.system = system; this.pendingKnownHashes = new PendingKnownHashesCollection(); - this.knownHashes = new FIFOSet(Blockchain.Singleton.MemPool.Capacity * 2); - this.sentHashes = new FIFOSet(Blockchain.Singleton.MemPool.Capacity * 2); + this.knownHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); + this.sentHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); } protected override void OnReceive(object message) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 818973ea4b..73fa49441a 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -32,7 +32,7 @@ private class Timer { } /// /// A set of known hashes, of inventories or payloads, already received. /// - private readonly FIFOSet knownHashes; + private readonly HashSetCache knownHashes; private readonly Dictionary globalTasks = new Dictionary(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); @@ -43,7 +43,7 @@ private class Timer { } public TaskManager(NeoSystem system) { this.system = system; - this.knownHashes = new FIFOSet(Blockchain.Singleton.MemPool.Capacity * 2); + this.knownHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); } private void OnHeaderTaskCompleted() diff --git a/tests/neo.UnitTests/IO/Caching/UT_FIFOSet.cs b/tests/neo.UnitTests/IO/Caching/UT_HashSetCache.cs similarity index 64% rename from tests/neo.UnitTests/IO/Caching/UT_FIFOSet.cs rename to tests/neo.UnitTests/IO/Caching/UT_HashSetCache.cs index d785127778..5f227c7e35 100644 --- a/tests/neo.UnitTests/IO/Caching/UT_FIFOSet.cs +++ b/tests/neo.UnitTests/IO/Caching/UT_HashSetCache.cs @@ -8,63 +8,50 @@ namespace Neo.UnitTests.IO.Caching { [TestClass] - public class UT_FIFOSet + public class UT_HashSetCache { [TestMethod] - public void FIFOSetTest() + public void TestHashSetCache() { - var a = UInt256.Zero; - var b = new UInt256(); - var c = new UInt256(new byte[32] { - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01 - }); - - var set = new FIFOSet(3); - - Assert.IsTrue(set.Add(a)); - Assert.IsFalse(set.Add(a)); - Assert.IsFalse(set.Add(b)); - Assert.IsTrue(set.Add(c)); - - CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c }); - - var d = new UInt256(new byte[32] { - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x02 - }); + var bucket = new HashSetCache(10); + for (int i = 1; i <= 100; i++) + { + bucket.Add(i); + } + bucket.Count.Should().Be(100); - // Testing Fifo max size - Assert.IsTrue(set.Add(d)); - CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c, d }); + int sum = 0; + foreach (var ele in bucket) + { + sum += ele; + } + sum.Should().Be(5050); - var e = new UInt256(new byte[32] { - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x03 - }); + bucket.Add(101); + bucket.Count.Should().Be(91); - Assert.IsTrue(set.Add(e)); - Assert.IsFalse(set.Add(e)); - CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { c, d, e }); + var items = new int[10]; + var value = 11; + for (int i = 0; i < 10; i++) + { + items[i] = value; + value += 2; + } + bucket.ExceptWith(items); + bucket.Count.Should().Be(81); + + bucket.Contains(13).Should().BeFalse(); + bucket.Contains(50).Should().BeTrue(); } [TestMethod] public void TestConstructor() { - Action action1 = () => new FIFOSet(-1); + Action action1 = () => new HashSetCache(-1); action1.Should().Throw(); - Action action2 = () => new FIFOSet(1, -1); + Action action2 = () => new HashSetCache(1, -1); action2.Should().Throw(); - - Action action3 = () => new FIFOSet(1, 2); - action3.Should().Throw(); } [TestMethod] @@ -82,7 +69,7 @@ public void TestAdd() 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02 }); - var set = new FIFOSet(1, 1) + var set = new HashSetCache(1, 1) { a, b @@ -105,7 +92,7 @@ public void TestGetEnumerator() 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02 }); - var set = new FIFOSet(1, 1) + var set = new HashSetCache(1, 1) { a, b @@ -136,7 +123,7 @@ public void TestExceptWith() 0x01, 0x03 }); - var set = new FIFOSet(10) + var set = new HashSetCache(10) { a, b, @@ -145,7 +132,7 @@ public void TestExceptWith() set.ExceptWith(new UInt256[] { b, c }); CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a }); - set = new FIFOSet(10) + set = new HashSetCache(10) { a, b, @@ -154,7 +141,7 @@ public void TestExceptWith() set.ExceptWith(new UInt256[] { a }); CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c }); - set = new FIFOSet(10) + set = new HashSetCache(10) { a, b,