Skip to content

Commit

Permalink
Use HashsetCache instead of FIFOSet for KnownHashes and SentHashes (n…
Browse files Browse the repository at this point in the history
…eo-project#1389)

* update

* change fifoset to hashsetcache for knownhashes and senthashes

* format

* optimise HashSetCache

* remove FIFOSet

* remove FIFOSet and update ut

* change List to LinkedList

* remove bucket count upper limit

* update exception message

* Adding some comments

* dotnet format

* rename maxBucketCount

* format

* Cache count

* Vitor suggestions

* Update HashSetCache.cs

* format

* Fix

* fix counting count

* optimise

* optimse

* update List

Co-authored-by: Shargon <[email protected]>
Co-authored-by: Vitor Nazário Coelho <[email protected]>
  • Loading branch information
3 people authored and Tommo-L committed Jun 22, 2020
1 parent 62fc232 commit 61208fd
Show file tree
Hide file tree
Showing 6 changed files with 148 additions and 122 deletions.
2 changes: 1 addition & 1 deletion src/neo/Helper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ internal static void Remove<T>(this HashSet<T> set, ISet<T> other)
}
}

internal static void Remove<T>(this HashSet<T> set, FIFOSet<T> other)
internal static void Remove<T>(this HashSet<T> set, HashSetCache<T> other)
where T : IEquatable<T>
{
if (set.Count > other.Count)
Expand Down
67 changes: 0 additions & 67 deletions src/neo/IO/Caching/FIFOSet.cs

This file was deleted.

106 changes: 106 additions & 0 deletions src/neo/IO/Caching/HashSetCache.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
using System;
using System.Collections;
using System.Collections.Generic;

namespace Neo.IO.Caching
{
public class HashSetCache<T> : IReadOnlyCollection<T> where T : IEquatable<T>
{
/// <summary>
/// Sets where the Hashes are stored
/// </summary>
private readonly LinkedList<HashSet<T>> sets = new LinkedList<HashSet<T>>();

/// <summary>
/// Maximum capacity of each bucket inside each HashSet of <see cref="sets"/>.
/// </summary>
private readonly int bucketCapacity;

/// <summary>
/// Maximum number of buckets for the LinkedList, meaning its maximum cardinality.
/// </summary>
private readonly int maxBucketCount;

/// <summary>
/// Entry count
/// </summary>
public int Count { get; private set; }

public HashSetCache(int bucketCapacity, int maxBucketCount = 10)
{
if (bucketCapacity <= 0) throw new ArgumentOutOfRangeException($"{nameof(bucketCapacity)} should be greater than 0");
if (maxBucketCount <= 0) throw new ArgumentOutOfRangeException($"{nameof(maxBucketCount)} should be greater than 0");

this.Count = 0;
this.bucketCapacity = bucketCapacity;
this.maxBucketCount = maxBucketCount;
sets.AddFirst(new HashSet<T>());
}

public bool Add(T item)
{
if (Contains(item)) return false;
Count++;
if (sets.First.Value.Count < bucketCapacity) return sets.First.Value.Add(item);
var newSet = new HashSet<T>
{
item
};
sets.AddFirst(newSet);
if (sets.Count > maxBucketCount)
{
Count -= sets.Last.Value.Count;
sets.RemoveLast();
}
return true;
}

public bool Contains(T item)
{
foreach (var set in sets)
{
if (set.Contains(item)) return true;
}
return false;
}

public void ExceptWith(IEnumerable<T> items)
{
List<HashSet<T>> removeList = null;
foreach (var item in items)
{
foreach (var set in sets)
{
if (set.Remove(item))
{
Count--;
if (set.Count == 0)
{
removeList ??= new List<HashSet<T>>();
removeList.Add(set);
}
break;
}
}
}
if (removeList == null) return;
foreach (var set in removeList)
{
sets.Remove(set);
}
}

public IEnumerator<T> GetEnumerator()
{
foreach (var set in sets)
{
foreach (var item in set)
{
yield return item;
}
}
}

IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
}
}
8 changes: 4 additions & 4 deletions src/neo/Network/P2P/ProtocolHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ protected override UInt256 GetKeyForItem((UInt256, DateTime) item)

private readonly NeoSystem system;
private readonly PendingKnownHashesCollection pendingKnownHashes;
private readonly FIFOSet<UInt256> knownHashes;
private readonly FIFOSet<UInt256> sentHashes;
private readonly HashSetCache<UInt256> knownHashes;
private readonly HashSetCache<UInt256> sentHashes;
private VersionPayload version;
private bool verack = false;
private BloomFilter bloom_filter;
Expand All @@ -47,8 +47,8 @@ public ProtocolHandler(NeoSystem system)
{
this.system = system;
this.pendingKnownHashes = new PendingKnownHashesCollection();
this.knownHashes = new FIFOSet<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2);
this.sentHashes = new FIFOSet<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2);
this.knownHashes = new HashSetCache<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2 / 5);
this.sentHashes = new HashSetCache<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2 / 5);
}

protected override void OnReceive(object message)
Expand Down
4 changes: 2 additions & 2 deletions src/neo/Network/P2P/TaskManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ private class Timer { }
/// <summary>
/// A set of known hashes, of inventories or payloads, already received.
/// </summary>
private readonly FIFOSet<UInt256> knownHashes;
private readonly HashSetCache<UInt256> knownHashes;
private readonly Dictionary<UInt256, int> globalTasks = new Dictionary<UInt256, int>();
private readonly Dictionary<IActorRef, TaskSession> sessions = new Dictionary<IActorRef, TaskSession>();
private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender);
Expand All @@ -43,7 +43,7 @@ private class Timer { }
public TaskManager(NeoSystem system)
{
this.system = system;
this.knownHashes = new FIFOSet<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2);
this.knownHashes = new HashSetCache<UInt256>(Blockchain.Singleton.MemPool.Capacity * 2 / 5);
}

private void OnHeaderTaskCompleted()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,63 +8,50 @@
namespace Neo.UnitTests.IO.Caching
{
[TestClass]
public class UT_FIFOSet
public class UT_HashSetCache
{
[TestMethod]
public void FIFOSetTest()
public void TestHashSetCache()
{
var a = UInt256.Zero;
var b = new UInt256();
var c = new UInt256(new byte[32] {
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01
});

var set = new FIFOSet<UInt256>(3);

Assert.IsTrue(set.Add(a));
Assert.IsFalse(set.Add(a));
Assert.IsFalse(set.Add(b));
Assert.IsTrue(set.Add(c));

CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c });

var d = new UInt256(new byte[32] {
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x02
});
var bucket = new HashSetCache<int>(10);
for (int i = 1; i <= 100; i++)
{
bucket.Add(i);
}
bucket.Count.Should().Be(100);

// Testing Fifo max size
Assert.IsTrue(set.Add(d));
CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, c, d });
int sum = 0;
foreach (var ele in bucket)
{
sum += ele;
}
sum.Should().Be(5050);

var e = new UInt256(new byte[32] {
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x03
});
bucket.Add(101);
bucket.Count.Should().Be(91);

Assert.IsTrue(set.Add(e));
Assert.IsFalse(set.Add(e));
CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { c, d, e });
var items = new int[10];
var value = 11;
for (int i = 0; i < 10; i++)
{
items[i] = value;
value += 2;
}
bucket.ExceptWith(items);
bucket.Count.Should().Be(81);

bucket.Contains(13).Should().BeFalse();
bucket.Contains(50).Should().BeTrue();
}

[TestMethod]
public void TestConstructor()
{
Action action1 = () => new FIFOSet<UInt256>(-1);
Action action1 = () => new HashSetCache<UInt256>(-1);
action1.Should().Throw<ArgumentOutOfRangeException>();

Action action2 = () => new FIFOSet<UInt256>(1, -1);
Action action2 = () => new HashSetCache<UInt256>(1, -1);
action2.Should().Throw<ArgumentOutOfRangeException>();

Action action3 = () => new FIFOSet<UInt256>(1, 2);
action3.Should().Throw<ArgumentOutOfRangeException>();
}

[TestMethod]
Expand All @@ -82,7 +69,7 @@ public void TestAdd()
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x02
});
var set = new FIFOSet<UInt256>(1, 1)
var set = new HashSetCache<UInt256>(1, 1)
{
a,
b
Expand All @@ -105,7 +92,7 @@ public void TestGetEnumerator()
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x02
});
var set = new FIFOSet<UInt256>(1, 1)
var set = new HashSetCache<UInt256>(1, 1)
{
a,
b
Expand Down Expand Up @@ -136,7 +123,7 @@ public void TestExceptWith()
0x01, 0x03
});

var set = new FIFOSet<UInt256>(10)
var set = new HashSetCache<UInt256>(10)
{
a,
b,
Expand All @@ -145,7 +132,7 @@ public void TestExceptWith()
set.ExceptWith(new UInt256[] { b, c });
CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a });

set = new FIFOSet<UInt256>(10)
set = new HashSetCache<UInt256>(10)
{
a,
b,
Expand All @@ -154,7 +141,7 @@ public void TestExceptWith()
set.ExceptWith(new UInt256[] { a });
CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c });

set = new FIFOSet<UInt256>(10)
set = new HashSetCache<UInt256>(10)
{
a,
b,
Expand Down

0 comments on commit 61208fd

Please sign in to comment.