From 82fde793f7a66ef766987f67fd92e92693854be2 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 24 Aug 2020 13:29:54 +0200 Subject: [PATCH 01/14] Add block cache limit --- src/neo/Ledger/Blockchain.cs | 48 +++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 0ec0c6ade4..794f870848 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -28,7 +28,9 @@ public class FillMemoryPool { public IEnumerable Transactions; } public class FillCompleted { } internal class PreverifyCompleted { public Transaction Transaction; public VerifyResult Result; public bool Relay; } public class RelayResult { public IInventory Inventory; public VerifyResult Result; } + private class UnverifiedBlocks { public LinkedList Blocks = new LinkedList(); public int Size; } + private const int MaxUnverifiedBlockSize = 100 * 1024 * 1024; public static readonly uint MillisecondsPerBlock = ProtocolSettings.Default.MillisecondsPerBlock; public static readonly TimeSpan TimePerBlock = TimeSpan.FromMilliseconds(MillisecondsPerBlock); public static readonly ECPoint[] StandbyCommittee = ProtocolSettings.Default.StandbyCommittee.Select(p => ECPoint.DecodePoint(p.HexToBytes(), ECCurve.Secp256r1)).ToArray(); @@ -61,7 +63,8 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private readonly List header_index = new List(); private uint stored_header_count = 0; private readonly Dictionary block_cache = new Dictionary(); - private readonly Dictionary> block_cache_unverified = new Dictionary>(); + private readonly Dictionary block_cache_unverified = new Dictionary(); + private int block_cache_unverified_size = 0; internal readonly RelayCache RelayCache = new RelayCache(100); private SnapshotView currentSnapshot; @@ -263,22 +266,49 @@ private void OnImport(IEnumerable blocks, bool verify) Sender.Tell(new ImportCompleted()); } + private void RemoveUnverifiedBlockToCache(uint index) + { + if (block_cache_unverified.Remove(index, out var entry)) + { + block_cache_unverified_size -= entry.Size; + } + } + private void AddUnverifiedBlockToCache(Block block) { + if (block_cache_unverified_size > MaxUnverifiedBlockSize && block.Index > Height + 10) + { + // Drop last entry if it's higher + + if (block_cache_unverified.Count > 0) + { + var max = block_cache_unverified.Keys.Max(); + if (max > Height + 10) + { + RemoveUnverifiedBlockToCache(max); + } + } + + // Drop block, we can't save it safely in memory + return; + } + // Check if any block proposal for height `block.Index` exists - if (!block_cache_unverified.TryGetValue(block.Index, out LinkedList blocks)) + if (!block_cache_unverified.TryGetValue(block.Index, out var blocks)) { // There are no blocks, a new LinkedList is created and, consequently, the current block is added to the list - blocks = new LinkedList(); + blocks = new UnverifiedBlocks(); block_cache_unverified.Add(block.Index, blocks); } // Check if any block with the hash being added already exists on possible candidates to be processed - foreach (var unverifiedBlock in blocks) + foreach (var unverifiedBlock in blocks.Blocks) { if (block.Hash == unverifiedBlock.Hash) return; } - blocks.AddLast(block); + blocks.Blocks.AddLast(block); + blocks.Size += block.Size; + block_cache_unverified_size += block.Size; } private void OnFillMemoryPool(IEnumerable transactions) @@ -328,16 +358,16 @@ private VerifyResult OnNewBlock(Block block) if (!block.Verify(currentSnapshot)) return VerifyResult.Invalid; block_cache.TryAdd(block.Hash, block); - block_cache_unverified.Remove(block.Index); + RemoveUnverifiedBlockToCache(block.Index); // We can store the new block in block_cache and tell the new height to other nodes before Persist(). system.LocalNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Singleton.Height + 1))); Persist(block); SaveHeaderHashList(); - if (block_cache_unverified.TryGetValue(Height + 1, out LinkedList unverifiedBlocks)) + if (block_cache_unverified.TryGetValue(Height + 1, out var unverifiedBlocks)) { - foreach (var unverifiedBlock in unverifiedBlocks) + foreach (var unverifiedBlock in unverifiedBlocks.Blocks) Self.Tell(unverifiedBlock, ActorRefs.NoSender); - block_cache_unverified.Remove(Height + 1); + RemoveUnverifiedBlockToCache(Height + 1); } } return VerifyResult.Succeed; From f8cad046e3862c3195697b9f7d9fd18e7ae2821b Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 25 Aug 2020 10:43:06 +0200 Subject: [PATCH 02/14] Remove known hashes --- src/neo/Ledger/Blockchain.cs | 16 ++++++++++++---- src/neo/Network/P2P/LocalNode.cs | 4 ++++ .../Network/P2P/RemoteNode.ProtocolHandler.cs | 5 +++++ src/neo/Network/P2P/RemoteNode.cs | 3 +++ src/neo/Network/P2P/TaskManager.cs | 9 +++++++++ 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 794f870848..7f93002855 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -266,11 +266,19 @@ private void OnImport(IEnumerable blocks, bool verify) Sender.Tell(new ImportCompleted()); } - private void RemoveUnverifiedBlockToCache(uint index) + private void RemoveUnverifiedBlockToCache(uint index, bool removeKnownHashes) { if (block_cache_unverified.Remove(index, out var entry)) { block_cache_unverified_size -= entry.Size; + + if (removeKnownHashes) + { + var hashes = entry.Blocks.Select(u => u.Hash).ToArray(); + + system.LocalNode.Tell(new TaskManager.RemoveKnownHashes { Hashes = hashes }); + system.TaskManager.Tell(new TaskManager.RemoveKnownHashes { Hashes = hashes }); + } } } @@ -285,7 +293,7 @@ private void AddUnverifiedBlockToCache(Block block) var max = block_cache_unverified.Keys.Max(); if (max > Height + 10) { - RemoveUnverifiedBlockToCache(max); + RemoveUnverifiedBlockToCache(max, true); } } @@ -358,7 +366,7 @@ private VerifyResult OnNewBlock(Block block) if (!block.Verify(currentSnapshot)) return VerifyResult.Invalid; block_cache.TryAdd(block.Hash, block); - RemoveUnverifiedBlockToCache(block.Index); + RemoveUnverifiedBlockToCache(block.Index, false); // We can store the new block in block_cache and tell the new height to other nodes before Persist(). system.LocalNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Singleton.Height + 1))); Persist(block); @@ -367,7 +375,7 @@ private VerifyResult OnNewBlock(Block block) { foreach (var unverifiedBlock in unverifiedBlocks.Blocks) Self.Tell(unverifiedBlock, ActorRefs.NoSender); - RemoveUnverifiedBlockToCache(Height + 1); + RemoveUnverifiedBlockToCache(Height + 1, false); } } return VerifyResult.Succeed; diff --git a/src/neo/Network/P2P/LocalNode.cs b/src/neo/Network/P2P/LocalNode.cs index c8bc937001..6498e56ecc 100644 --- a/src/neo/Network/P2P/LocalNode.cs +++ b/src/neo/Network/P2P/LocalNode.cs @@ -196,6 +196,9 @@ protected override void OnReceive(object message) case SendDirectly send: OnSendDirectly(send.Inventory); break; + case TaskManager.RemoveKnownHashes remove: + OnRemoveKnownHashes(remove); + break; } } @@ -216,6 +219,7 @@ private void OnRelayDirectly(IInventory inventory) SendToRemoteNodes(message); } + private void OnRemoveKnownHashes(TaskManager.RemoveKnownHashes remove) => SendToRemoteNodes(remove); private void OnSendDirectly(IInventory inventory) => SendToRemoteNodes(inventory); protected override void OnTcpConnected(IActorRef connection) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 0cddbd49d7..8444512eb2 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -37,6 +37,11 @@ protected override UInt256 GetKeyForItem((UInt256, DateTime) item) private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); + private void OnRemoveKnownHashes(UInt256[] hashes) + { + knownHashes.ExceptWith(hashes); + } + private void OnMessage(Message msg) { foreach (IP2PPlugin plugin in Plugin.P2PPlugins) diff --git a/src/neo/Network/P2P/RemoteNode.cs b/src/neo/Network/P2P/RemoteNode.cs index b1457a8522..0dcc48878b 100644 --- a/src/neo/Network/P2P/RemoteNode.cs +++ b/src/neo/Network/P2P/RemoteNode.cs @@ -123,6 +123,9 @@ protected override void OnReceive(object message) case Timer _: RefreshPendingKnownHashes(); break; + case TaskManager.RemoveKnownHashes remove: + OnRemoveKnownHashes(remove.Hashes); + break; case Message msg: EnqueueMessage(msg); break; diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 5bea005a36..c71d35ee14 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -19,6 +19,7 @@ public class Update { public uint LastBlockIndex; public bool RequestTasks; } public class NewTasks { public InvPayload Payload; } public class RestartTasks { public InvPayload Payload; } private class Timer { } + internal class RemoveKnownHashes { public UInt256[] Hashes; } private static readonly TimeSpan TimerInterval = TimeSpan.FromSeconds(30); private static readonly TimeSpan TaskTimeout = TimeSpan.FromMinutes(1); @@ -121,10 +122,18 @@ private void OnPersistCompleted(Block block) receivedBlockIndex.Remove(block.Index); } + private void OnRemoveKnownHashes(UInt256[] hashes) + { + knownHashes.ExceptWith(hashes); + } + protected override void OnReceive(object message) { switch (message) { + case RemoveKnownHashes remove: + OnRemoveKnownHashes(remove.Hashes); + break; case Register register: OnRegister(register.Version); break; From 7ad05307a3eb2bc52e5ca774d71e75207a2cdbe3 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Aug 2020 12:13:47 +0200 Subject: [PATCH 03/14] Optimize --- src/neo/Ledger/Blockchain.cs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 7f93002855..18873d0c0d 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -284,21 +284,11 @@ private void RemoveUnverifiedBlockToCache(uint index, bool removeKnownHashes) private void AddUnverifiedBlockToCache(Block block) { - if (block_cache_unverified_size > MaxUnverifiedBlockSize && block.Index > Height + 10) + while (block_cache_unverified_size > MaxUnverifiedBlockSize && block_cache_unverified.Count > 0) { - // Drop last entry if it's higher - - if (block_cache_unverified.Count > 0) - { - var max = block_cache_unverified.Keys.Max(); - if (max > Height + 10) - { - RemoveUnverifiedBlockToCache(max, true); - } - } - - // Drop block, we can't save it safely in memory - return; + // Drop last entry + var max = block_cache_unverified.Keys.Max(); + RemoveUnverifiedBlockToCache(max, true); } // Check if any block proposal for height `block.Index` exists From dfbff17ca79e3aa964b09df53717770e4ef8f1cc Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Aug 2020 12:29:53 +0200 Subject: [PATCH 04/14] Update Blockchain.cs --- src/neo/Ledger/Blockchain.cs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 18873d0c0d..665dfce4d2 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -284,13 +284,6 @@ private void RemoveUnverifiedBlockToCache(uint index, bool removeKnownHashes) private void AddUnverifiedBlockToCache(Block block) { - while (block_cache_unverified_size > MaxUnverifiedBlockSize && block_cache_unverified.Count > 0) - { - // Drop last entry - var max = block_cache_unverified.Keys.Max(); - RemoveUnverifiedBlockToCache(max, true); - } - // Check if any block proposal for height `block.Index` exists if (!block_cache_unverified.TryGetValue(block.Index, out var blocks)) { @@ -298,15 +291,26 @@ private void AddUnverifiedBlockToCache(Block block) blocks = new UnverifiedBlocks(); block_cache_unverified.Add(block.Index, blocks); } - // Check if any block with the hash being added already exists on possible candidates to be processed - foreach (var unverifiedBlock in blocks.Blocks) + else { - if (block.Hash == unverifiedBlock.Hash) - return; + // Check if any block with the hash being added already exists on possible candidates to be processed + foreach (var unverifiedBlock in blocks.Blocks) + { + if (block.Hash == unverifiedBlock.Hash) + return; + } } + blocks.Blocks.AddLast(block); blocks.Size += block.Size; block_cache_unverified_size += block.Size; + + while (block_cache_unverified_size > MaxUnverifiedBlockSize && block_cache_unverified.Count > 0) + { + // Drop last entry + var max = block_cache_unverified.Keys.Max(); + RemoveUnverifiedBlockToCache(max, true); + } } private void OnFillMemoryPool(IEnumerable transactions) From 8d27f5a6c877f535df077a3fb3fb920c47c59daa Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Aug 2020 12:32:24 +0200 Subject: [PATCH 05/14] Clean code --- src/neo/Ledger/Blockchain.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 665dfce4d2..77c729ec31 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -305,11 +305,10 @@ private void AddUnverifiedBlockToCache(Block block) blocks.Size += block.Size; block_cache_unverified_size += block.Size; - while (block_cache_unverified_size > MaxUnverifiedBlockSize && block_cache_unverified.Count > 0) + while (block_cache_unverified_size > MaxUnverifiedBlockSize) { // Drop last entry - var max = block_cache_unverified.Keys.Max(); - RemoveUnverifiedBlockToCache(max, true); + RemoveUnverifiedBlockToCache(block_cache_unverified.Keys.Max(), true); } } From 9fff15badc5cf2b2bff773b31cb839eeefcc7e70 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Aug 2020 12:33:49 +0200 Subject: [PATCH 06/14] Rename method --- src/neo/Ledger/Blockchain.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 77c729ec31..991bad8e97 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -266,7 +266,7 @@ private void OnImport(IEnumerable blocks, bool verify) Sender.Tell(new ImportCompleted()); } - private void RemoveUnverifiedBlockToCache(uint index, bool removeKnownHashes) + private void RemoveUnverifiedBlockFromCache(uint index, bool removeKnownHashes) { if (block_cache_unverified.Remove(index, out var entry)) { @@ -308,7 +308,7 @@ private void AddUnverifiedBlockToCache(Block block) while (block_cache_unverified_size > MaxUnverifiedBlockSize) { // Drop last entry - RemoveUnverifiedBlockToCache(block_cache_unverified.Keys.Max(), true); + RemoveUnverifiedBlockFromCache(block_cache_unverified.Keys.Max(), true); } } @@ -359,7 +359,7 @@ private VerifyResult OnNewBlock(Block block) if (!block.Verify(currentSnapshot)) return VerifyResult.Invalid; block_cache.TryAdd(block.Hash, block); - RemoveUnverifiedBlockToCache(block.Index, false); + RemoveUnverifiedBlockFromCache(block.Index, false); // We can store the new block in block_cache and tell the new height to other nodes before Persist(). system.LocalNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Singleton.Height + 1))); Persist(block); @@ -368,7 +368,7 @@ private VerifyResult OnNewBlock(Block block) { foreach (var unverifiedBlock in unverifiedBlocks.Blocks) Self.Tell(unverifiedBlock, ActorRefs.NoSender); - RemoveUnverifiedBlockToCache(Height + 1, false); + RemoveUnverifiedBlockFromCache(Height + 1, false); } } return VerifyResult.Succeed; From 15c4b57f0755ea7789a53d76fb13bb0f20c2d9c9 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Aug 2020 13:29:04 +0200 Subject: [PATCH 07/14] Prevent process far blocks --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 8444512eb2..1bea649eb9 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -292,18 +292,20 @@ private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) private void OnInventoryReceived(IInventory inventory) { pendingKnownHashes.Remove(inventory.Hash); - knownHashes.Add(inventory.Hash); - system.TaskManager.Tell(inventory); - system.Blockchain.Tell(inventory, ActorRefs.NoSender); switch (inventory) { case Transaction transaction: + knownHashes.Add(inventory.Hash); system.Consensus?.Tell(transaction); break; case Block block: + if (block.Index > Blockchain.Singleton.Height + 500) return; + knownHashes.Add(inventory.Hash); UpdateLastBlockIndex(block.Index, false); break; } + system.TaskManager.Tell(inventory); + system.Blockchain.Tell(inventory, ActorRefs.NoSender); } private void OnInvMessageReceived(InvPayload payload) From 596470ab69c811672ed5bc690f665c3561a5f8b1 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 26 Aug 2020 13:38:27 +0200 Subject: [PATCH 08/14] Block ACL --- src/neo/Ledger/Blockchain.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 991bad8e97..c51ea3163d 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -1,5 +1,6 @@ using Akka.Actor; using Akka.Configuration; +using Akka.IO; using Neo.Cryptography.ECC; using Neo.IO; using Neo.IO.Actors; @@ -28,7 +29,7 @@ public class FillMemoryPool { public IEnumerable Transactions; } public class FillCompleted { } internal class PreverifyCompleted { public Transaction Transaction; public VerifyResult Result; public bool Relay; } public class RelayResult { public IInventory Inventory; public VerifyResult Result; } - private class UnverifiedBlocks { public LinkedList Blocks = new LinkedList(); public int Size; } + private class UnverifiedBlocks { public LinkedList Blocks = new LinkedList(); public HashSet Nodes = new HashSet(); public int Size; } private const int MaxUnverifiedBlockSize = 100 * 1024 * 1024; public static readonly uint MillisecondsPerBlock = ProtocolSettings.Default.MillisecondsPerBlock; @@ -299,6 +300,13 @@ private void AddUnverifiedBlockToCache(Block block) if (block.Hash == unverifiedBlock.Hash) return; } + + if (!blocks.Nodes.Add(Sender)) + { + // Same index with different hash + Sender.Tell(Tcp.Abort.Instance); + return; + } } blocks.Blocks.AddLast(block); From 93ebf53406793bc93d552847b7d72e01a0bc2264 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 1 Sep 2020 12:17:35 +0200 Subject: [PATCH 09/14] Remove size control https://github.com/neo-project/neo/pull/1870#discussion_r480079809 --- src/neo/Ledger/Blockchain.cs | 32 ++----------------- src/neo/Network/P2P/LocalNode.cs | 4 --- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 5 --- src/neo/Network/P2P/RemoteNode.cs | 3 -- src/neo/Network/P2P/TaskManager.cs | 9 ------ 5 files changed, 3 insertions(+), 50 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index c51ea3163d..8b7127677e 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -29,9 +29,8 @@ public class FillMemoryPool { public IEnumerable Transactions; } public class FillCompleted { } internal class PreverifyCompleted { public Transaction Transaction; public VerifyResult Result; public bool Relay; } public class RelayResult { public IInventory Inventory; public VerifyResult Result; } - private class UnverifiedBlocks { public LinkedList Blocks = new LinkedList(); public HashSet Nodes = new HashSet(); public int Size; } + private class UnverifiedBlocks { public LinkedList Blocks = new LinkedList(); public HashSet Nodes = new HashSet(); } - private const int MaxUnverifiedBlockSize = 100 * 1024 * 1024; public static readonly uint MillisecondsPerBlock = ProtocolSettings.Default.MillisecondsPerBlock; public static readonly TimeSpan TimePerBlock = TimeSpan.FromMilliseconds(MillisecondsPerBlock); public static readonly ECPoint[] StandbyCommittee = ProtocolSettings.Default.StandbyCommittee.Select(p => ECPoint.DecodePoint(p.HexToBytes(), ECCurve.Secp256r1)).ToArray(); @@ -65,7 +64,6 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private uint stored_header_count = 0; private readonly Dictionary block_cache = new Dictionary(); private readonly Dictionary block_cache_unverified = new Dictionary(); - private int block_cache_unverified_size = 0; internal readonly RelayCache RelayCache = new RelayCache(100); private SnapshotView currentSnapshot; @@ -267,22 +265,6 @@ private void OnImport(IEnumerable blocks, bool verify) Sender.Tell(new ImportCompleted()); } - private void RemoveUnverifiedBlockFromCache(uint index, bool removeKnownHashes) - { - if (block_cache_unverified.Remove(index, out var entry)) - { - block_cache_unverified_size -= entry.Size; - - if (removeKnownHashes) - { - var hashes = entry.Blocks.Select(u => u.Hash).ToArray(); - - system.LocalNode.Tell(new TaskManager.RemoveKnownHashes { Hashes = hashes }); - system.TaskManager.Tell(new TaskManager.RemoveKnownHashes { Hashes = hashes }); - } - } - } - private void AddUnverifiedBlockToCache(Block block) { // Check if any block proposal for height `block.Index` exists @@ -310,14 +292,6 @@ private void AddUnverifiedBlockToCache(Block block) } blocks.Blocks.AddLast(block); - blocks.Size += block.Size; - block_cache_unverified_size += block.Size; - - while (block_cache_unverified_size > MaxUnverifiedBlockSize) - { - // Drop last entry - RemoveUnverifiedBlockFromCache(block_cache_unverified.Keys.Max(), true); - } } private void OnFillMemoryPool(IEnumerable transactions) @@ -367,7 +341,7 @@ private VerifyResult OnNewBlock(Block block) if (!block.Verify(currentSnapshot)) return VerifyResult.Invalid; block_cache.TryAdd(block.Hash, block); - RemoveUnverifiedBlockFromCache(block.Index, false); + block_cache_unverified.Remove(block.Index); // We can store the new block in block_cache and tell the new height to other nodes before Persist(). system.LocalNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Singleton.Height + 1))); Persist(block); @@ -376,7 +350,7 @@ private VerifyResult OnNewBlock(Block block) { foreach (var unverifiedBlock in unverifiedBlocks.Blocks) Self.Tell(unverifiedBlock, ActorRefs.NoSender); - RemoveUnverifiedBlockFromCache(Height + 1, false); + block_cache_unverified.Remove(Height + 1); } } return VerifyResult.Succeed; diff --git a/src/neo/Network/P2P/LocalNode.cs b/src/neo/Network/P2P/LocalNode.cs index 6498e56ecc..c8bc937001 100644 --- a/src/neo/Network/P2P/LocalNode.cs +++ b/src/neo/Network/P2P/LocalNode.cs @@ -196,9 +196,6 @@ protected override void OnReceive(object message) case SendDirectly send: OnSendDirectly(send.Inventory); break; - case TaskManager.RemoveKnownHashes remove: - OnRemoveKnownHashes(remove); - break; } } @@ -219,7 +216,6 @@ private void OnRelayDirectly(IInventory inventory) SendToRemoteNodes(message); } - private void OnRemoveKnownHashes(TaskManager.RemoveKnownHashes remove) => SendToRemoteNodes(remove); private void OnSendDirectly(IInventory inventory) => SendToRemoteNodes(inventory); protected override void OnTcpConnected(IActorRef connection) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 1bea649eb9..f569946dc7 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -37,11 +37,6 @@ protected override UInt256 GetKeyForItem((UInt256, DateTime) item) private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); - private void OnRemoveKnownHashes(UInt256[] hashes) - { - knownHashes.ExceptWith(hashes); - } - private void OnMessage(Message msg) { foreach (IP2PPlugin plugin in Plugin.P2PPlugins) diff --git a/src/neo/Network/P2P/RemoteNode.cs b/src/neo/Network/P2P/RemoteNode.cs index 0dcc48878b..b1457a8522 100644 --- a/src/neo/Network/P2P/RemoteNode.cs +++ b/src/neo/Network/P2P/RemoteNode.cs @@ -123,9 +123,6 @@ protected override void OnReceive(object message) case Timer _: RefreshPendingKnownHashes(); break; - case TaskManager.RemoveKnownHashes remove: - OnRemoveKnownHashes(remove.Hashes); - break; case Message msg: EnqueueMessage(msg); break; diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index c71d35ee14..5bea005a36 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -19,7 +19,6 @@ public class Update { public uint LastBlockIndex; public bool RequestTasks; } public class NewTasks { public InvPayload Payload; } public class RestartTasks { public InvPayload Payload; } private class Timer { } - internal class RemoveKnownHashes { public UInt256[] Hashes; } private static readonly TimeSpan TimerInterval = TimeSpan.FromSeconds(30); private static readonly TimeSpan TaskTimeout = TimeSpan.FromMinutes(1); @@ -122,18 +121,10 @@ private void OnPersistCompleted(Block block) receivedBlockIndex.Remove(block.Index); } - private void OnRemoveKnownHashes(UInt256[] hashes) - { - knownHashes.ExceptWith(hashes); - } - protected override void OnReceive(object message) { switch (message) { - case RemoveKnownHashes remove: - OnRemoveKnownHashes(remove.Hashes); - break; case Register register: OnRegister(register.Version); break; From 44a20a3b21c1c4d85ce20094a4a59c7abbd82fa1 Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 3 Sep 2020 16:08:39 +0200 Subject: [PATCH 10/14] Move knownHashes --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index f569946dc7..c8f0f77e44 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -290,15 +290,14 @@ private void OnInventoryReceived(IInventory inventory) switch (inventory) { case Transaction transaction: - knownHashes.Add(inventory.Hash); system.Consensus?.Tell(transaction); break; case Block block: if (block.Index > Blockchain.Singleton.Height + 500) return; - knownHashes.Add(inventory.Hash); UpdateLastBlockIndex(block.Index, false); break; } + knownHashes.Add(inventory.Hash); system.TaskManager.Tell(inventory); system.Blockchain.Tell(inventory, ActorRefs.NoSender); } From 899b404330ab564266f74933bfc88fc8a0f28e85 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 6 Sep 2020 12:17:03 +0200 Subject: [PATCH 11/14] Remove no sender --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index c8f0f77e44..d1d37d7f5e 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -299,7 +299,7 @@ private void OnInventoryReceived(IInventory inventory) } knownHashes.Add(inventory.Hash); system.TaskManager.Tell(inventory); - system.Blockchain.Tell(inventory, ActorRefs.NoSender); + system.Blockchain.Tell(inventory); } private void OnInvMessageReceived(InvPayload payload) From 6164e8f936b99419023093f7bebc4939abe44490 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Sun, 6 Sep 2020 18:46:21 +0800 Subject: [PATCH 12/14] Rename --- src/neo/Ledger/Blockchain.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 01653dcbaf..ba797186d7 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -29,7 +29,7 @@ public class FillMemoryPool { public IEnumerable Transactions; } public class FillCompleted { } internal class PreverifyCompleted { public Transaction Transaction; public VerifyResult Result; public bool Relay; } public class RelayResult { public IInventory Inventory; public VerifyResult Result; } - private class UnverifiedBlocks { public LinkedList Blocks = new LinkedList(); public HashSet Nodes = new HashSet(); } + private class UnverifiedBlocksList { public LinkedList Blocks = new LinkedList(); public HashSet Nodes = new HashSet(); } public static readonly uint MillisecondsPerBlock = ProtocolSettings.Default.MillisecondsPerBlock; public static readonly TimeSpan TimePerBlock = TimeSpan.FromMilliseconds(MillisecondsPerBlock); @@ -63,7 +63,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private readonly List header_index = new List(); private uint stored_header_count = 0; private readonly Dictionary block_cache = new Dictionary(); - private readonly Dictionary block_cache_unverified = new Dictionary(); + private readonly Dictionary block_cache_unverified = new Dictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); private SnapshotView currentSnapshot; @@ -278,7 +278,7 @@ private void AddUnverifiedBlockToCache(Block block) if (!block_cache_unverified.TryGetValue(block.Index, out var blocks)) { // There are no blocks, a new LinkedList is created and, consequently, the current block is added to the list - blocks = new UnverifiedBlocks(); + blocks = new UnverifiedBlocksList(); block_cache_unverified.Add(block.Index, blocks); } else From 4def3f27ca5cffcc367738a87aedbce874f5f1d9 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Sun, 6 Sep 2020 18:51:55 +0800 Subject: [PATCH 13/14] Rename --- src/neo/Ledger/Blockchain.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index ba797186d7..3c27ecfacc 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -275,22 +275,22 @@ private void OnImport(IEnumerable blocks, bool verify) private void AddUnverifiedBlockToCache(Block block) { // Check if any block proposal for height `block.Index` exists - if (!block_cache_unverified.TryGetValue(block.Index, out var blocks)) + if (!block_cache_unverified.TryGetValue(block.Index, out var list)) { - // There are no blocks, a new LinkedList is created and, consequently, the current block is added to the list - blocks = new UnverifiedBlocksList(); - block_cache_unverified.Add(block.Index, blocks); + // There are no blocks, a new UnverifiedBlocksList is created and, consequently, the current block is added to the list + list = new UnverifiedBlocksList(); + block_cache_unverified.Add(block.Index, list); } else { // Check if any block with the hash being added already exists on possible candidates to be processed - foreach (var unverifiedBlock in blocks.Blocks) + foreach (var unverifiedBlock in list.Blocks) { if (block.Hash == unverifiedBlock.Hash) return; } - if (!blocks.Nodes.Add(Sender)) + if (!list.Nodes.Add(Sender)) { // Same index with different hash Sender.Tell(Tcp.Abort.Instance); @@ -298,7 +298,7 @@ private void AddUnverifiedBlockToCache(Block block) } } - blocks.Blocks.AddLast(block); + list.Blocks.AddLast(block); } private void OnFillMemoryPool(IEnumerable transactions) From 52ef2747a0d97af70f7dabc7133a129471094740 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Sun, 6 Sep 2020 18:58:36 +0800 Subject: [PATCH 14/14] Update RemoteNode.ProtocolHandler.cs --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index d1d37d7f5e..2c3a6c5682 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -293,7 +293,7 @@ private void OnInventoryReceived(IInventory inventory) system.Consensus?.Tell(transaction); break; case Block block: - if (block.Index > Blockchain.Singleton.Height + 500) return; + if (block.Index > Blockchain.Singleton.Height + InvPayload.MaxHashesCount) return; UpdateLastBlockIndex(block.Index, false); break; }