From 5a8cf21f5e960a07f7001bcad035da1be6fadc65 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 8 Jul 2020 17:09:04 +0800 Subject: [PATCH 1/7] Remove the lock from SendersFeeMonitor and rename it to TransactionVerificationContext --- src/neo/Consensus/ConsensusContext.cs | 10 +- src/neo/Consensus/ConsensusService.cs | 28 +- src/neo/Ledger/Blockchain.cs | 13 +- src/neo/Ledger/MemoryPool.cs | 48 +- src/neo/Ledger/SendersFeeMonitor.cs | 43 - .../Ledger/TransactionVerificationContext.cs | 37 + src/neo/Network/P2P/Payloads/Transaction.cs | 13 +- tests/neo.UnitTests/Ledger/UT_MemoryPool.cs | 1040 ++++++++--------- ...s => UT_TransactionVerificationContext.cs} | 27 +- 9 files changed, 624 insertions(+), 635 deletions(-) delete mode 100644 src/neo/Ledger/SendersFeeMonitor.cs create mode 100644 src/neo/Ledger/TransactionVerificationContext.cs rename tests/neo.UnitTests/Ledger/{UT_SendersFeeMonitor.cs => UT_TransactionVerificationContext.cs} (58%) diff --git a/src/neo/Consensus/ConsensusContext.cs b/src/neo/Consensus/ConsensusContext.cs index 3d3a4fcb5d..7b1cc8b3d0 100644 --- a/src/neo/Consensus/ConsensusContext.cs +++ b/src/neo/Consensus/ConsensusContext.cs @@ -40,7 +40,7 @@ internal class ConsensusContext : IDisposable, ISerializable /// /// Store all verified unsorted transactions' senders' fee currently in the consensus context. /// - public SendersFeeMonitor SendersFeeMonitor = new SendersFeeMonitor(); + public TransactionVerificationContext VerificationContext = new TransactionVerificationContext(); public SnapshotView Snapshot { get; private set; } private KeyPair keyPair; @@ -116,11 +116,11 @@ public void Deserialize(BinaryReader reader) if (TransactionHashes.Length == 0 && !RequestSentOrReceived) TransactionHashes = null; Transactions = transactions.Length == 0 && !RequestSentOrReceived ? null : transactions.ToDictionary(p => p.Hash); - SendersFeeMonitor = new SendersFeeMonitor(); + VerificationContext = new TransactionVerificationContext(); if (Transactions != null) { foreach (Transaction tx in Transactions.Values) - SendersFeeMonitor.AddSenderFee(tx); + VerificationContext.AddTransaction(tx); } } @@ -266,7 +266,7 @@ internal void EnsureMaxBlockLimitation(IEnumerable txs) txs = txs.Take((int)maxTransactionsPerBlock); List hashes = new List(); Transactions = new Dictionary(); - SendersFeeMonitor = new SendersFeeMonitor(); + VerificationContext = new TransactionVerificationContext(); // Expected block size var blockSize = GetExpectedBlockSizeWithoutTransactions(txs.Count()); @@ -285,7 +285,7 @@ internal void EnsureMaxBlockLimitation(IEnumerable txs) hashes.Add(tx.Hash); Transactions.Add(tx.Hash, tx); - SendersFeeMonitor.AddSenderFee(tx); + VerificationContext.AddTransaction(tx); } TransactionHashes = hashes.ToArray(); diff --git a/src/neo/Consensus/ConsensusService.cs b/src/neo/Consensus/ConsensusService.cs index 26648c4429..9caca4b651 100644 --- a/src/neo/Consensus/ConsensusService.cs +++ b/src/neo/Consensus/ConsensusService.cs @@ -64,20 +64,24 @@ internal ConsensusService(IActorRef localNode, IActorRef taskManager, IActorRef private bool AddTransaction(Transaction tx, bool verify) { - if (verify && tx.Verify(context.Snapshot, context.SendersFeeMonitor.GetSenderFee(tx.Sender)) != VerifyResult.Succeed) + if (verify) { - Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); - RequestChangeView(ChangeViewReason.TxInvalid); - return false; - } - if (!NativeContract.Policy.CheckPolicy(tx, context.Snapshot)) - { - Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); - RequestChangeView(ChangeViewReason.TxRejectedByPolicy); - return false; + VerifyResult result = tx.Verify(context.Snapshot, context.VerificationContext); + if (result == VerifyResult.PolicyFail) + { + Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); + RequestChangeView(ChangeViewReason.TxRejectedByPolicy); + return false; + } + else if (result != VerifyResult.Succeed) + { + Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); + RequestChangeView(ChangeViewReason.TxInvalid); + return false; + } } context.Transactions[tx.Hash] = tx; - context.SendersFeeMonitor.AddSenderFee(tx); + context.VerificationContext.AddTransaction(tx); return CheckPrepareResponse(); } @@ -433,7 +437,7 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m context.Block.ConsensusData.Nonce = message.Nonce; context.TransactionHashes = message.TransactionHashes; context.Transactions = new Dictionary(); - context.SendersFeeMonitor = new SendersFeeMonitor(); + context.VerificationContext = new TransactionVerificationContext(); for (int i = 0; i < context.PreparationPayloads.Length; i++) if (context.PreparationPayloads[i] != null) if (!context.PreparationPayloads[i].GetDeserializedMessage().PreparationHash.Equals(payload.Hash)) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 50cee9509d..a1ff8f91ab 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -283,15 +283,10 @@ private void OnFillMemoryPool(IEnumerable transactions) { if (View.ContainsTransaction(tx.Hash)) continue; - if (!NativeContract.Policy.CheckPolicy(tx, currentSnapshot)) - continue; // First remove the tx if it is unverified in the pool. MemPool.TryRemoveUnVerified(tx.Hash, out _); - // Verify the the transaction - if (tx.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(tx.Sender)) != VerifyResult.Succeed) - continue; // Add to the memory pool - MemPool.TryAdd(tx.Hash, tx); + MemPool.TryAdd(tx, currentSnapshot); } // Transactions originally in the pool will automatically be reverified based on their priority. @@ -355,11 +350,7 @@ private VerifyResult OnNewInventory(IInventory inventory) private VerifyResult OnNewTransaction(Transaction transaction) { if (ContainsTransaction(transaction.Hash)) return VerifyResult.AlreadyExists; - if (!MemPool.CanTransactionFitInPool(transaction)) return VerifyResult.OutOfMemory; - VerifyResult reason = transaction.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(transaction.Sender)); - if (reason != VerifyResult.Succeed) return reason; - if (!MemPool.TryAdd(transaction.Hash, transaction)) return VerifyResult.OutOfMemory; - return VerifyResult.Succeed; + return MemPool.TryAdd(transaction, currentSnapshot); } protected override void OnReceive(object message) diff --git a/src/neo/Ledger/MemoryPool.cs b/src/neo/Ledger/MemoryPool.cs index 62466d65e8..ad7410000c 100644 --- a/src/neo/Ledger/MemoryPool.cs +++ b/src/neo/Ledger/MemoryPool.cs @@ -71,7 +71,7 @@ public class MemoryPool : IReadOnlyCollection /// /// Store all verified unsorted transactions' senders' fee currently in the memory pool. /// - public SendersFeeMonitor SendersFeeMonitor = new SendersFeeMonitor(); + private TransactionVerificationContext VerificationContext = new TransactionVerificationContext(); /// /// Total count of transactions in the pool. @@ -261,18 +261,21 @@ internal bool CanTransactionFitInPool(Transaction tx) /// /// /// - internal bool TryAdd(UInt256 hash, Transaction tx) + internal VerifyResult TryAdd(Transaction tx, StoreView snapshot) { var poolItem = new PoolItem(tx); - if (_unsortedTransactions.ContainsKey(hash)) return false; + if (_unsortedTransactions.ContainsKey(tx.Hash)) return VerifyResult.AlreadyExists; List removedTransactions = null; _txRwLock.EnterWriteLock(); try { - _unsortedTransactions.Add(hash, poolItem); - SendersFeeMonitor.AddSenderFee(tx); + VerifyResult result = tx.Verify(snapshot, VerificationContext); + if (result != VerifyResult.Succeed) return result; + + _unsortedTransactions.Add(tx.Hash, poolItem); + VerificationContext.AddTransaction(tx); _sortedTransactions.Add(poolItem); if (Count > Capacity) @@ -290,7 +293,8 @@ internal bool TryAdd(UInt256 hash, Transaction tx) plugin.TransactionsRemoved(MemoryPoolTxRemovalReason.CapacityExceeded, removedTransactions); } - return _unsortedTransactions.ContainsKey(hash); + if (!_unsortedTransactions.ContainsKey(tx.Hash)) return VerifyResult.OutOfMemory; + return VerifyResult.Succeed; } private List RemoveOverCapacity() @@ -315,7 +319,7 @@ private bool TryRemoveVerified(UInt256 hash, out PoolItem item) return false; _unsortedTransactions.Remove(hash); - SendersFeeMonitor.RemoveSenderFee(item.Tx); + VerificationContext.RemoveTransaction(item.Tx); _sortedTransactions.Remove(item); return true; @@ -343,7 +347,7 @@ internal void InvalidateVerifiedTransactions() // Clear the verified transactions now, since they all must be reverified. _unsortedTransactions.Clear(); - SendersFeeMonitor = new SendersFeeMonitor(); + VerificationContext = new TransactionVerificationContext(); _sortedTransactions.Clear(); } @@ -413,23 +417,23 @@ private int ReverifyTransactions(SortedSet verifiedSortedTxPool, List reverifiedItems = new List(count); List invalidItems = new List(); - // Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end. - foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count)) + _txRwLock.EnterWriteLock(); + try { - if (item.Tx.VerifyForEachBlock(snapshot, SendersFeeMonitor.GetSenderFee(item.Tx.Sender)) == VerifyResult.Succeed) + // Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end. + foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count)) { - reverifiedItems.Add(item); - SendersFeeMonitor.AddSenderFee(item.Tx); - } - else // Transaction no longer valid -- it will be removed from unverifiedTxPool. - invalidItems.Add(item); + if (item.Tx.VerifyForEachBlock(snapshot, VerificationContext) == VerifyResult.Succeed) + { + reverifiedItems.Add(item); + VerificationContext.AddTransaction(item.Tx); + } + else // Transaction no longer valid -- it will be removed from unverifiedTxPool. + invalidItems.Add(item); - if (DateTime.UtcNow > reverifyCutOffTimeStamp) break; - } + if (DateTime.UtcNow > reverifyCutOffTimeStamp) break; + } - _txRwLock.EnterWriteLock(); - try - { int blocksTillRebroadcast = BlocksTillRebroadcast; // Increases, proportionally, blocksTillRebroadcast if mempool has more items than threshold bigger RebroadcastMultiplierThreshold if (Count > RebroadcastMultiplierThreshold) @@ -450,7 +454,7 @@ private int ReverifyTransactions(SortedSet verifiedSortedTxPool, } } else - SendersFeeMonitor.RemoveSenderFee(item.Tx); + VerificationContext.RemoveTransaction(item.Tx); _unverifiedTransactions.Remove(item.Tx.Hash); unverifiedSortedTxPool.Remove(item); diff --git a/src/neo/Ledger/SendersFeeMonitor.cs b/src/neo/Ledger/SendersFeeMonitor.cs deleted file mode 100644 index efe3ac6ebe..0000000000 --- a/src/neo/Ledger/SendersFeeMonitor.cs +++ /dev/null @@ -1,43 +0,0 @@ -using Neo.Network.P2P.Payloads; -using System.Collections.Generic; -using System.Numerics; -using System.Threading; - -namespace Neo.Ledger -{ - public class SendersFeeMonitor - { - private readonly ReaderWriterLockSlim _senderFeeRwLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); - - /// - /// Store all verified unsorted transactions' senders' fee currently in the memory pool. - /// - private readonly Dictionary _senderFee = new Dictionary(); - - public BigInteger GetSenderFee(UInt160 sender) - { - _senderFeeRwLock.EnterReadLock(); - if (!_senderFee.TryGetValue(sender, out var value)) - value = BigInteger.Zero; - _senderFeeRwLock.ExitReadLock(); - return value; - } - - public void AddSenderFee(Transaction tx) - { - _senderFeeRwLock.EnterWriteLock(); - if (_senderFee.TryGetValue(tx.Sender, out var value)) - _senderFee[tx.Sender] = value + tx.SystemFee + tx.NetworkFee; - else - _senderFee.Add(tx.Sender, tx.SystemFee + tx.NetworkFee); - _senderFeeRwLock.ExitWriteLock(); - } - - public void RemoveSenderFee(Transaction tx) - { - _senderFeeRwLock.EnterWriteLock(); - if ((_senderFee[tx.Sender] -= tx.SystemFee + tx.NetworkFee) == 0) _senderFee.Remove(tx.Sender); - _senderFeeRwLock.ExitWriteLock(); - } - } -} diff --git a/src/neo/Ledger/TransactionVerificationContext.cs b/src/neo/Ledger/TransactionVerificationContext.cs new file mode 100644 index 0000000000..a2bae45b0b --- /dev/null +++ b/src/neo/Ledger/TransactionVerificationContext.cs @@ -0,0 +1,37 @@ +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract.Native; +using System.Collections.Generic; +using System.Numerics; + +namespace Neo.Ledger +{ + public class TransactionVerificationContext + { + /// + /// Store all verified unsorted transactions' senders' fee currently in the memory pool. + /// + private readonly Dictionary senderFee = new Dictionary(); + + public void AddTransaction(Transaction tx) + { + if (senderFee.TryGetValue(tx.Sender, out var value)) + senderFee[tx.Sender] = value + tx.SystemFee + tx.NetworkFee; + else + senderFee.Add(tx.Sender, tx.SystemFee + tx.NetworkFee); + } + + public bool CheckTransaction(Transaction tx, StoreView snapshot) + { + BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, tx.Sender); + senderFee.TryGetValue(tx.Sender, out var totalSenderFeeFromPool); + BigInteger fee = tx.SystemFee + tx.NetworkFee + totalSenderFeeFromPool; + return balance >= fee; + } + + public void RemoveTransaction(Transaction tx) + { + if ((senderFee[tx.Sender] -= tx.SystemFee + tx.NetworkFee) == 0) senderFee.Remove(tx.Sender); + } + } +} diff --git a/src/neo/Network/P2P/Payloads/Transaction.cs b/src/neo/Network/P2P/Payloads/Transaction.cs index 127f56cb17..0f14e62908 100644 --- a/src/neo/Network/P2P/Payloads/Transaction.cs +++ b/src/neo/Network/P2P/Payloads/Transaction.cs @@ -12,7 +12,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Numerics; using Array = Neo.VM.Types.Array; namespace Neo.Network.P2P.Payloads @@ -258,10 +257,10 @@ public JObject ToJson() bool IInventory.Verify(StoreView snapshot) { - return Verify(snapshot, BigInteger.Zero) == VerifyResult.Succeed; + return Verify(snapshot, null) == VerifyResult.Succeed; } - public virtual VerifyResult VerifyForEachBlock(StoreView snapshot, BigInteger totalSenderFeeFromPool) + public virtual VerifyResult VerifyForEachBlock(StoreView snapshot, TransactionVerificationContext context) { if (ValidUntilBlock <= snapshot.Height || ValidUntilBlock > snapshot.Height + MaxValidUntilBlockIncrement) return VerifyResult.Expired; @@ -270,9 +269,7 @@ public virtual VerifyResult VerifyForEachBlock(StoreView snapshot, BigInteger to return VerifyResult.PolicyFail; if (NativeContract.Policy.GetMaxBlockSystemFee(snapshot) < SystemFee) return VerifyResult.PolicyFail; - BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, Sender); - BigInteger fee = SystemFee + NetworkFee + totalSenderFeeFromPool; - if (balance < fee) return VerifyResult.InsufficientFunds; + if (!(context?.CheckTransaction(this, snapshot) ?? true)) return VerifyResult.InsufficientFunds; if (hashes.Length != Witnesses.Length) return VerifyResult.Invalid; for (int i = 0; i < hashes.Length; i++) { @@ -282,9 +279,9 @@ public virtual VerifyResult VerifyForEachBlock(StoreView snapshot, BigInteger to return VerifyResult.Succeed; } - public virtual VerifyResult Verify(StoreView snapshot, BigInteger totalSenderFeeFromPool) + public virtual VerifyResult Verify(StoreView snapshot, TransactionVerificationContext context) { - VerifyResult result = VerifyForEachBlock(snapshot, totalSenderFeeFromPool); + VerifyResult result = VerifyForEachBlock(snapshot, context); if (result != VerifyResult.Succeed) return result; int size = Size; if (size > MaxTransactionSize) return VerifyResult.Invalid; diff --git a/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs b/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs index 4e96605b7b..cd93f9533a 100644 --- a/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs +++ b/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs @@ -1,520 +1,520 @@ -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; -using Neo.Cryptography; -using Neo.IO; -using Neo.Ledger; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.Plugins; -using Neo.SmartContract; -using Neo.SmartContract.Native; -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Numerics; - -namespace Neo.UnitTests.Ledger -{ - internal class TestIMemoryPoolTxObserverPlugin : Plugin, IMemoryPoolTxObserverPlugin - { - protected override void Configure() { } - public void TransactionAdded(Transaction tx) { } - public void TransactionsRemoved(MemoryPoolTxRemovalReason reason, IEnumerable transactions) { } - } - - [TestClass] - public class UT_MemoryPool - { - private const byte Prefix_MaxTransactionsPerBlock = 23; - private const byte Prefix_FeePerByte = 10; - private MemoryPool _unit; - private MemoryPool _unit2; - private TestIMemoryPoolTxObserverPlugin plugin; - - [TestInitialize] - public void TestSetup() - { - // protect against external changes on TimeProvider - TimeProvider.ResetToDefault(); - - TestBlockchain.InitializeMockNeoSystem(); - - // Create a MemoryPool with capacity of 100 - _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 100); - _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); - - // Verify capacity equals the amount specified - _unit.Capacity.Should().Be(100); - - _unit.VerifiedCount.Should().Be(0); - _unit.UnVerifiedCount.Should().Be(0); - _unit.Count.Should().Be(0); - _unit2 = new MemoryPool(TestBlockchain.TheNeoSystem, 0); - plugin = new TestIMemoryPoolTxObserverPlugin(); - } - - [TestCleanup] - public void CleanUp() - { - Plugin.TxObserverPlugins.Remove(plugin); - } - - long LongRandom(long min, long max, Random rand) - { - // Only returns positive random long values. - long longRand = (long)rand.NextBigInteger(63); - return longRand % (max - min) + min; - } - - private Transaction CreateTransactionWithFee(long fee) - { - Random random = new Random(); - var randomBytes = new byte[16]; - random.NextBytes(randomBytes); - Mock mock = new Mock(); - mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); - mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); - mock.Object.Script = randomBytes; - mock.Object.Sender = UInt160.Zero; - mock.Object.NetworkFee = fee; - mock.Object.Attributes = Array.Empty(); - mock.Object.Witnesses = new[] - { - new Witness - { - InvocationScript = new byte[0], - VerificationScript = new byte[0] - } - }; - return mock.Object; - } - - private Transaction CreateTransactionWithFeeAndBalanceVerify(long fee) - { - Random random = new Random(); - var randomBytes = new byte[16]; - random.NextBytes(randomBytes); - Mock mock = new Mock(); - UInt160 sender = UInt160.Zero; - mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns((StoreView snapshot, BigInteger amount) => NativeContract.GAS.BalanceOf(snapshot, sender) >= amount + fee ? VerifyResult.Succeed : VerifyResult.InsufficientFunds); - mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); - mock.Object.Script = randomBytes; - mock.Object.Sender = sender; - mock.Object.NetworkFee = fee; - mock.Object.Attributes = Array.Empty(); - mock.Object.Witnesses = new[] - { - new Witness - { - InvocationScript = new byte[0], - VerificationScript = new byte[0] - } - }; - return mock.Object; - } - - private Transaction CreateTransaction(long fee = -1) - { - if (fee != -1) - return CreateTransactionWithFee(fee); - return CreateTransactionWithFee(LongRandom(100000, 100000000, TestUtils.TestRandom)); - } - - private void AddTransactions(int count) - { - for (int i = 0; i < count; i++) - { - var txToAdd = CreateTransaction(); - _unit.TryAdd(txToAdd.Hash, txToAdd); - } - - Console.WriteLine($"created {count} tx"); - } - - private void AddTransaction(Transaction txToAdd) - { - _unit.TryAdd(txToAdd.Hash, txToAdd); - } - - private void AddTransactionsWithBalanceVerify(int count, long fee) - { - for (int i = 0; i < count; i++) - { - var txToAdd = CreateTransactionWithFeeAndBalanceVerify(fee); - _unit.TryAdd(txToAdd.Hash, txToAdd); - } - - Console.WriteLine($"created {count} tx"); - } - - [TestMethod] - public void CapacityTest() - { - // Add over the capacity items, verify that the verified count increases each time - AddTransactions(101); - - Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} Count {_unit.SortedTxCount}"); - - _unit.SortedTxCount.Should().Be(100); - _unit.VerifiedCount.Should().Be(100); - _unit.UnVerifiedCount.Should().Be(0); - _unit.Count.Should().Be(100); - } - - [TestMethod] - public void BlockPersistMovesTxToUnverifiedAndReverification() - { - AddTransactions(70); - - _unit.SortedTxCount.Should().Be(70); - - var block = new Block - { - Transactions = _unit.GetSortedVerifiedTransactions().Take(10) - .Concat(_unit.GetSortedVerifiedTransactions().Take(5)).ToArray() - }; - _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); - _unit.InvalidateVerifiedTransactions(); - _unit.SortedTxCount.Should().Be(0); - _unit.UnverifiedSortedTxCount.Should().Be(60); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(10); - _unit.UnverifiedSortedTxCount.Should().Be(50); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(20); - _unit.UnverifiedSortedTxCount.Should().Be(40); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(30); - _unit.UnverifiedSortedTxCount.Should().Be(30); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(40); - _unit.UnverifiedSortedTxCount.Should().Be(20); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(50); - _unit.UnverifiedSortedTxCount.Should().Be(10); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(60); - _unit.UnverifiedSortedTxCount.Should().Be(0); - } - - [TestMethod] - public void BlockPersistAndReverificationWillAbandonTxAsBalanceTransfered() - { - long txFee = 1; - AddTransactionsWithBalanceVerify(70, txFee); - - _unit.SortedTxCount.Should().Be(70); - - var block = new Block - { - Transactions = _unit.GetSortedVerifiedTransactions().Take(10).ToArray() - }; - - // Simulate the transfer process in tx by burning the balance - UInt160 sender = block.Transactions[0].Sender; - SnapshotView snapshot = Blockchain.Singleton.GetSnapshot(); - BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, sender); - - ApplicationEngine applicationEngine = new ApplicationEngine(TriggerType.All, block, snapshot, (long)balance); - NativeContract.GAS.Burn(applicationEngine, sender, balance); - NativeContract.GAS.Mint(applicationEngine, sender, txFee * 30); // Set the balance to meet 30 txs only - - // Persist block and reverify all the txs in mempool, but half of the txs will be discarded - _unit.UpdatePoolForBlockPersisted(block, snapshot); - _unit.SortedTxCount.Should().Be(30); - _unit.UnverifiedSortedTxCount.Should().Be(0); - - // Revert the balance - NativeContract.GAS.Burn(applicationEngine, sender, txFee * 30); - NativeContract.GAS.Mint(applicationEngine, sender, balance); - } - - private void VerifyTransactionsSortedDescending(IEnumerable transactions) - { - Transaction lastTransaction = null; - foreach (var tx in transactions) - { - if (lastTransaction != null) - { - if (lastTransaction.FeePerByte == tx.FeePerByte) - { - if (lastTransaction.NetworkFee == tx.NetworkFee) - lastTransaction.Hash.Should().BeLessThan(tx.Hash); - else - lastTransaction.NetworkFee.Should().BeGreaterThan(tx.NetworkFee); - } - else - { - lastTransaction.FeePerByte.Should().BeGreaterThan(tx.FeePerByte); - } - } - lastTransaction = tx; - } - } - - [TestMethod] - public void VerifySortOrderAndThatHighetFeeTransactionsAreReverifiedFirst() - { - AddTransactions(100); - - var sortedVerifiedTxs = _unit.GetSortedVerifiedTransactions().ToList(); - // verify all 100 transactions are returned in sorted order - sortedVerifiedTxs.Count.Should().Be(100); - VerifyTransactionsSortedDescending(sortedVerifiedTxs); - - // move all to unverified - var block = new Block { Transactions = new Transaction[0] }; - _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); - _unit.InvalidateVerifiedTransactions(); - _unit.SortedTxCount.Should().Be(0); - _unit.UnverifiedSortedTxCount.Should().Be(100); - - // We can verify the order they are re-verified by reverifying 2 at a time - while (_unit.UnVerifiedCount > 0) - { - _unit.GetVerifiedAndUnverifiedTransactions(out var sortedVerifiedTransactions, out var sortedUnverifiedTransactions); - sortedVerifiedTransactions.Count().Should().Be(0); - var sortedUnverifiedArray = sortedUnverifiedTransactions.ToArray(); - VerifyTransactionsSortedDescending(sortedUnverifiedArray); - var maxTransaction = sortedUnverifiedArray.First(); - var minTransaction = sortedUnverifiedArray.Last(); - - // reverify 1 high priority and 1 low priority transaction - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); - var verifiedTxs = _unit.GetSortedVerifiedTransactions().ToArray(); - verifiedTxs.Length.Should().Be(1); - verifiedTxs[0].Should().BeEquivalentTo(maxTransaction); - var blockWith2Tx = new Block { Transactions = new[] { maxTransaction, minTransaction } }; - // verify and remove the 2 transactions from the verified pool - _unit.UpdatePoolForBlockPersisted(blockWith2Tx, Blockchain.Singleton.GetSnapshot()); - _unit.InvalidateVerifiedTransactions(); - _unit.SortedTxCount.Should().Be(0); - } - _unit.UnverifiedSortedTxCount.Should().Be(0); - } - - void VerifyCapacityThresholdForAttemptingToAddATransaction() - { - var sortedVerified = _unit.GetSortedVerifiedTransactions().ToArray(); - - var txBarelyWontFit = CreateTransactionWithFee(sortedVerified.Last().NetworkFee - 1); - _unit.CanTransactionFitInPool(txBarelyWontFit).Should().Be(false); - var txBarelyFits = CreateTransactionWithFee(sortedVerified.Last().NetworkFee + 1); - _unit.CanTransactionFitInPool(txBarelyFits).Should().Be(true); - } - - [TestMethod] - public void VerifyCanTransactionFitInPoolWorksAsIntended() - { - AddTransactions(100); - VerifyCapacityThresholdForAttemptingToAddATransaction(); - AddTransactions(50); - VerifyCapacityThresholdForAttemptingToAddATransaction(); - AddTransactions(50); - VerifyCapacityThresholdForAttemptingToAddATransaction(); - } - - [TestMethod] - public void CapacityTestWithUnverifiedHighProirtyTransactions() - { - // Verify that unverified high priority transactions will not be pushed out of the queue by incoming - // low priority transactions - - // Fill pool with high priority transactions - AddTransactions(99); - - // move all to unverified - var block = new Block { Transactions = new Transaction[0] }; - _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); - - _unit.CanTransactionFitInPool(CreateTransaction()).Should().Be(true); - AddTransactions(1); - _unit.CanTransactionFitInPool(CreateTransactionWithFee(0)).Should().Be(false); - } - - [TestMethod] - public void TestInvalidateAll() - { - AddTransactions(30); - - _unit.UnverifiedSortedTxCount.Should().Be(0); - _unit.SortedTxCount.Should().Be(30); - _unit.InvalidateAllTransactions(); - _unit.UnverifiedSortedTxCount.Should().Be(30); - _unit.SortedTxCount.Should().Be(0); - } - - [TestMethod] - public void TestContainsKey() - { - AddTransactions(10); - - var txToAdd = CreateTransaction(); - _unit.TryAdd(txToAdd.Hash, txToAdd); - _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); - _unit.InvalidateVerifiedTransactions(); - _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); - } - - [TestMethod] - public void TestGetEnumerator() - { - AddTransactions(10); - _unit.InvalidateVerifiedTransactions(); - IEnumerator enumerator = _unit.GetEnumerator(); - foreach (Transaction tx in _unit) - { - enumerator.MoveNext(); - enumerator.Current.Should().BeSameAs(tx); - } - } - - [TestMethod] - public void TestIEnumerableGetEnumerator() - { - AddTransactions(10); - _unit.InvalidateVerifiedTransactions(); - IEnumerable enumerable = _unit; - var enumerator = enumerable.GetEnumerator(); - foreach (Transaction tx in _unit) - { - enumerator.MoveNext(); - enumerator.Current.Should().BeSameAs(tx); - } - } - - [TestMethod] - public void TestGetVerifiedTransactions() - { - var tx1 = CreateTransaction(); - var tx2 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1); - _unit.InvalidateVerifiedTransactions(); - _unit.TryAdd(tx2.Hash, tx2); - IEnumerable enumerable = _unit.GetVerifiedTransactions(); - enumerable.Count().Should().Be(1); - var enumerator = enumerable.GetEnumerator(); - enumerator.MoveNext(); - enumerator.Current.Should().BeSameAs(tx2); - } - - [TestMethod] - public void TestReVerifyTopUnverifiedTransactionsIfNeeded() - { - _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 600); - _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); - AddTransaction(CreateTransaction(100000001)); - AddTransaction(CreateTransaction(100000001)); - AddTransaction(CreateTransaction(100000001)); - AddTransaction(CreateTransaction(1)); - _unit.VerifiedCount.Should().Be(4); - _unit.UnVerifiedCount.Should().Be(0); - - _unit.InvalidateVerifiedTransactions(); - _unit.VerifiedCount.Should().Be(0); - _unit.UnVerifiedCount.Should().Be(4); - - AddTransactions(511); // Max per block currently is 512 - _unit.VerifiedCount.Should().Be(511); - _unit.UnVerifiedCount.Should().Be(4); - - var result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); - result.Should().BeTrue(); - _unit.VerifiedCount.Should().Be(512); - _unit.UnVerifiedCount.Should().Be(3); - - result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(2, Blockchain.Singleton.GetSnapshot()); - result.Should().BeTrue(); - _unit.VerifiedCount.Should().Be(514); - _unit.UnVerifiedCount.Should().Be(1); - - result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(3, Blockchain.Singleton.GetSnapshot()); - result.Should().BeFalse(); - _unit.VerifiedCount.Should().Be(515); - _unit.UnVerifiedCount.Should().Be(0); - } - - [TestMethod] - public void TestTryAdd() - { - var tx1 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1).Should().BeTrue(); - _unit.TryAdd(tx1.Hash, tx1).Should().BeFalse(); - _unit2.TryAdd(tx1.Hash, tx1).Should().BeFalse(); - } - - [TestMethod] - public void TestTryGetValue() - { - var tx1 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1); - _unit.TryGetValue(tx1.Hash, out Transaction tx).Should().BeTrue(); - tx.Should().BeEquivalentTo(tx1); - - _unit.InvalidateVerifiedTransactions(); - _unit.TryGetValue(tx1.Hash, out tx).Should().BeTrue(); - tx.Should().BeEquivalentTo(tx1); - - var tx2 = CreateTransaction(); - _unit.TryGetValue(tx2.Hash, out tx).Should().BeFalse(); - } - - [TestMethod] - public void TestUpdatePoolForBlockPersisted() - { - var snapshot = Blockchain.Singleton.GetSnapshot(); - byte[] transactionsPerBlock = { 0x18, 0x00, 0x00, 0x00 }; // 24 - byte[] feePerByte = { 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00 }; // 1048576 - StorageItem item1 = new StorageItem - { - Value = transactionsPerBlock - }; - StorageItem item2 = new StorageItem - { - Value = feePerByte - }; - var key1 = CreateStorageKey(Prefix_MaxTransactionsPerBlock); - var key2 = CreateStorageKey(Prefix_FeePerByte); - key1.Id = NativeContract.Policy.Id; - key2.Id = NativeContract.Policy.Id; - snapshot.Storages.Add(key1, item1); - snapshot.Storages.Add(key2, item2); - - var tx1 = CreateTransaction(); - var tx2 = CreateTransaction(); - Transaction[] transactions = { tx1, tx2 }; - _unit.TryAdd(tx1.Hash, tx1); - - var block = new Block { Transactions = transactions }; - - _unit.UnVerifiedCount.Should().Be(0); - _unit.VerifiedCount.Should().Be(1); - - _unit.UpdatePoolForBlockPersisted(block, snapshot); - - _unit.UnVerifiedCount.Should().Be(0); - _unit.VerifiedCount.Should().Be(0); - } - - public StorageKey CreateStorageKey(byte prefix, byte[] key = null) - { - StorageKey storageKey = new StorageKey - { - Id = 0, - Key = new byte[sizeof(byte) + (key?.Length ?? 0)] - }; - storageKey.Key[0] = prefix; - if (key != null) - Buffer.BlockCopy(key, 0, storageKey.Key, 1, key.Length); - return storageKey; - } - } -} +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Cryptography; +using Neo.IO; +using Neo.Ledger; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Plugins; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; + +namespace Neo.UnitTests.Ledger +{ + internal class TestIMemoryPoolTxObserverPlugin : Plugin, IMemoryPoolTxObserverPlugin + { + protected override void Configure() { } + public void TransactionAdded(Transaction tx) { } + public void TransactionsRemoved(MemoryPoolTxRemovalReason reason, IEnumerable transactions) { } + } + + [TestClass] + public class UT_MemoryPool + { + private const byte Prefix_MaxTransactionsPerBlock = 23; + private const byte Prefix_FeePerByte = 10; + private MemoryPool _unit; + private MemoryPool _unit2; + private TestIMemoryPoolTxObserverPlugin plugin; + + [TestInitialize] + public void TestSetup() + { + // protect against external changes on TimeProvider + TimeProvider.ResetToDefault(); + + TestBlockchain.InitializeMockNeoSystem(); + + // Create a MemoryPool with capacity of 100 + _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 100); + _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); + + // Verify capacity equals the amount specified + _unit.Capacity.Should().Be(100); + + _unit.VerifiedCount.Should().Be(0); + _unit.UnVerifiedCount.Should().Be(0); + _unit.Count.Should().Be(0); + _unit2 = new MemoryPool(TestBlockchain.TheNeoSystem, 0); + plugin = new TestIMemoryPoolTxObserverPlugin(); + } + + [TestCleanup] + public void CleanUp() + { + Plugin.TxObserverPlugins.Remove(plugin); + } + + long LongRandom(long min, long max, Random rand) + { + // Only returns positive random long values. + long longRand = (long)rand.NextBigInteger(63); + return longRand % (max - min) + min; + } + + private Transaction CreateTransactionWithFee(long fee) + { + Random random = new Random(); + var randomBytes = new byte[16]; + random.NextBytes(randomBytes); + Mock mock = new Mock(); + mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Object.Script = randomBytes; + mock.Object.Sender = UInt160.Zero; + mock.Object.NetworkFee = fee; + mock.Object.Attributes = Array.Empty(); + mock.Object.Witnesses = new[] + { + new Witness + { + InvocationScript = new byte[0], + VerificationScript = new byte[0] + } + }; + return mock.Object; + } + + private Transaction CreateTransactionWithFeeAndBalanceVerify(long fee) + { + Random random = new Random(); + var randomBytes = new byte[16]; + random.NextBytes(randomBytes); + Mock mock = new Mock(); + UInt160 sender = UInt160.Zero; + mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns((StoreView snapshot, BigInteger amount) => NativeContract.GAS.BalanceOf(snapshot, sender) >= amount + fee ? VerifyResult.Succeed : VerifyResult.InsufficientFunds); + mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Object.Script = randomBytes; + mock.Object.Sender = sender; + mock.Object.NetworkFee = fee; + mock.Object.Attributes = Array.Empty(); + mock.Object.Witnesses = new[] + { + new Witness + { + InvocationScript = new byte[0], + VerificationScript = new byte[0] + } + }; + return mock.Object; + } + + private Transaction CreateTransaction(long fee = -1) + { + if (fee != -1) + return CreateTransactionWithFee(fee); + return CreateTransactionWithFee(LongRandom(100000, 100000000, TestUtils.TestRandom)); + } + + private void AddTransactions(int count) + { + for (int i = 0; i < count; i++) + { + var txToAdd = CreateTransaction(); + _unit.TryAdd(txToAdd.Hash, txToAdd); + } + + Console.WriteLine($"created {count} tx"); + } + + private void AddTransaction(Transaction txToAdd) + { + _unit.TryAdd(txToAdd.Hash, txToAdd); + } + + private void AddTransactionsWithBalanceVerify(int count, long fee) + { + for (int i = 0; i < count; i++) + { + var txToAdd = CreateTransactionWithFeeAndBalanceVerify(fee); + _unit.TryAdd(txToAdd.Hash, txToAdd); + } + + Console.WriteLine($"created {count} tx"); + } + + [TestMethod] + public void CapacityTest() + { + // Add over the capacity items, verify that the verified count increases each time + AddTransactions(101); + + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} Count {_unit.SortedTxCount}"); + + _unit.SortedTxCount.Should().Be(100); + _unit.VerifiedCount.Should().Be(100); + _unit.UnVerifiedCount.Should().Be(0); + _unit.Count.Should().Be(100); + } + + [TestMethod] + public void BlockPersistMovesTxToUnverifiedAndReverification() + { + AddTransactions(70); + + _unit.SortedTxCount.Should().Be(70); + + var block = new Block + { + Transactions = _unit.GetSortedVerifiedTransactions().Take(10) + .Concat(_unit.GetSortedVerifiedTransactions().Take(5)).ToArray() + }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + _unit.InvalidateVerifiedTransactions(); + _unit.SortedTxCount.Should().Be(0); + _unit.UnverifiedSortedTxCount.Should().Be(60); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(10); + _unit.UnverifiedSortedTxCount.Should().Be(50); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(20); + _unit.UnverifiedSortedTxCount.Should().Be(40); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(30); + _unit.UnverifiedSortedTxCount.Should().Be(30); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(40); + _unit.UnverifiedSortedTxCount.Should().Be(20); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(50); + _unit.UnverifiedSortedTxCount.Should().Be(10); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(60); + _unit.UnverifiedSortedTxCount.Should().Be(0); + } + + [TestMethod] + public void BlockPersistAndReverificationWillAbandonTxAsBalanceTransfered() + { + long txFee = 1; + AddTransactionsWithBalanceVerify(70, txFee); + + _unit.SortedTxCount.Should().Be(70); + + var block = new Block + { + Transactions = _unit.GetSortedVerifiedTransactions().Take(10).ToArray() + }; + + // Simulate the transfer process in tx by burning the balance + UInt160 sender = block.Transactions[0].Sender; + SnapshotView snapshot = Blockchain.Singleton.GetSnapshot(); + BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, sender); + + ApplicationEngine applicationEngine = new ApplicationEngine(TriggerType.All, block, snapshot, (long)balance); + NativeContract.GAS.Burn(applicationEngine, sender, balance); + NativeContract.GAS.Mint(applicationEngine, sender, txFee * 30); // Set the balance to meet 30 txs only + + // Persist block and reverify all the txs in mempool, but half of the txs will be discarded + _unit.UpdatePoolForBlockPersisted(block, snapshot); + _unit.SortedTxCount.Should().Be(30); + _unit.UnverifiedSortedTxCount.Should().Be(0); + + // Revert the balance + NativeContract.GAS.Burn(applicationEngine, sender, txFee * 30); + NativeContract.GAS.Mint(applicationEngine, sender, balance); + } + + private void VerifyTransactionsSortedDescending(IEnumerable transactions) + { + Transaction lastTransaction = null; + foreach (var tx in transactions) + { + if (lastTransaction != null) + { + if (lastTransaction.FeePerByte == tx.FeePerByte) + { + if (lastTransaction.NetworkFee == tx.NetworkFee) + lastTransaction.Hash.Should().BeLessThan(tx.Hash); + else + lastTransaction.NetworkFee.Should().BeGreaterThan(tx.NetworkFee); + } + else + { + lastTransaction.FeePerByte.Should().BeGreaterThan(tx.FeePerByte); + } + } + lastTransaction = tx; + } + } + + [TestMethod] + public void VerifySortOrderAndThatHighetFeeTransactionsAreReverifiedFirst() + { + AddTransactions(100); + + var sortedVerifiedTxs = _unit.GetSortedVerifiedTransactions().ToList(); + // verify all 100 transactions are returned in sorted order + sortedVerifiedTxs.Count.Should().Be(100); + VerifyTransactionsSortedDescending(sortedVerifiedTxs); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + _unit.InvalidateVerifiedTransactions(); + _unit.SortedTxCount.Should().Be(0); + _unit.UnverifiedSortedTxCount.Should().Be(100); + + // We can verify the order they are re-verified by reverifying 2 at a time + while (_unit.UnVerifiedCount > 0) + { + _unit.GetVerifiedAndUnverifiedTransactions(out var sortedVerifiedTransactions, out var sortedUnverifiedTransactions); + sortedVerifiedTransactions.Count().Should().Be(0); + var sortedUnverifiedArray = sortedUnverifiedTransactions.ToArray(); + VerifyTransactionsSortedDescending(sortedUnverifiedArray); + var maxTransaction = sortedUnverifiedArray.First(); + var minTransaction = sortedUnverifiedArray.Last(); + + // reverify 1 high priority and 1 low priority transaction + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); + var verifiedTxs = _unit.GetSortedVerifiedTransactions().ToArray(); + verifiedTxs.Length.Should().Be(1); + verifiedTxs[0].Should().BeEquivalentTo(maxTransaction); + var blockWith2Tx = new Block { Transactions = new[] { maxTransaction, minTransaction } }; + // verify and remove the 2 transactions from the verified pool + _unit.UpdatePoolForBlockPersisted(blockWith2Tx, Blockchain.Singleton.GetSnapshot()); + _unit.InvalidateVerifiedTransactions(); + _unit.SortedTxCount.Should().Be(0); + } + _unit.UnverifiedSortedTxCount.Should().Be(0); + } + + void VerifyCapacityThresholdForAttemptingToAddATransaction() + { + var sortedVerified = _unit.GetSortedVerifiedTransactions().ToArray(); + + var txBarelyWontFit = CreateTransactionWithFee(sortedVerified.Last().NetworkFee - 1); + _unit.CanTransactionFitInPool(txBarelyWontFit).Should().Be(false); + var txBarelyFits = CreateTransactionWithFee(sortedVerified.Last().NetworkFee + 1); + _unit.CanTransactionFitInPool(txBarelyFits).Should().Be(true); + } + + [TestMethod] + public void VerifyCanTransactionFitInPoolWorksAsIntended() + { + AddTransactions(100); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + } + + [TestMethod] + public void CapacityTestWithUnverifiedHighProirtyTransactions() + { + // Verify that unverified high priority transactions will not be pushed out of the queue by incoming + // low priority transactions + + // Fill pool with high priority transactions + AddTransactions(99); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + + _unit.CanTransactionFitInPool(CreateTransaction()).Should().Be(true); + AddTransactions(1); + _unit.CanTransactionFitInPool(CreateTransactionWithFee(0)).Should().Be(false); + } + + [TestMethod] + public void TestInvalidateAll() + { + AddTransactions(30); + + _unit.UnverifiedSortedTxCount.Should().Be(0); + _unit.SortedTxCount.Should().Be(30); + _unit.InvalidateAllTransactions(); + _unit.UnverifiedSortedTxCount.Should().Be(30); + _unit.SortedTxCount.Should().Be(0); + } + + [TestMethod] + public void TestContainsKey() + { + AddTransactions(10); + + var txToAdd = CreateTransaction(); + _unit.TryAdd(txToAdd.Hash, txToAdd); + _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); + _unit.InvalidateVerifiedTransactions(); + _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); + } + + [TestMethod] + public void TestGetEnumerator() + { + AddTransactions(10); + _unit.InvalidateVerifiedTransactions(); + IEnumerator enumerator = _unit.GetEnumerator(); + foreach (Transaction tx in _unit) + { + enumerator.MoveNext(); + enumerator.Current.Should().BeSameAs(tx); + } + } + + [TestMethod] + public void TestIEnumerableGetEnumerator() + { + AddTransactions(10); + _unit.InvalidateVerifiedTransactions(); + IEnumerable enumerable = _unit; + var enumerator = enumerable.GetEnumerator(); + foreach (Transaction tx in _unit) + { + enumerator.MoveNext(); + enumerator.Current.Should().BeSameAs(tx); + } + } + + [TestMethod] + public void TestGetVerifiedTransactions() + { + var tx1 = CreateTransaction(); + var tx2 = CreateTransaction(); + _unit.TryAdd(tx1.Hash, tx1); + _unit.InvalidateVerifiedTransactions(); + _unit.TryAdd(tx2.Hash, tx2); + IEnumerable enumerable = _unit.GetVerifiedTransactions(); + enumerable.Count().Should().Be(1); + var enumerator = enumerable.GetEnumerator(); + enumerator.MoveNext(); + enumerator.Current.Should().BeSameAs(tx2); + } + + [TestMethod] + public void TestReVerifyTopUnverifiedTransactionsIfNeeded() + { + _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 600); + _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(1)); + _unit.VerifiedCount.Should().Be(4); + _unit.UnVerifiedCount.Should().Be(0); + + _unit.InvalidateVerifiedTransactions(); + _unit.VerifiedCount.Should().Be(0); + _unit.UnVerifiedCount.Should().Be(4); + + AddTransactions(511); // Max per block currently is 512 + _unit.VerifiedCount.Should().Be(511); + _unit.UnVerifiedCount.Should().Be(4); + + var result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); + result.Should().BeTrue(); + _unit.VerifiedCount.Should().Be(512); + _unit.UnVerifiedCount.Should().Be(3); + + result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(2, Blockchain.Singleton.GetSnapshot()); + result.Should().BeTrue(); + _unit.VerifiedCount.Should().Be(514); + _unit.UnVerifiedCount.Should().Be(1); + + result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(3, Blockchain.Singleton.GetSnapshot()); + result.Should().BeFalse(); + _unit.VerifiedCount.Should().Be(515); + _unit.UnVerifiedCount.Should().Be(0); + } + + [TestMethod] + public void TestTryAdd() + { + var tx1 = CreateTransaction(); + _unit.TryAdd(tx1.Hash, tx1).Should().BeTrue(); + _unit.TryAdd(tx1.Hash, tx1).Should().BeFalse(); + _unit2.TryAdd(tx1.Hash, tx1).Should().BeFalse(); + } + + [TestMethod] + public void TestTryGetValue() + { + var tx1 = CreateTransaction(); + _unit.TryAdd(tx1.Hash, tx1); + _unit.TryGetValue(tx1.Hash, out Transaction tx).Should().BeTrue(); + tx.Should().BeEquivalentTo(tx1); + + _unit.InvalidateVerifiedTransactions(); + _unit.TryGetValue(tx1.Hash, out tx).Should().BeTrue(); + tx.Should().BeEquivalentTo(tx1); + + var tx2 = CreateTransaction(); + _unit.TryGetValue(tx2.Hash, out tx).Should().BeFalse(); + } + + [TestMethod] + public void TestUpdatePoolForBlockPersisted() + { + var snapshot = Blockchain.Singleton.GetSnapshot(); + byte[] transactionsPerBlock = { 0x18, 0x00, 0x00, 0x00 }; // 24 + byte[] feePerByte = { 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00 }; // 1048576 + StorageItem item1 = new StorageItem + { + Value = transactionsPerBlock + }; + StorageItem item2 = new StorageItem + { + Value = feePerByte + }; + var key1 = CreateStorageKey(Prefix_MaxTransactionsPerBlock); + var key2 = CreateStorageKey(Prefix_FeePerByte); + key1.Id = NativeContract.Policy.Id; + key2.Id = NativeContract.Policy.Id; + snapshot.Storages.Add(key1, item1); + snapshot.Storages.Add(key2, item2); + + var tx1 = CreateTransaction(); + var tx2 = CreateTransaction(); + Transaction[] transactions = { tx1, tx2 }; + _unit.TryAdd(tx1.Hash, tx1); + + var block = new Block { Transactions = transactions }; + + _unit.UnVerifiedCount.Should().Be(0); + _unit.VerifiedCount.Should().Be(1); + + _unit.UpdatePoolForBlockPersisted(block, snapshot); + + _unit.UnVerifiedCount.Should().Be(0); + _unit.VerifiedCount.Should().Be(0); + } + + public StorageKey CreateStorageKey(byte prefix, byte[] key = null) + { + StorageKey storageKey = new StorageKey + { + Id = 0, + Key = new byte[sizeof(byte) + (key?.Length ?? 0)] + }; + storageKey.Key[0] = prefix; + if (key != null) + Buffer.BlockCopy(key, 0, storageKey.Key, 1, key.Length); + return storageKey; + } + } +} diff --git a/tests/neo.UnitTests/Ledger/UT_SendersFeeMonitor.cs b/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs similarity index 58% rename from tests/neo.UnitTests/Ledger/UT_SendersFeeMonitor.cs rename to tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs index cd8a5af463..6f8fb5d62a 100644 --- a/tests/neo.UnitTests/Ledger/UT_SendersFeeMonitor.cs +++ b/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs @@ -5,12 +5,11 @@ using Neo.Network.P2P.Payloads; using Neo.Persistence; using System; -using System.Numerics; namespace Neo.UnitTests.Ledger { [TestClass] - public class UT_SendersFeeMonitor + public class UT_TransactionVerificationContext { private Transaction CreateTransactionWithFee(long networkFee, long systemFee) { @@ -18,8 +17,8 @@ private Transaction CreateTransactionWithFee(long networkFee, long systemFee) var randomBytes = new byte[16]; random.NextBytes(randomBytes); Mock mock = new Mock(); - mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); - mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); mock.Object.Script = randomBytes; mock.Object.Sender = UInt160.Zero; mock.Object.NetworkFee = networkFee; @@ -40,16 +39,16 @@ private Transaction CreateTransactionWithFee(long networkFee, long systemFee) public void TestMemPoolSenderFee() { Transaction transaction = CreateTransactionWithFee(1, 2); - SendersFeeMonitor sendersFeeMonitor = new SendersFeeMonitor(); - sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(0); - sendersFeeMonitor.AddSenderFee(transaction); - sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(3); - sendersFeeMonitor.AddSenderFee(transaction); - sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(6); - sendersFeeMonitor.RemoveSenderFee(transaction); - sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(3); - sendersFeeMonitor.RemoveSenderFee(transaction); - sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(0); + TransactionVerificationContext verificationContext = new TransactionVerificationContext(); + verificationContext.GetSenderFee(transaction.Sender).Should().Be(0); + verificationContext.AddTransaction(transaction); + verificationContext.GetSenderFee(transaction.Sender).Should().Be(3); + verificationContext.AddTransaction(transaction); + verificationContext.GetSenderFee(transaction.Sender).Should().Be(6); + verificationContext.RemoveTransaction(transaction); + verificationContext.GetSenderFee(transaction.Sender).Should().Be(3); + verificationContext.RemoveTransaction(transaction); + verificationContext.GetSenderFee(transaction.Sender).Should().Be(0); } } } From 04b5728dd7ede2a5a65d7164faa42c5f194b3063 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 8 Jul 2020 17:25:56 +0800 Subject: [PATCH 2/7] format --- tests/neo.UnitTests/Ledger/UT_MemoryPool.cs | 1040 +++++++++---------- 1 file changed, 520 insertions(+), 520 deletions(-) diff --git a/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs b/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs index cd93f9533a..dbfab6864f 100644 --- a/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs +++ b/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs @@ -1,520 +1,520 @@ -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; -using Neo.Cryptography; -using Neo.IO; -using Neo.Ledger; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.Plugins; -using Neo.SmartContract; -using Neo.SmartContract.Native; -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Numerics; - -namespace Neo.UnitTests.Ledger -{ - internal class TestIMemoryPoolTxObserverPlugin : Plugin, IMemoryPoolTxObserverPlugin - { - protected override void Configure() { } - public void TransactionAdded(Transaction tx) { } - public void TransactionsRemoved(MemoryPoolTxRemovalReason reason, IEnumerable transactions) { } - } - - [TestClass] - public class UT_MemoryPool - { - private const byte Prefix_MaxTransactionsPerBlock = 23; - private const byte Prefix_FeePerByte = 10; - private MemoryPool _unit; - private MemoryPool _unit2; - private TestIMemoryPoolTxObserverPlugin plugin; - - [TestInitialize] - public void TestSetup() - { - // protect against external changes on TimeProvider - TimeProvider.ResetToDefault(); - - TestBlockchain.InitializeMockNeoSystem(); - - // Create a MemoryPool with capacity of 100 - _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 100); - _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); - - // Verify capacity equals the amount specified - _unit.Capacity.Should().Be(100); - - _unit.VerifiedCount.Should().Be(0); - _unit.UnVerifiedCount.Should().Be(0); - _unit.Count.Should().Be(0); - _unit2 = new MemoryPool(TestBlockchain.TheNeoSystem, 0); - plugin = new TestIMemoryPoolTxObserverPlugin(); - } - - [TestCleanup] - public void CleanUp() - { - Plugin.TxObserverPlugins.Remove(plugin); - } - - long LongRandom(long min, long max, Random rand) - { - // Only returns positive random long values. - long longRand = (long)rand.NextBigInteger(63); - return longRand % (max - min) + min; - } - - private Transaction CreateTransactionWithFee(long fee) - { - Random random = new Random(); - var randomBytes = new byte[16]; - random.NextBytes(randomBytes); - Mock mock = new Mock(); - mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); - mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); - mock.Object.Script = randomBytes; - mock.Object.Sender = UInt160.Zero; - mock.Object.NetworkFee = fee; - mock.Object.Attributes = Array.Empty(); - mock.Object.Witnesses = new[] - { - new Witness - { - InvocationScript = new byte[0], - VerificationScript = new byte[0] - } - }; - return mock.Object; - } - - private Transaction CreateTransactionWithFeeAndBalanceVerify(long fee) - { - Random random = new Random(); - var randomBytes = new byte[16]; - random.NextBytes(randomBytes); - Mock mock = new Mock(); - UInt160 sender = UInt160.Zero; - mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns((StoreView snapshot, BigInteger amount) => NativeContract.GAS.BalanceOf(snapshot, sender) >= amount + fee ? VerifyResult.Succeed : VerifyResult.InsufficientFunds); - mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); - mock.Object.Script = randomBytes; - mock.Object.Sender = sender; - mock.Object.NetworkFee = fee; - mock.Object.Attributes = Array.Empty(); - mock.Object.Witnesses = new[] - { - new Witness - { - InvocationScript = new byte[0], - VerificationScript = new byte[0] - } - }; - return mock.Object; - } - - private Transaction CreateTransaction(long fee = -1) - { - if (fee != -1) - return CreateTransactionWithFee(fee); - return CreateTransactionWithFee(LongRandom(100000, 100000000, TestUtils.TestRandom)); - } - - private void AddTransactions(int count) - { - for (int i = 0; i < count; i++) - { - var txToAdd = CreateTransaction(); - _unit.TryAdd(txToAdd.Hash, txToAdd); - } - - Console.WriteLine($"created {count} tx"); - } - - private void AddTransaction(Transaction txToAdd) - { - _unit.TryAdd(txToAdd.Hash, txToAdd); - } - - private void AddTransactionsWithBalanceVerify(int count, long fee) - { - for (int i = 0; i < count; i++) - { - var txToAdd = CreateTransactionWithFeeAndBalanceVerify(fee); - _unit.TryAdd(txToAdd.Hash, txToAdd); - } - - Console.WriteLine($"created {count} tx"); - } - - [TestMethod] - public void CapacityTest() - { - // Add over the capacity items, verify that the verified count increases each time - AddTransactions(101); - - Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} Count {_unit.SortedTxCount}"); - - _unit.SortedTxCount.Should().Be(100); - _unit.VerifiedCount.Should().Be(100); - _unit.UnVerifiedCount.Should().Be(0); - _unit.Count.Should().Be(100); - } - - [TestMethod] - public void BlockPersistMovesTxToUnverifiedAndReverification() - { - AddTransactions(70); - - _unit.SortedTxCount.Should().Be(70); - - var block = new Block - { - Transactions = _unit.GetSortedVerifiedTransactions().Take(10) - .Concat(_unit.GetSortedVerifiedTransactions().Take(5)).ToArray() - }; - _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); - _unit.InvalidateVerifiedTransactions(); - _unit.SortedTxCount.Should().Be(0); - _unit.UnverifiedSortedTxCount.Should().Be(60); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(10); - _unit.UnverifiedSortedTxCount.Should().Be(50); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(20); - _unit.UnverifiedSortedTxCount.Should().Be(40); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(30); - _unit.UnverifiedSortedTxCount.Should().Be(30); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(40); - _unit.UnverifiedSortedTxCount.Should().Be(20); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(50); - _unit.UnverifiedSortedTxCount.Should().Be(10); - - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); - _unit.SortedTxCount.Should().Be(60); - _unit.UnverifiedSortedTxCount.Should().Be(0); - } - - [TestMethod] - public void BlockPersistAndReverificationWillAbandonTxAsBalanceTransfered() - { - long txFee = 1; - AddTransactionsWithBalanceVerify(70, txFee); - - _unit.SortedTxCount.Should().Be(70); - - var block = new Block - { - Transactions = _unit.GetSortedVerifiedTransactions().Take(10).ToArray() - }; - - // Simulate the transfer process in tx by burning the balance - UInt160 sender = block.Transactions[0].Sender; - SnapshotView snapshot = Blockchain.Singleton.GetSnapshot(); - BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, sender); - - ApplicationEngine applicationEngine = new ApplicationEngine(TriggerType.All, block, snapshot, (long)balance); - NativeContract.GAS.Burn(applicationEngine, sender, balance); - NativeContract.GAS.Mint(applicationEngine, sender, txFee * 30); // Set the balance to meet 30 txs only - - // Persist block and reverify all the txs in mempool, but half of the txs will be discarded - _unit.UpdatePoolForBlockPersisted(block, snapshot); - _unit.SortedTxCount.Should().Be(30); - _unit.UnverifiedSortedTxCount.Should().Be(0); - - // Revert the balance - NativeContract.GAS.Burn(applicationEngine, sender, txFee * 30); - NativeContract.GAS.Mint(applicationEngine, sender, balance); - } - - private void VerifyTransactionsSortedDescending(IEnumerable transactions) - { - Transaction lastTransaction = null; - foreach (var tx in transactions) - { - if (lastTransaction != null) - { - if (lastTransaction.FeePerByte == tx.FeePerByte) - { - if (lastTransaction.NetworkFee == tx.NetworkFee) - lastTransaction.Hash.Should().BeLessThan(tx.Hash); - else - lastTransaction.NetworkFee.Should().BeGreaterThan(tx.NetworkFee); - } - else - { - lastTransaction.FeePerByte.Should().BeGreaterThan(tx.FeePerByte); - } - } - lastTransaction = tx; - } - } - - [TestMethod] - public void VerifySortOrderAndThatHighetFeeTransactionsAreReverifiedFirst() - { - AddTransactions(100); - - var sortedVerifiedTxs = _unit.GetSortedVerifiedTransactions().ToList(); - // verify all 100 transactions are returned in sorted order - sortedVerifiedTxs.Count.Should().Be(100); - VerifyTransactionsSortedDescending(sortedVerifiedTxs); - - // move all to unverified - var block = new Block { Transactions = new Transaction[0] }; - _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); - _unit.InvalidateVerifiedTransactions(); - _unit.SortedTxCount.Should().Be(0); - _unit.UnverifiedSortedTxCount.Should().Be(100); - - // We can verify the order they are re-verified by reverifying 2 at a time - while (_unit.UnVerifiedCount > 0) - { - _unit.GetVerifiedAndUnverifiedTransactions(out var sortedVerifiedTransactions, out var sortedUnverifiedTransactions); - sortedVerifiedTransactions.Count().Should().Be(0); - var sortedUnverifiedArray = sortedUnverifiedTransactions.ToArray(); - VerifyTransactionsSortedDescending(sortedUnverifiedArray); - var maxTransaction = sortedUnverifiedArray.First(); - var minTransaction = sortedUnverifiedArray.Last(); - - // reverify 1 high priority and 1 low priority transaction - _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); - var verifiedTxs = _unit.GetSortedVerifiedTransactions().ToArray(); - verifiedTxs.Length.Should().Be(1); - verifiedTxs[0].Should().BeEquivalentTo(maxTransaction); - var blockWith2Tx = new Block { Transactions = new[] { maxTransaction, minTransaction } }; - // verify and remove the 2 transactions from the verified pool - _unit.UpdatePoolForBlockPersisted(blockWith2Tx, Blockchain.Singleton.GetSnapshot()); - _unit.InvalidateVerifiedTransactions(); - _unit.SortedTxCount.Should().Be(0); - } - _unit.UnverifiedSortedTxCount.Should().Be(0); - } - - void VerifyCapacityThresholdForAttemptingToAddATransaction() - { - var sortedVerified = _unit.GetSortedVerifiedTransactions().ToArray(); - - var txBarelyWontFit = CreateTransactionWithFee(sortedVerified.Last().NetworkFee - 1); - _unit.CanTransactionFitInPool(txBarelyWontFit).Should().Be(false); - var txBarelyFits = CreateTransactionWithFee(sortedVerified.Last().NetworkFee + 1); - _unit.CanTransactionFitInPool(txBarelyFits).Should().Be(true); - } - - [TestMethod] - public void VerifyCanTransactionFitInPoolWorksAsIntended() - { - AddTransactions(100); - VerifyCapacityThresholdForAttemptingToAddATransaction(); - AddTransactions(50); - VerifyCapacityThresholdForAttemptingToAddATransaction(); - AddTransactions(50); - VerifyCapacityThresholdForAttemptingToAddATransaction(); - } - - [TestMethod] - public void CapacityTestWithUnverifiedHighProirtyTransactions() - { - // Verify that unverified high priority transactions will not be pushed out of the queue by incoming - // low priority transactions - - // Fill pool with high priority transactions - AddTransactions(99); - - // move all to unverified - var block = new Block { Transactions = new Transaction[0] }; - _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); - - _unit.CanTransactionFitInPool(CreateTransaction()).Should().Be(true); - AddTransactions(1); - _unit.CanTransactionFitInPool(CreateTransactionWithFee(0)).Should().Be(false); - } - - [TestMethod] - public void TestInvalidateAll() - { - AddTransactions(30); - - _unit.UnverifiedSortedTxCount.Should().Be(0); - _unit.SortedTxCount.Should().Be(30); - _unit.InvalidateAllTransactions(); - _unit.UnverifiedSortedTxCount.Should().Be(30); - _unit.SortedTxCount.Should().Be(0); - } - - [TestMethod] - public void TestContainsKey() - { - AddTransactions(10); - - var txToAdd = CreateTransaction(); - _unit.TryAdd(txToAdd.Hash, txToAdd); - _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); - _unit.InvalidateVerifiedTransactions(); - _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); - } - - [TestMethod] - public void TestGetEnumerator() - { - AddTransactions(10); - _unit.InvalidateVerifiedTransactions(); - IEnumerator enumerator = _unit.GetEnumerator(); - foreach (Transaction tx in _unit) - { - enumerator.MoveNext(); - enumerator.Current.Should().BeSameAs(tx); - } - } - - [TestMethod] - public void TestIEnumerableGetEnumerator() - { - AddTransactions(10); - _unit.InvalidateVerifiedTransactions(); - IEnumerable enumerable = _unit; - var enumerator = enumerable.GetEnumerator(); - foreach (Transaction tx in _unit) - { - enumerator.MoveNext(); - enumerator.Current.Should().BeSameAs(tx); - } - } - - [TestMethod] - public void TestGetVerifiedTransactions() - { - var tx1 = CreateTransaction(); - var tx2 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1); - _unit.InvalidateVerifiedTransactions(); - _unit.TryAdd(tx2.Hash, tx2); - IEnumerable enumerable = _unit.GetVerifiedTransactions(); - enumerable.Count().Should().Be(1); - var enumerator = enumerable.GetEnumerator(); - enumerator.MoveNext(); - enumerator.Current.Should().BeSameAs(tx2); - } - - [TestMethod] - public void TestReVerifyTopUnverifiedTransactionsIfNeeded() - { - _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 600); - _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); - AddTransaction(CreateTransaction(100000001)); - AddTransaction(CreateTransaction(100000001)); - AddTransaction(CreateTransaction(100000001)); - AddTransaction(CreateTransaction(1)); - _unit.VerifiedCount.Should().Be(4); - _unit.UnVerifiedCount.Should().Be(0); - - _unit.InvalidateVerifiedTransactions(); - _unit.VerifiedCount.Should().Be(0); - _unit.UnVerifiedCount.Should().Be(4); - - AddTransactions(511); // Max per block currently is 512 - _unit.VerifiedCount.Should().Be(511); - _unit.UnVerifiedCount.Should().Be(4); - - var result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); - result.Should().BeTrue(); - _unit.VerifiedCount.Should().Be(512); - _unit.UnVerifiedCount.Should().Be(3); - - result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(2, Blockchain.Singleton.GetSnapshot()); - result.Should().BeTrue(); - _unit.VerifiedCount.Should().Be(514); - _unit.UnVerifiedCount.Should().Be(1); - - result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(3, Blockchain.Singleton.GetSnapshot()); - result.Should().BeFalse(); - _unit.VerifiedCount.Should().Be(515); - _unit.UnVerifiedCount.Should().Be(0); - } - - [TestMethod] - public void TestTryAdd() - { - var tx1 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1).Should().BeTrue(); - _unit.TryAdd(tx1.Hash, tx1).Should().BeFalse(); - _unit2.TryAdd(tx1.Hash, tx1).Should().BeFalse(); - } - - [TestMethod] - public void TestTryGetValue() - { - var tx1 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1); - _unit.TryGetValue(tx1.Hash, out Transaction tx).Should().BeTrue(); - tx.Should().BeEquivalentTo(tx1); - - _unit.InvalidateVerifiedTransactions(); - _unit.TryGetValue(tx1.Hash, out tx).Should().BeTrue(); - tx.Should().BeEquivalentTo(tx1); - - var tx2 = CreateTransaction(); - _unit.TryGetValue(tx2.Hash, out tx).Should().BeFalse(); - } - - [TestMethod] - public void TestUpdatePoolForBlockPersisted() - { - var snapshot = Blockchain.Singleton.GetSnapshot(); - byte[] transactionsPerBlock = { 0x18, 0x00, 0x00, 0x00 }; // 24 - byte[] feePerByte = { 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00 }; // 1048576 - StorageItem item1 = new StorageItem - { - Value = transactionsPerBlock - }; - StorageItem item2 = new StorageItem - { - Value = feePerByte - }; - var key1 = CreateStorageKey(Prefix_MaxTransactionsPerBlock); - var key2 = CreateStorageKey(Prefix_FeePerByte); - key1.Id = NativeContract.Policy.Id; - key2.Id = NativeContract.Policy.Id; - snapshot.Storages.Add(key1, item1); - snapshot.Storages.Add(key2, item2); - - var tx1 = CreateTransaction(); - var tx2 = CreateTransaction(); - Transaction[] transactions = { tx1, tx2 }; - _unit.TryAdd(tx1.Hash, tx1); - - var block = new Block { Transactions = transactions }; - - _unit.UnVerifiedCount.Should().Be(0); - _unit.VerifiedCount.Should().Be(1); - - _unit.UpdatePoolForBlockPersisted(block, snapshot); - - _unit.UnVerifiedCount.Should().Be(0); - _unit.VerifiedCount.Should().Be(0); - } - - public StorageKey CreateStorageKey(byte prefix, byte[] key = null) - { - StorageKey storageKey = new StorageKey - { - Id = 0, - Key = new byte[sizeof(byte) + (key?.Length ?? 0)] - }; - storageKey.Key[0] = prefix; - if (key != null) - Buffer.BlockCopy(key, 0, storageKey.Key, 1, key.Length); - return storageKey; - } - } -} +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Cryptography; +using Neo.IO; +using Neo.Ledger; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Plugins; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; + +namespace Neo.UnitTests.Ledger +{ + internal class TestIMemoryPoolTxObserverPlugin : Plugin, IMemoryPoolTxObserverPlugin + { + protected override void Configure() { } + public void TransactionAdded(Transaction tx) { } + public void TransactionsRemoved(MemoryPoolTxRemovalReason reason, IEnumerable transactions) { } + } + + [TestClass] + public class UT_MemoryPool + { + private const byte Prefix_MaxTransactionsPerBlock = 23; + private const byte Prefix_FeePerByte = 10; + private MemoryPool _unit; + private MemoryPool _unit2; + private TestIMemoryPoolTxObserverPlugin plugin; + + [TestInitialize] + public void TestSetup() + { + // protect against external changes on TimeProvider + TimeProvider.ResetToDefault(); + + TestBlockchain.InitializeMockNeoSystem(); + + // Create a MemoryPool with capacity of 100 + _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 100); + _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); + + // Verify capacity equals the amount specified + _unit.Capacity.Should().Be(100); + + _unit.VerifiedCount.Should().Be(0); + _unit.UnVerifiedCount.Should().Be(0); + _unit.Count.Should().Be(0); + _unit2 = new MemoryPool(TestBlockchain.TheNeoSystem, 0); + plugin = new TestIMemoryPoolTxObserverPlugin(); + } + + [TestCleanup] + public void CleanUp() + { + Plugin.TxObserverPlugins.Remove(plugin); + } + + long LongRandom(long min, long max, Random rand) + { + // Only returns positive random long values. + long longRand = (long)rand.NextBigInteger(63); + return longRand % (max - min) + min; + } + + private Transaction CreateTransactionWithFee(long fee) + { + Random random = new Random(); + var randomBytes = new byte[16]; + random.NextBytes(randomBytes); + Mock mock = new Mock(); + mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Object.Script = randomBytes; + mock.Object.Sender = UInt160.Zero; + mock.Object.NetworkFee = fee; + mock.Object.Attributes = Array.Empty(); + mock.Object.Witnesses = new[] + { + new Witness + { + InvocationScript = new byte[0], + VerificationScript = new byte[0] + } + }; + return mock.Object; + } + + private Transaction CreateTransactionWithFeeAndBalanceVerify(long fee) + { + Random random = new Random(); + var randomBytes = new byte[16]; + random.NextBytes(randomBytes); + Mock mock = new Mock(); + UInt160 sender = UInt160.Zero; + mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns((StoreView snapshot, BigInteger amount) => NativeContract.GAS.BalanceOf(snapshot, sender) >= amount + fee ? VerifyResult.Succeed : VerifyResult.InsufficientFunds); + mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); + mock.Object.Script = randomBytes; + mock.Object.Sender = sender; + mock.Object.NetworkFee = fee; + mock.Object.Attributes = Array.Empty(); + mock.Object.Witnesses = new[] + { + new Witness + { + InvocationScript = new byte[0], + VerificationScript = new byte[0] + } + }; + return mock.Object; + } + + private Transaction CreateTransaction(long fee = -1) + { + if (fee != -1) + return CreateTransactionWithFee(fee); + return CreateTransactionWithFee(LongRandom(100000, 100000000, TestUtils.TestRandom)); + } + + private void AddTransactions(int count) + { + for (int i = 0; i < count; i++) + { + var txToAdd = CreateTransaction(); + _unit.TryAdd(txToAdd.Hash, txToAdd); + } + + Console.WriteLine($"created {count} tx"); + } + + private void AddTransaction(Transaction txToAdd) + { + _unit.TryAdd(txToAdd.Hash, txToAdd); + } + + private void AddTransactionsWithBalanceVerify(int count, long fee) + { + for (int i = 0; i < count; i++) + { + var txToAdd = CreateTransactionWithFeeAndBalanceVerify(fee); + _unit.TryAdd(txToAdd.Hash, txToAdd); + } + + Console.WriteLine($"created {count} tx"); + } + + [TestMethod] + public void CapacityTest() + { + // Add over the capacity items, verify that the verified count increases each time + AddTransactions(101); + + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} Count {_unit.SortedTxCount}"); + + _unit.SortedTxCount.Should().Be(100); + _unit.VerifiedCount.Should().Be(100); + _unit.UnVerifiedCount.Should().Be(0); + _unit.Count.Should().Be(100); + } + + [TestMethod] + public void BlockPersistMovesTxToUnverifiedAndReverification() + { + AddTransactions(70); + + _unit.SortedTxCount.Should().Be(70); + + var block = new Block + { + Transactions = _unit.GetSortedVerifiedTransactions().Take(10) + .Concat(_unit.GetSortedVerifiedTransactions().Take(5)).ToArray() + }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + _unit.InvalidateVerifiedTransactions(); + _unit.SortedTxCount.Should().Be(0); + _unit.UnverifiedSortedTxCount.Should().Be(60); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(10); + _unit.UnverifiedSortedTxCount.Should().Be(50); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(20); + _unit.UnverifiedSortedTxCount.Should().Be(40); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(30); + _unit.UnverifiedSortedTxCount.Should().Be(30); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(40); + _unit.UnverifiedSortedTxCount.Should().Be(20); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(50); + _unit.UnverifiedSortedTxCount.Should().Be(10); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedTxCount.Should().Be(60); + _unit.UnverifiedSortedTxCount.Should().Be(0); + } + + [TestMethod] + public void BlockPersistAndReverificationWillAbandonTxAsBalanceTransfered() + { + long txFee = 1; + AddTransactionsWithBalanceVerify(70, txFee); + + _unit.SortedTxCount.Should().Be(70); + + var block = new Block + { + Transactions = _unit.GetSortedVerifiedTransactions().Take(10).ToArray() + }; + + // Simulate the transfer process in tx by burning the balance + UInt160 sender = block.Transactions[0].Sender; + SnapshotView snapshot = Blockchain.Singleton.GetSnapshot(); + BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, sender); + + ApplicationEngine applicationEngine = new ApplicationEngine(TriggerType.All, block, snapshot, (long)balance); + NativeContract.GAS.Burn(applicationEngine, sender, balance); + NativeContract.GAS.Mint(applicationEngine, sender, txFee * 30); // Set the balance to meet 30 txs only + + // Persist block and reverify all the txs in mempool, but half of the txs will be discarded + _unit.UpdatePoolForBlockPersisted(block, snapshot); + _unit.SortedTxCount.Should().Be(30); + _unit.UnverifiedSortedTxCount.Should().Be(0); + + // Revert the balance + NativeContract.GAS.Burn(applicationEngine, sender, txFee * 30); + NativeContract.GAS.Mint(applicationEngine, sender, balance); + } + + private void VerifyTransactionsSortedDescending(IEnumerable transactions) + { + Transaction lastTransaction = null; + foreach (var tx in transactions) + { + if (lastTransaction != null) + { + if (lastTransaction.FeePerByte == tx.FeePerByte) + { + if (lastTransaction.NetworkFee == tx.NetworkFee) + lastTransaction.Hash.Should().BeLessThan(tx.Hash); + else + lastTransaction.NetworkFee.Should().BeGreaterThan(tx.NetworkFee); + } + else + { + lastTransaction.FeePerByte.Should().BeGreaterThan(tx.FeePerByte); + } + } + lastTransaction = tx; + } + } + + [TestMethod] + public void VerifySortOrderAndThatHighetFeeTransactionsAreReverifiedFirst() + { + AddTransactions(100); + + var sortedVerifiedTxs = _unit.GetSortedVerifiedTransactions().ToList(); + // verify all 100 transactions are returned in sorted order + sortedVerifiedTxs.Count.Should().Be(100); + VerifyTransactionsSortedDescending(sortedVerifiedTxs); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + _unit.InvalidateVerifiedTransactions(); + _unit.SortedTxCount.Should().Be(0); + _unit.UnverifiedSortedTxCount.Should().Be(100); + + // We can verify the order they are re-verified by reverifying 2 at a time + while (_unit.UnVerifiedCount > 0) + { + _unit.GetVerifiedAndUnverifiedTransactions(out var sortedVerifiedTransactions, out var sortedUnverifiedTransactions); + sortedVerifiedTransactions.Count().Should().Be(0); + var sortedUnverifiedArray = sortedUnverifiedTransactions.ToArray(); + VerifyTransactionsSortedDescending(sortedUnverifiedArray); + var maxTransaction = sortedUnverifiedArray.First(); + var minTransaction = sortedUnverifiedArray.Last(); + + // reverify 1 high priority and 1 low priority transaction + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); + var verifiedTxs = _unit.GetSortedVerifiedTransactions().ToArray(); + verifiedTxs.Length.Should().Be(1); + verifiedTxs[0].Should().BeEquivalentTo(maxTransaction); + var blockWith2Tx = new Block { Transactions = new[] { maxTransaction, minTransaction } }; + // verify and remove the 2 transactions from the verified pool + _unit.UpdatePoolForBlockPersisted(blockWith2Tx, Blockchain.Singleton.GetSnapshot()); + _unit.InvalidateVerifiedTransactions(); + _unit.SortedTxCount.Should().Be(0); + } + _unit.UnverifiedSortedTxCount.Should().Be(0); + } + + void VerifyCapacityThresholdForAttemptingToAddATransaction() + { + var sortedVerified = _unit.GetSortedVerifiedTransactions().ToArray(); + + var txBarelyWontFit = CreateTransactionWithFee(sortedVerified.Last().NetworkFee - 1); + _unit.CanTransactionFitInPool(txBarelyWontFit).Should().Be(false); + var txBarelyFits = CreateTransactionWithFee(sortedVerified.Last().NetworkFee + 1); + _unit.CanTransactionFitInPool(txBarelyFits).Should().Be(true); + } + + [TestMethod] + public void VerifyCanTransactionFitInPoolWorksAsIntended() + { + AddTransactions(100); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + } + + [TestMethod] + public void CapacityTestWithUnverifiedHighProirtyTransactions() + { + // Verify that unverified high priority transactions will not be pushed out of the queue by incoming + // low priority transactions + + // Fill pool with high priority transactions + AddTransactions(99); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + + _unit.CanTransactionFitInPool(CreateTransaction()).Should().Be(true); + AddTransactions(1); + _unit.CanTransactionFitInPool(CreateTransactionWithFee(0)).Should().Be(false); + } + + [TestMethod] + public void TestInvalidateAll() + { + AddTransactions(30); + + _unit.UnverifiedSortedTxCount.Should().Be(0); + _unit.SortedTxCount.Should().Be(30); + _unit.InvalidateAllTransactions(); + _unit.UnverifiedSortedTxCount.Should().Be(30); + _unit.SortedTxCount.Should().Be(0); + } + + [TestMethod] + public void TestContainsKey() + { + AddTransactions(10); + + var txToAdd = CreateTransaction(); + _unit.TryAdd(txToAdd.Hash, txToAdd); + _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); + _unit.InvalidateVerifiedTransactions(); + _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); + } + + [TestMethod] + public void TestGetEnumerator() + { + AddTransactions(10); + _unit.InvalidateVerifiedTransactions(); + IEnumerator enumerator = _unit.GetEnumerator(); + foreach (Transaction tx in _unit) + { + enumerator.MoveNext(); + enumerator.Current.Should().BeSameAs(tx); + } + } + + [TestMethod] + public void TestIEnumerableGetEnumerator() + { + AddTransactions(10); + _unit.InvalidateVerifiedTransactions(); + IEnumerable enumerable = _unit; + var enumerator = enumerable.GetEnumerator(); + foreach (Transaction tx in _unit) + { + enumerator.MoveNext(); + enumerator.Current.Should().BeSameAs(tx); + } + } + + [TestMethod] + public void TestGetVerifiedTransactions() + { + var tx1 = CreateTransaction(); + var tx2 = CreateTransaction(); + _unit.TryAdd(tx1.Hash, tx1); + _unit.InvalidateVerifiedTransactions(); + _unit.TryAdd(tx2.Hash, tx2); + IEnumerable enumerable = _unit.GetVerifiedTransactions(); + enumerable.Count().Should().Be(1); + var enumerator = enumerable.GetEnumerator(); + enumerator.MoveNext(); + enumerator.Current.Should().BeSameAs(tx2); + } + + [TestMethod] + public void TestReVerifyTopUnverifiedTransactionsIfNeeded() + { + _unit = new MemoryPool(TestBlockchain.TheNeoSystem, 600); + _unit.LoadPolicy(Blockchain.Singleton.GetSnapshot()); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(100000001)); + AddTransaction(CreateTransaction(1)); + _unit.VerifiedCount.Should().Be(4); + _unit.UnVerifiedCount.Should().Be(0); + + _unit.InvalidateVerifiedTransactions(); + _unit.VerifiedCount.Should().Be(0); + _unit.UnVerifiedCount.Should().Be(4); + + AddTransactions(511); // Max per block currently is 512 + _unit.VerifiedCount.Should().Be(511); + _unit.UnVerifiedCount.Should().Be(4); + + var result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(1, Blockchain.Singleton.GetSnapshot()); + result.Should().BeTrue(); + _unit.VerifiedCount.Should().Be(512); + _unit.UnVerifiedCount.Should().Be(3); + + result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(2, Blockchain.Singleton.GetSnapshot()); + result.Should().BeTrue(); + _unit.VerifiedCount.Should().Be(514); + _unit.UnVerifiedCount.Should().Be(1); + + result = _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(3, Blockchain.Singleton.GetSnapshot()); + result.Should().BeFalse(); + _unit.VerifiedCount.Should().Be(515); + _unit.UnVerifiedCount.Should().Be(0); + } + + [TestMethod] + public void TestTryAdd() + { + var tx1 = CreateTransaction(); + _unit.TryAdd(tx1.Hash, tx1).Should().BeTrue(); + _unit.TryAdd(tx1.Hash, tx1).Should().BeFalse(); + _unit2.TryAdd(tx1.Hash, tx1).Should().BeFalse(); + } + + [TestMethod] + public void TestTryGetValue() + { + var tx1 = CreateTransaction(); + _unit.TryAdd(tx1.Hash, tx1); + _unit.TryGetValue(tx1.Hash, out Transaction tx).Should().BeTrue(); + tx.Should().BeEquivalentTo(tx1); + + _unit.InvalidateVerifiedTransactions(); + _unit.TryGetValue(tx1.Hash, out tx).Should().BeTrue(); + tx.Should().BeEquivalentTo(tx1); + + var tx2 = CreateTransaction(); + _unit.TryGetValue(tx2.Hash, out tx).Should().BeFalse(); + } + + [TestMethod] + public void TestUpdatePoolForBlockPersisted() + { + var snapshot = Blockchain.Singleton.GetSnapshot(); + byte[] transactionsPerBlock = { 0x18, 0x00, 0x00, 0x00 }; // 24 + byte[] feePerByte = { 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00 }; // 1048576 + StorageItem item1 = new StorageItem + { + Value = transactionsPerBlock + }; + StorageItem item2 = new StorageItem + { + Value = feePerByte + }; + var key1 = CreateStorageKey(Prefix_MaxTransactionsPerBlock); + var key2 = CreateStorageKey(Prefix_FeePerByte); + key1.Id = NativeContract.Policy.Id; + key2.Id = NativeContract.Policy.Id; + snapshot.Storages.Add(key1, item1); + snapshot.Storages.Add(key2, item2); + + var tx1 = CreateTransaction(); + var tx2 = CreateTransaction(); + Transaction[] transactions = { tx1, tx2 }; + _unit.TryAdd(tx1.Hash, tx1); + + var block = new Block { Transactions = transactions }; + + _unit.UnVerifiedCount.Should().Be(0); + _unit.VerifiedCount.Should().Be(1); + + _unit.UpdatePoolForBlockPersisted(block, snapshot); + + _unit.UnVerifiedCount.Should().Be(0); + _unit.VerifiedCount.Should().Be(0); + } + + public StorageKey CreateStorageKey(byte prefix, byte[] key = null) + { + StorageKey storageKey = new StorageKey + { + Id = 0, + Key = new byte[sizeof(byte) + (key?.Length ?? 0)] + }; + storageKey.Key[0] = prefix; + if (key != null) + Buffer.BlockCopy(key, 0, storageKey.Key, 1, key.Length); + return storageKey; + } + } +} From 4e8d550b9894270fd401506c812395569da3b829 Mon Sep 17 00:00:00 2001 From: Luchuan Date: Thu, 9 Jul 2020 14:09:52 +0800 Subject: [PATCH 3/7] fix ut (#1759) --- tests/neo.UnitTests/Ledger/UT_Blockchain.cs | 2 +- tests/neo.UnitTests/Ledger/UT_MemoryPool.cs | 39 +++++++++++++------ .../UT_TransactionVerificationContext.cs | 39 +++++++++++++------ .../Network/P2P/Payloads/UT_Transaction.cs | 2 +- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/tests/neo.UnitTests/Ledger/UT_Blockchain.cs b/tests/neo.UnitTests/Ledger/UT_Blockchain.cs index 7e04dfe229..06cf435956 100644 --- a/tests/neo.UnitTests/Ledger/UT_Blockchain.cs +++ b/tests/neo.UnitTests/Ledger/UT_Blockchain.cs @@ -54,7 +54,7 @@ public class UT_Blockchain : TestKit public void Initialize() { system = TestBlockchain.TheNeoSystem; - Blockchain.Singleton.MemPool.TryAdd(txSample.Hash, txSample); + Blockchain.Singleton.MemPool.TryAdd(txSample, Blockchain.Singleton.GetSnapshot()); } [TestMethod] diff --git a/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs b/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs index dbfab6864f..eee44960cf 100644 --- a/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs +++ b/tests/neo.UnitTests/Ledger/UT_MemoryPool.cs @@ -27,12 +27,20 @@ public void TransactionsRemoved(MemoryPoolTxRemovalReason reason, IEnumerable mock = new Mock(); UInt160 sender = UInt160.Zero; - mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns((StoreView snapshot, BigInteger amount) => NativeContract.GAS.BalanceOf(snapshot, sender) >= amount + fee ? VerifyResult.Succeed : VerifyResult.InsufficientFunds); + mock.Setup(p => p.VerifyForEachBlock(It.IsAny(), It.IsAny())).Returns((StoreView snapshot, TransactionVerificationContext context) => context.CheckTransaction(mock.Object, snapshot) ? VerifyResult.Succeed : VerifyResult.InsufficientFunds); mock.Setup(p => p.Verify(It.IsAny(), It.IsAny())).Returns(VerifyResult.Succeed); mock.Object.Script = randomBytes; mock.Object.Sender = sender; @@ -124,10 +132,11 @@ private Transaction CreateTransaction(long fee = -1) private void AddTransactions(int count) { + var snapshot = Blockchain.Singleton.GetSnapshot(); for (int i = 0; i < count; i++) { var txToAdd = CreateTransaction(); - _unit.TryAdd(txToAdd.Hash, txToAdd); + _unit.TryAdd(txToAdd, snapshot); } Console.WriteLine($"created {count} tx"); @@ -135,15 +144,17 @@ private void AddTransactions(int count) private void AddTransaction(Transaction txToAdd) { - _unit.TryAdd(txToAdd.Hash, txToAdd); + var snapshot = Blockchain.Singleton.GetSnapshot(); + _unit.TryAdd(txToAdd, snapshot); } private void AddTransactionsWithBalanceVerify(int count, long fee) { + var snapshot = Blockchain.Singleton.GetSnapshot(); for (int i = 0; i < count; i++) { var txToAdd = CreateTransactionWithFeeAndBalanceVerify(fee); - _unit.TryAdd(txToAdd.Hash, txToAdd); + _unit.TryAdd(txToAdd, snapshot); } Console.WriteLine($"created {count} tx"); @@ -355,10 +366,11 @@ public void TestInvalidateAll() [TestMethod] public void TestContainsKey() { + var snapshot = Blockchain.Singleton.GetSnapshot(); AddTransactions(10); var txToAdd = CreateTransaction(); - _unit.TryAdd(txToAdd.Hash, txToAdd); + _unit.TryAdd(txToAdd, snapshot); _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); _unit.InvalidateVerifiedTransactions(); _unit.ContainsKey(txToAdd.Hash).Should().BeTrue(); @@ -394,11 +406,12 @@ public void TestIEnumerableGetEnumerator() [TestMethod] public void TestGetVerifiedTransactions() { + var snapshot = Blockchain.Singleton.GetSnapshot(); var tx1 = CreateTransaction(); var tx2 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1); + _unit.TryAdd(tx1, snapshot); _unit.InvalidateVerifiedTransactions(); - _unit.TryAdd(tx2.Hash, tx2); + _unit.TryAdd(tx2, snapshot); IEnumerable enumerable = _unit.GetVerifiedTransactions(); enumerable.Count().Should().Be(1); var enumerator = enumerable.GetEnumerator(); @@ -445,17 +458,19 @@ public void TestReVerifyTopUnverifiedTransactionsIfNeeded() [TestMethod] public void TestTryAdd() { + var snapshot = Blockchain.Singleton.GetSnapshot(); var tx1 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1).Should().BeTrue(); - _unit.TryAdd(tx1.Hash, tx1).Should().BeFalse(); - _unit2.TryAdd(tx1.Hash, tx1).Should().BeFalse(); + _unit.TryAdd(tx1, snapshot).Should().Be(VerifyResult.Succeed); + _unit.TryAdd(tx1, snapshot).Should().NotBe(VerifyResult.Succeed); + _unit2.TryAdd(tx1, snapshot).Should().NotBe(VerifyResult.Succeed); } [TestMethod] public void TestTryGetValue() { + var snapshot = Blockchain.Singleton.GetSnapshot(); var tx1 = CreateTransaction(); - _unit.TryAdd(tx1.Hash, tx1); + _unit.TryAdd(tx1, snapshot); _unit.TryGetValue(tx1.Hash, out Transaction tx).Should().BeTrue(); tx.Should().BeEquivalentTo(tx1); @@ -491,7 +506,7 @@ public void TestUpdatePoolForBlockPersisted() var tx1 = CreateTransaction(); var tx2 = CreateTransaction(); Transaction[] transactions = { tx1, tx2 }; - _unit.TryAdd(tx1.Hash, tx1); + _unit.TryAdd(tx1, snapshot); var block = new Block { Transactions = transactions }; diff --git a/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs b/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs index 6f8fb5d62a..e188935846 100644 --- a/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs +++ b/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs @@ -4,13 +4,24 @@ using Neo.Ledger; using Neo.Network.P2P.Payloads; using Neo.Persistence; +using Neo.SmartContract; +using Neo.SmartContract.Native; using System; +using System.Numerics; namespace Neo.UnitTests.Ledger { [TestClass] public class UT_TransactionVerificationContext { + private static NeoSystem testBlockchain; + + [ClassInitialize] + public static void TestSetup(TestContext ctx) + { + testBlockchain = TestBlockchain.TheNeoSystem; + } + private Transaction CreateTransactionWithFee(long networkFee, long systemFee) { Random random = new Random(); @@ -36,19 +47,25 @@ private Transaction CreateTransactionWithFee(long networkFee, long systemFee) } [TestMethod] - public void TestMemPoolSenderFee() + public void TestTransactionSenderFee() { - Transaction transaction = CreateTransactionWithFee(1, 2); + var snapshot = Blockchain.Singleton.GetSnapshot(); + ApplicationEngine engine = new ApplicationEngine(TriggerType.Application, null, snapshot, long.MaxValue); + BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, UInt160.Zero); + NativeContract.GAS.Burn(engine, UInt160.Zero, balance); + NativeContract.GAS.Mint(engine, UInt160.Zero, 8); + TransactionVerificationContext verificationContext = new TransactionVerificationContext(); - verificationContext.GetSenderFee(transaction.Sender).Should().Be(0); - verificationContext.AddTransaction(transaction); - verificationContext.GetSenderFee(transaction.Sender).Should().Be(3); - verificationContext.AddTransaction(transaction); - verificationContext.GetSenderFee(transaction.Sender).Should().Be(6); - verificationContext.RemoveTransaction(transaction); - verificationContext.GetSenderFee(transaction.Sender).Should().Be(3); - verificationContext.RemoveTransaction(transaction); - verificationContext.GetSenderFee(transaction.Sender).Should().Be(0); + var tx = CreateTransactionWithFee(1, 2); + verificationContext.CheckTransaction(tx, snapshot).Should().BeTrue(); + verificationContext.AddTransaction(tx); + verificationContext.CheckTransaction(tx, snapshot).Should().BeTrue(); + verificationContext.AddTransaction(tx); + verificationContext.CheckTransaction(tx, snapshot).Should().BeFalse(); + verificationContext.RemoveTransaction(tx); + verificationContext.CheckTransaction(tx, snapshot).Should().BeTrue(); + verificationContext.AddTransaction(tx); + verificationContext.CheckTransaction(tx, snapshot).Should().BeFalse(); } } } diff --git a/tests/neo.UnitTests/Network/P2P/Payloads/UT_Transaction.cs b/tests/neo.UnitTests/Network/P2P/Payloads/UT_Transaction.cs index 61ea913867..723a06b68b 100644 --- a/tests/neo.UnitTests/Network/P2P/Payloads/UT_Transaction.cs +++ b/tests/neo.UnitTests/Network/P2P/Payloads/UT_Transaction.cs @@ -748,7 +748,7 @@ public void Transaction_Reverify_Hashes_Length_Unequal_To_Witnesses_Length() }; UInt160[] hashes = txSimple.GetScriptHashesForVerifying(snapshot); Assert.AreEqual(2, hashes.Length); - Assert.AreNotEqual(VerifyResult.Succeed, txSimple.VerifyForEachBlock(snapshot, BigInteger.Zero)); + Assert.AreNotEqual(VerifyResult.Succeed, txSimple.VerifyForEachBlock(snapshot, new TransactionVerificationContext())); } [TestMethod] From 34cd06ee48dd86dfcb2862172e731c37837ae9bd Mon Sep 17 00:00:00 2001 From: erikzhang Date: Fri, 10 Jul 2020 16:36:32 +0800 Subject: [PATCH 4/7] Fix --- src/neo/Ledger/MemoryPool.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/neo/Ledger/MemoryPool.cs b/src/neo/Ledger/MemoryPool.cs index ad7410000c..5ca24c13bd 100644 --- a/src/neo/Ledger/MemoryPool.cs +++ b/src/neo/Ledger/MemoryPool.cs @@ -307,6 +307,9 @@ private List RemoveOverCapacity() unsortedPool.Remove(minItem.Tx.Hash); sortedPool.Remove(minItem); removedTransactions.Add(minItem.Tx); + + if (ReferenceEquals(sortedPool, _sortedTransactions)) + VerificationContext.RemoveTransaction(minItem.Tx); } while (Count > Capacity); return removedTransactions; From b8862b9ab5acfab10759f11c86a949bc4b4be1b6 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Fri, 10 Jul 2020 22:35:03 +0800 Subject: [PATCH 5/7] Remove CheckPolicy() --- .../SmartContract/Native/PolicyContract.cs | 10 -------- .../SmartContract/Native/UT_PolicyContract.cs | 23 ------------------- 2 files changed, 33 deletions(-) diff --git a/src/neo/SmartContract/Native/PolicyContract.cs b/src/neo/SmartContract/Native/PolicyContract.cs index 6fd173b409..65c690424e 100644 --- a/src/neo/SmartContract/Native/PolicyContract.cs +++ b/src/neo/SmartContract/Native/PolicyContract.cs @@ -2,12 +2,10 @@ using Neo.IO; using Neo.Ledger; -using Neo.Network.P2P.Payloads; using Neo.Persistence; using Neo.SmartContract.Manifest; using System; using System.Collections.Generic; -using System.Linq; namespace Neo.SmartContract.Native { @@ -27,14 +25,6 @@ public PolicyContract() Manifest.Features = ContractFeatures.HasStorage; } - internal bool CheckPolicy(Transaction tx, StoreView snapshot) - { - UInt160[] blockedAccounts = GetBlockedAccounts(snapshot); - if (blockedAccounts.Intersect(tx.GetScriptHashesForVerifying(snapshot)).Any()) - return false; - return true; - } - private bool CheckCommittees(ApplicationEngine engine) { UInt160 committeeMultiSigAddr = NEO.GetCommitteeAddress(engine.Snapshot); diff --git a/tests/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs b/tests/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs index ac3b1dd2e9..7eda18dcab 100644 --- a/tests/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs +++ b/tests/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs @@ -276,28 +276,5 @@ public void Check_Block_UnblockAccount() ret.Should().BeOfType(); ((VM.Types.Array)ret).Count.Should().Be(0); } - - [TestMethod] - public void TestCheckPolicy() - { - Transaction tx = Blockchain.GenesisBlock.Transactions[0]; - var snapshot = Blockchain.Singleton.GetSnapshot(); - - StorageKey storageKey = new StorageKey - { - Id = NativeContract.Policy.Id, - Key = new byte[sizeof(byte)] - }; - storageKey.Key[0] = 15; - snapshot.Storages.Add(storageKey, new StorageItem - { - Value = new UInt160[] { tx.Sender }.ToByteArray(), - }); - - NativeContract.Policy.CheckPolicy(tx, snapshot).Should().BeFalse(); - - snapshot = Blockchain.Singleton.GetSnapshot(); - NativeContract.Policy.CheckPolicy(tx, snapshot).Should().BeTrue(); - } } } From 79b710f6f0c2b6cc795994f33e816218d4879a04 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Sun, 12 Jul 2020 20:21:00 +0800 Subject: [PATCH 6/7] Fix UT --- tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs b/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs index 542c63d05d..a3636a4c91 100644 --- a/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs +++ b/tests/neo.UnitTests/Ledger/UT_TransactionVerificationContext.cs @@ -50,7 +50,7 @@ private Transaction CreateTransactionWithFee(long networkFee, long systemFee) public void TestTransactionSenderFee() { var snapshot = Blockchain.Singleton.GetSnapshot(); - ApplicationEngine engine = new ApplicationEngine(TriggerType.Application, null, snapshot, long.MaxValue); + ApplicationEngine engine = ApplicationEngine.Create(TriggerType.Application, null, snapshot, long.MaxValue); BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, UInt160.Zero); NativeContract.GAS.Burn(engine, UInt160.Zero, balance); NativeContract.GAS.Mint(engine, UInt160.Zero, 8); From 921acc8b260713270c6db91b5c22e32bb3915315 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Sun, 12 Jul 2020 20:36:34 +0800 Subject: [PATCH 7/7] Optimize --- src/neo/Ledger/MemoryPool.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/neo/Ledger/MemoryPool.cs b/src/neo/Ledger/MemoryPool.cs index 5ca24c13bd..9a8b42b6e6 100644 --- a/src/neo/Ledger/MemoryPool.cs +++ b/src/neo/Ledger/MemoryPool.cs @@ -322,7 +322,6 @@ private bool TryRemoveVerified(UInt256 hash, out PoolItem item) return false; _unsortedTransactions.Remove(hash); - VerificationContext.RemoveTransaction(item.Tx); _sortedTransactions.Remove(item); return true;