Skip to content

Commit

Permalink
[neox] Merge master (#1788)
Browse files Browse the repository at this point in the history
* Allow call from native contract (#1700)

* Fix MethodCallback (#1723)

* Fix #1714 (#1715)

* Add base64 SYSCALLs (#1717)

* Neo.VM.3.0.0-CI00230 (#1725)

* Allow to iterate buffer (#1726)

* Buffer iterator

* Rename

* Remove using

Co-authored-by: Erik Zhang <[email protected]>

* Update StorageContext.cs (#1728)

StorageContext needs to be public so it can be accessed by Neo Debugger

Co-authored-by: Erik Zhang <[email protected]>

* Fix return value check (#1730)

Co-authored-by: Shargon <[email protected]>

* Fix ContractEventDescriptor (#1733)

* fix enumerator (#1744)

Co-authored-by: Tommo-L <[email protected]>

* Change json fields to all lower case for consistency (#1736)

* Replace DataCache.Find by DataCache.Seek (#1740)

* replace DataCache.Find by DataCache.Seek

* fix order

* optimize

* Update ByteArrayComparer.cs

* Update ByteArrayComparer.cs

* Update DataCache.cs

* fix comments

* fix comments

* fix comments

* Update DataCache.cs

* Update DataCache.cs

* Reorder methods

Co-authored-by: Tommo-L <[email protected]>
Co-authored-by: erikzhang <[email protected]>
Co-authored-by: Shargon <[email protected]>

* Add MaxVerificationGas (#1745)

Co-authored-by: Shargon <[email protected]>

* Create KeyBuilder (#1748)

* Speed up the initialization of ApplicationEngine. (#1749)

* Change nef checksum to double SHA256 (#1751)

* Change to double SHA256

* Clean code

* Revert change

* Clean code

* Fix checksum

* Update NefFile.cs

* Fix compile

* Clean code

Clean code
Fix checksum
Update NefFile.cs
Fix compile

* Optimize

* Optimize

* Fix

Co-authored-by: erikzhang <[email protected]>

* Check witnesses on isStandard (#1754)

* Check witness on isStandard

* Add IsDeployed

* Add IsPayable

* Fix UT

* format

* Add coverage

* Remove IsPayable, IsDeployed

* Move NEP10 to manifest (#1729)

* NEP10 abi

* To lower case

* Move to Manifest

* Clean code

* Update ContractManifest.cs

* Update ContractManifest.cs

* Fix native contracts

Co-authored-by: Erik Zhang <[email protected]>

* Capture fault Exception (#1761)

* Capture faultException

* Update Wallet error

* Remove flag check

* Clean code

* Sender from signers (#1752)

* Sender from signers

* Remove co-

* Move signers outside attributes

* Fix UT and remove Signers class

* Fix UT

* Add FeeOnly scope

* Remove orderBy

* Remove _signersCache

* Fix Signers

* Fix WitnessScope

* Fix Sender

* Update TransactionAttributeType.cs

* Update Wallet.cs

* Fix Wallet

* Rename

* Update Wallet.cs

* Update Wallet.cs

* Partial UT fix

* More UT fixes

* Fix Sender's WitnessScope

* Fix Wallet

* Fix UT

* Explicit FeeOnly for DeployNativeContracts

* Same order as serialization

* Test FeeOnly

* dotnet format

* format

Co-authored-by: Erik Zhang <[email protected]>

* IApplicationEngineProvider (#1758)

* Remove the lock from SendersFeeMonitor and rename it to TransactionVerificationContext (#1756)

* Add EffectiveVoterTurnout (#1762)

* Remove AllowedTriggers from SYSCALLs (#1755)

* fix validatorscount (#1770)

Co-authored-by: Tommo-L <[email protected]>

* impl SeekInternal

Co-authored-by: Erik Zhang <[email protected]>
Co-authored-by: Shargon <[email protected]>
Co-authored-by: Harry Pierson <[email protected]>
Co-authored-by: Luchuan <[email protected]>
Co-authored-by: Tommo-L <[email protected]>
Co-authored-by: joeqian <[email protected]>
  • Loading branch information
7 people authored Jul 22, 2020
1 parent 9563ff3 commit 3a0a732
Show file tree
Hide file tree
Showing 105 changed files with 1,536 additions and 1,257 deletions.
18 changes: 9 additions & 9 deletions src/neo/Consensus/ConsensusContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ internal class ConsensusContext : IDisposable, ISerializable
/// <summary>
/// Store all verified unsorted transactions' senders' fee currently in the consensus context.
/// </summary>
public SendersFeeMonitor SendersFeeMonitor = new SendersFeeMonitor();
public TransactionVerificationContext VerificationContext = new TransactionVerificationContext();

public SnapshotView Snapshot { get; private set; }
private KeyPair keyPair;
Expand Down Expand Up @@ -130,18 +130,18 @@ public void Deserialize(BinaryReader reader)
ViewNumber = reader.ReadByte();
TransactionHashes = reader.ReadSerializableArray<UInt256>();
Transaction[] transactions = reader.ReadSerializableArray<Transaction>(Block.MaxTransactionsPerBlock);
PreparationPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
CommitPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
ChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
LastChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.MaxValidatorsCount);
PreparationPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
CommitPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
ChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
LastChangeViewPayloads = reader.ReadNullableArray<ConsensusPayload>(ProtocolSettings.Default.ValidatorsCount);
if (TransactionHashes.Length == 0 && !RequestSentOrReceived)
TransactionHashes = null;
Transactions = transactions.Length == 0 && !RequestSentOrReceived ? null : transactions.ToDictionary(p => p.Hash);
SendersFeeMonitor = new SendersFeeMonitor();
VerificationContext = new TransactionVerificationContext();
if (Transactions != null)
{
foreach (Transaction tx in Transactions.Values)
SendersFeeMonitor.AddSenderFee(tx);
VerificationContext.AddTransaction(tx);
}
}

Expand Down Expand Up @@ -301,7 +301,7 @@ internal void EnsureMaxBlockLimitation(IEnumerable<Transaction> txs)
txs = txs.Take((int)maxTransactionsPerBlock);
List<UInt256> hashes = new List<UInt256>();
Transactions = new Dictionary<UInt256, Transaction>();
SendersFeeMonitor = new SendersFeeMonitor();
VerificationContext = new TransactionVerificationContext();

// Expected block size
var blockSize = GetExpectedBlockSizeWithoutTransactions(txs.Count());
Expand All @@ -320,7 +320,7 @@ internal void EnsureMaxBlockLimitation(IEnumerable<Transaction> txs)

hashes.Add(tx.Hash);
Transactions.Add(tx.Hash, tx);
SendersFeeMonitor.AddSenderFee(tx);
VerificationContext.AddTransaction(tx);
}

TransactionHashes = hashes.ToArray();
Expand Down
28 changes: 16 additions & 12 deletions src/neo/Consensus/ConsensusService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -64,20 +64,24 @@ internal ConsensusService(IActorRef localNode, IActorRef taskManager, IActorRef

private bool AddTransaction(Transaction tx, bool verify)
{
if (verify && tx.Verify(context.Snapshot, context.SendersFeeMonitor.GetSenderFee(tx.Sender)) != VerifyResult.Succeed)
if (verify)
{
Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
RequestChangeView(ChangeViewReason.TxInvalid);
return false;
}
if (!NativeContract.Policy.CheckPolicy(tx, context.Snapshot))
{
Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
RequestChangeView(ChangeViewReason.TxRejectedByPolicy);
return false;
VerifyResult result = tx.Verify(context.Snapshot, context.VerificationContext);
if (result == VerifyResult.PolicyFail)
{
Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
RequestChangeView(ChangeViewReason.TxRejectedByPolicy);
return false;
}
else if (result != VerifyResult.Succeed)
{
Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
RequestChangeView(ChangeViewReason.TxInvalid);
return false;
}
}
context.Transactions[tx.Hash] = tx;
context.SendersFeeMonitor.AddSenderFee(tx);
context.VerificationContext.AddTransaction(tx);
return CheckPrepareResponse();
}

Expand Down Expand Up @@ -443,7 +447,7 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m
context.Block.ConsensusData.Nonce = message.Nonce;
context.TransactionHashes = message.TransactionHashes;
context.Transactions = new Dictionary<UInt256, Transaction>();
context.SendersFeeMonitor = new SendersFeeMonitor();
context.VerificationContext = new TransactionVerificationContext();
for (int i = 0; i < context.PreparationPayloads.Length; i++)
if (context.PreparationPayloads[i] != null)
if (!context.PreparationPayloads[i].GetDeserializedMessage<PrepareResponse>().PreparationHash.Equals(payload.Hash))
Expand Down
6 changes: 3 additions & 3 deletions src/neo/Consensus/RecoveryMessage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public RecoveryMessage() : base(ConsensusMessageType.RecoveryMessage)
public override void Deserialize(BinaryReader reader)
{
base.Deserialize(reader);
ChangeViewMessages = reader.ReadSerializableArray<ChangeViewPayloadCompact>(ProtocolSettings.Default.MaxValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
ChangeViewMessages = reader.ReadSerializableArray<ChangeViewPayloadCompact>(ProtocolSettings.Default.ValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
if (reader.ReadBoolean())
PrepareRequestMessage = reader.ReadSerializable<PrepareRequest>();
else
Expand All @@ -41,8 +41,8 @@ public override void Deserialize(BinaryReader reader)
PreparationHash = new UInt256(reader.ReadFixedBytes(preparationHashSize));
}

PreparationMessages = reader.ReadSerializableArray<PreparationPayloadCompact>(ProtocolSettings.Default.MaxValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
CommitMessages = reader.ReadSerializableArray<CommitPayloadCompact>(ProtocolSettings.Default.MaxValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
PreparationMessages = reader.ReadSerializableArray<PreparationPayloadCompact>(ProtocolSettings.Default.ValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
CommitMessages = reader.ReadSerializableArray<CommitPayloadCompact>(ProtocolSettings.Default.ValidatorsCount).ToDictionary(p => (int)p.ValidatorIndex);
}

internal ConsensusPayload[] GetChangeViewPayloads(ConsensusContext context, ConsensusPayload payload)
Expand Down
19 changes: 18 additions & 1 deletion src/neo/IO/ByteArrayComparer.cs
Original file line number Diff line number Diff line change
@@ -1,13 +1,30 @@
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;

namespace Neo.IO
{
internal class ByteArrayComparer : IComparer<byte[]>
{
public static readonly ByteArrayComparer Default = new ByteArrayComparer();
public static readonly ByteArrayComparer Default = new ByteArrayComparer(1);
public static readonly ByteArrayComparer Reverse = new ByteArrayComparer(-1);

private readonly int direction;

private ByteArrayComparer(int direction)
{
this.direction = direction;
}

public int Compare(byte[] x, byte[] y)
{
return direction > 0
? CompareInternal(x, y)
: -CompareInternal(x, y);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int CompareInternal(byte[] x, byte[] y)
{
int length = Math.Min(x.Length, y.Length);
for (int i = 0; i < length; i++)
Expand Down
10 changes: 5 additions & 5 deletions src/neo/IO/Caching/CloneCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ protected override void DeleteInternal(TKey key)
innerCache.Delete(key);
}

protected override IEnumerable<(TKey, TValue)> FindInternal(byte[] key_prefix)
protected override TValue GetInternal(TKey key)
{
foreach (var (key, value) in innerCache.Find(key_prefix))
yield return (key, value.Clone());
return innerCache[key].Clone();
}

protected override TValue GetInternal(TKey key)
protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPreifx, SeekDirection direction)
{
return innerCache[key].Clone();
foreach (var (key, value) in innerCache.Seek(keyOrPreifx, direction))
yield return (key, value.Clone());
}

protected override TValue TryGetInternal(TKey key)
Expand Down
120 changes: 71 additions & 49 deletions src/neo/IO/Caching/DataCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -151,57 +151,18 @@ public void Delete(TKey key)
/// <returns>Entries found with the desired prefix</returns>
public IEnumerable<(TKey Key, TValue Value)> Find(byte[] key_prefix = null)
{
IEnumerable<(byte[], TKey, TValue)> cached;
HashSet<TKey> cachedKeySet;
lock (dictionary)
{
cached = dictionary
.Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || p.Key.ToArray().AsSpan().StartsWith(key_prefix)))
.Select(p =>
(
KeyBytes: p.Key.ToArray(),
p.Key,
p.Value.Item
))
.OrderBy(p => p.KeyBytes, ByteArrayComparer.Default)
.ToArray();
cachedKeySet = new HashSet<TKey>(dictionary.Keys);
}
var uncached = FindInternal(key_prefix ?? Array.Empty<byte>())
.Where(p => !cachedKeySet.Contains(p.Key))
.Select(p =>
(
KeyBytes: p.Key.ToArray(),
p.Key,
p.Value
));
using (var e1 = cached.GetEnumerator())
using (var e2 = uncached.GetEnumerator())
{
(byte[] KeyBytes, TKey Key, TValue Item) i1, i2;
bool c1 = e1.MoveNext();
bool c2 = e2.MoveNext();
i1 = c1 ? e1.Current : default;
i2 = c2 ? e2.Current : default;
while (c1 || c2)
{
if (!c2 || (c1 && ByteArrayComparer.Default.Compare(i1.KeyBytes, i2.KeyBytes) < 0))
{
yield return (i1.Key, i1.Item);
c1 = e1.MoveNext();
i1 = c1 ? e1.Current : default;
}
else
{
yield return (i2.Key, i2.Item);
c2 = e2.MoveNext();
i2 = c2 ? e2.Current : default;
}
}
}
foreach (var (key, value) in Seek(key_prefix, SeekDirection.Forward))
if (key.ToArray().AsSpan().StartsWith(key_prefix))
yield return (key, value);
}

protected abstract IEnumerable<(TKey Key, TValue Value)> FindInternal(byte[] key_prefix);
public IEnumerable<(TKey Key, TValue Value)> FindRange(TKey start, TKey end)
{
var endKey = end.ToArray();
foreach (var (key, value) in Seek(start.ToArray(), SeekDirection.Forward))
if (ByteArrayComparer.Default.Compare(key.ToArray(), endKey) < 0)
yield return (key, value);
}

public IEnumerable<Trackable> GetChangeSet()
{
Expand Down Expand Up @@ -299,6 +260,67 @@ public TValue GetOrAdd(TKey key, Func<TValue> factory)
}
}

/// <summary>
/// Seek to the entry with specific key
/// </summary>
/// <param name="keyOrPrefix">The key to be sought</param>
/// <param name="direction">The direction of seek</param>
/// <returns>An enumerator containing all the entries after seeking.</returns>
public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward)
{
IEnumerable<(byte[], TKey, TValue)> cached;
HashSet<TKey> cachedKeySet;
ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse;
lock (dictionary)
{
cached = dictionary
.Where(p => p.Value.State != TrackState.Deleted && (keyOrPrefix == null || comparer.Compare(p.Key.ToArray(), keyOrPrefix) >= 0))
.Select(p =>
(
KeyBytes: p.Key.ToArray(),
p.Key,
p.Value.Item
))
.OrderBy(p => p.KeyBytes, comparer)
.ToArray();
cachedKeySet = new HashSet<TKey>(dictionary.Keys);
}
var uncached = SeekInternal(keyOrPrefix ?? Array.Empty<byte>(), direction)
.Where(p => !cachedKeySet.Contains(p.Key))
.Select(p =>
(
KeyBytes: p.Key.ToArray(),
p.Key,
p.Value
));
using (var e1 = cached.GetEnumerator())
using (var e2 = uncached.GetEnumerator())
{
(byte[] KeyBytes, TKey Key, TValue Item) i1, i2;
bool c1 = e1.MoveNext();
bool c2 = e2.MoveNext();
i1 = c1 ? e1.Current : default;
i2 = c2 ? e2.Current : default;
while (c1 || c2)
{
if (!c2 || (c1 && comparer.Compare(i1.KeyBytes, i2.KeyBytes) < 0))
{
yield return (i1.Key, i1.Item);
c1 = e1.MoveNext();
i1 = c1 ? e1.Current : default;
}
else
{
yield return (i2.Key, i2.Item);
c2 = e2.MoveNext();
i2 = c2 ? e2.Current : default;
}
}
}
}

protected abstract IEnumerable<(TKey Key, TValue Value)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction);

public TValue TryGet(TKey key)
{
lock (dictionary)
Expand Down
8 changes: 8 additions & 0 deletions src/neo/IO/Caching/SeekDirection.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace Neo.IO.Caching
{
public enum SeekDirection : sbyte
{
Forward = 1,
Backward = -1
}
}
30 changes: 14 additions & 16 deletions src/neo/Ledger/Blockchain.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu
public const uint DecrementInterval = 2000000;
public static readonly uint[] GenerationAmount = { 6, 5, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
public static readonly TimeSpan TimePerBlock = TimeSpan.FromMilliseconds(MillisecondsPerBlock);
public static readonly ECPoint[] StandbyCommittee = ProtocolSettings.Default.StandbyCommittee.Take(ProtocolSettings.Default.MaxCommitteeMembersCount).Select(p => ECPoint.DecodePoint(p.HexToBytes(), ECCurve.Secp256r1)).ToArray();
public static readonly ECPoint[] StandbyValidators = StandbyCommittee.Take(ProtocolSettings.Default.MaxValidatorsCount).ToArray();
public static readonly ECPoint[] StandbyCommittee = ProtocolSettings.Default.StandbyCommittee.Select(p => ECPoint.DecodePoint(p.HexToBytes(), ECCurve.Secp256r1)).ToArray();
public static readonly ECPoint[] StandbyValidators = StandbyCommittee[0..ProtocolSettings.Default.ValidatorsCount];

public static readonly Block GenesisBlock = new Block
{
Expand Down Expand Up @@ -166,8 +166,15 @@ private static Transaction DeployNativeContracts()
{
Version = 0,
Script = script,
Sender = (new[] { (byte)OpCode.PUSH1 }).ToScriptHash(),
SystemFee = 0,
Signers = new[]
{
new Signer
{
Account = (new[] { (byte)OpCode.PUSH1 }).ToScriptHash(),
Scopes = WitnessScope.FeeOnly
}
},
Attributes = Array.Empty<TransactionAttribute>(),
Witnesses = new[]
{
Expand Down Expand Up @@ -283,15 +290,10 @@ private void OnFillMemoryPool(IEnumerable<Transaction> transactions)
{
if (View.ContainsTransaction(tx.Hash))
continue;
if (!NativeContract.Policy.CheckPolicy(tx, currentSnapshot))
continue;
// First remove the tx if it is unverified in the pool.
MemPool.TryRemoveUnVerified(tx.Hash, out _);
// Verify the the transaction
if (tx.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(tx.Sender)) != VerifyResult.Succeed)
continue;
// Add to the memory pool
MemPool.TryAdd(tx.Hash, tx);
MemPool.TryAdd(tx, currentSnapshot);
}
// Transactions originally in the pool will automatically be reverified based on their priority.

Expand Down Expand Up @@ -355,11 +357,7 @@ private VerifyResult OnNewInventory(IInventory inventory)
private VerifyResult OnNewTransaction(Transaction transaction)
{
if (ContainsTransaction(transaction.Hash)) return VerifyResult.AlreadyExists;
if (!MemPool.CanTransactionFitInPool(transaction)) return VerifyResult.OutOfMemory;
VerifyResult reason = transaction.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(transaction.Sender));
if (reason != VerifyResult.Succeed) return reason;
if (!MemPool.TryAdd(transaction.Hash, transaction)) return VerifyResult.OutOfMemory;
return VerifyResult.Succeed;
return MemPool.TryAdd(transaction, currentSnapshot);
}

protected override void OnReceive(object message)
Expand Down Expand Up @@ -404,7 +402,7 @@ private void Persist(Block block)
snapshot.PersistingBlock = block;
if (block.Index > 0)
{
using (ApplicationEngine engine = new ApplicationEngine(TriggerType.System, null, snapshot, 0, true))
using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.System, null, snapshot, 0, true))
{
engine.LoadScript(onPersistNativeContractScript);
if (engine.Execute() != VMState.HALT) throw new InvalidOperationException();
Expand All @@ -427,7 +425,7 @@ private void Persist(Block block)
clonedSnapshot.Transactions.Add(tx.Hash, state);
clonedSnapshot.Transactions.Commit();

using (ApplicationEngine engine = new ApplicationEngine(TriggerType.Application, tx, clonedSnapshot, tx.SystemFee))
using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.Application, tx, clonedSnapshot, tx.SystemFee))
{
engine.LoadScript(tx.Script);
state.VMState = engine.Execute();
Expand Down
Loading

0 comments on commit 3a0a732

Please sign in to comment.