From 96f6bab9f48b630dedf0dcc866782302653d0303 Mon Sep 17 00:00:00 2001 From: jsolman Date: Mon, 11 Mar 2019 12:53:58 -0700 Subject: [PATCH 01/18] Add Plugin for UTXO tracking by account. --- RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs | 182 ++++++++++++++++++ .../RpcSystemAssetPlugin.csproj | 17 ++ .../RpcSystemAssetTracker/config.json | 6 + .../UserUnspentCoinOutputs.cs | 77 ++++++++ .../UserUnspentCoinOutputsKey.cs | 81 ++++++++ 5 files changed, 363 insertions(+) create mode 100644 RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs create mode 100644 RpcSystemAssetPlugin/RpcSystemAssetPlugin.csproj create mode 100644 RpcSystemAssetPlugin/RpcSystemAssetTracker/config.json create mode 100644 RpcSystemAssetPlugin/UserUnspentCoinOutputs.cs create mode 100644 RpcSystemAssetPlugin/UserUnspentCoinOutputsKey.cs diff --git a/RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs b/RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs new file mode 100644 index 000000000..cd976c727 --- /dev/null +++ b/RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs @@ -0,0 +1,182 @@ +using Microsoft.AspNetCore.Http; +using Neo.IO.Caching; +using Neo.IO.Data.LevelDB; +using Neo.IO.Json; +using Neo.Network.P2P.Payloads; +using Neo.Persistence.LevelDB; +using Neo.Wallets; +using System; +using System.Collections.Generic; +using System.Linq; +using Neo.Ledger; +using Snapshot = Neo.Persistence.Snapshot; + +namespace Neo.Plugins +{ + public class RpcSystemAssetPlugin : Plugin, IPersistencePlugin, IRpcPlugin + { + private const byte SystemAssetUnspentCoinsPrefix = 0xfb; + private DB _db; + private DataCache _userUnspentCoins; + private WriteBatch _writeBatch; + private int _rpcMaxUnspents; + + public override void Configure() + { + if (_db == null) + { + var dbPath = GetConfiguration().GetSection("DBPath").Value ?? "SystemAssetBalanceData"; + _db = DB.Open(dbPath, new Options { CreateIfMissing = true }); + _rpcMaxUnspents = int.Parse(GetConfiguration().GetSection("MaxReturnedUnspents").Value ?? "0"); + + } + } + + private void ResetBatch() + { + _writeBatch = new WriteBatch(); + var balancesSnapshot = _db.GetSnapshot(); + ReadOptions dbOptions = new ReadOptions { FillCache = false, Snapshot = balancesSnapshot }; + _userUnspentCoins = new DbCache(_db, dbOptions, + _writeBatch, SystemAssetUnspentCoinsPrefix); + } + + + public void OnPersist(Snapshot snapshot, IReadOnlyList applicationExecutedList) + { + // Start freshly with a new DBCache for each block. + ResetBatch(); + + foreach (Transaction tx in snapshot.PersistingBlock.Transactions) + { + ushort outputIndex = 0; + foreach (TransactionOutput output in tx.Outputs) + { + bool isGoverningToken = output.AssetId.Equals(Blockchain.GoverningToken.Hash); + if (isGoverningToken || output.AssetId.Equals(Blockchain.UtilityToken.Hash)) + { + // Add new unspent UTXOs by account script hash. + UserUnspentCoinOutputs outputs = _userUnspentCoins.GetAndChange( + new UserUnspentCoinOutputsKey(isGoverningToken, output.ScriptHash, tx.Hash), + () => new UserUnspentCoinOutputs()); + outputs.AddTxIndex(outputIndex, output.Value); + } + outputIndex++; + } + + // Iterate all input Transactions by grouping by common input hashes. + foreach (var group in tx.Inputs.GroupBy(p => p.PrevHash)) + { + TransactionState txPrev = snapshot.Transactions[group.Key]; + // For each input being spent by this transaction. + foreach (CoinReference input in group) + { + // Get the output from the the previous transaction that is now being spent. + var outPrev = txPrev.Transaction.Outputs[input.PrevIndex]; + + bool isGoverningToken = outPrev.AssetId.Equals(Blockchain.GoverningToken.Hash); + if (isGoverningToken || outPrev.AssetId.Equals(Blockchain.UtilityToken.Hash)) + { + // Remove spent UTXOs for unspent outputs by account script hash. + var userUnspentCoinOutputsKey = + new UserUnspentCoinOutputsKey(isGoverningToken, outPrev.ScriptHash, input.PrevHash); + UserUnspentCoinOutputs outputs = _userUnspentCoins.GetAndChange( + userUnspentCoinOutputsKey, () => new UserUnspentCoinOutputs()); + outputs.RemoveTxIndex(input.PrevIndex); + if (outputs.AmountByTxIndex.Count == 0) + _userUnspentCoins.Delete(userUnspentCoinOutputsKey); + } + } + } + } + } + + public void OnCommit(Snapshot snapshot) + { + _userUnspentCoins.Commit(); + _db.Write(WriteOptions.Default, _writeBatch); + } + + public bool ShouldThrowExceptionFromCommit(Exception ex) + { + return true; + } + + public void PreProcess(HttpContext context, string method, JArray _params) + { + } + + public JObject OnProcess(HttpContext context, string method, JArray _params) + { + if (method != "getunspents") return null; + + UInt160 scriptHash = UInt160.Parse(_params[0].AsString()); + + string[] nativeAssetNames = {"GAS", "NEO"}; + UInt256[] nativeAssetIds = {Blockchain.UtilityToken.Hash, Blockchain.GoverningToken.Hash}; + + byte startingToken = 0; // 0 = Utility Token (GAS), 1 = Governing Token (NEO) + int maxIterations = 2; + + if (_params.Count > 1) + { + maxIterations = 1; + bool isGoverningToken = _params[1].AsBoolean(); + if (isGoverningToken) startingToken = 1; + } + + var unspentsCache = new DbCache( + _db, null, null, SystemAssetUnspentCoinsPrefix); + + (JArray, Fixed8) RetreiveUnspentsForPrefix(byte[] prefix) + { + var unspents = new JArray(); + Fixed8 total = new Fixed8(0); + + foreach (var unspentInTx in unspentsCache.Find(prefix)) + { + var txId = unspentInTx.Key.TxHash.ToString().Substring(2); + foreach (var unspent in unspentInTx.Value.AmountByTxIndex) + { + var utxo = new JObject(); + utxo["txid"] = txId; + utxo["n"] = unspent.Key; + utxo["value"] = (double) (decimal) unspent.Value; + total += unspent.Value; + + unspents.Add(utxo); + if (unspents.Count > _rpcMaxUnspents) + return (unspents, total); + } + } + return (unspents, total); + } + + JObject json = new JObject(); + JArray balances = new JArray(); + json["balance"] = balances; + json["address"] = scriptHash.ToAddress(); + for (byte tokenIndex = startingToken; maxIterations-- > 0; tokenIndex++) + { + byte[] prefix = new [] { tokenIndex }.Concat(scriptHash.ToArray()).ToArray(); + + var (unspents, total) = RetreiveUnspentsForPrefix(prefix); + + if (unspents.Count <= 0) continue; + + var balance = new JObject(); + balance["unspent"] = unspents; + balance["asset_hash"] = nativeAssetIds[tokenIndex].ToString().Substring(2); + balance["asset_symbol"] = balance["asset"] = nativeAssetNames[tokenIndex]; + balance["amount"] = new JNumber((double) (decimal) total); ; + balances.Add(balance); + } + + return json; + } + + public void PostProcess(HttpContext context, string method, JArray _params, JObject result) + { + } + } +} diff --git a/RpcSystemAssetPlugin/RpcSystemAssetPlugin.csproj b/RpcSystemAssetPlugin/RpcSystemAssetPlugin.csproj new file mode 100644 index 000000000..c5b745a37 --- /dev/null +++ b/RpcSystemAssetPlugin/RpcSystemAssetPlugin.csproj @@ -0,0 +1,17 @@ + + + 2.10.0 + netstandard2.0 + Neo.Plugins + latest + + + + PreserveNewest + PreserveNewest + + + + + + diff --git a/RpcSystemAssetPlugin/RpcSystemAssetTracker/config.json b/RpcSystemAssetPlugin/RpcSystemAssetTracker/config.json new file mode 100644 index 000000000..fdabf0015 --- /dev/null +++ b/RpcSystemAssetPlugin/RpcSystemAssetTracker/config.json @@ -0,0 +1,6 @@ +{ + "PluginConfiguration": { + "DBPath": "SystemAssetBalanceData", + "MaxReturnedUnspents": 1000 + } +} diff --git a/RpcSystemAssetPlugin/UserUnspentCoinOutputs.cs b/RpcSystemAssetPlugin/UserUnspentCoinOutputs.cs new file mode 100644 index 000000000..8dee4b741 --- /dev/null +++ b/RpcSystemAssetPlugin/UserUnspentCoinOutputs.cs @@ -0,0 +1,77 @@ +using System.Collections.Generic; +using System.IO; +using Neo.IO; +using Neo.Ledger; + +namespace Neo.Plugins +{ + public class UserUnspentCoinOutputs : StateBase, ICloneable + { + public Fixed8 TotalAmount; + public Dictionary AmountByTxIndex; + + + public UserUnspentCoinOutputs() + { + TotalAmount = new Fixed8(0); + AmountByTxIndex = new Dictionary(); + } + + public void AddTxIndex(ushort index, Fixed8 amount) + { + TotalAmount += amount; + AmountByTxIndex.Add(index, amount); + } + + public bool RemoveTxIndex(ushort index) + { + if(AmountByTxIndex.TryGetValue(index, out Fixed8 amount)) + { + AmountByTxIndex.Remove(index); + TotalAmount -= amount; + return true; + } + + return false; + } + public UserUnspentCoinOutputs Clone() + { + return new UserUnspentCoinOutputs() + { + TotalAmount = TotalAmount, + AmountByTxIndex = new Dictionary(AmountByTxIndex) + }; + } + + public void FromReplica(UserUnspentCoinOutputs replica) + { + TotalAmount = replica.TotalAmount; + AmountByTxIndex = replica.AmountByTxIndex; + } + + public override void Serialize(BinaryWriter writer) + { + base.Serialize(writer); + writer.Write(TotalAmount); + writer.Write((ushort)AmountByTxIndex.Count); + foreach (KeyValuePair txIndex in AmountByTxIndex) + { + writer.Write(txIndex.Key); + writer.Write(txIndex.Value); + } + } + + public override void Deserialize(BinaryReader reader) + { + base.Deserialize(reader); + ((ISerializable)TotalAmount).Deserialize(reader); + ushort count = reader.ReadUInt16(); + for (int i = 0; i < count; i++) + { + ushort txIndex = reader.ReadUInt16(); + Fixed8 amount = reader.ReadSerializable(); + AmountByTxIndex.Add(txIndex, amount); + } + } + } +} \ No newline at end of file diff --git a/RpcSystemAssetPlugin/UserUnspentCoinOutputsKey.cs b/RpcSystemAssetPlugin/UserUnspentCoinOutputsKey.cs new file mode 100644 index 000000000..93475ce76 --- /dev/null +++ b/RpcSystemAssetPlugin/UserUnspentCoinOutputsKey.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Neo.IO; + +namespace Neo.Plugins +{ + public class UserUnspentCoinOutputsKey : IComparable, IEquatable, + ISerializable + { + public bool IsGoverningToken; // It's either the governing token or the utility token + public readonly UInt160 UserAddress; + public readonly UInt256 TxHash; + + public int Size => 1 + UserAddress.Size + TxHash.Size; + + public bool Equals(UserUnspentCoinOutputsKey other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return IsGoverningToken == other.IsGoverningToken && Equals(UserAddress, other.UserAddress) && Equals(TxHash, other.TxHash); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((UserUnspentCoinOutputsKey) obj); + } + + public int CompareTo(UserUnspentCoinOutputsKey other) + { + if (ReferenceEquals(this, other)) return 0; + if (ReferenceEquals(null, other)) return 1; + var isGoverningTokenComparison = IsGoverningToken.CompareTo(other.IsGoverningToken); + if (isGoverningTokenComparison != 0) return isGoverningTokenComparison; + var userAddressComparison = Comparer.Default.Compare(UserAddress, other.UserAddress); + if (userAddressComparison != 0) return userAddressComparison; + return Comparer.Default.Compare(TxHash, other.TxHash); + } + + public override int GetHashCode() + { + unchecked + { + var hashCode = IsGoverningToken.GetHashCode(); + hashCode = (hashCode * 397) ^ (UserAddress != null ? UserAddress.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (TxHash != null ? TxHash.GetHashCode() : 0); + return hashCode; + } + } + + public UserUnspentCoinOutputsKey() + { + UserAddress = new UInt160(); + TxHash = new UInt256(); + } + + public UserUnspentCoinOutputsKey(bool isGoverningToken, UInt160 userAddress, UInt256 txHash) + { + IsGoverningToken = isGoverningToken; + UserAddress = userAddress; + TxHash = txHash; + } + + public void Serialize(BinaryWriter writer) + { + writer.Write(IsGoverningToken); + writer.Write(UserAddress.ToArray()); + writer.Write(TxHash.ToArray()); + } + + public void Deserialize(BinaryReader reader) + { + IsGoverningToken = reader.ReadBoolean(); + ((ISerializable) UserAddress).Deserialize(reader); + ((ISerializable) TxHash).Deserialize(reader); + } + } +} \ No newline at end of file From 0a80ceb905727f1263b768b20011369b4b60db41 Mon Sep 17 00:00:00 2001 From: jsolman Date: Mon, 11 Mar 2019 13:53:29 -0700 Subject: [PATCH 02/18] Rename RpcSystemAssetPlugin -> RpcSystemAssetTracker. --- .../RpcSystemAssetTracker.csproj | 0 .../RpcSystemAssetTracker/config.json | 0 .../RpcSystemAssetTrackerPlugin.cs | 2 +- .../UserUnspentCoinOutputs.cs | 0 .../UserUnspentCoinOutputsKey.cs | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename RpcSystemAssetPlugin/RpcSystemAssetPlugin.csproj => RpcSystemAssetTracker/RpcSystemAssetTracker.csproj (100%) rename {RpcSystemAssetPlugin => RpcSystemAssetTracker}/RpcSystemAssetTracker/config.json (100%) rename RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs => RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs (98%) rename {RpcSystemAssetPlugin => RpcSystemAssetTracker}/UserUnspentCoinOutputs.cs (100%) rename {RpcSystemAssetPlugin => RpcSystemAssetTracker}/UserUnspentCoinOutputsKey.cs (100%) diff --git a/RpcSystemAssetPlugin/RpcSystemAssetPlugin.csproj b/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj similarity index 100% rename from RpcSystemAssetPlugin/RpcSystemAssetPlugin.csproj rename to RpcSystemAssetTracker/RpcSystemAssetTracker.csproj diff --git a/RpcSystemAssetPlugin/RpcSystemAssetTracker/config.json b/RpcSystemAssetTracker/RpcSystemAssetTracker/config.json similarity index 100% rename from RpcSystemAssetPlugin/RpcSystemAssetTracker/config.json rename to RpcSystemAssetTracker/RpcSystemAssetTracker/config.json diff --git a/RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs similarity index 98% rename from RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs rename to RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index cd976c727..6f567487a 100644 --- a/RpcSystemAssetPlugin/RpcSystemAssetPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -13,7 +13,7 @@ namespace Neo.Plugins { - public class RpcSystemAssetPlugin : Plugin, IPersistencePlugin, IRpcPlugin + public class RpcSystemAssetTrackerPlugin : Plugin, IPersistencePlugin, IRpcPlugin { private const byte SystemAssetUnspentCoinsPrefix = 0xfb; private DB _db; diff --git a/RpcSystemAssetPlugin/UserUnspentCoinOutputs.cs b/RpcSystemAssetTracker/UserUnspentCoinOutputs.cs similarity index 100% rename from RpcSystemAssetPlugin/UserUnspentCoinOutputs.cs rename to RpcSystemAssetTracker/UserUnspentCoinOutputs.cs diff --git a/RpcSystemAssetPlugin/UserUnspentCoinOutputsKey.cs b/RpcSystemAssetTracker/UserUnspentCoinOutputsKey.cs similarity index 100% rename from RpcSystemAssetPlugin/UserUnspentCoinOutputsKey.cs rename to RpcSystemAssetTracker/UserUnspentCoinOutputsKey.cs From 87aea4f5cb099aea552533891bcf5f038844b21c Mon Sep 17 00:00:00 2001 From: jsolman Date: Mon, 11 Mar 2019 14:53:37 -0700 Subject: [PATCH 03/18] Accept address or script hash. --- RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 6f567487a..114a7d61d 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -106,11 +106,17 @@ public void PreProcess(HttpContext context, string method, JArray _params) { } + private UInt160 GetScriptHashFromParam(string addressOrScriptHash) + { + return addressOrScriptHash.Length < 40 ? + addressOrScriptHash.ToScriptHash() : UInt160.Parse(addressOrScriptHash); + } + public JObject OnProcess(HttpContext context, string method, JArray _params) { if (method != "getunspents") return null; - UInt160 scriptHash = UInt160.Parse(_params[0].AsString()); + UInt160 scriptHash = GetScriptHashFromParam(_params[0].AsString()); string[] nativeAssetNames = {"GAS", "NEO"}; UInt256[] nativeAssetIds = {Blockchain.UtilityToken.Hash, Blockchain.GoverningToken.Hash}; From fc297be7fb1906c4973e23989fcb3084bb9f4034 Mon Sep 17 00:00:00 2001 From: jsolman Date: Mon, 11 Mar 2019 15:20:22 -0700 Subject: [PATCH 04/18] Remove unnecessary white space. --- RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs | 1 - RpcSystemAssetTracker/UserUnspentCoinOutputs.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 114a7d61d..a2e7af1ac 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -41,7 +41,6 @@ private void ResetBatch() _writeBatch, SystemAssetUnspentCoinsPrefix); } - public void OnPersist(Snapshot snapshot, IReadOnlyList applicationExecutedList) { // Start freshly with a new DBCache for each block. diff --git a/RpcSystemAssetTracker/UserUnspentCoinOutputs.cs b/RpcSystemAssetTracker/UserUnspentCoinOutputs.cs index 8dee4b741..f00ce9d7b 100644 --- a/RpcSystemAssetTracker/UserUnspentCoinOutputs.cs +++ b/RpcSystemAssetTracker/UserUnspentCoinOutputs.cs @@ -10,7 +10,6 @@ public class UserUnspentCoinOutputs : StateBase, ICloneable AmountByTxIndex; - public UserUnspentCoinOutputs() { TotalAmount = new Fixed8(0); From 1b9870a2729d730fc609196b42b383357653b542 Mon Sep 17 00:00:00 2001 From: jsolman Date: Mon, 11 Mar 2019 21:37:24 -0700 Subject: [PATCH 05/18] Update project ref to Neo 2.10.1-CI00002. --- RpcSystemAssetTracker/RpcSystemAssetTracker.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj b/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj index c5b745a37..9b12f2523 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj +++ b/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj @@ -1,6 +1,6 @@  - 2.10.0 + 2.10.1 netstandard2.0 Neo.Plugins latest @@ -12,6 +12,6 @@ - + From 44d358a2e89237d8d68b78ab0fecd0609cd8ec02 Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 12 Mar 2019 12:31:41 -0700 Subject: [PATCH 06/18] Ensure consistency in case of crash or kill. --- .../RpcSystemAssetTrackerPlugin.cs | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index a2e7af1ac..4c42bc64e 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using System.Linq; using Neo.Ledger; +using Neo.Persistence; using Snapshot = Neo.Persistence.Snapshot; namespace Neo.Plugins @@ -20,6 +21,7 @@ public class RpcSystemAssetTrackerPlugin : Plugin, IPersistencePlugin, IRpcPlugi private DataCache _userUnspentCoins; private WriteBatch _writeBatch; private int _rpcMaxUnspents; + private uint _lastPersistedBlock; public override void Configure() { @@ -28,7 +30,16 @@ public override void Configure() var dbPath = GetConfiguration().GetSection("DBPath").Value ?? "SystemAssetBalanceData"; _db = DB.Open(dbPath, new Options { CreateIfMissing = true }); _rpcMaxUnspents = int.Parse(GetConfiguration().GetSection("MaxReturnedUnspents").Value ?? "0"); - + try + { + _lastPersistedBlock = _db.Get(ReadOptions.Default, SystemAssetUnspentCoinsPrefix).ToUInt32(); + } + catch (LevelDBException ex) + { + if (!ex.Message.Contains("not found")) + throw; + _lastPersistedBlock = 0; + } } } @@ -41,12 +52,9 @@ private void ResetBatch() _writeBatch, SystemAssetUnspentCoinsPrefix); } - public void OnPersist(Snapshot snapshot, IReadOnlyList applicationExecutedList) + private void ProcessBlock(Snapshot snapshot, Block block) { - // Start freshly with a new DBCache for each block. - ResetBatch(); - - foreach (Transaction tx in snapshot.PersistingBlock.Transactions) + foreach (Transaction tx in block.Transactions) { ushort outputIndex = 0; foreach (TransactionOutput output in tx.Outputs) @@ -88,6 +96,31 @@ public void OnPersist(Snapshot snapshot, IReadOnlyList applicationExecutedList) + { + if (snapshot.PersistingBlock.Index > _lastPersistedBlock + 1) + ProcessSkippedBlocks(snapshot); + + ResetBatch(); + ProcessBlock(snapshot, snapshot.PersistingBlock); } public void OnCommit(Snapshot snapshot) From f21450a1d6aec9fc2ad3439db8856c2133cf495c Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 12 Mar 2019 12:48:34 -0700 Subject: [PATCH 07/18] Fix setting lastPersistedBlock after each block is persisted. --- RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 4c42bc64e..b22abeb3d 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -99,11 +99,11 @@ private void ProcessBlock(Snapshot snapshot, Block block) // Write the current height into the key of the prefix itself _writeBatch.Put(SystemAssetUnspentCoinsPrefix, block.Index); + _lastPersistedBlock = block.Index; } private void ProcessSkippedBlocks(Snapshot snapshot) { - // Process an for (uint blockIndex = _lastPersistedBlock + 1; blockIndex < snapshot.PersistingBlock.Index; blockIndex++) { var skippedBlock = Blockchain.Singleton.Store.GetBlock(blockIndex); From c7e09221b7b9951a0de59d0cd58a32b04fa2d78e Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 12 Mar 2019 12:58:29 -0700 Subject: [PATCH 08/18] Performance optimization: Don't write block number for empty blocks. --- .../RpcSystemAssetTrackerPlugin.cs | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index b22abeb3d..5d7b3dbf7 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -22,6 +22,7 @@ public class RpcSystemAssetTrackerPlugin : Plugin, IPersistencePlugin, IRpcPlugi private WriteBatch _writeBatch; private int _rpcMaxUnspents; private uint _lastPersistedBlock; + private bool _shouldPersistBlock; public override void Configure() { @@ -52,8 +53,11 @@ private void ResetBatch() _writeBatch, SystemAssetUnspentCoinsPrefix); } - private void ProcessBlock(Snapshot snapshot, Block block) + private bool ProcessBlock(Snapshot snapshot, Block block) { + if (block.Transactions.Length <= 1) return false; + ResetBatch(); + foreach (Transaction tx in block.Transactions) { ushort outputIndex = 0; @@ -100,6 +104,7 @@ private void ProcessBlock(Snapshot snapshot, Block block) // Write the current height into the key of the prefix itself _writeBatch.Put(SystemAssetUnspentCoinsPrefix, block.Index); _lastPersistedBlock = block.Index; + return true; } private void ProcessSkippedBlocks(Snapshot snapshot) @@ -107,10 +112,17 @@ private void ProcessSkippedBlocks(Snapshot snapshot) for (uint blockIndex = _lastPersistedBlock + 1; blockIndex < snapshot.PersistingBlock.Index; blockIndex++) { var skippedBlock = Blockchain.Singleton.Store.GetBlock(blockIndex); - ResetBatch(); - ProcessBlock(snapshot, skippedBlock); - _userUnspentCoins.Commit(); - _db.Write(WriteOptions.Default, _writeBatch); + if (skippedBlock.Transactions.Length <= 1) + { + _lastPersistedBlock = skippedBlock.Index; + continue; + } + + if (ProcessBlock(snapshot, skippedBlock)) + { + _userUnspentCoins.Commit(); + _db.Write(WriteOptions.Default, _writeBatch); + } } } @@ -119,12 +131,12 @@ public void OnPersist(Snapshot snapshot, IReadOnlyList _lastPersistedBlock + 1) ProcessSkippedBlocks(snapshot); - ResetBatch(); - ProcessBlock(snapshot, snapshot.PersistingBlock); + _shouldPersistBlock = ProcessBlock(snapshot, snapshot.PersistingBlock); } public void OnCommit(Snapshot snapshot) { + if (!_shouldPersistBlock) return; _userUnspentCoins.Commit(); _db.Write(WriteOptions.Default, _writeBatch); } From 0777eee9e17a1b37c049620f682324180441df30 Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 12 Mar 2019 13:04:00 -0700 Subject: [PATCH 09/18] Optimize. --- RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 5d7b3dbf7..40a0cd104 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -55,7 +55,12 @@ private void ResetBatch() private bool ProcessBlock(Snapshot snapshot, Block block) { - if (block.Transactions.Length <= 1) return false; + if (block.Transactions.Length <= 1) + { + _lastPersistedBlock = block.Index; + return false; + } + ResetBatch(); foreach (Transaction tx in block.Transactions) From ab8a4d72e4ee2453363cc23241551022314f24e3 Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 12 Mar 2019 14:42:02 -0700 Subject: [PATCH 10/18] Add support for tracking unclaimed spent outputs. --- .../RpcSystemAssetTracker/config.json | 3 +- .../RpcSystemAssetTrackerPlugin.cs | 122 +++++++++++++----- ...tputs.cs => UserSystemAssetCoinOutputs.cs} | 10 +- ...ey.cs => UserSystemAssetCoinOutputsKey.cs} | 12 +- 4 files changed, 100 insertions(+), 47 deletions(-) rename RpcSystemAssetTracker/{UserUnspentCoinOutputs.cs => UserSystemAssetCoinOutputs.cs} (87%) rename RpcSystemAssetTracker/{UserUnspentCoinOutputsKey.cs => UserSystemAssetCoinOutputsKey.cs} (84%) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTracker/config.json b/RpcSystemAssetTracker/RpcSystemAssetTracker/config.json index fdabf0015..214c08939 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTracker/config.json +++ b/RpcSystemAssetTracker/RpcSystemAssetTracker/config.json @@ -1,6 +1,7 @@ { "PluginConfiguration": { "DBPath": "SystemAssetBalanceData", - "MaxReturnedUnspents": 1000 + "MaxReturnedUnspents": 1000, + "TrackUnclaimed": true } } diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 40a0cd104..acd6c5388 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -7,6 +7,7 @@ using Neo.Wallets; using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using Neo.Ledger; using Neo.Persistence; @@ -17,8 +18,11 @@ namespace Neo.Plugins public class RpcSystemAssetTrackerPlugin : Plugin, IPersistencePlugin, IRpcPlugin { private const byte SystemAssetUnspentCoinsPrefix = 0xfb; + private const byte SystemAssetSpentUnclaimedCoinsPrefix = 0xfc; private DB _db; - private DataCache _userUnspentCoins; + private DataCache _userUnspentCoins; + private bool _shouldTrackUnclaimed; + private DataCache _userSpentUnclaimedCoins; private WriteBatch _writeBatch; private int _rpcMaxUnspents; private uint _lastPersistedBlock; @@ -31,6 +35,7 @@ public override void Configure() var dbPath = GetConfiguration().GetSection("DBPath").Value ?? "SystemAssetBalanceData"; _db = DB.Open(dbPath, new Options { CreateIfMissing = true }); _rpcMaxUnspents = int.Parse(GetConfiguration().GetSection("MaxReturnedUnspents").Value ?? "0"); + _shouldTrackUnclaimed = (GetConfiguration().GetSection("TrackUnclaimed").Value ?? true.ToString()) != false.ToString(); try { _lastPersistedBlock = _db.Get(ReadOptions.Default, SystemAssetUnspentCoinsPrefix).ToUInt32(); @@ -49,8 +54,11 @@ private void ResetBatch() _writeBatch = new WriteBatch(); var balancesSnapshot = _db.GetSnapshot(); ReadOptions dbOptions = new ReadOptions { FillCache = false, Snapshot = balancesSnapshot }; - _userUnspentCoins = new DbCache(_db, dbOptions, + _userUnspentCoins = new DbCache(_db, dbOptions, _writeBatch, SystemAssetUnspentCoinsPrefix); + if (!_shouldTrackUnclaimed) return; + _userSpentUnclaimedCoins = new DbCache(_db, dbOptions, + _writeBatch, SystemAssetSpentUnclaimedCoinsPrefix); } private bool ProcessBlock(Snapshot snapshot, Block block) @@ -63,6 +71,7 @@ private bool ProcessBlock(Snapshot snapshot, Block block) ResetBatch(); + var transactionsCache = snapshot.Transactions; foreach (Transaction tx in block.Transactions) { ushort outputIndex = 0; @@ -72,9 +81,9 @@ private bool ProcessBlock(Snapshot snapshot, Block block) if (isGoverningToken || output.AssetId.Equals(Blockchain.UtilityToken.Hash)) { // Add new unspent UTXOs by account script hash. - UserUnspentCoinOutputs outputs = _userUnspentCoins.GetAndChange( - new UserUnspentCoinOutputsKey(isGoverningToken, output.ScriptHash, tx.Hash), - () => new UserUnspentCoinOutputs()); + UserSystemAssetCoinOutputs outputs = _userUnspentCoins.GetAndChange( + new UserSystemAssetCoinOutputsKey(isGoverningToken, output.ScriptHash, tx.Hash), + () => new UserSystemAssetCoinOutputs()); outputs.AddTxIndex(outputIndex, output.Value); } outputIndex++; @@ -83,7 +92,7 @@ private bool ProcessBlock(Snapshot snapshot, Block block) // Iterate all input Transactions by grouping by common input hashes. foreach (var group in tx.Inputs.GroupBy(p => p.PrevHash)) { - TransactionState txPrev = snapshot.Transactions[group.Key]; + TransactionState txPrev = transactionsCache[group.Key]; // For each input being spent by this transaction. foreach (CoinReference input in group) { @@ -94,16 +103,43 @@ private bool ProcessBlock(Snapshot snapshot, Block block) if (isGoverningToken || outPrev.AssetId.Equals(Blockchain.UtilityToken.Hash)) { // Remove spent UTXOs for unspent outputs by account script hash. - var userUnspentCoinOutputsKey = - new UserUnspentCoinOutputsKey(isGoverningToken, outPrev.ScriptHash, input.PrevHash); - UserUnspentCoinOutputs outputs = _userUnspentCoins.GetAndChange( - userUnspentCoinOutputsKey, () => new UserUnspentCoinOutputs()); + var userCoinOutputsKey = + new UserSystemAssetCoinOutputsKey(isGoverningToken, outPrev.ScriptHash, input.PrevHash); + UserSystemAssetCoinOutputs outputs = _userUnspentCoins.GetAndChange( + userCoinOutputsKey, () => new UserSystemAssetCoinOutputs()); outputs.RemoveTxIndex(input.PrevIndex); if (outputs.AmountByTxIndex.Count == 0) - _userUnspentCoins.Delete(userUnspentCoinOutputsKey); + _userUnspentCoins.Delete(userCoinOutputsKey); + + if (_shouldTrackUnclaimed && isGoverningToken) + { + UserSystemAssetCoinOutputs spentUnclaimedOutputs = _userSpentUnclaimedCoins.GetAndChange( + userCoinOutputsKey, () => new UserSystemAssetCoinOutputs()); + spentUnclaimedOutputs.AddTxIndex(input.PrevIndex, outPrev.Value); + } } } } + + if (_shouldTrackUnclaimed && tx is ClaimTransaction claimTransaction) + { + foreach (CoinReference input in claimTransaction.Claims) + { + TransactionState txPrev = transactionsCache[input.PrevHash]; + var outPrev = txPrev.Transaction.Outputs[input.PrevIndex]; + + var claimedCoinKey = + new UserSystemAssetCoinOutputsKey(true, outPrev.ScriptHash, input.PrevHash); + UserSystemAssetCoinOutputs spentUnclaimedOutputs = _userSpentUnclaimedCoins.GetAndChange( + claimedCoinKey, () => new UserSystemAssetCoinOutputs()); + spentUnclaimedOutputs.RemoveTxIndex(input.PrevIndex); + if (spentUnclaimedOutputs.AmountByTxIndex.Count == 0) + _userSpentUnclaimedCoins.Delete(claimedCoinKey); + + if (snapshot.SpentCoins.TryGet(input.PrevHash)?.Items.Remove(input.PrevIndex) == true) + snapshot.SpentCoins.GetAndChange(input.PrevHash); + } + } } // Write the current height into the key of the prefix itself @@ -112,6 +148,7 @@ private bool ProcessBlock(Snapshot snapshot, Block block) return true; } + private void ProcessSkippedBlocks(Snapshot snapshot) { for (uint blockIndex = _lastPersistedBlock + 1; blockIndex < snapshot.PersistingBlock.Index; blockIndex++) @@ -123,11 +160,8 @@ private void ProcessSkippedBlocks(Snapshot snapshot) continue; } - if (ProcessBlock(snapshot, skippedBlock)) - { - _userUnspentCoins.Commit(); - _db.Write(WriteOptions.Default, _writeBatch); - } + _shouldPersistBlock = ProcessBlock(snapshot, skippedBlock); + OnCommit(snapshot); } } @@ -143,6 +177,7 @@ public void OnCommit(Snapshot snapshot) { if (!_shouldPersistBlock) return; _userUnspentCoins.Commit(); + if (_shouldTrackUnclaimed) _userSpentUnclaimedCoins.Commit(); _db.Write(WriteOptions.Default, _writeBatch); } @@ -161,34 +196,18 @@ private UInt160 GetScriptHashFromParam(string addressOrScriptHash) addressOrScriptHash.ToScriptHash() : UInt160.Parse(addressOrScriptHash); } - public JObject OnProcess(HttpContext context, string method, JArray _params) + private JObject GenerateUtxoResponse(UInt160 scriptHash, byte startingToken, int maxIterations, + DbCache dbCache) { - if (method != "getunspents") return null; - - UInt160 scriptHash = GetScriptHashFromParam(_params[0].AsString()); - string[] nativeAssetNames = {"GAS", "NEO"}; UInt256[] nativeAssetIds = {Blockchain.UtilityToken.Hash, Blockchain.GoverningToken.Hash}; - byte startingToken = 0; // 0 = Utility Token (GAS), 1 = Governing Token (NEO) - int maxIterations = 2; - - if (_params.Count > 1) - { - maxIterations = 1; - bool isGoverningToken = _params[1].AsBoolean(); - if (isGoverningToken) startingToken = 1; - } - - var unspentsCache = new DbCache( - _db, null, null, SystemAssetUnspentCoinsPrefix); - (JArray, Fixed8) RetreiveUnspentsForPrefix(byte[] prefix) { var unspents = new JArray(); Fixed8 total = new Fixed8(0); - foreach (var unspentInTx in unspentsCache.Find(prefix)) + foreach (var unspentInTx in dbCache.Find(prefix)) { var txId = unspentInTx.Key.TxHash.ToString().Substring(2); foreach (var unspent in unspentInTx.Value.AmountByTxIndex) @@ -230,6 +249,39 @@ public JObject OnProcess(HttpContext context, string method, JArray _params) return json; } + private JObject ProcessGetUnclaimedSpents(UInt160 scriptHash) + { + var dbCache = new DbCache( + _db, null, null, SystemAssetSpentUnclaimedCoinsPrefix); + return GenerateUtxoResponse(scriptHash, 1, 1, dbCache); + } + + private JObject ProcessGetUnspents(JArray _params) + { + UInt160 scriptHash = GetScriptHashFromParam(_params[0].AsString()); + byte startingToken = 0; // 0 = Utility Token (GAS), 1 = Governing Token (NEO) + int maxIterations = 2; + + if (_params.Count > 1) + { + maxIterations = 1; + bool isGoverningToken = _params[1].AsBoolean(); + if (isGoverningToken) startingToken = 1; + } + + var unspentsCache = new DbCache( + _db, null, null, SystemAssetUnspentCoinsPrefix); + + return GenerateUtxoResponse(scriptHash, startingToken, maxIterations, unspentsCache); + } + + public JObject OnProcess(HttpContext context, string method, JArray _params) + { + if (_shouldTrackUnclaimed && method == "getunclaimedspents") + return ProcessGetUnclaimedSpents(GetScriptHashFromParam(_params[0].AsString())); + return method != "getunspents" ? null : ProcessGetUnspents(_params); + } + public void PostProcess(HttpContext context, string method, JArray _params, JObject result) { } diff --git a/RpcSystemAssetTracker/UserUnspentCoinOutputs.cs b/RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs similarity index 87% rename from RpcSystemAssetTracker/UserUnspentCoinOutputs.cs rename to RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs index f00ce9d7b..08507c587 100644 --- a/RpcSystemAssetTracker/UserUnspentCoinOutputs.cs +++ b/RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs @@ -5,12 +5,12 @@ namespace Neo.Plugins { - public class UserUnspentCoinOutputs : StateBase, ICloneable + public class UserSystemAssetCoinOutputs : StateBase, ICloneable { public Fixed8 TotalAmount; public Dictionary AmountByTxIndex; - public UserUnspentCoinOutputs() + public UserSystemAssetCoinOutputs() { TotalAmount = new Fixed8(0); AmountByTxIndex = new Dictionary(); @@ -33,16 +33,16 @@ public bool RemoveTxIndex(ushort index) return false; } - public UserUnspentCoinOutputs Clone() + public UserSystemAssetCoinOutputs Clone() { - return new UserUnspentCoinOutputs() + return new UserSystemAssetCoinOutputs() { TotalAmount = TotalAmount, AmountByTxIndex = new Dictionary(AmountByTxIndex) }; } - public void FromReplica(UserUnspentCoinOutputs replica) + public void FromReplica(UserSystemAssetCoinOutputs replica) { TotalAmount = replica.TotalAmount; AmountByTxIndex = replica.AmountByTxIndex; diff --git a/RpcSystemAssetTracker/UserUnspentCoinOutputsKey.cs b/RpcSystemAssetTracker/UserSystemAssetCoinOutputsKey.cs similarity index 84% rename from RpcSystemAssetTracker/UserUnspentCoinOutputsKey.cs rename to RpcSystemAssetTracker/UserSystemAssetCoinOutputsKey.cs index 93475ce76..7f0cad859 100644 --- a/RpcSystemAssetTracker/UserUnspentCoinOutputsKey.cs +++ b/RpcSystemAssetTracker/UserSystemAssetCoinOutputsKey.cs @@ -5,7 +5,7 @@ namespace Neo.Plugins { - public class UserUnspentCoinOutputsKey : IComparable, IEquatable, + public class UserSystemAssetCoinOutputsKey : IComparable, IEquatable, ISerializable { public bool IsGoverningToken; // It's either the governing token or the utility token @@ -14,7 +14,7 @@ public class UserUnspentCoinOutputsKey : IComparable, public int Size => 1 + UserAddress.Size + TxHash.Size; - public bool Equals(UserUnspentCoinOutputsKey other) + public bool Equals(UserSystemAssetCoinOutputsKey other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; @@ -26,10 +26,10 @@ public override bool Equals(object obj) if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((UserUnspentCoinOutputsKey) obj); + return Equals((UserSystemAssetCoinOutputsKey) obj); } - public int CompareTo(UserUnspentCoinOutputsKey other) + public int CompareTo(UserSystemAssetCoinOutputsKey other) { if (ReferenceEquals(this, other)) return 0; if (ReferenceEquals(null, other)) return 1; @@ -51,13 +51,13 @@ public override int GetHashCode() } } - public UserUnspentCoinOutputsKey() + public UserSystemAssetCoinOutputsKey() { UserAddress = new UInt160(); TxHash = new UInt256(); } - public UserUnspentCoinOutputsKey(bool isGoverningToken, UInt160 userAddress, UInt256 txHash) + public UserSystemAssetCoinOutputsKey(bool isGoverningToken, UInt160 userAddress, UInt256 txHash) { IsGoverningToken = isGoverningToken; UserAddress = userAddress; From 1544902d14fab2d459480c42ac8bac48a87f79f9 Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 12 Mar 2019 22:50:16 -0700 Subject: [PATCH 11/18] Provide getclaimable RPC call for getting available claim spent outputs. --- .../RpcSystemAssetTrackerPlugin.cs | 147 ++++++++++++++---- 1 file changed, 114 insertions(+), 33 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index acd6c5388..a58fb27b0 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -196,9 +196,116 @@ private UInt160 GetScriptHashFromParam(string addressOrScriptHash) addressOrScriptHash.ToScriptHash() : UInt160.Parse(addressOrScriptHash); } - private JObject GenerateUtxoResponse(UInt160 scriptHash, byte startingToken, int maxIterations, - DbCache dbCache) + + private long GetSysFeeAmountForHeight(DataCache blocks, uint height) { + return blocks.TryGet(Blockchain.Singleton.GetBlockHash(height)).SystemFeeAmount; + } + + private void CalculateClaimable(Snapshot snapshot, Fixed8 value, uint startHeight, uint endHeight, out Fixed8 generated, out Fixed8 sysFee) + { + uint amount = 0; + uint ustart = startHeight / Blockchain.DecrementInterval; + if (ustart < Blockchain.GenerationAmount.Length) + { + uint istart = startHeight % Blockchain.DecrementInterval; + uint uend = endHeight / Blockchain.DecrementInterval; + uint iend = endHeight % Blockchain.DecrementInterval; + if (uend >= Blockchain.GenerationAmount.Length) + { + uend = (uint)Blockchain.GenerationAmount.Length; + iend = 0; + } + if (iend == 0) + { + uend--; + iend = Blockchain.DecrementInterval; + } + while (ustart < uend) + { + amount += (Blockchain.DecrementInterval - istart) * Blockchain.GenerationAmount[ustart]; + ustart++; + istart = 0; + } + amount += (iend - istart) * Blockchain.GenerationAmount[ustart]; + } + + Fixed8 fractionalShare = value / 100000000; + generated = fractionalShare * amount; + sysFee = fractionalShare * (GetSysFeeAmountForHeight(snapshot.Blocks, endHeight - 1) - + (startHeight == 0 ? 0 : GetSysFeeAmountForHeight(snapshot.Blocks, startHeight - 1))); + } + + private bool AddClaims(JArray claimableOutput, ref Fixed8 runningTotal, int maxClaims, + Snapshot snapshot, DataCache storeSpentCoins, + KeyValuePair claimableInTx) + { + foreach (var claimTransaction in claimableInTx.Value.AmountByTxIndex) + { + var utxo = new JObject(); + var txId = claimableInTx.Key.TxHash.ToString().Substring(2); + utxo["txid"] = txId; + utxo["n"] = claimTransaction.Key; + var spentCoinState = storeSpentCoins.TryGet(claimableInTx.Key.TxHash); + var startHeight = spentCoinState.TransactionHeight; + var endHeight = spentCoinState.Items[claimTransaction.Key]; + CalculateClaimable(snapshot, claimTransaction.Value, startHeight, endHeight, out var generated, + out var sysFee); + var unclaimed = generated + sysFee; + utxo["value"] = (double) (decimal) claimTransaction.Value; + utxo["start_height"] = startHeight; + utxo["end_height"] = endHeight; + utxo["generated"] = (double) (decimal) generated; + utxo["sys_fee"] = (double) (decimal) sysFee; + utxo["unclaimed"] = (double) (decimal) unclaimed; + runningTotal += unclaimed; + claimableOutput.Add(utxo); + if (claimableOutput.Count > maxClaims) + return false; + } + + return true; + } + + private JObject ProcessGetClaimableSpents(JArray parameters) + { + UInt160 scriptHash = GetScriptHashFromParam(parameters[0].AsString()); + var dbCache = new DbCache( + _db, null, null, SystemAssetSpentUnclaimedCoinsPrefix); + + JObject json = new JObject(); + JArray claimable = new JArray(); + json["claimable"] = claimable; + json["address"] = scriptHash.ToAddress(); + byte[] prefix = new [] { (byte) 1 }.Concat(scriptHash.ToArray()).ToArray(); + + var snapshot = Blockchain.Singleton.GetSnapshot(); + var storeSpentCoins = snapshot.SpentCoins; + + Fixed8 totalUnclaimed = Fixed8.Zero; + foreach (var claimableInTx in dbCache.Find(prefix)) + if (!AddClaims(claimable, ref totalUnclaimed, _rpcMaxUnspents, snapshot, storeSpentCoins, claimableInTx)) break; + + json["unclaimed"] = (double) (decimal) totalUnclaimed; + return json; + } + + private JObject ProcessGetUnspents(JArray _params) + { + UInt160 scriptHash = GetScriptHashFromParam(_params[0].AsString()); + byte startingToken = 0; // 0 = Utility Token (GAS), 1 = Governing Token (NEO) + int maxIterations = 2; + + if (_params.Count > 1) + { + maxIterations = 1; + bool isGoverningToken = _params[1].AsBoolean(); + if (isGoverningToken) startingToken = 1; + } + + var unspentsCache = new DbCache( + _db, null, null, SystemAssetUnspentCoinsPrefix); + string[] nativeAssetNames = {"GAS", "NEO"}; UInt256[] nativeAssetIds = {Blockchain.UtilityToken.Hash, Blockchain.GoverningToken.Hash}; @@ -207,7 +314,7 @@ private JObject GenerateUtxoResponse(UInt160 scriptHash, byte startingToken, int var unspents = new JArray(); Fixed8 total = new Fixed8(0); - foreach (var unspentInTx in dbCache.Find(prefix)) + foreach (var unspentInTx in unspentsCache.Find(prefix)) { var txId = unspentInTx.Key.TxHash.ToString().Substring(2); foreach (var unspent in unspentInTx.Value.AmountByTxIndex) @@ -249,37 +356,11 @@ private JObject GenerateUtxoResponse(UInt160 scriptHash, byte startingToken, int return json; } - private JObject ProcessGetUnclaimedSpents(UInt160 scriptHash) - { - var dbCache = new DbCache( - _db, null, null, SystemAssetSpentUnclaimedCoinsPrefix); - return GenerateUtxoResponse(scriptHash, 1, 1, dbCache); - } - - private JObject ProcessGetUnspents(JArray _params) - { - UInt160 scriptHash = GetScriptHashFromParam(_params[0].AsString()); - byte startingToken = 0; // 0 = Utility Token (GAS), 1 = Governing Token (NEO) - int maxIterations = 2; - - if (_params.Count > 1) - { - maxIterations = 1; - bool isGoverningToken = _params[1].AsBoolean(); - if (isGoverningToken) startingToken = 1; - } - - var unspentsCache = new DbCache( - _db, null, null, SystemAssetUnspentCoinsPrefix); - - return GenerateUtxoResponse(scriptHash, startingToken, maxIterations, unspentsCache); - } - - public JObject OnProcess(HttpContext context, string method, JArray _params) + public JObject OnProcess(HttpContext context, string method, JArray parameters) { - if (_shouldTrackUnclaimed && method == "getunclaimedspents") - return ProcessGetUnclaimedSpents(GetScriptHashFromParam(_params[0].AsString())); - return method != "getunspents" ? null : ProcessGetUnspents(_params); + if (_shouldTrackUnclaimed && method == "getclaimable") + return ProcessGetClaimableSpents(parameters); + return method != "getunspents" ? null : ProcessGetUnspents(parameters); } public void PostProcess(HttpContext context, string method, JArray _params, JObject result) From d2722ad71bc2a0a568568baee4156371ae3a5274 Mon Sep 17 00:00:00 2001 From: jsolman Date: Wed, 13 Mar 2019 10:48:55 -0700 Subject: [PATCH 12/18] Clean-up. --- .../RpcSystemAssetTrackerPlugin.cs | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index a58fb27b0..6e97b8051 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -290,6 +290,25 @@ private JObject ProcessGetClaimableSpents(JArray parameters) return json; } + private bool AddUnspents(JArray unspents, ref Fixed8 runningTotal, + KeyValuePair unspentInTx) + { + var txId = unspentInTx.Key.TxHash.ToString().Substring(2); + foreach (var unspent in unspentInTx.Value.AmountByTxIndex) + { + var utxo = new JObject(); + utxo["txid"] = txId; + utxo["n"] = unspent.Key; + utxo["value"] = (double) (decimal) unspent.Value; + runningTotal += unspent.Value; + + unspents.Add(utxo); + if (unspents.Count > _rpcMaxUnspents) + return false; + } + return true; + } + private JObject ProcessGetUnspents(JArray _params) { UInt160 scriptHash = GetScriptHashFromParam(_params[0].AsString()); @@ -309,30 +328,6 @@ private JObject ProcessGetUnspents(JArray _params) string[] nativeAssetNames = {"GAS", "NEO"}; UInt256[] nativeAssetIds = {Blockchain.UtilityToken.Hash, Blockchain.GoverningToken.Hash}; - (JArray, Fixed8) RetreiveUnspentsForPrefix(byte[] prefix) - { - var unspents = new JArray(); - Fixed8 total = new Fixed8(0); - - foreach (var unspentInTx in unspentsCache.Find(prefix)) - { - var txId = unspentInTx.Key.TxHash.ToString().Substring(2); - foreach (var unspent in unspentInTx.Value.AmountByTxIndex) - { - var utxo = new JObject(); - utxo["txid"] = txId; - utxo["n"] = unspent.Key; - utxo["value"] = (double) (decimal) unspent.Value; - total += unspent.Value; - - unspents.Add(utxo); - if (unspents.Count > _rpcMaxUnspents) - return (unspents, total); - } - } - return (unspents, total); - } - JObject json = new JObject(); JArray balances = new JArray(); json["balance"] = balances; @@ -341,7 +336,11 @@ private JObject ProcessGetUnspents(JArray _params) { byte[] prefix = new [] { tokenIndex }.Concat(scriptHash.ToArray()).ToArray(); - var (unspents, total) = RetreiveUnspentsForPrefix(prefix); + var unspents = new JArray(); + Fixed8 total = new Fixed8(0); + + foreach (var unspentInTx in unspentsCache.Find(prefix)) + if (!AddUnspents(unspents, ref total, unspentInTx)) break; if (unspents.Count <= 0) continue; From ee6ff9c7dee84cf03361da9cbb607eafbd89de72 Mon Sep 17 00:00:00 2001 From: jsolman Date: Wed, 13 Mar 2019 12:05:57 -0700 Subject: [PATCH 13/18] Don't leak memory. --- .../RpcSystemAssetTrackerPlugin.cs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 6e97b8051..1ffee30dc 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -27,6 +27,7 @@ public class RpcSystemAssetTrackerPlugin : Plugin, IPersistencePlugin, IRpcPlugi private int _rpcMaxUnspents; private uint _lastPersistedBlock; private bool _shouldPersistBlock; + private Neo.IO.Data.LevelDB.Snapshot _levelDbSnapshot; public override void Configure() { @@ -52,8 +53,9 @@ public override void Configure() private void ResetBatch() { _writeBatch = new WriteBatch(); - var balancesSnapshot = _db.GetSnapshot(); - ReadOptions dbOptions = new ReadOptions { FillCache = false, Snapshot = balancesSnapshot }; + _levelDbSnapshot?.Dispose(); + _levelDbSnapshot = _db.GetSnapshot(); + var dbOptions = new ReadOptions { FillCache = false, Snapshot = _levelDbSnapshot }; _userUnspentCoins = new DbCache(_db, dbOptions, _writeBatch, SystemAssetUnspentCoinsPrefix); if (!_shouldTrackUnclaimed) return; @@ -279,17 +281,20 @@ private JObject ProcessGetClaimableSpents(JArray parameters) json["address"] = scriptHash.ToAddress(); byte[] prefix = new [] { (byte) 1 }.Concat(scriptHash.ToArray()).ToArray(); - var snapshot = Blockchain.Singleton.GetSnapshot(); - var storeSpentCoins = snapshot.SpentCoins; - Fixed8 totalUnclaimed = Fixed8.Zero; - foreach (var claimableInTx in dbCache.Find(prefix)) - if (!AddClaims(claimable, ref totalUnclaimed, _rpcMaxUnspents, snapshot, storeSpentCoins, claimableInTx)) break; - + using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot()) + { + var storeSpentCoins = snapshot.SpentCoins; + foreach (var claimableInTx in dbCache.Find(prefix)) + if (!AddClaims(claimable, ref totalUnclaimed, _rpcMaxUnspents, snapshot, storeSpentCoins, + claimableInTx)) + break; + } json["unclaimed"] = (double) (decimal) totalUnclaimed; return json; } + private bool AddUnspents(JArray unspents, ref Fixed8 runningTotal, KeyValuePair unspentInTx) { @@ -357,8 +362,7 @@ private JObject ProcessGetUnspents(JArray _params) public JObject OnProcess(HttpContext context, string method, JArray parameters) { - if (_shouldTrackUnclaimed && method == "getclaimable") - return ProcessGetClaimableSpents(parameters); + if (method == "getclaimable") return ProcessGetClaimableSpents(parameters); return method != "getunspents" ? null : ProcessGetUnspents(parameters); } From 008087ea7bc44f407e188b8b3357e06027655162 Mon Sep 17 00:00:00 2001 From: jsolman Date: Wed, 13 Mar 2019 13:15:16 -0700 Subject: [PATCH 14/18] Implement getclaimable to get get total unclaimed amount for an address. --- .../RpcSystemAssetTrackerPlugin.cs | 58 ++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 1ffee30dc..cea7fb5aa 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -279,12 +279,12 @@ private JObject ProcessGetClaimableSpents(JArray parameters) JArray claimable = new JArray(); json["claimable"] = claimable; json["address"] = scriptHash.ToAddress(); - byte[] prefix = new [] { (byte) 1 }.Concat(scriptHash.ToArray()).ToArray(); Fixed8 totalUnclaimed = Fixed8.Zero; using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot()) { var storeSpentCoins = snapshot.SpentCoins; + byte[] prefix = new [] { (byte) 1 }.Concat(scriptHash.ToArray()).ToArray(); foreach (var claimableInTx in dbCache.Find(prefix)) if (!AddClaims(claimable, ref totalUnclaimed, _rpcMaxUnspents, snapshot, storeSpentCoins, claimableInTx)) @@ -294,6 +294,56 @@ private JObject ProcessGetClaimableSpents(JArray parameters) return json; } + private JObject ProcessGetUnclaimed(JArray parameters) + { + UInt160 scriptHash = GetScriptHashFromParam(parameters[0].AsString()); + JObject json = new JObject(); + + Fixed8 available = Fixed8.Zero; + Fixed8 unavailable = Fixed8.Zero; + var spentsCache = new DbCache( + _db, null, null, SystemAssetSpentUnclaimedCoinsPrefix); + var unspentsCache = new DbCache( + _db, null, null, SystemAssetUnspentCoinsPrefix); + using (Snapshot snapshot = Blockchain.Singleton.GetSnapshot()) + { + var storeSpentCoins = snapshot.SpentCoins; + byte[] prefix = new [] { (byte) 1 }.Concat(scriptHash.ToArray()).ToArray(); + foreach (var claimableInTx in spentsCache.Find(prefix)) + { + var spentCoinState = storeSpentCoins.TryGet(claimableInTx.Key.TxHash); + foreach (var claimTxIndex in claimableInTx.Value.AmountByTxIndex) + { + var startHeight = spentCoinState.TransactionHeight; + var endHeight = spentCoinState.Items[claimTxIndex.Key]; + CalculateClaimable(snapshot, claimTxIndex.Value, startHeight, endHeight, out var generated, + out var sysFee); + available += generated + sysFee; + } + } + + var transactionsCache = snapshot.Transactions; + foreach (var claimableInTx in unspentsCache.Find(prefix)) + { + var transaction = transactionsCache.TryGet(claimableInTx.Key.TxHash); // Blockchain.Singleton.GetTransaction(claimableInTx.Key.TxHash); + + foreach (var claimTxIndex in claimableInTx.Value.AmountByTxIndex) + { + var startHeight = transaction.BlockIndex; + var endHeight = Blockchain.Singleton.Height; + CalculateClaimable(snapshot, claimTxIndex.Value, startHeight, endHeight, + out var generated, + out var sysFee); + unavailable += generated + sysFee; + } + } + } + + json["available"] = (double) (decimal) available; + json["unavailable"] = (double) (decimal) unavailable; + json["unclaimed"] = (double) (decimal) (available + unavailable); + return json; + } private bool AddUnspents(JArray unspents, ref Fixed8 runningTotal, KeyValuePair unspentInTx) @@ -362,7 +412,11 @@ private JObject ProcessGetUnspents(JArray _params) public JObject OnProcess(HttpContext context, string method, JArray parameters) { - if (method == "getclaimable") return ProcessGetClaimableSpents(parameters); + if (_shouldTrackUnclaimed) + { + if (method == "getclaimable") return ProcessGetClaimableSpents(parameters); + if (method == "getunclaimed") return ProcessGetUnclaimed(parameters); + } return method != "getunspents" ? null : ProcessGetUnspents(parameters); } From 01fb48ef2ede95545d749cf28a60aa0cfb5f3448 Mon Sep 17 00:00:00 2001 From: jsolman Date: Thu, 14 Mar 2019 11:15:25 -0700 Subject: [PATCH 15/18] Remove unnecessary comment. --- RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index cea7fb5aa..2291aa757 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -325,7 +325,7 @@ private JObject ProcessGetUnclaimed(JArray parameters) var transactionsCache = snapshot.Transactions; foreach (var claimableInTx in unspentsCache.Find(prefix)) { - var transaction = transactionsCache.TryGet(claimableInTx.Key.TxHash); // Blockchain.Singleton.GetTransaction(claimableInTx.Key.TxHash); + var transaction = transactionsCache.TryGet(claimableInTx.Key.TxHash); foreach (var claimTxIndex in claimableInTx.Value.AmountByTxIndex) { From 9a5a87e15fdc8a5044eede1bb5b39d0fb478cebb Mon Sep 17 00:00:00 2001 From: jsolman Date: Mon, 18 Mar 2019 13:54:40 -0700 Subject: [PATCH 16/18] Fix size of UserSystemAssetCoinOutputs. Remove unused using statement. --- RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs | 1 - RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index 2291aa757..bc4d6ae5a 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -7,7 +7,6 @@ using Neo.Wallets; using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using Neo.Ledger; using Neo.Persistence; diff --git a/RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs b/RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs index 08507c587..9d6251ea6 100644 --- a/RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs +++ b/RpcSystemAssetTracker/UserSystemAssetCoinOutputs.cs @@ -10,6 +10,9 @@ public class UserSystemAssetCoinOutputs : StateBase, ICloneable AmountByTxIndex; + public override int Size => base.Size + TotalAmount.Size + sizeof(ushort) + + (AmountByTxIndex.Count * (sizeof(ushort) + sizeof(ulong))); + public UserSystemAssetCoinOutputs() { TotalAmount = new Fixed8(0); From ce54a8a9210a79ca24c7faab360f5fb3dce4651f Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 19 Mar 2019 18:13:48 -0700 Subject: [PATCH 17/18] Reload max utxo's from configuration if it changes. --- RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs index bc4d6ae5a..076bb4f6a 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs +++ b/RpcSystemAssetTracker/RpcSystemAssetTrackerPlugin.cs @@ -34,7 +34,6 @@ public override void Configure() { var dbPath = GetConfiguration().GetSection("DBPath").Value ?? "SystemAssetBalanceData"; _db = DB.Open(dbPath, new Options { CreateIfMissing = true }); - _rpcMaxUnspents = int.Parse(GetConfiguration().GetSection("MaxReturnedUnspents").Value ?? "0"); _shouldTrackUnclaimed = (GetConfiguration().GetSection("TrackUnclaimed").Value ?? true.ToString()) != false.ToString(); try { @@ -47,6 +46,7 @@ public override void Configure() _lastPersistedBlock = 0; } } + _rpcMaxUnspents = int.Parse(GetConfiguration().GetSection("MaxReturnedUnspents").Value ?? "0"); } private void ResetBatch() From 84df11c2b5ed440640040fa2dbc2128477c68aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Fri, 5 Apr 2019 09:53:41 -0300 Subject: [PATCH 18/18] Upgrade dependency --- RpcSystemAssetTracker/RpcSystemAssetTracker.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj b/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj index 9b12f2523..4d03ef295 100644 --- a/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj +++ b/RpcSystemAssetTracker/RpcSystemAssetTracker.csproj @@ -12,6 +12,6 @@ - +