[StateService] Storage and P2P (#431)

* make leveldb public * storage * rename folder * update neo * Fix end of line marker * file encoding * Update src/StateService/Network/StateRoot.cs Co-authored-by: Luchuan <[email protected]> * fix some * recover levelDB * use neo LoadStore * clean project file * Update MPTNode.cs * add p2p * internal payload category * update neo and remove OnVerifiedInventory * null check and clean StateStore and clean settings * remove IP2PPlugin interface impl * clean settings * Update Settings.cs * Remove StatePlugin.Persistence.cs * clean actor system * Fix StatePlugin * Optimize * format * remove StoreDataCache and StoreMetaDataCache * remove StoreDataCache from mpt * Optimize * sync OnPersist Co-authored-by: Shargon <[email protected]> Co-authored-by: Erik Zhang <[email protected]> Co-authored-by: Luchuan <[email protected]> Co-authored-by: Owen Zhang <[email protected]>
neo-project · Jan 18, 2021 · 9bee8fa · 9bee8fa
1 parent 79aede4
commit 9bee8fa
Show file tree

Hide file tree

Showing 10 changed files with 532 additions and 10 deletions.
diff --git a/src/StateService/MPT/MPTCache.cs b/src/StateService/MPT/MPTCache.cs
@@ -1,20 +1,48 @@
-using Neo.IO.Caching;
+using Neo.IO;
 using Neo.Persistence;
+using System.Collections.Generic;
 
 namespace Neo.Plugins.MPT
 {
     public class MPTCache
     {
-        private readonly DataCache<UInt256, MPTNode> cache;
+        private enum TrackState : byte
+        {
+            None,
+            Added,
+            Changed,
+            Deleted
+        }
+
+        private class Trackable
+        {
+            public MPTNode Node;
+            public TrackState State;
+        }
+
+        private readonly ISnapshot store;
+        private readonly byte prefix;
+        private readonly Dictionary<UInt256, Trackable> cache = new Dictionary<UInt256, Trackable>();
 
         public MPTCache(ISnapshot store, byte prefix)
         {
-            cache = new StoreDataCache<UInt256, MPTNode>(store, prefix);
+            this.store = store;
+            this.prefix = prefix;
         }
 
         public MPTNode Resolve(UInt256 hash)
         {
-            return cache.TryGet(hash)?.Clone();
+            if (cache.TryGetValue(hash, out Trackable t))
+            {
+                return t.Node?.Clone();
+            }
+            var n = store.TryGet(prefix, hash.ToArray())?.AsSerializable<MPTNode>();
+            cache.Add(hash, new Trackable
+            {
+                Node = n,
+                State = TrackState.None,
+            });
+            return n?.Clone();
         }
 
         public void PutNode(MPTNode np)
@@ -23,10 +51,16 @@ public void PutNode(MPTNode np)
             if (n is null)
             {
                 np.Reference = 1;
-                cache.Add(np.Hash, np.Clone());
+                cache[np.Hash] = new Trackable
+                {
+                    Node = np.Clone(),
+                    State = TrackState.Added,
+                };
                 return;
             }
-            cache.GetAndChange(np.Hash).Reference++;
+            var entry = cache[np.Hash];
+            entry.Node.Reference++;
+            entry.State = TrackState.Changed;
         }
 
         public void DeleteNode(UInt256 hash)
@@ -35,15 +69,34 @@ public void DeleteNode(UInt256 hash)
             if (n is null) return;
             if (1 < n.Reference)
             {
-                cache.GetAndChange(hash).Reference--;
+                var entry = cache[hash];
+                entry.Node.Reference--;
+                entry.State = TrackState.Changed;
                 return;
             }
-            cache.Delete(hash);
+            cache[hash] = new Trackable
+            {
+                Node = null,
+                State = TrackState.Deleted,
+            };
         }
 
         public void Commit()
         {
-            cache.Commit();
+            foreach (var item in cache)
+            {
+                switch (item.Value.State)
+                {
+                    case TrackState.Added:
+                    case TrackState.Changed:
+                        store.Put(prefix, item.Key.ToArray(), item.Value.Node.ToArray());
+                        break;
+                    case TrackState.Deleted:
+                        store.Delete(prefix, item.Key.ToArray());
+                        break;
+                }
+            }
+            cache.Clear();
         }
     }
 }
diff --git a/src/StateService/Settings.cs b/src/StateService/Settings.cs
@@ -0,0 +1,23 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Neo.Plugins.StateService
+{
+    internal class Settings
+    {
+        public string Path { get; }
+        public bool FullState { get; }
+
+        public static Settings Default { get; private set; }
+
+        private Settings(IConfigurationSection section)
+        {
+            Path = string.Format(section.GetValue("Path", "Data_MPT_{0}"), ProtocolSettings.Default.Magic.ToString("X8"));
+            FullState = section.GetValue("FullState", false);
+        }
+
+        public static void Load(IConfigurationSection section)
+        {
+            Default = new Settings(section);
+        }
+    }
+}
diff --git a/src/StateService/StatePlugin.cs b/src/StateService/StatePlugin.cs
@@ -0,0 +1,41 @@
+using Akka.Actor;
+using Neo.IO.Caching;
+using Neo.Network.P2P.Payloads;
+using Neo.Persistence;
+using Neo.Plugins.StateService.Storage;
+using System.Collections.Generic;
+using System.Linq;
+using static Neo.Ledger.Blockchain;
+
+namespace Neo.Plugins.StateService
+{
+    public class StatePlugin : Plugin, IPersistencePlugin
+    {
+        public const string StatePayloadCategory = "StateService";
+        public override string Name => "StateService";
+        public override string Description => "Enables MPT for the node";
+
+        private IActorRef store;
+
+        protected override void Configure()
+        {
+            Settings.Load(GetConfiguration());
+        }
+
+        protected override void OnPluginsLoaded()
+        {
+            store = System.ActorSystem.ActorOf(StateStore.Props(System, Settings.Default.Path));
+        }
+
+        public override void Dispose()
+        {
+            base.Dispose();
+            System.EnsureStoped(store);
+        }
+
+        void IPersistencePlugin.OnPersist(Block block, StoreView snapshot, IReadOnlyList<ApplicationExecuted> applicationExecutedList)
+        {
+            StateStore.Singleton.UpdateLocalStateRoot(block.Index, snapshot.Storages.GetChangeSet().Where(p => p.State != TrackState.None).ToList());
+        }
+    }
+}
diff --git a/src/StateService/StateService.csproj b/src/StateService/StateService.csproj
@@ -5,5 +5,12 @@
     <RootNamespace>Neo.Plugins</RootNamespace>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
+
+  <ItemGroup>
+    <None Update="StateService\config.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
+    </None>
+  </ItemGroup>
 
 </Project>
diff --git a/src/StateService/StateService/config.json b/src/StateService/StateService/config.json
@@ -0,0 +1,6 @@
+{
+  "PluginConfiguration": {
+    "Path": "Data_MPT_{0}",
+    "FullState": false
+  }
+}
diff --git a/src/StateService/Storage/Prefixs.cs b/src/StateService/Storage/Prefixs.cs
@@ -0,0 +1,10 @@
+
+namespace Neo.Plugins.StateService.Storage
+{
+    public static class Prefixs
+    {
+        public const byte StateRoot = 0x01;
+        public const byte CurrentLocalRootIndex = 0x02;
+        public const byte CurrentValidatedRootIndex = 0x04;
+    }
+}
diff --git a/src/StateService/Storage/StateRoot.cs b/src/StateService/Storage/StateRoot.cs
@@ -0,0 +1,130 @@
+using Neo.Cryptography;
+using Neo.Cryptography.ECC;
+using Neo.IO;
+using Neo.IO.Json;
+using Neo.Ledger;
+using Neo.Network.P2P;
+using Neo.Network.P2P.Payloads;
+using Neo.Persistence;
+using Neo.SmartContract;
+using Neo.SmartContract.Native;
+using System;
+using System.IO;
+
+namespace Neo.Plugins.StateService.Storage
+{
+    public class StateRoot : ICloneable<StateRoot>, IVerifiable, ISerializable
+    {
+        public byte Version;
+        public uint Index;
+        public UInt256 RootHash;
+        public Witness Witness;
+
+        private UInt256 _hash = null;
+
+        public UInt256 Hash
+        {
+            get
+            {
+                if (_hash is null)
+                {
+                    _hash = new UInt256(Crypto.Hash256(this.GetHashData()));
+                }
+                return _hash;
+            }
+        }
+
+        Witness[] IVerifiable.Witnesses
+        {
+            get
+            {
+                return new[] { Witness };
+            }
+            set
+            {
+                if (value is null || value.Length != 1) throw new ArgumentException();
+                Witness = value[0];
+            }
+        }
+
+        public int Size =>
+            sizeof(byte) +      //Version
+            sizeof(uint) +      //Index
+            UInt256.Length +    //RootHash
+            (Witness is null ? 1 : 1 + Witness.Size); //Witness
+
+        public StateRoot Clone()
+        {
+            return new StateRoot
+            {
+                Version = Version,
+                Index = Index,
+                RootHash = RootHash,
+                Witness = Witness,
+            };
+        }
+
+        public void FromReplica(StateRoot replica)
+        {
+            Version = replica.Version;
+            Index = replica.Index;
+            RootHash = replica.RootHash;
+            Witness = replica.Witness;
+        }
+
+        public void Deserialize(BinaryReader reader)
+        {
+            this.DeserializeUnsigned(reader);
+            Witness[] arr = reader.ReadSerializableArray<Witness>();
+            if (arr.Length < 1)
+                Witness = null;
+            else
+                Witness = arr[0];
+        }
+
+        public void DeserializeUnsigned(BinaryReader reader)
+        {
+            Version = reader.ReadByte();
+            Index = reader.ReadUInt32();
+            RootHash = reader.ReadSerializable<UInt256>();
+        }
+
+        public void Serialize(BinaryWriter writer)
+        {
+            this.SerializeUnsigned(writer);
+            if (Witness is null)
+                writer.WriteVarInt(0);
+            else
+                writer.Write(new Witness[] { Witness });
+        }
+
+        public void SerializeUnsigned(BinaryWriter writer)
+        {
+            writer.Write(Version);
+            writer.Write(Index);
+            writer.Write(RootHash);
+        }
+
+        public bool Verify(StoreView snapshot)
+        {
+            return this.VerifyWitnesses(snapshot, 1_00000000);
+        }
+
+        public UInt160[] GetScriptHashesForVerifying(StoreView snapshot)
+        {
+            ECPoint[] validators = NativeContract.RoleManagement.GetDesignatedByRole(snapshot, Role.StateValidator, Index);
+            if (validators.Length < 1) throw new InvalidOperationException("No script hash for state root verifying");
+            return new UInt160[] { Blockchain.GetConsensusAddress(validators) };
+        }
+
+        public JObject ToJson()
+        {
+            var json = new JObject();
+            json["version"] = Version;
+            json["index"] = Index;
+            json["roothash"] = RootHash.ToString();
+            json["witness"] = Witness?.ToJson();
+            return json;
+        }
+    }
+}