Merge pull request #406 from CityOfZion/feature-storage-iterator

Feature storage iterator

CHANGELOG.rst

@@ -5,7 +5,12 @@ All notable changes to this project are documented in this file.
 
 [0.6.9] in progress
 -----------------------
--
+- alter logging
+- fix issue with dispatching transfer events when ``from_addr`` is ``False``
+- add TPS monitor to ``prompt`` ``state`` command
+- add check for db schema changes
+- add support for ``StorageIterator`` and ``Storage.Find`` from smart contracts
+- update to ``neocore==0.4.3``
 
 
 [0.6.8] 2018-04-26
@@ -15,6 +20,7 @@ All notable changes to this project are documented in this file.
 - fix for token_delete command not removing tokens from wallet file
 - fixed sc-events and notification DB showing previous block height instead of final block height of event
 - persist refund() notify events in notification DB
+- add smart contract storage searching using a prefix.
 - add Runtime.Serialize/Deserialize support for MAP
 - fix for debug breakpoints not being cleared.
 - add VERIFY op to ExecutionEngine
@@ -23,9 +29,11 @@ All notable changes to this project are documented in this file.
 - fix asset amount rounding for very small amounts
 - fix storage commit routine for failed contract executions
 
+
 [0.6.7] 2018-04-06
 -----------------------
 - Update all the requirements
+- Networking changes
 - added ``--maxpeers`` option for ``np-prompt`` and ``np-api-server``.  This allows p2p discovery of new nodes up to the value specified
 - added ``--host`` option for ``np-api-server`` in order to specify a hostname for the server
 - added more testing for ``neo.Network`` module

fixtures/.gitignore

@@ -7,4 +7,5 @@
 !withdraw_wallet.db3
 !withdraw_wallet2.db3
 !797966.txt
-!empty_fixture.tar.gz
+!empty_fixture.tar.gz
+!storage_find.avm

neo/Core/State/StorageKey.py

@@ -3,7 +3,7 @@
 from neocore.BigInteger import BigInteger
 
 
-class StorageKey(SerializableMixin):
+class StorageKey():
     ScriptHash = None
     Key = None
 
@@ -46,30 +46,19 @@ def GetHashCodeBytes(self):
         bigint = BigInteger(self.GetHashCode())
         return bigint.ToByteArray()
 
-    def Deserialize(self, reader):
-        """
-        Deserialize full object.
-
-        Args:
-            reader (neocore.IO.BinaryReader):
-        """
-        self.ScriptHash = reader.ReadUInt160()
-        self.Key = reader.ReadBytes()
-
-    def Serialize(self, writer):
-        """
-        Serialize full object.
-
-        Args:
-            writer (neo.IO.BinaryWriter):
-        """
-        writer.WriteUInt160(self.ScriptHash)
-        writer.WriteVarBytes(self.Key)
-
     def __eq__(self, other):
         if other is None:
             return False
         if other is self:
             return True
 
         return self.ScriptHash == other.ScriptHash and self.Key == other.Key
+
+    def ToArray(self):
+        """
+        Convert object members to bytes and a concatenate them.
+
+        Returns:
+            bytes:
+        """
+        return bytes(self.ScriptHash.ToArray()) + bytes(self.Key)

neo/Implementations/Blockchains/LevelDB/DBCollection.py

@@ -1,11 +1,11 @@
 import binascii
 from logzero import logger
+from neo.SmartContract.StorageIterator import StorageIterator
 
 
 class DBCollection:
-
     DB = None
-#    SN = None
+    #    SN = None
     Prefix = None
 
     ClassRef = None
@@ -119,7 +119,6 @@ def TryGet(self, keyval):
 
         # if the key is there, get the item
         if key is not None:
-
             self.MarkChanged(keyval)
 
             item = self._GetItem(keyval)
@@ -156,17 +155,35 @@ def MarkChanged(self, keyval):
         if keyval not in self.Changed:
             self.Changed.append(keyval)
 
-    # @TODO This has not been tested or verified to work.
+    def TryFind(self, key_prefix):
+        candidates = {}
+        for keyval in self.Collection.keys():
+            # See if we find a partial match in the keys that not have been committed yet, excluding those that are to be deleted
+            if key_prefix in keyval and keyval not in self.Deleted:
+                candidates[keyval[20:]] = self.Collection[keyval].Value
+
+        db_results = self.Find(key_prefix)
+
+        # {**x, **y} merges two dictionaries, with the values of y overwriting the vals of x
+        # withouth this merge, you sometimes get 2 results for each key
+        # then take the dict and make a list of tuples
+        final_collection = [(k, v) for k, v in {**db_results, **candidates}.items()]
+
+        return StorageIterator(iter(final_collection))
+
     def Find(self, key_prefix):
         key_prefix = self.Prefix + key_prefix
-        res = []
+        res = {}
         for key, val in self.DB.iterator(prefix=key_prefix):
-            res.append({key: val})
+            # we want the storage item, not the raw bytes
+            item = self.ClassRef.DeserializeFromDB(binascii.unhexlify(val)).Value
+            # also here we need to skip the 1 byte storage prefix
+            res[key[21:]] = item
         return res
 
     def Destroy(self):
         self.DB = None
-#        self.SN = None
+        #        self.SN = None
         self.Collection = None
         self.ClassRef = None
         self.Prefix = None

neo/Implementations/Blockchains/LevelDB/LevelDBBlockchain.py

@@ -34,6 +34,8 @@
 from neocore.BigInteger import BigInteger
 from neo.EventHub import events
 
+from prompt_toolkit import prompt
+
 
 class LevelDBBlockchain(Blockchain):
     _path = None
@@ -51,10 +53,12 @@ class LevelDBBlockchain(Blockchain):
 
     # this is the version of the database
     # should not be updated for network version changes
-    _sysversion = b'/NEO:2.0.1/'
+    _sysversion = b'schema v.0.6.9'
 
     _persisting_block = None
 
+    TXProcessed = 0
+
     @property
     def CurrentBlockHash(self):
         try:
@@ -95,13 +99,15 @@ def CurrentBlock(self):
     def Path(self):
         return self._path
 
-    def __init__(self, path):
+    def __init__(self, path, skip_version_check=False):
         super(LevelDBBlockchain, self).__init__()
         self._path = path
 
         self._header_index = []
         self._header_index.append(Blockchain.GenesisBlock().Header.Hash.ToBytes())
 
+        self.TXProcessed = 0
+
         try:
             self._db = plyvel.DB(self._path, create_if_missing=True)
         #            self._db = plyvel.DB(self._path, create_if_missing=True, bloom_filter_bits=16, compression=None)
@@ -112,6 +118,10 @@ def __init__(self, path):
 
         version = self._db.get(DBPrefix.SYS_Version)
 
+        if skip_version_check:
+            self._db.put(DBPrefix.SYS_Version, self._sysversion)
+            version = self._sysversion
+
         if version == self._sysversion:  # or in the future, if version doesn't equal the current version...
 
             ba = bytearray(self._db.get(DBPrefix.SYS_CurrentBlock, 0))
@@ -174,13 +184,28 @@ def __init__(self, path):
                     self.AddHeaders(newhashes)
                 except Exception as e:
                     pass
-        else:
-            with self._db.write_batch() as wb:
-                for key, value in self._db.iterator():
-                    wb.delete(key)
 
+        elif version is None:
             self.Persist(Blockchain.GenesisBlock())
             self._db.put(DBPrefix.SYS_Version, self._sysversion)
+        else:
+
+            logger.error("\n\n")
+            logger.warning("Database schema has changed from %s to %s.\n" % (version, self._sysversion))
+            logger.warning("You must either resync from scratch, or use the np-bootstrap command to bootstrap the chain.")
+
+            res = prompt("Type 'continue' to erase your current database and sync from new. Otherwise this program will exit:\n> ")
+            if res == 'continue':
+
+                with self._db.write_batch() as wb:
+                    for key, value in self._db.iterator():
+                        wb.delete(key)
+
+                self.Persist(Blockchain.GenesisBlock())
+                self._db.put(DBPrefix.SYS_Version, self._sysversion)
+
+            else:
+                raise Exception("Database schema changed")
 
     def GetAccountState(self, script_hash, print_all_accounts=False):
 
@@ -202,7 +227,7 @@ def GetAccountState(self, script_hash, print_all_accounts=False):
     def GetStorageItem(self, storage_key):
         sn = self._db.snapshot()
         storages = DBCollection(self._db, sn, DBPrefix.ST_Storage, StorageItem)
-        item = storages.TryGet(storage_key.GetHashCodeBytes())
+        item = storages.TryGet(storage_key.ToArray())
         sn.close()
         return item
 
@@ -783,10 +808,13 @@ def Persist(self, block):
             self._current_block_height = block.Index
             self._persisting_block = None
 
+            self.TXProcessed += len(block.Transactions)
+
             for event in to_dispatch:
                 events.emit(event.event_type, event)
 
     def PersistBlocks(self):
+
         if not self._paused:
             #            logger.info("PERRRRRSISST:: Hheight, b height, cache: %s/%s %s  --%s %s" % (self.Height, self.HeaderHeight, len(self._block_cache), self.CurrentHeaderHash, self.BlockSearchTries))
 
@@ -813,6 +841,7 @@ def PersistBlocks(self):
                     raise e
 
     def Resume(self):
+        self._currently_persisting = False
         super(LevelDBBlockchain, self).Resume()
         self.PersistBlocks()
 

neo/Implementations/Blockchains/LevelDB/test_LevelDBBlockchain.py

@@ -9,7 +9,7 @@ def leveldb_testpath(self):
         return os.path.join(settings.DATA_DIR_PATH, 'fixtures/test_chain')
 
     def test_01_initial_setup(self):
-        self.assertEqual(self._blockchain.Height, 758715)
+        self.assertEqual(self._blockchain.Height, 758986)
 
     def test_02_GetBlockHash(self):
         # test requested block height exceeding blockchain current_height

neo/Implementations/Blockchains/LevelDB/tests/test_initial_db.py

@@ -21,7 +21,7 @@ class LevelDBTest(NeoTestCase):
 
     @classmethod
     def setUpClass(self):
-        self._blockchain = LevelDBBlockchain(path=self.LEVELDB_TESTPATH)
+        self._blockchain = LevelDBBlockchain(path=self.LEVELDB_TESTPATH, skip_version_check=True)
         Blockchain.RegisterBlockchain(self._blockchain)
         self._genesis = Blockchain.GenesisBlock()
 

neo/Implementations/Blockchains/LevelDB/tests/test_leveldb.py

@@ -10,5 +10,4 @@ def leveldb_testpath(self):
         return os.path.join(settings.DATA_DIR_PATH, 'fixtures/test_chain')
 
     def test_a_initial_setup(self):
-
-        self.assertEqual(self._blockchain.Height, 758715)
+        self.assertEqual(self._blockchain.Height, 758986)

neo/Implementations/Notifications/LevelDB/NotificationDB.py

@@ -204,7 +204,7 @@ def on_persist_completed(self, block):
 
                 hash_data = token_event.ToByteArray()
                 hash_key = token_event.contract.Code.ScriptHash().ToBytes()
-                logger.info("persist new NEP5 contract: %s " % (hash_key))
+#                logger.info("persist new NEP5 contract: %s " % (hash_key))
                 token_write_batch.put(hash_key, hash_data)
 
             token_write_batch.write()

neo/Network/NeoNode.py

@@ -18,7 +18,7 @@
 
 
 MODE_MAINTAIN = 7
-MODE_CATCHUP = 1
+MODE_CATCHUP = 2
 
 
 class NeoNode(Protocol):
@@ -362,11 +362,12 @@ def HandleInvMessage(self, payload):
             inventory (neo.Network.Payloads.InvPayload):
         """
 
-        inventory = IOHelper.AsSerializableWithType(payload, 'neo.Network.Payloads.InvPayload.InvPayload')
+        if self.sync_mode != MODE_MAINTAIN:
+            return
 
-#        self.Log("INVENTORy %s " % inventory.Type)
+        inventory = IOHelper.AsSerializableWithType(payload, 'neo.Network.Payloads.InvPayload.InvPayload')
 
-        if inventory.Type == InventoryType.BlockInt and self.sync_mode == MODE_MAINTAIN:
+        if inventory.Type == InventoryType.BlockInt:
 
             ok_hashes = []
             for hash in inventory.Hashes:

neo/Network/NodeLeader.py

@@ -36,8 +36,8 @@ class NodeLeader:
 
     _MissedBlocks = []
 
-    BREQPART = 20
-    NREQMAX = 100
+    BREQPART = 50
+    NREQMAX = 250
     BREQMAX = 10000
 
     KnownHashes = []

neo/Prompt/Commands/BuildNRun.py

@@ -112,6 +112,9 @@ def TestBuild(script, invoke_args, wallet, plist='05', ret='05', dynamic=False):
     if dynamic:
         properties += ContractPropertyState.HasDynamicInvoke
 
+    if not isinstance(ret, bytearray):
+        ret = bytearray(binascii.unhexlify(str(ret).encode('utf-8')))
+
     script = generate_deploy_script(script, contract_properties=int(properties), parameter_list=plist, return_type=ret)
 
     return test_deploy_and_invoke(script, invoke_args, wallet)

neo/Prompt/Commands/Invoke.py

@@ -405,17 +405,13 @@ def test_deploy_and_invoke(deploy_script, invoke_args, wallet, from_addr=None, m
         shash = contract_state.Code.ScriptHash()
 
         invoke_args, neo_to_attach, gas_to_attach = get_asset_attachments(invoke_args)
-
         invoke_args.reverse()
 
-        # print("neo, gas %s %s " % (neo_to_attach,gas_to_attach.ToString()))
-
         sb = ScriptBuilder()
 
         for p in invoke_args:
 
             item = parse_param(p, wallet)
-
             if type(item) is list:
                 item.reverse()
                 listlength = len(item)