Enable state dumping and loading (#40)

* Add dumping/loading

* Add more dumping tests

* Remove hash disclaimer

* Add --no-compile flag to test_plugin

* Set pylint min-public-methods to 1

.circleci/config.yml

@@ -41,6 +41,9 @@ jobs:
       - run:
           name: Test postman
           command: poetry run pytest test/test_postman.py
+      - run:
+          name: Test dumping/loading
+          command: poetry run pytest test/test_dump.py
       - run:
           name: Test plugin - dockerized
           command: ./test/test_plugin.sh

.pylintrc

@@ -1,2 +1,5 @@
 [FORMAT]
 max-line-length=150
+
+[BASIC]
+min-public-methods=1

README.md

@@ -23,7 +23,6 @@ brew install gmp
 
 ## Disclaimer
 - Devnet should not be used as a replacement for Alpha testnet. After testing on Devnet, be sure to test on testnet!
-- Hash calculation of transactions and blocks differs from the one used in Alpha testnet.
 - Specifying a block by its hash/number is not supported. All interaction is done with the latest block.
 - Read more in [interaction](#interaction-api).
 
@@ -39,14 +38,21 @@ optional arguments:
   -v, --version         Print the version
   --host HOST           Specify the address to listen at; defaults to localhost (use the address the program outputs on start)
   --port PORT, -p PORT  Specify the port to listen at; defaults to 5000
+  --load-path LOAD_PATH
+                        Specify the path from which the state is loaded on
+                        startup
+  --dump-path DUMP_PATH
+                        Specify the path to dump to
+  --dump-on DUMP_ON     Specify when to dump; can dump on: exit, transaction
 ```
 
-## Run - Docker
+## Run with Docker
 Devnet is available as a Docker container ([shardlabs/starknet-devnet](https://hub.docker.com/repository/docker/shardlabs/starknet-devnet)):
 ```text
 docker pull shardlabs/starknet-devnet
 ```
 
+### Host and port with Docker
 The server inside the container listens to the port 5000, which you need to publish to a desired `<PORT>` on your host machine:
 ```text
 docker run -it -p [HOST:]<PORT>:5000 shardlabs/starknet-devnet
@@ -104,6 +110,60 @@ constructor(MockStarknetMessaging mockStarknetMessaging_) public {
 }
 ```
 
+## Dumping
+To preserve your Devnet instance for future use, there are several options:
+
+- Dumping on exit (handles Ctrl+C, i.e. SIGINT, doesn't handle SIGKILL):
+```
+starknet-devnet --dump-on exit --dump-path <PATH>
+```
+
+- Dumping after each transaction (done in background, doesn't block):
+```
+starknet-devnet --dump-on transaction --dump-path <PATH>
+```
+
+- Dumping on request (replace `<HOST>`, `<PORT>` and `<PATH>` with your own):
+```
+curl -X POST http://<HOST>:<PORT>/dump -d '{ "path": <PATH> }' -H "Content-Type: application/json"
+```
+
+## Loading
+To load a preserved Devnet instance, run:
+```
+starknet-devnet --load-path <PATH>
+```
+
+## Enabling dumping and loading with Docker
+To enable dumping and loading if running Devnet in a Docker container, you must bind the container path with the path on your host machine.
+
+This example:
+- Relies on [Docker bind mount](https://docs.docker.com/storage/bind-mounts/); try [Docker volume](https://docs.docker.com/storage/volumes/) instead.
+- Assumes that `/actual/dumpdir` exists. If unsure, use absolute paths.
+- Assumes you are listening on `127.0.0.1:5000`. However, leave the `--host 0.0.0.0` part as it is.
+
+If there is `dump.pkl` inside `/actual/dumpdir`, you can load it with:
+```
+docker run -it \
+  -p 127.0.0.1:5000:5000 \
+  --mount type=bind,source=/actual/dumpdir,target=/dumpdir \
+  shardlabs/starknet-devnet \
+  poetry run starknet-devnet \
+  --host 0.0.0.0 --port 5000 \
+  --load-path /dumpdir/dump.pkl
+```
+
+To dump to `/actual/dumpdir/dump.pkl` on Devnet shutdown, run:
+```
+docker run -it \
+  -p 127.0.0.1:5000:5000 \
+  --mount type=bind,source=/actual/dumpdir,target=/dumpdir \
+  shardlabs/starknet-devnet \
+  poetry run starknet-devnet \
+  --host 0.0.0.0 --port 5000 \
+  --dump-on exit --dump-path /dumpdir/dump.pkl
+```
+
 ## Development - Prerequisite
 If you're a developer willing to contribute, be sure to have installed [Poetry](https://pypi.org/project/poetry/).
 

pyproject.toml

@@ -15,6 +15,7 @@ python = "^3.7"
 Flask = {extras = ["async"], version = "^2.0.2"}
 flask-cors = "^3.0.10"
 cairo-lang = "0.7.1"
+dill = "^0.3.4"
 
 [tool.poetry.dev-dependencies]
 pylint = "^2.12.2"

starknet_devnet/dump.py

@@ -0,0 +1,38 @@
+"""Dumping utilities."""
+
+import multiprocessing
+
+import dill as pickle
+
+from .util import DumpOn
+
+class Dumper:
+    """Class for dumping objects."""
+
+    def __init__(self, dumpable):
+        """Specify the `dumpable` object to be dumped."""
+
+        self.dumpable = dumpable
+
+        self.dump_path: str = None
+        """Where to dump."""
+
+        self.dump_on: DumpOn = None
+        """When to dump."""
+
+    def __write_file(self, path):
+        """Writes the dump to disk."""
+        with open(path, "wb") as file:
+            pickle.dump(self.dumpable, file)
+
+    def dump(self, path: str=None):
+        """Dump to `path`."""
+        path = path or self.dump_path
+        assert path, "No dump_path defined"
+        print("Dumping Devnet to:", path)
+
+        multiprocessing.Process(
+            target=self.__write_file,
+            args=[path]
+        ).start()
+        # don't .join(), let it run in background

starknet_devnet/server.py

@@ -4,6 +4,9 @@
 
 import os
 import json
+import signal
+import sys
+import dill as pickle
 
 from flask import Flask, request, jsonify, abort
 from flask.wrappers import Response
@@ -15,8 +18,9 @@
 from werkzeug.datastructures import MultiDict
 
 from .constants import CAIRO_LANG_VERSION
+from .dump import Dumper
 from .starknet_wrapper import StarknetWrapper
-from .util import custom_int, fixed_length_hex, parse_args
+from .util import DumpOn, custom_int, fixed_length_hex, parse_args
 
 app = Flask(__name__)
 CORS(app)
@@ -43,7 +47,10 @@ async def add_transaction():
     else:
         abort(Response(f"Invalid tx_type: {tx_type}.", 400))
 
+    # after tx
     await starknet_wrapper.postman_flush()
+    if dumper.dump_on == DumpOn.TRANSACTION:
+        dumper.dump()
 
     return jsonify({
         "code": StarkErrorCode.TRANSACTION_RECEIVED.name,
@@ -204,19 +211,54 @@ def validate_load_messaging_contract(request_dict: dict):
         abort(Response(error_message, 400))
     return network_url
 
+@app.route("/dump", methods=["POST"])
+def dump():
+    """Dumps the starknet_wrapper"""
+
+    request_dict = request.json or {}
+    dump_path = request_dict.get("path") or dumper.dump_path
+    if not dump_path:
+        abort(Response("No path provided", 400))
+
+    dumper.dump(dump_path)
+    return Response(status=200)
+
+def dump_on_exit(_signum, _frame):
+    """Dumps on exit."""
+    dumper.dump(dumper.dump_path)
+    sys.exit(0)
+
 starknet_wrapper = StarknetWrapper()
+dumper = Dumper(starknet_wrapper)
 
 def main():
     """Runs the server."""
 
+    # pylint: disable=global-statement, invalid-name
+    global starknet_wrapper
+
     # reduce startup logging
     os.environ["WERKZEUG_RUN_MAIN"] = "true"
 
     args = parse_args()
+
     # Uncomment this once fork support is added
     # origin = Origin(args.fork) if args.fork else NullOrigin()
     # starknet_wrapper.set_origin(origin)
 
+    if args.load_path:
+        try:
+            starknet_wrapper = StarknetWrapper.load(args.load_path)
+        except (FileNotFoundError, pickle.UnpicklingError):
+            sys.exit(f"Error: Cannot load from {args.load_path}. Make sure the file exists and contains a Devnet dump.")
+
+    if args.dump_on == DumpOn.EXIT:
+        for sig in [signal.SIGTERM, signal.SIGINT]:
+            signal.signal(sig, dump_on_exit)
+
+    dumper.dump_path = args.dump_path
+    dumper.dump_on = args.dump_on
+
     app.run(host=args.host, port=args.port)
 
 if __name__ == "__main__":

starknet_devnet/starknet_wrapper.py

@@ -7,6 +7,7 @@
 from copy import deepcopy
 from typing import Dict
 
+import dill as pickle
 from starkware.starknet.business_logic.internal_transaction import InternalInvokeFunction
 from starkware.starknet.business_logic.state import CarriedState
 from starkware.starknet.definitions.transaction_type import TransactionType
@@ -18,13 +19,16 @@
 from starkware.starknet.services.api.feeder_gateway.block_hash import calculate_block_hash
 
 from .origin import NullOrigin, Origin
-from .util import Choice, StarknetDevnetException, TxStatus, fixed_length_hex, DummyExecutionInfo
+from .util import Choice, StarknetDevnetException, TxStatus, fixed_length_hex, DummyExecutionInfo, enable_pickling
 from .contract_wrapper import ContractWrapper
 from .transaction_wrapper import TransactionWrapper, DeployTransactionWrapper, InvokeTransactionWrapper
 from .postman_wrapper import GanachePostmanWrapper
 from .constants import FAILURE_REASON_KEY
 
-class StarknetWrapper: # pylint: disable=too-many-instance-attributes
+enable_pickling()
+
+#pylint: disable=too-many-instance-attributes
+class StarknetWrapper:
     """
     Wraps a Starknet instance and stores data to be returned by the server:
     contract states, transactions, blocks, storages.
@@ -55,6 +59,12 @@ def __init__(self):
         self.__l1_provider = None
         """Saves the L1 URL being used for L1 <> L2 communication."""
 
+    @staticmethod
+    def load(path: str) -> "StarknetWrapper":
+        """Load a serialized instance of this class from `path`."""
+        with open(path, "rb") as file:
+            return pickle.load(file)
+
     async def __preserve_current_state(self, state: CarriedState):
         self.__current_carried_state = deepcopy(state)
         self.__current_carried_state.shared_state = state.shared_state