Make ClientIoMessage generic over the Client (#10981)

* Add client-traits crate
Move the BlockInfo trait to new crate

* New crate `machine`
Contains code extracted from ethcore that defines `Machine`, `Externalities` and other execution related code.

* Use new machine and client-traits crates in ethcore

* Use new crates machine and client-traits instead of ethcore where appropriate

* Fix tests

* Don't re-export so many types from ethcore::client

* Fixing more fallout from removing re-export

* fix test

* More fallout from not re-exporting types

* Add some docs

* cleanup

* import the macro edition style

* Tweak docs

* Add missing import

* remove unused ethabi_derive imports

* Use latest ethabi-contract

* Move many traits from ethcore/client/traits to client-traits crate
Initial version of extracted Engine trait

* Move snapshot related traits to the engine crate (eew)

* Move a few snapshot related types to common_types
Cleanup Executed as exported from machine crate

* fix warning

* Gradually introduce new engine crate: snapshot

* ethcore typechecks with new engine crate

* Sort out types outside ethcore

* Add an EpochVerifier to ethash and use that in Engine.epoch_verifier()
Cleanup

* Document pub members

* Sort out tests
Sort out default impls for EpochVerifier

* Add test-helpers feature and move EngineSigner impl to the right place

* Sort out tests

* Sort out tests and refactor verification types

* Fix missing traits

* More missing traits
Fix Histogram

* Fix tests and cleanup

* cleanup

* Put back needed logger import

* Don't rexport common_types from ethcore/src/client
Don't export ethcore::client::*

* Remove files no longer used
Use types from the engine crate
Explicit exports from engine::engine

* Get rid of itertools

* Move a few more traits from ethcore to client-traits: BlockChainReset, ScheduleInfo, StateClient

* Move ProvingBlockChainClient to client-traits

* Don't re-export ForkChoice and Transition from ethcore

* Address grumbles: sort imports, remove commented out code

* Fix merge resolution error

* Extract the Clique engine to own crate

* Extract NullEngine and the block_reward module from ethcore

* Extract InstantSeal engine to own crate

* Extract remaining engines

* Extract executive_state to own crate so it can be used by engine crates

* Remove snapshot stuff from the engine crate

* Put snapshot traits back in ethcore

* cleanup

* Remove stuff from ethcore

* Don't use itertools

* itertools in aura is legit-ish

* More post-merge fixes

* Re-export less types in client

* cleanup

* Extract spec to own crate

* Put back the test-helpers from basic-authority

* Fix ethcore benchmarks

* Reduce the public api of ethcore/verification

* WIP

* Add Cargo.toml

* Fix compilation outside ethcore

* Audit uses of import_verified_blocks() and remove unneeded calls
Cleanup

* cleanup

* Remove unused imports from ethcore

* Cleanup

* remove double semi-colons

* Add missing generic param

* More missing generics

* Update ethcore/block-reward/Cargo.toml

Co-Authored-By: Tomasz Drwięga <[email protected]>

* Update ethcore/engines/basic-authority/Cargo.toml

Co-Authored-By: Tomasz Drwięga <[email protected]>

* Update ethcore/engines/ethash/Cargo.toml

Co-Authored-By: Tomasz Drwięga <[email protected]>

* Update ethcore/engines/clique/src/lib.rs

Co-Authored-By: Tomasz Drwięga <[email protected]>

* signers is already a ref

* Add an EngineType enum to tighten up Engine.name()

* Introduce Snapshotting enum to distinguish the type of snapshots a chain uses

* Rename supports_warp to snapshot_mode

* Missing import

* Update ethcore/src/snapshot/consensus/mod.rs

Co-Authored-By: Tomasz Drwięga <[email protected]>

* missing import

* Fix import

* double semi

* Fix merge problem

* cleanup

* Parametrise `ClientIoMessage` with `()` for the light client

* Add impl Tick for ()

* Address review feedback

* Move ClientIoMessage to common-types

* remove superseeded fixme

* fix merge conflict errors

Cargo.toml

@@ -73,6 +73,7 @@ spec = { path = "ethcore/spec" }
 term_size = "0.3"
 textwrap = "0.9"
 toml = "0.4"
+verification = { path = "ethcore/verification" }
 
 [build-dependencies]
 rustc_version = "0.2"

ethcore/Cargo.toml

@@ -35,16 +35,14 @@ evm = { path = "evm" }
 executive-state = { path = "executive-state" }
 futures = "0.1"
 hash-db = "0.15.0"
-parity-util-mem = "0.2.0"
 itertools = "0.5"
 journaldb = { path = "../util/journaldb" }
 keccak-hash = "0.2.0"
 keccak-hasher = { path = "../util/keccak-hasher" }
 kvdb = "0.1"
-kvdb-memorydb = "0.1"
+kvdb-memorydb = { version = "0.1", optional = true }
 kvdb-rocksdb = { version = "0.1.3", optional = true }
-lazy_static = "1.3.0"
-len-caching-lock = { path = "../util/len-caching-lock" }
+lazy_static = { version = "1.3", optional = true }
 log = "0.4"
 macros = { path = "../util/macros", optional = true }
 machine = { path = "./machine" }
@@ -61,20 +59,19 @@ rand_xorshift = "0.1.1"
 rayon = "1.1"
 rlp = "0.4.0"
 rlp_derive = { path = "../util/rlp-derive" }
-rustc-hex = "1.0"
+rustc-hex = { version = "1", optional = true }
 serde = "1.0"
 serde_derive = "1.0"
 spec = { path = "spec" }
 state-db = { path = "state-db" }
-stats = { path = "../util/stats" }
 tempdir = { version = "0.3", optional = true }
-time-utils = { path = "../util/time-utils" }
 trace = { path = "trace" }
 trace-time = "0.1"
 trie-vm-factories = { path = "trie-vm-factories" }
 triehash-ethereum = { version = "0.2",  path = "../util/triehash-ethereum" }
 unexpected = { path = "../util/unexpected" }
 using_queue = { path = "../miner/using-queue" }
+verification = { path = "./verification" }
 vm = { path = "vm" }
 
 [dev-dependencies]
@@ -88,14 +85,18 @@ ethcore-accounts = { path = "../accounts" }
 ethjson = { path = "../json" }
 ethkey = { path = "../accounts/ethkey" }
 fetch = { path = "../util/fetch" }
+kvdb-memorydb = "0.1"
 kvdb-rocksdb = "0.1.3"
+lazy_static = { version = "1.3" }
 machine = { path = "./machine", features = ["test-helpers"] }
 macros = { path = "../util/macros" }
 null-engine = { path = "./engines/null-engine" }
 parity-runtime = { path = "../util/runtime" }
 pod = { path = "pod" }
 rlp_compress = { path = "../util/rlp-compress" }
+rustc-hex = "1"
 serde_json = "1.0"
+stats = { path = "../util/stats" }
 tempdir = "0.3"
 trie-standardmap = "0.15.0"
 
@@ -121,14 +122,14 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
 # EVM debug traces are printed.
 slow-blocks = []
 # Run JSON consensus tests.
-json-tests = ["env_logger", "test-helpers", "machine/test-helpers"]
+json-tests = ["env_logger", "test-helpers", "lazy_static", "machine/test-helpers"]
 # Skip JSON consensus tests with pending issues.
 ci-skip-tests = []
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile test helpers
 # note[dvdplm]: "basic-authority/test-helpers" is needed so that `generate_dummy_client_with_spec` works
-test-helpers = ["tempdir", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "basic-authority/test-helpers"]
+test-helpers = ["tempdir", "kvdb-memorydb", "kvdb-rocksdb", "blooms-db", "ethash", "ethjson", "ethkey", "macros", "pod", "rustc-hex", "basic-authority/test-helpers"]
 
 [[bench]]
 name = "builtin"

ethcore/client-traits/src/lib.rs

@@ -32,7 +32,7 @@ use common_types::{
 	client_types::Mode,
 	encoded,
 	engines::{epoch::Transition as EpochTransition, machine::Executed},
-	errors::EthcoreResult,
+	errors::{EthcoreError, EthcoreResult},
 	filter::Filter,
 	header::Header,
 	ids::{BlockId, TransactionId, TraceId, UncleId},
@@ -56,6 +56,8 @@ use trace::{
 };
 use vm::{LastHashes, Schedule};
 
+use common_types::snapshot::Progress;
+
 /// State information to be used during client query
 pub enum StateOrBlock {
 	/// State to be used, may be pending
@@ -168,11 +170,14 @@ pub trait EngineClient: Sync + Send + ChainInfo {
 	fn block_header(&self, id: BlockId) -> Option<encoded::Header>;
 }
 
-// FIXME Why these methods belong to BlockChainClient and not MiningBlockChainClient?
 /// Provides methods to import block into blockchain
 pub trait ImportBlock {
 	/// Import a block into the blockchain.
 	fn import_block(&self, block: Unverified) -> EthcoreResult<H256>;
+
+	/// Triggered by a message from a block queue when the block is ready for insertion.
+	/// Returns the number of blocks imported.
+	fn import_verified_blocks(&self) -> usize;
 }
 
 /// IO operations that should off-load heavy work to another thread.
@@ -187,6 +192,14 @@ pub trait IoClient: Sync + Send {
 	fn queue_consensus_message(&self, message: Bytes);
 }
 
+/// Implement this for clients that need logic to decide when/how to advance.
+pub trait Tick {
+	/// Tick the client
+	fn tick(&self, _prevent_sleep: bool) {}
+}
+
+impl Tick for () {}
+
 /// Provides recently seen bad blocks.
 pub trait BadBlocks {
 	/// Returns a list of blocks that were recently not imported because they were invalid.
@@ -377,6 +390,9 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra
 pub trait BlockChainReset {
 	/// reset to best_block - n
 	fn reset(&self, num: u32) -> Result<(), String>;
+
+	/// Number of eras kept in a journal before they are pruned
+	fn pruning_history(&self) -> u64;
 }
 
 
@@ -423,3 +439,38 @@ pub trait ProvingBlockChainClient: BlockChainClient {
 	/// Get an epoch change signal by block hash.
 	fn epoch_signal(&self, hash: H256) -> Option<Vec<u8>>;
 }
+
+/// External database restoration handler
+pub trait DatabaseRestore: Send + Sync {
+	/// Restart with a new backend. Takes ownership of passed database and moves it to a new location.
+	fn restore_db(&self, new_db: &str) -> Result<(), EthcoreError>;
+}
+
+/// Snapshot related functionality
+pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore + BlockChainReset {
+	/// Take a snapshot at the given block.
+	/// If the ID given is "latest", this will default to 1000 blocks behind.
+	fn take_snapshot<W: SnapshotWriter + Send>(
+		&self,
+		writer: W,
+		at: BlockId,
+		p: &Progress,
+	) -> Result<(), EthcoreError>;
+}
+
+
+// todo[dvdplm] move this back to snapshot once extracted from ethcore
+/// Something which can write snapshots.
+/// Writing the same chunk multiple times will lead to implementation-defined
+/// behavior, and is not advised.
+pub trait SnapshotWriter {
+	/// Write a compressed state chunk.
+	fn write_state_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>;
+
+	/// Write a compressed block chunk.
+	fn write_block_chunk(&mut self, hash: H256, chunk: &[u8]) -> std::io::Result<()>;
+
+	/// Complete writing. The manifest's chunk lists must be consistent
+	/// with the chunks written.
+	fn finish(self, manifest: common_types::snapshot::ManifestData) -> std::io::Result<()> where Self: Sized;
+}

ethcore/light/Cargo.toml

@@ -13,7 +13,6 @@ client-traits = { path = "../client-traits" }
 common-types = { path = "../types" }
 derive_more = "0.14.0"
 engine = { path = "../engine" }
-ethcore = { path = ".."}
 ethcore-db = { path = "../db" }
 ethcore-blockchain = { path = "../blockchain" }
 ethereum-types = "0.6.0"
@@ -49,6 +48,7 @@ kvdb = "0.1"
 memory-cache = { path = "../../util/memory-cache" }
 error-chain = { version = "0.12", default-features = false }
 journaldb = { path = "../../util/journaldb" }
+verification = { path = "../verification" }
 
 [dev-dependencies]
 ethcore = { path = "..", features = ["test-helpers"] }

ethcore/light/src/client/mod.rs

@@ -19,8 +19,7 @@
 use std::sync::{Weak, Arc};
 
 use engine::{Engine, EpochChange, Proof};
-use ethcore::client::{ClientReport, ClientIoMessage};
-use ethcore::verification::queue::{self, HeaderQueue};
+use verification::queue::{self, HeaderQueue};
 use spec::{Spec, SpecHardcodedSync};
 use io::IoChannel;
 use parking_lot::{Mutex, RwLock};
@@ -30,12 +29,14 @@ use common_types::{
 	BlockNumber,
 	block_status::BlockStatus,
 	blockchain_info::BlockChainInfo,
+	client_types::ClientReport,
 	encoded,
 	engines::epoch::{Transition as EpochTransition, PendingTransition},
 	errors::EthcoreError as Error,
 	errors::EthcoreResult,
 	header::Header,
 	ids::BlockId,
+	io_message::ClientIoMessage,
 	verification::VerificationQueueInfo as BlockQueueInfo,
 };
 use kvdb::KeyValueDB;
@@ -162,7 +163,7 @@ impl<T: LightChainClient> AsLightClient for T {
 
 /// Light client implementation.
 pub struct Client<T> {
-	queue: HeaderQueue,
+	queue: HeaderQueue<()>,
 	engine: Arc<dyn Engine>,
 	chain: HeaderChain,
 	report: RwLock<ClientReport>,
@@ -183,7 +184,7 @@ impl<T: ChainDataFetcher> Client<T> {
 		chain_col: Option<u32>,
 		spec: &Spec,
 		fetcher: T,
-		io_channel: IoChannel<ClientIoMessage>,
+		io_channel: IoChannel<ClientIoMessage<()>>,
 		cache: Arc<Mutex<Cache>>
 	) -> Result<Self, Error> {
 		Ok(Self {
@@ -649,3 +650,5 @@ impl<T: ChainDataFetcher> client_traits::EngineClient for Client<T> {
 		Client::block_header(self, id)
 	}
 }
+
+impl<T> client_traits::Tick for Client<T> {}