From 8d8d10267f45e0a5800f8fd6cb4e112beb11affc Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 6 Sep 2019 15:45:53 +0200 Subject: [PATCH] Consensus: Proof of Work (#3473) * consensus-pow: init primtives and verifier * consensus-pow: add total difficulty auxiliary * consensus-pow: implement total difficulty chain selection * consensus-pow: implement pow import queue * consensus-pow-primitives: add mine into PowApi * consensus-pow: implement mining * Update lock file * Style fixes No run-on expressions allowed. * consensus-pow: refactor register_pow_inherent_data_provider * consensus-pow: make PowApi::mine yieldable * consensus-pow: better mining loop * Add missing license header * consensus-pow-primitives: clarify the meaning of None for PowApi::verify * consensus-pow: changing total difficulty addition to use saturating add * consensus-pow: change mine-loop error to log on error! level * consensus-pow: allow inserting arbitrary preruntime digest for pow The preruntime digest can be intepreted by the runtime as the block author/coinbase. * Fix line width * More line width fixes * consensus-pow: separate difficulty, verify API This makes it more apparent that currently in PoW engine, `difficulty` should be input, not output. * srml-pow: implementation of average_span difficulty adjustment * srml-pow: basic blake2 algo example * srml-pow-average-span: make it not require genesis config * srml-pow: add support for authorship * Missing license headers * consensus-pow: PowAlgorithm trait generalization * Missing docs for consensus-pow * More docs * node-runtime: bump impl_version * Add rationale for difficulty type * consensus-pow: refactor aux_key * Update lock file * Update core/consensus/pow/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/primitives/src/lib.rs Co-Authored-By: Sergei Pepyakin * Update core/consensus/pow/primitives/src/lib.rs Co-Authored-By: Sergei Pepyakin * Remove PowRuntimeAlgorithm * block_id -> parent_block_id * Auxiliary data -> auxiliary storage data * Fix error message * Fix compile * Update core/consensus/pow/primitives/src/lib.rs Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com> * Update core/consensus/pow/src/lib.rs Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com> * Update core/consensus/pow/primitives/src/lib.rs Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com> * Update core/consensus/pow/src/lib.rs Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com> * Fix crate description * More docs * Address grumbles 1. Make preruntime Optional. 2. Add more docs on what is `preruntie` and `round`. 3. Replace `Default::default` with the approriate type. --- Cargo.lock | 26 ++ Cargo.toml | 1 + core/consensus/aura/primitives/Cargo.toml | 2 +- core/consensus/pow/Cargo.toml | 18 + core/consensus/pow/primitives/Cargo.toml | 21 ++ core/consensus/pow/primitives/src/lib.rs | 36 ++ core/consensus/pow/src/lib.rs | 417 ++++++++++++++++++++++ 7 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 core/consensus/pow/Cargo.toml create mode 100644 core/consensus/pow/primitives/Cargo.toml create mode 100644 core/consensus/pow/primitives/src/lib.rs create mode 100644 core/consensus/pow/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 4a21f644331b7..e6579b29293f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4691,6 +4691,32 @@ dependencies = [ "substrate-test-runtime-client 2.0.0", ] +[[package]] +name = "substrate-consensus-pow" +version = "2.0.0" +dependencies = [ + "futures-preview 0.3.0-alpha.17 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sr-primitives 2.0.0", + "srml-timestamp 2.0.0", + "substrate-client 2.0.0", + "substrate-consensus-common 2.0.0", + "substrate-consensus-pow-primitives 2.0.0", + "substrate-inherents 2.0.0", + "substrate-primitives 2.0.0", +] + +[[package]] +name = "substrate-consensus-pow-primitives" +version = "2.0.0" +dependencies = [ + "sr-primitives 2.0.0", + "sr-std 2.0.0", + "substrate-client 2.0.0", + "substrate-primitives 2.0.0", +] + [[package]] name = "substrate-consensus-rhd" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 048bfb7629060..aaa8c372fb182 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "core/consensus/rhd", "core/consensus/slots", "core/consensus/uncles", + "core/consensus/pow", "core/executor", "core/executor/runtime-test", "core/finality-grandpa", diff --git a/core/consensus/aura/primitives/Cargo.toml b/core/consensus/aura/primitives/Cargo.toml index ac2c2c791b2b1..7fd3f3d05d9bf 100644 --- a/core/consensus/aura/primitives/Cargo.toml +++ b/core/consensus/aura/primitives/Cargo.toml @@ -10,7 +10,7 @@ codec = { package = "parity-scale-codec", version = "1.0.0", default-features = substrate-client = { path = "../../../client", default-features = false } app-crypto = { package = "substrate-application-crypto", path = "../../../application-crypto", default-features = false } rstd = { package = "sr-std", path = "../../../sr-std", default-features = false } -sr-primitives = { path = "../../../sr-primitives", default-features = false } +sr-primitives = { path = "../../../sr-primitives", default-features = false } [features] default = ["std"] diff --git a/core/consensus/pow/Cargo.toml b/core/consensus/pow/Cargo.toml new file mode 100644 index 0000000000000..5ddc0f478dbbc --- /dev/null +++ b/core/consensus/pow/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substrate-consensus-pow" +version = "2.0.0" +authors = ["Parity Technologies "] +description = "PoW consensus algorithm for substrate" +edition = "2018" + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } +primitives = { package = "substrate-primitives", path = "../../primitives" } +sr-primitives = { path = "../../sr-primitives" } +client = { package = "substrate-client", path = "../../client" } +srml-timestamp = { path = "../../../srml/timestamp" } +inherents = { package = "substrate-inherents", path = "../../inherents" } +pow-primitives = { package = "substrate-consensus-pow-primitives", path = "primitives" } +consensus-common = { package = "substrate-consensus-common", path = "../common" } +log = "0.4" +futures-preview = { version = "=0.3.0-alpha.17", features = ["compat"] } diff --git a/core/consensus/pow/primitives/Cargo.toml b/core/consensus/pow/primitives/Cargo.toml new file mode 100644 index 0000000000000..a7e0d284b09fa --- /dev/null +++ b/core/consensus/pow/primitives/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "substrate-consensus-pow-primitives" +version = "2.0.0" +authors = ["Parity Technologies "] +description = "Primitives for Aura consensus" +edition = "2018" + +[dependencies] +substrate-client = { path = "../../../client", default-features = false } +rstd = { package = "sr-std", path = "../../../sr-std", default-features = false } +sr-primitives = { path = "../../../sr-primitives", default-features = false } +primitives = { package = "substrate-primitives", path = "../../../primitives", default-features = false } + +[features] +default = ["std"] +std = [ + "rstd/std", + "substrate-client/std", + "sr-primitives/std", + "primitives/std", +] diff --git a/core/consensus/pow/primitives/src/lib.rs b/core/consensus/pow/primitives/src/lib.rs new file mode 100644 index 0000000000000..807a7b2df2c90 --- /dev/null +++ b/core/consensus/pow/primitives/src/lib.rs @@ -0,0 +1,36 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Primitives for Substrate Proof-of-Work (PoW) consensus. + +#![cfg_attr(not(feature = "std"), no_std)] + +use rstd::vec::Vec; +use sr_primitives::ConsensusEngineId; + +/// The `ConsensusEngineId` of PoW. +pub const POW_ENGINE_ID: ConsensusEngineId = [b'p', b'o', b'w', b'_']; + +/// Type of difficulty. +/// +/// For runtime designed for Substrate, it's always possible to fit its total +/// difficulty range under `u128::max_value()` because it can be freely scaled +/// up or scaled down. Very few PoW chains use difficulty values +/// larger than `u128::max_value()`. +pub type Difficulty = u128; + +/// Type of seal. +pub type Seal = Vec; diff --git a/core/consensus/pow/src/lib.rs b/core/consensus/pow/src/lib.rs new file mode 100644 index 0000000000000..343c7b14c6af1 --- /dev/null +++ b/core/consensus/pow/src/lib.rs @@ -0,0 +1,417 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Proof of work consensus for Substrate. +//! +//! To use this engine, you can need to have a struct that implements +//! `PowAlgorithm`. After that, pass an instance of the struct, along +//! with other necessary client references to `import_queue` to setup +//! the queue. Use the `start_mine` function for basic CPU mining. +//! +//! The auxiliary storage for PoW engine only stores the total difficulty. +//! For other storage requirements for particular PoW algorithm (such as +//! the actual difficulty for each particular blocks), you can take a client +//! reference in your `PowAlgorithm` implementation, and use a separate prefix +//! for the auxiliary storage. It is also possible to just use the runtime +//! as the storage, but it is not recommended as it won't work well with light +//! clients. + +use std::sync::Arc; +use std::thread; +use std::collections::HashMap; +use client::{ + BlockOf, blockchain::{HeaderBackend, ProvideCache}, + block_builder::api::BlockBuilder as BlockBuilderApi, backend::AuxStore, +}; +use sr_primitives::Justification; +use sr_primitives::generic::{BlockId, Digest, DigestItem}; +use sr_primitives::traits::{Block as BlockT, Header as HeaderT, ProvideRuntimeApi}; +use srml_timestamp::{TimestampInherentData, InherentError as TIError}; +use pow_primitives::{Difficulty, Seal, POW_ENGINE_ID}; +use primitives::H256; +use inherents::{InherentDataProviders, InherentData}; +use consensus_common::{ + BlockImportParams, BlockOrigin, ForkChoiceStrategy, + well_known_cache_keys::Id as CacheKeyId, Environment, Proposer, +}; +use consensus_common::import_queue::{BoxBlockImport, BasicQueue, Verifier}; +use codec::{Encode, Decode}; +use log::*; + +/// Auxiliary storage prefix for PoW engine. +pub const POW_AUX_PREFIX: [u8; 4] = *b"PoW:"; + +/// Get the auxiliary storage key used by engine to store total difficulty. +fn aux_key(hash: &H256) -> Vec { + POW_AUX_PREFIX.iter().chain(&hash[..]) + .cloned().collect::>() +} + +/// Auxiliary storage data for PoW. +#[derive(Encode, Decode, Clone, Debug, Default)] +pub struct PowAux { + /// Total difficulty. + pub total_difficulty: Difficulty, +} + +impl PowAux { + /// Read the auxiliary from client. + pub fn read(client: &C, hash: &H256) -> Result { + let key = aux_key(hash); + + match client.get_aux(&key).map_err(|e| format!("{:?}", e))? { + Some(bytes) => PowAux::decode(&mut &bytes[..]).map_err(|e| format!("{:?}", e)), + None => Ok(PowAux::default()), + } + } +} + +/// Algorithm used for proof of work. +pub trait PowAlgorithm { + /// Get the next block's difficulty. + fn difficulty(&self, parent: &BlockId) -> Result; + /// Verify proof of work against the given difficulty. + fn verify( + &self, + parent: &BlockId, + pre_hash: &H256, + seal: &Seal, + difficulty: Difficulty, + ) -> Result; + /// Mine a seal that satisfy the given difficulty. + fn mine( + &self, + parent: &BlockId, + pre_hash: &H256, + seed: &H256, + difficulty: Difficulty, + round: u32, + ) -> Result, String>; +} + +/// A verifier for PoW blocks. +pub struct PowVerifier { + client: Arc, + algorithm: Algorithm, + inherent_data_providers: inherents::InherentDataProviders, +} + +impl PowVerifier { + fn check_header>( + &self, + mut header: B::Header, + parent_block_id: BlockId, + ) -> Result<(B::Header, Difficulty, DigestItem), String> where + Algorithm: PowAlgorithm, + { + let hash = header.hash(); + + let (seal, inner_seal) = match header.digest_mut().pop() { + Some(DigestItem::Seal(id, seal)) => { + if id == POW_ENGINE_ID { + (DigestItem::Seal(id, seal.clone()), seal) + } else { + return Err(format!("Header uses the wrong engine {:?}", id)) + } + }, + _ => return Err(format!("Header {:?} is unsealed", hash)), + }; + + let pre_hash = header.hash(); + let difficulty = self.algorithm.difficulty(&parent_block_id)?; + + if !self.algorithm.verify( + &parent_block_id, + &pre_hash, + &inner_seal, + difficulty, + )? { + return Err("PoW validation error: invalid seal".into()); + } + + Ok((header, difficulty, seal)) + } + + fn check_inherents>( + &self, + block: B, + block_id: BlockId, + inherent_data: InherentData, + timestamp_now: u64, + ) -> Result<(), String> where + C: ProvideRuntimeApi, C::Api: BlockBuilderApi + { + const MAX_TIMESTAMP_DRIFT_SECS: u64 = 60; + + let inherent_res = self.client.runtime_api().check_inherents( + &block_id, + block, + inherent_data, + ).map_err(|e| format!("{:?}", e))?; + + if !inherent_res.ok() { + inherent_res + .into_errors() + .try_for_each(|(i, e)| match TIError::try_from(&i, &e) { + Some(TIError::ValidAtTimestamp(timestamp)) => { + if timestamp > timestamp_now + MAX_TIMESTAMP_DRIFT_SECS { + return Err("Rejecting block too far in future".into()); + } + + Ok(()) + }, + Some(TIError::Other(e)) => Err(e.into()), + None => Err(self.inherent_data_providers.error_to_string(&i, &e)), + }) + } else { + Ok(()) + } + } +} + +impl, C, Algorithm> Verifier for PowVerifier where + C: ProvideRuntimeApi + Send + Sync + HeaderBackend + AuxStore + ProvideCache + BlockOf, + C::Api: BlockBuilderApi, + Algorithm: PowAlgorithm + Send + Sync, +{ + fn verify( + &mut self, + origin: BlockOrigin, + header: B::Header, + justification: Option, + mut body: Option>, + ) -> Result<(BlockImportParams, Option)>>), String> { + let inherent_data = self.inherent_data_providers + .create_inherent_data().map_err(String::from)?; + let timestamp_now = inherent_data.timestamp_inherent_data().map_err(String::from)?; + + let best_hash = self.client.info().best_hash; + let hash = header.hash(); + let parent_hash = *header.parent_hash(); + let best_aux = PowAux::read(self.client.as_ref(), &best_hash)?; + let mut aux = PowAux::read(self.client.as_ref(), &parent_hash)?; + + let (checked_header, difficulty, seal) = self.check_header::( + header, + BlockId::Hash(parent_hash), + )?; + aux.total_difficulty = aux.total_difficulty.saturating_add(difficulty); + + if let Some(inner_body) = body.take() { + let block = B::new(checked_header.clone(), inner_body); + + self.check_inherents( + block.clone(), + BlockId::Hash(parent_hash), + inherent_data, + timestamp_now + )?; + + let (_, inner_body) = block.deconstruct(); + body = Some(inner_body); + } + let key = aux_key(&hash); + let import_block = BlockImportParams { + origin, + header: checked_header, + post_digests: vec![seal], + body, + finalized: false, + justification, + auxiliary: vec![(key, Some(aux.encode()))], + fork_choice: ForkChoiceStrategy::Custom(aux.total_difficulty > best_aux.total_difficulty), + }; + + Ok((import_block, None)) + } +} + +/// Register the PoW inherent data provider, if not registered already. +fn register_pow_inherent_data_provider( + inherent_data_providers: &InherentDataProviders, +) -> Result<(), consensus_common::Error> { + if !inherent_data_providers.has_provider(&srml_timestamp::INHERENT_IDENTIFIER) { + inherent_data_providers + .register_provider(srml_timestamp::InherentDataProvider) + .map_err(Into::into) + .map_err(consensus_common::Error::InherentData) + } else { + Ok(()) + } +} + +/// The PoW import queue type. +pub type PowImportQueue = BasicQueue; + +/// Import queue for PoW engine. +pub fn import_queue( + block_import: BoxBlockImport, + client: Arc, + algorithm: Algorithm, + inherent_data_providers: InherentDataProviders, +) -> Result, consensus_common::Error> where + B: BlockT, + C: ProvideRuntimeApi + HeaderBackend + BlockOf + ProvideCache + AuxStore, + C: Send + Sync + AuxStore + 'static, + C::Api: BlockBuilderApi, + Algorithm: PowAlgorithm + Send + Sync + 'static, +{ + register_pow_inherent_data_provider(&inherent_data_providers)?; + + let verifier = PowVerifier { + client: client.clone(), + algorithm, + inherent_data_providers, + }; + + Ok(BasicQueue::new( + verifier, + block_import, + None, + None + )) +} + +/// Start the background mining thread for PoW. Note that because PoW mining +/// is CPU-intensive, it is not possible to use an async future to define this. +/// However, it's not recommended to use background threads in the rest of the +/// codebase. +/// +/// `preruntime` is a parameter that allows a custom additional pre-runtime +/// digest to be inserted for blocks being built. This can encode authorship +/// information, or just be a graffiti. `round` is for number of rounds the +/// CPU miner runs each time. This parameter should be tweaked so that each +/// mining round is within sub-second time. +pub fn start_mine, C, Algorithm, E>( + mut block_import: BoxBlockImport, + client: Arc, + algorithm: Algorithm, + mut env: E, + preruntime: Option>, + round: u32, + inherent_data_providers: inherents::InherentDataProviders, +) where + C: HeaderBackend + AuxStore + 'static, + Algorithm: PowAlgorithm + Send + Sync + 'static, + E: Environment + Send + Sync + 'static, + E::Error: std::fmt::Debug, +{ + if let Err(_) = register_pow_inherent_data_provider(&inherent_data_providers) { + warn!("Registering inherent data provider for timestamp failed"); + } + + thread::spawn(move || { + loop { + match mine_loop( + &mut block_import, + client.as_ref(), + &algorithm, + &mut env, + preruntime.as_ref(), + round, + &inherent_data_providers + ) { + Ok(()) => (), + Err(e) => error!( + "Mining block failed with {:?}. Sleep for 1 second before restarting...", + e + ), + } + + std::thread::sleep(std::time::Duration::new(1, 0)); + } + }); +} + +fn mine_loop, C, Algorithm, E>( + block_import: &mut BoxBlockImport, + client: &C, + algorithm: &Algorithm, + env: &mut E, + preruntime: Option<&Vec>, + round: u32, + inherent_data_providers: &inherents::InherentDataProviders, +) -> Result<(), String> where + C: HeaderBackend + AuxStore, + Algorithm: PowAlgorithm, + E: Environment, + E::Error: std::fmt::Debug, +{ + 'outer: loop { + let best_hash = client.info().best_hash; + let best_header = client.header(BlockId::Hash(best_hash)) + .map_err(|e| format!("Fetching best header failed: {:?}", e))? + .ok_or("Best header does not exist")?; + let mut aux = PowAux::read(client, &best_hash)?; + let mut proposer = env.init(&best_header).map_err(|e| format!("{:?}", e))?; + + let inherent_data = inherent_data_providers + .create_inherent_data().map_err(String::from)?; + let mut inherent_digest = Digest::default(); + if let Some(preruntime) = &preruntime { + inherent_digest.push(DigestItem::PreRuntime(POW_ENGINE_ID, preruntime.to_vec())); + } + let block = futures::executor::block_on(proposer.propose( + inherent_data, + inherent_digest, + std::time::Duration::new(0, 0) + )).map_err(|e| format!("Block proposing error: {:?}", e))?; + + let (header, body) = block.deconstruct(); + let seed = H256::random(); + let (difficulty, seal) = { + loop { + let difficulty = algorithm.difficulty( + &BlockId::Hash(best_hash), + )?; + + let seal = algorithm.mine( + &BlockId::Hash(best_hash), + &header.hash(), + &seed, + difficulty, + round, + )?; + + if let Some(seal) = seal { + break (difficulty, seal) + } + + if best_hash != client.info().best_hash { + continue 'outer + } + } + }; + + aux.total_difficulty = aux.total_difficulty.saturating_add(difficulty); + let hash = header.hash(); + + let key = aux_key(&hash); + let import_block = BlockImportParams { + origin: BlockOrigin::Own, + header, + justification: None, + post_digests: vec![DigestItem::Seal(POW_ENGINE_ID, seal)], + body: Some(body), + finalized: false, + auxiliary: vec![(key, Some(aux.encode()))], + fork_choice: ForkChoiceStrategy::Custom(true), + }; + + block_import.import_block(import_block, HashMap::default()) + .map_err(|e| format!("Error with block built on {:?}: {:?}", best_hash, e))?; + } +}