From e0c064a33e65a1ca7cc918d83b830558a7041957 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 26 May 2018 22:23:38 +0200 Subject: [PATCH 001/160] WIP Make HashDB generic --- util/hashdb/src/lib.rs | 78 ++++++++++++++++++++---------------- util/memorydb/Cargo.toml | 1 + util/memorydb/src/lib.rs | 86 +++++++++++++++++++++++++++------------- 3 files changed, 103 insertions(+), 62 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index b65f304e423..15b1fd36b92 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -16,63 +16,71 @@ //! Database of byte-slices keyed to their Keccak hash. extern crate elastic_array; -extern crate ethereum_types; +//extern crate ethereum_types; use std::collections::HashMap; use elastic_array::ElasticArray128; -use ethereum_types::H256; +//use ethereum_types::H256; +use std::{fmt::Debug, hash::Hash}; + +pub trait Hasher: Sync + Send { + type Out: Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync; + fn hash(x: &[u8]) -> Self::Out; +} /// `HashDB` value type. pub type DBValue = ElasticArray128; /// Trait modelling datastore keyed by a 32-byte Keccak hash. -pub trait HashDB: AsHashDB + Send + Sync { +pub trait HashDB: Send + Sync { + type H: Hasher; /// Get the keys in the database together with number of underlying references. - fn keys(&self) -> HashMap; +// fn keys(&self) -> HashMap; + fn keys(&self) -> HashMap<::Out, i32>; /// Look up a given hash into the bytes that hash to it, returning None if the /// hash is not known. - fn get(&self, key: &H256) -> Option; + fn get(&self, key: &::Out) -> Option; /// Check for the existance of a hash-key. - fn contains(&self, key: &H256) -> bool; + fn contains(&self, key: &::Out) -> bool; /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. - fn insert(&mut self, value: &[u8]) -> H256; + fn insert(&mut self, value: &[u8]) -> ::Out; /// Like `insert()` , except you provide the key and the data is all moved. - fn emplace(&mut self, key: H256, value: DBValue); + fn emplace(&mut self, key: ::Out, value: DBValue); /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may /// happen without the data being eventually being inserted into the DB. It can be "owed" more than once. - fn remove(&mut self, key: &H256); -} - -/// Upcast trait. -pub trait AsHashDB { - /// Perform upcast to HashDB for anything that derives from HashDB. - fn as_hashdb(&self) -> &HashDB; - /// Perform mutable upcast to HashDB for anything that derives from HashDB. - fn as_hashdb_mut(&mut self) -> &mut HashDB; + fn remove(&mut self, key: &::Out); } -impl AsHashDB for T { - fn as_hashdb(&self) -> &HashDB { - self - } - fn as_hashdb_mut(&mut self) -> &mut HashDB { - self - } -} - -impl<'a> AsHashDB for &'a mut HashDB { - fn as_hashdb(&self) -> &HashDB { - &**self - } - - fn as_hashdb_mut(&mut self) -> &mut HashDB { - &mut **self - } -} +///// Upcast trait. +//pub trait AsHashDB { +// /// Perform upcast to HashDB for anything that derives from HashDB. +// fn as_hashdb(&self) -> &HashDB; +// /// Perform mutable upcast to HashDB for anything that derives from HashDB. +// fn as_hashdb_mut(&mut self) -> &mut HashDB; +//} +// +//impl> AsHashDB for T { +// fn as_hashdb(&self) -> &HashDB { +// self +// } +// fn as_hashdb_mut(&mut self) -> &mut HashDB { +// self +// } +//} +// +//impl<'a> AsHashDB for &'a mut HashDB { +// fn as_hashdb(&self) -> &HashDB { +// &**self +// } +// +// fn as_hashdb_mut(&mut self) -> &mut HashDB { +// &mut **self +// } +//} diff --git a/util/memorydb/Cargo.toml b/util/memorydb/Cargo.toml index 41c41bb628e..c2d8c6566dd 100644 --- a/util/memorydb/Cargo.toml +++ b/util/memorydb/Cargo.toml @@ -10,6 +10,7 @@ elastic-array = "0.10" heapsize = "0.4" ethereum-types = "0.3" keccak-hash = { version = "0.1.0", path = "../hash" } +tiny-keccak = "1.4.2" hashdb = { version = "0.1.1", path = "../hashdb" } plain_hasher = { path = "../plain_hasher" } rlp = { version = "0.2.1", path = "../rlp" } diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 12eb62e0574..74d5c5567ed 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -15,23 +15,26 @@ // along with Parity. If not, see . //! Reference-counted memory-based `HashDB` implementation. -extern crate heapsize; +extern crate elastic_array; extern crate ethereum_types; extern crate hashdb; +extern crate heapsize; extern crate keccak_hash as keccak; +extern crate tiny_keccak; extern crate plain_hasher; extern crate rlp; -extern crate elastic_array; -use std::mem; -use std::collections::HashMap; -use std::collections::hash_map::Entry; -use heapsize::HeapSizeOf; use ethereum_types::H256; -use hashdb::{HashDB, DBValue}; -use keccak::{KECCAK_NULL_RLP, keccak}; +use hashdb::{HashDB, Hasher, DBValue}; +use heapsize::HeapSizeOf; +use keccak::{KECCAK_NULL_RLP, keccak}; // TODO: probably not needed, prefer straight tiny_keccak +use tiny_keccak::Keccak as Kecc; use plain_hasher::H256FastMap; use rlp::NULL_RLP; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::marker::PhantomData; +use std::mem; /// Reference-counted memory-based `HashDB` implementation. /// /// Use `new()` to create a new database. Insert items with `insert()`, remove items @@ -77,15 +80,16 @@ use rlp::NULL_RLP; /// } /// ``` #[derive(Default, Clone, PartialEq)] -pub struct MemoryDB { +pub struct MemoryDB { data: H256FastMap<(DBValue, i32)>, + marker: PhantomData } -impl MemoryDB { +impl MemoryDB { /// Create a new instance of the memory DB. - pub fn new() -> MemoryDB { + pub fn new() -> MemoryDB { MemoryDB { - data: H256FastMap::default(), + data: H256FastMap::default(), marker: PhantomData } } @@ -177,19 +181,21 @@ impl MemoryDB { } } -impl HashDB for MemoryDB { - fn get(&self, key: &H256) -> Option { - if key == &KECCAK_NULL_RLP { - return Some(DBValue::from_slice(&NULL_RLP)); - } - - match self.data.get(key) { - Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), - _ => None - } +#[derive(Debug)] +pub struct KeccakHasher; +impl Hasher for KeccakHasher { + type Out = H256; + fn hash(x: &[u8]) -> Self::Out { + let mut out = [0;32]; + Kecc::keccak256(x, &mut out); + out.into() } +} - fn keys(&self) -> HashMap { +impl HashDB for MemoryDB { + type H = KeccakHasher; + + fn keys(&self) -> HashMap<::Out, i32> { self.data.iter() .filter_map(|(k, v)| if v.1 != 0 { Some((*k, v.1)) @@ -199,7 +205,18 @@ impl HashDB for MemoryDB { .collect() } - fn contains(&self, key: &H256) -> bool { + fn get(&self, key: &::Out) -> Option { + if key == &KECCAK_NULL_RLP { + return Some(DBValue::from_slice(&NULL_RLP)); + } + + match self.data.get(key) { + Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + _ => None + } + } + + fn contains(&self, key: &::Out) -> bool { if key == &KECCAK_NULL_RLP { return true; } @@ -210,7 +227,7 @@ impl HashDB for MemoryDB { } } - fn insert(&mut self, value: &[u8]) -> H256 { + fn insert(&mut self, value: &[u8]) -> ::Out { if value == &NULL_RLP { return KECCAK_NULL_RLP.clone(); } @@ -230,7 +247,7 @@ impl HashDB for MemoryDB { key } - fn emplace(&mut self, key: H256, value: DBValue) { + fn emplace(&mut self, key:::Out, value: DBValue) { if &*value == &NULL_RLP { return; } @@ -249,7 +266,7 @@ impl HashDB for MemoryDB { } } - fn remove(&mut self, key: &H256) { + fn remove(&mut self, key: &::Out) { if key == &KECCAK_NULL_RLP { return; } @@ -320,4 +337,19 @@ mod tests { assert_eq!(overlay.get(&insert_key).unwrap(), &(DBValue::from_slice(b"arf"), 2)); assert_eq!(overlay.get(&negative_remove_key).unwrap(), &(DBValue::from_slice(b"negative"), -2)); } + +// #[test] +// fn other_hashers() { +// struct DummyHasher; +// impl Hasher for DummyHasher { +// type Out = u8; +// fn hash(x:&[u8]) -> Self::Out { +// x[0] +// } +// } +// let db = MemoryDB::::new(); +// let key = db.insert(b"321"); +// +// assert_eq!(key, 3u8); +// } } From 1cd7dedd060a59fee604086cf3b6ef54aada4f58 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 27 May 2018 22:03:45 +0200 Subject: [PATCH 002/160] Add HeapSizeOf bound to the output type of a Hasher Add associated const to Hasher for the fast-path Make HashDB impl for MemoryDB generic Make H256FastMap generic (eew) --- util/hashdb/Cargo.toml | 1 + util/hashdb/src/lib.rs | 5 +++- util/memorydb/src/lib.rs | 56 +++++++++++++++++++----------------- util/plain_hasher/Cargo.toml | 1 + util/plain_hasher/src/lib.rs | 4 ++- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/util/hashdb/Cargo.toml b/util/hashdb/Cargo.toml index d4e055f9ff0..48ee52bc221 100644 --- a/util/hashdb/Cargo.toml +++ b/util/hashdb/Cargo.toml @@ -8,3 +8,4 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" ethereum-types = "0.3" +heapsize = "0.4" \ No newline at end of file diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 15b1fd36b92..c03a036b681 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -17,15 +17,18 @@ //! Database of byte-slices keyed to their Keccak hash. extern crate elastic_array; //extern crate ethereum_types; +extern crate heapsize; use std::collections::HashMap; use elastic_array::ElasticArray128; //use ethereum_types::H256; use std::{fmt::Debug, hash::Hash}; +use heapsize::HeapSizeOf; pub trait Hasher: Sync + Send { - type Out: Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync; + type Out: Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync + HeapSizeOf; + const HASHED_NULL_RLP: Self::Out; fn hash(x: &[u8]) -> Self::Out; } /// `HashDB` value type. diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 74d5c5567ed..fc3bd414bf9 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -27,13 +27,11 @@ extern crate rlp; use ethereum_types::H256; use hashdb::{HashDB, Hasher, DBValue}; use heapsize::HeapSizeOf; -use keccak::{KECCAK_NULL_RLP, keccak}; // TODO: probably not needed, prefer straight tiny_keccak -use tiny_keccak::Keccak as Kecc; +use tiny_keccak::Keccak; use plain_hasher::H256FastMap; use rlp::NULL_RLP; use std::collections::hash_map::Entry; use std::collections::HashMap; -use std::marker::PhantomData; use std::mem; /// Reference-counted memory-based `HashDB` implementation. /// @@ -49,7 +47,7 @@ use std::mem; /// use hashdb::*; /// use memorydb::*; /// fn main() { -/// let mut m = MemoryDB::new(); +/// let mut m = MemoryDB::::new(); /// let d = "Hello world!".as_bytes(); /// /// let k = m.insert(d); @@ -81,15 +79,14 @@ use std::mem; /// ``` #[derive(Default, Clone, PartialEq)] pub struct MemoryDB { - data: H256FastMap<(DBValue, i32)>, - marker: PhantomData + data: H256FastMap, } impl MemoryDB { /// Create a new instance of the memory DB. pub fn new() -> MemoryDB { MemoryDB { - data: H256FastMap::default(), marker: PhantomData + data: H256FastMap::::default() } } @@ -102,7 +99,7 @@ impl MemoryDB { /// use hashdb::*; /// use memorydb::*; /// fn main() { - /// let mut m = MemoryDB::new(); + /// let mut m = MemoryDB::::new(); /// let hello_bytes = "Hello world!".as_bytes(); /// let hash = m.insert(hello_bytes); /// assert!(m.contains(&hash)); @@ -120,8 +117,8 @@ impl MemoryDB { } /// Return the internal map of hashes to data, clearing the current state. - pub fn drain(&mut self) -> H256FastMap<(DBValue, i32)> { - mem::replace(&mut self.data, H256FastMap::default()) + pub fn drain(&mut self) -> H256FastMap { + mem::replace(&mut self.data, H256FastMap::::default()) } /// Grab the raw information associated with a key. Returns None if the key @@ -129,8 +126,8 @@ impl MemoryDB { /// /// Even when Some is returned, the data is only guaranteed to be useful /// when the refs > 0. - pub fn raw(&self, key: &H256) -> Option<(DBValue, i32)> { - if key == &KECCAK_NULL_RLP { + pub fn raw(&self, key: &::Out) -> Option<(DBValue, i32)> { + if key == &H::HASHED_NULL_RLP { return Some((DBValue::from_slice(&NULL_RLP), 1)); } self.data.get(key).cloned() @@ -143,8 +140,8 @@ impl MemoryDB { /// Remove an element and delete it from storage if reference count reaches zero. /// If the value was purged, return the old value. - pub fn remove_and_purge(&mut self, key: &H256) -> Option { - if key == &KECCAK_NULL_RLP { + pub fn remove_and_purge(&mut self, key: &::Out) -> Option { + if key == &H::HASHED_NULL_RLP { return None; } match self.data.entry(key.clone()) { @@ -182,19 +179,26 @@ impl MemoryDB { } #[derive(Debug)] +// REVIEW: Where do the concrete Hasher implementations go? Own crate? pub struct KeccakHasher; impl Hasher for KeccakHasher { type Out = H256; + const HASHED_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); + fn hash(x: &[u8]) -> Self::Out { let mut out = [0;32]; - Kecc::keccak256(x, &mut out); + Keccak::keccak256(x, &mut out); out.into() } } -impl HashDB for MemoryDB { - type H = KeccakHasher; +impl HashDB for MemoryDB { + type H = H; // REVIEW this is a bit confusing, but not sure what is better (e.g. using `impl … … type H = HH;`) + // REVIEW: this method is what made it necessary to add a type param to H256FastMap, which I'd rather have avoided. + // The problem is that the keys returned are `H256` and type inference fails on the `collect()` call. + // I could not make it work without parameterizing H256FastMap too. It all sort of adds up as I could + // avoid adding PhantomData to MemoryDB, but still quite annoying. What's a better way? fn keys(&self) -> HashMap<::Out, i32> { self.data.iter() .filter_map(|(k, v)| if v.1 != 0 { @@ -206,7 +210,7 @@ impl HashDB for MemoryDB { } fn get(&self, key: &::Out) -> Option { - if key == &KECCAK_NULL_RLP { + if key == &Self::H::HASHED_NULL_RLP { return Some(DBValue::from_slice(&NULL_RLP)); } @@ -217,7 +221,7 @@ impl HashDB for MemoryDB { } fn contains(&self, key: &::Out) -> bool { - if key == &KECCAK_NULL_RLP { + if key == &Self::H::HASHED_NULL_RLP { return true; } @@ -229,9 +233,9 @@ impl HashDB for MemoryDB { fn insert(&mut self, value: &[u8]) -> ::Out { if value == &NULL_RLP { - return KECCAK_NULL_RLP.clone(); + return Self::H::HASHED_NULL_RLP.clone(); } - let key = keccak(value); + let key = Self::H::hash(value); match self.data.entry(key) { Entry::Occupied(mut entry) => { let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); @@ -267,7 +271,7 @@ impl HashDB for MemoryDB { } fn remove(&mut self, key: &::Out) { - if key == &KECCAK_NULL_RLP { + if key == &Self::H::HASHED_NULL_RLP { return; } @@ -293,7 +297,7 @@ mod tests { let hello_bytes = b"Hello world!"; let hello_key = keccak(hello_bytes); - let mut m = MemoryDB::new(); + let mut m = MemoryDB::::new(); m.remove(&hello_key); assert_eq!(m.raw(&hello_key).unwrap().1, -1); m.purge(); @@ -303,7 +307,7 @@ mod tests { m.purge(); assert_eq!(m.raw(&hello_key), None); - let mut m = MemoryDB::new(); + let mut m = MemoryDB::::new(); assert!(m.remove_and_purge(&hello_key).is_none()); assert_eq!(m.raw(&hello_key).unwrap().1, -1); m.insert(hello_bytes); @@ -316,8 +320,8 @@ mod tests { #[test] fn consolidate() { - let mut main = MemoryDB::new(); - let mut other = MemoryDB::new(); + let mut main = MemoryDB::::new(); + let mut other = MemoryDB::::new(); let remove_key = other.insert(b"doggo"); main.remove(&remove_key); diff --git a/util/plain_hasher/Cargo.toml b/util/plain_hasher/Cargo.toml index d909f8f9d6b..f23c823dcf4 100644 --- a/util/plain_hasher/Cargo.toml +++ b/util/plain_hasher/Cargo.toml @@ -10,3 +10,4 @@ homepage = "https://github.com/paritytech/plain_hasher" [dependencies] crunchy = "0.1.6" ethereum-types = "0.3" +hashdb = { version = "0.1.1", path = "../hashdb" } \ No newline at end of file diff --git a/util/plain_hasher/src/lib.rs b/util/plain_hasher/src/lib.rs index 54bad92f47e..9cf675575c2 100644 --- a/util/plain_hasher/src/lib.rs +++ b/util/plain_hasher/src/lib.rs @@ -1,13 +1,15 @@ #[macro_use] extern crate crunchy; extern crate ethereum_types; +extern crate hashdb; use std::{hash, mem}; use std::collections::{HashMap, HashSet}; use ethereum_types::H256; +use hashdb::Hasher; /// Specialized version of `HashMap` with H256 keys and fast hashing function. -pub type H256FastMap = HashMap>; +pub type H256FastMap = HashMap<::Out, T, hash::BuildHasherDefault>; /// Specialized version of `HashSet` with H256 keys and fast hashing function. pub type H256FastSet = HashSet>; From 4bf7f6dfaed7603b035d85c1a5ab54c842d6cd10 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 27 May 2018 22:51:53 +0200 Subject: [PATCH 003/160] Add comments and todos --- util/hashdb/src/lib.rs | 3 +-- util/memorydb/src/lib.rs | 36 ++++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index c03a036b681..3c6f976bd19 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -27,7 +27,7 @@ use std::{fmt::Debug, hash::Hash}; use heapsize::HeapSizeOf; pub trait Hasher: Sync + Send { - type Out: Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync + HeapSizeOf; + type Out: Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? */ + HeapSizeOf; const HASHED_NULL_RLP: Self::Out; fn hash(x: &[u8]) -> Self::Out; } @@ -38,7 +38,6 @@ pub type DBValue = ElasticArray128; pub trait HashDB: Send + Sync { type H: Hasher; /// Get the keys in the database together with number of underlying references. -// fn keys(&self) -> HashMap; fn keys(&self) -> HashMap<::Out, i32>; /// Look up a given hash into the bytes that hash to it, returning None if the diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index fc3bd414bf9..e1b63f77b42 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -342,18 +342,26 @@ mod tests { assert_eq!(overlay.get(&negative_remove_key).unwrap(), &(DBValue::from_slice(b"negative"), -2)); } -// #[test] -// fn other_hashers() { -// struct DummyHasher; -// impl Hasher for DummyHasher { -// type Out = u8; -// fn hash(x:&[u8]) -> Self::Out { -// x[0] -// } -// } -// let db = MemoryDB::::new(); -// let key = db.insert(b"321"); -// -// assert_eq!(key, 3u8); -// } + #[test] + fn other_hashers() { + struct DummyHasher; + impl Hasher for DummyHasher { + // TODO: Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) + // It looks like this work must touch H256FastMap as well. + type Out = ethereum_types::H264; + const HASHED_NULL_RLP: ethereum_types::H264= ethereum_types::H264([0; 33]); + fn hash(_x: &[u8]) -> Self::Out { + ethereum_types::H264(*b"010102020101020201010202010102025") + + } + } + + impl HeapSizeOf for DummyHasher { + fn heap_size_of_children(&self) -> usize { 0 } + } + let mut db = MemoryDB::::new(); +// let key = db.insert(b"32103210321032103210321032103210"); + // Fails +// assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); + } } From fd666d6618b825ca7080572e9f76bb38066bf04a Mon Sep 17 00:00:00 2001 From: David Palm Date: Sun, 27 May 2018 22:54:14 +0200 Subject: [PATCH 004/160] Cleanup and more todos --- util/hashdb/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 3c6f976bd19..6c764c3e43a 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -21,7 +21,6 @@ extern crate heapsize; use std::collections::HashMap; use elastic_array::ElasticArray128; -//use ethereum_types::H256; use std::{fmt::Debug, hash::Hash}; use heapsize::HeapSizeOf; @@ -60,6 +59,7 @@ pub trait HashDB: Send + Sync { fn remove(&mut self, key: &::Out); } +// TODO: Figure out what these do and if they're needed ///// Upcast trait. //pub trait AsHashDB { // /// Perform upcast to HashDB for anything that derives from HashDB. From 5325a7091dd5f5ba0202cf9d73834847aab8b057 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 08:43:55 +0200 Subject: [PATCH 005/160] =?UTF-8?q?Move=20concrete=20hasher=20impl=20to=20?= =?UTF-8?q?hashdb=20=E2=80=93=20TODO:=20where=20do=20these=20go=3F=20Make?= =?UTF-8?q?=20AsHashDB=20generic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/hashdb/Cargo.toml | 3 +- util/hashdb/src/lib.rs | 78 +++++++++++++++++++++++++----------------- 2 files changed, 49 insertions(+), 32 deletions(-) diff --git a/util/hashdb/Cargo.toml b/util/hashdb/Cargo.toml index 48ee52bc221..792f8afa1a8 100644 --- a/util/hashdb/Cargo.toml +++ b/util/hashdb/Cargo.toml @@ -8,4 +8,5 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" ethereum-types = "0.3" -heapsize = "0.4" \ No newline at end of file +heapsize = "0.4" +tiny-keccak = "1.4.2" \ No newline at end of file diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 6c764c3e43a..e9f40c6e53c 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -16,20 +16,37 @@ //! Database of byte-slices keyed to their Keccak hash. extern crate elastic_array; -//extern crate ethereum_types; +extern crate ethereum_types; extern crate heapsize; +extern crate tiny_keccak; -use std::collections::HashMap; use elastic_array::ElasticArray128; - -use std::{fmt::Debug, hash::Hash}; +use ethereum_types::H256; use heapsize::HeapSizeOf; +use std::collections::HashMap; +use std::{fmt::Debug, hash::Hash}; +use tiny_keccak::Keccak; pub trait Hasher: Sync + Send { type Out: Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? */ + HeapSizeOf; const HASHED_NULL_RLP: Self::Out; fn hash(x: &[u8]) -> Self::Out; } + +#[derive(Debug)] +// REVIEW: Where do the concrete Hasher implementations go? Own crate? +pub struct KeccakHasher; +impl Hasher for KeccakHasher { + type Out = H256; + const HASHED_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); + + fn hash(x: &[u8]) -> Self::Out { + let mut out = [0;32]; + Keccak::keccak256(x, &mut out); + out.into() + } +} + /// `HashDB` value type. pub type DBValue = ElasticArray128; @@ -59,30 +76,29 @@ pub trait HashDB: Send + Sync { fn remove(&mut self, key: &::Out); } -// TODO: Figure out what these do and if they're needed -///// Upcast trait. -//pub trait AsHashDB { -// /// Perform upcast to HashDB for anything that derives from HashDB. -// fn as_hashdb(&self) -> &HashDB; -// /// Perform mutable upcast to HashDB for anything that derives from HashDB. -// fn as_hashdb_mut(&mut self) -> &mut HashDB; -//} -// -//impl> AsHashDB for T { -// fn as_hashdb(&self) -> &HashDB { -// self -// } -// fn as_hashdb_mut(&mut self) -> &mut HashDB { -// self -// } -//} -// -//impl<'a> AsHashDB for &'a mut HashDB { -// fn as_hashdb(&self) -> &HashDB { -// &**self -// } -// -// fn as_hashdb_mut(&mut self) -> &mut HashDB { -// &mut **self -// } -//} +/// Upcast trait. +pub trait AsHashDB { + /// Perform upcast to HashDB for anything that derives from HashDB. + fn as_hashdb(&self) -> &HashDB; + /// Perform mutable upcast to HashDB for anything that derives from HashDB. + fn as_hashdb_mut(&mut self) -> &mut HashDB; +} + +impl> AsHashDB for T { + fn as_hashdb(&self) -> &HashDB { + self + } + fn as_hashdb_mut(&mut self) -> &mut HashDB { + self + } +} + +impl<'a, HF: Hasher> AsHashDB for &'a mut HashDB { + fn as_hashdb(&self) -> &HashDB { + &**self + } + + fn as_hashdb_mut(&mut self) -> &mut HashDB { + &mut **self + } +} From 041d6a26db49fadde6dbbdd127c82d69f38b6052 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 08:46:12 +0200 Subject: [PATCH 006/160] Remove unused crates/uses --- util/memorydb/Cargo.toml | 7 ++-- util/memorydb/src/lib.rs | 73 +++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 46 deletions(-) diff --git a/util/memorydb/Cargo.toml b/util/memorydb/Cargo.toml index c2d8c6566dd..bef8570da25 100644 --- a/util/memorydb/Cargo.toml +++ b/util/memorydb/Cargo.toml @@ -8,9 +8,10 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" heapsize = "0.4" -ethereum-types = "0.3" -keccak-hash = { version = "0.1.0", path = "../hash" } -tiny-keccak = "1.4.2" hashdb = { version = "0.1.1", path = "../hashdb" } plain_hasher = { path = "../plain_hasher" } rlp = { version = "0.2.1", path = "../rlp" } + +[dev-dependencies] +tiny-keccak = "1.4.2" +ethereum-types = "0.3" \ No newline at end of file diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index e1b63f77b42..2604dcc387d 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -16,18 +16,15 @@ //! Reference-counted memory-based `HashDB` implementation. extern crate elastic_array; -extern crate ethereum_types; extern crate hashdb; extern crate heapsize; -extern crate keccak_hash as keccak; -extern crate tiny_keccak; extern crate plain_hasher; extern crate rlp; +#[cfg(test)] extern crate tiny_keccak; +#[cfg(test)] extern crate ethereum_types; -use ethereum_types::H256; use hashdb::{HashDB, Hasher, DBValue}; use heapsize::HeapSizeOf; -use tiny_keccak::Keccak; use plain_hasher::H256FastMap; use rlp::NULL_RLP; use std::collections::hash_map::Entry; @@ -178,20 +175,6 @@ impl MemoryDB { } } -#[derive(Debug)] -// REVIEW: Where do the concrete Hasher implementations go? Own crate? -pub struct KeccakHasher; -impl Hasher for KeccakHasher { - type Out = H256; - const HASHED_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); - - fn hash(x: &[u8]) -> Self::Out { - let mut out = [0;32]; - Keccak::keccak256(x, &mut out); - out.into() - } -} - impl HashDB for MemoryDB { type H = H; // REVIEW this is a bit confusing, but not sure what is better (e.g. using `impl … … type H = HH;`) @@ -289,13 +272,17 @@ impl HashDB for MemoryDB { #[cfg(test)] mod tests { - use keccak::keccak; use super::*; + use tiny_keccak::Keccak; + use ethereum_types::H256; + use hashdb::KeccakHasher; #[test] fn memorydb_remove_and_purge() { let hello_bytes = b"Hello world!"; - let hello_key = keccak(hello_bytes); + let mut hello_key = [0;32]; + Keccak::keccak256(hello_bytes, &mut hello_key); + let hello_key = H256(hello_key); let mut m = MemoryDB::::new(); m.remove(&hello_key); @@ -342,26 +329,26 @@ mod tests { assert_eq!(overlay.get(&negative_remove_key).unwrap(), &(DBValue::from_slice(b"negative"), -2)); } - #[test] - fn other_hashers() { - struct DummyHasher; - impl Hasher for DummyHasher { - // TODO: Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) - // It looks like this work must touch H256FastMap as well. - type Out = ethereum_types::H264; - const HASHED_NULL_RLP: ethereum_types::H264= ethereum_types::H264([0; 33]); - fn hash(_x: &[u8]) -> Self::Out { - ethereum_types::H264(*b"010102020101020201010202010102025") - - } - } - - impl HeapSizeOf for DummyHasher { - fn heap_size_of_children(&self) -> usize { 0 } - } - let mut db = MemoryDB::::new(); -// let key = db.insert(b"32103210321032103210321032103210"); - // Fails -// assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); - } +// #[test] +// fn other_hashers() { +// struct DummyHasher; +// impl Hasher for DummyHasher { +// // TODO: Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) +// // It looks like this work must touch H256FastMap as well. +// type Out = ethereum_types::H264; +// const HASHED_NULL_RLP: ethereum_types::H264= ethereum_types::H264([0; 33]); +// fn hash(_x: &[u8]) -> Self::Out { +// ethereum_types::H264(*b"010102020101020201010202010102025") +// +// } +// } +// +// impl HeapSizeOf for DummyHasher { +// fn heap_size_of_children(&self) -> usize { 0 } +// } +// let mut db = MemoryDB::::new(); +//// let key = db.insert(b"32103210321032103210321032103210"); +// // Fails +//// assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); +// } } From f716105a7480851404e911bfeff6a24cf6190e8d Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 08:57:13 +0200 Subject: [PATCH 007/160] =?UTF-8?q?Version=20bump=20=E2=80=93=20breaking?= =?UTF-8?q?=20change=20=3D>=200.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/memorydb/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/memorydb/Cargo.toml b/util/memorydb/Cargo.toml index bef8570da25..6bf29e5722f 100644 --- a/util/memorydb/Cargo.toml +++ b/util/memorydb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "memorydb" -version = "0.1.1" +version = "0.2.0" authors = ["Parity Technologies "] description = "in-memory implementation of hashdb" license = "GPL-3.0" @@ -8,7 +8,7 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" heapsize = "0.4" -hashdb = { version = "0.1.1", path = "../hashdb" } +hashdb = { version = "0.2.0", path = "../hashdb" } plain_hasher = { path = "../plain_hasher" } rlp = { version = "0.2.1", path = "../rlp" } From 8ce39ebc229ecfa27a6442445fd675056b67cc80 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 08:57:37 +0200 Subject: [PATCH 008/160] Version bump --- util/hashdb/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/hashdb/Cargo.toml b/util/hashdb/Cargo.toml index 792f8afa1a8..15520c1921f 100644 --- a/util/hashdb/Cargo.toml +++ b/util/hashdb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hashdb" -version = "0.1.1" +version = "0.2.0" authors = ["Parity Technologies "] description = "trait for hash-keyed databases." license = "GPL-3.0" From a7ff7c2b1997ece357b7a9a479634e327c7d5f66 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 15:31:50 +0200 Subject: [PATCH 009/160] Derive Clone and PartialEq for KeccakHasher to satisfy the code in journaldb --- util/hashdb/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index e9f40c6e53c..fd4da5ceb7a 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -33,7 +33,7 @@ pub trait Hasher: Sync + Send { fn hash(x: &[u8]) -> Self::Out; } -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq)] // REVIEW: Where do the concrete Hasher implementations go? Own crate? pub struct KeccakHasher; impl Hasher for KeccakHasher { From ef1e48a0c431f21a31820bbe990718f867d00afb Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 15:33:10 +0200 Subject: [PATCH 010/160] =?UTF-8?q?Thread=20`Hasher`=20throughout=20the=20?= =?UTF-8?q?crate=20=E2=80=93=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/patricia_trie/src/fatdb.rs | 42 +++++++++-------- util/patricia_trie/src/fatdbmut.rs | 27 +++++------ util/patricia_trie/src/lib.rs | 64 ++++++++++++++------------ util/patricia_trie/src/lookup.rs | 12 ++--- util/patricia_trie/src/recorder.rs | 23 +++++---- util/patricia_trie/src/sectriedb.rs | 30 ++++++------ util/patricia_trie/src/sectriedbmut.rs | 27 +++++------ util/patricia_trie/src/triedb.rs | 51 ++++++++++---------- util/patricia_trie/src/triedbmut.rs | 36 ++++++++------- 9 files changed, 165 insertions(+), 147 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index d428ff8116b..4cdd131c95a 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -16,22 +16,22 @@ use ethereum_types::H256; use keccak::keccak; -use hashdb::HashDB; +use hashdb::{HashDB, Hasher}; use super::{TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query}; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// Additionaly it stores inserted hash-key mappings for later retrieval. /// /// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDB<'db> { - raw: TrieDB<'db>, +pub struct FatDB<'db, H: Hasher + 'db> { + raw: TrieDB<'db, H>, } -impl<'db> FatDB<'db> { +impl<'db, H: Hasher> FatDB<'db, H> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { let fatdb = FatDB { raw: TrieDB::new(db, root)? }; @@ -40,40 +40,42 @@ impl<'db> FatDB<'db> { } /// Get the backing database. - pub fn db(&self) -> &HashDB { + pub fn db(&self) -> &HashDB { self.raw.db() } } -impl<'db> Trie for FatDB<'db> { - fn iter<'a>(&'a self) -> super::Result + 'a>> { - FatDBIterator::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) - } +impl<'db, H: Hasher> Trie for FatDB<'db, H> { + type H = H; - fn root(&self) -> &H256 { + fn root(&self) -> &::Out { self.raw.root() } fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(&keccak(key)) + self.raw.contains(&keccak(key)) // TODO } fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result> where 'a: 'key { - self.raw.get_with(&keccak(key), query) + self.raw.get_with(&keccak(key), query) // TODO + } + + fn iter<'a>(&'a self) -> super::Result + 'a>> { + FatDBIterator::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) } } /// Itarator over inserted pairs of key values. -pub struct FatDBIterator<'db> { - trie_iterator: TrieDBIterator<'db>, - trie: &'db TrieDB<'db>, +pub struct FatDBIterator<'db, H: Hasher + 'db> { + trie_iterator: TrieDBIterator<'db, H>, + trie: &'db TrieDB<'db, H>, } -impl<'db> FatDBIterator<'db> { +impl<'db, H: Hasher> FatDBIterator<'db, H> { /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> super::Result { + pub fn new(trie: &'db TrieDB) -> super::Result { Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie: trie, @@ -81,13 +83,13 @@ impl<'db> FatDBIterator<'db> { } } -impl<'db> TrieIterator for FatDBIterator<'db> { +impl<'db, H: Hasher> TrieIterator for FatDBIterator<'db, H> { fn seek(&mut self, key: &[u8]) -> super::Result<()> { self.trie_iterator.seek(&keccak(key)) } } -impl<'db> Iterator for FatDBIterator<'db> { +impl<'db, H: Hasher> Iterator for FatDBIterator<'db, H> { type Item = TrieItem<'db>; fn next(&mut self) -> Option { diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 4b7f2de0632..1d007586365 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -16,49 +16,50 @@ use ethereum_types::H256; use keccak::keccak; -use hashdb::{HashDB, DBValue}; +use hashdb::{HashDB, DBValue, Hasher}; use super::{TrieDBMut, TrieMut}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// Additionaly it stores inserted hash-key mappings for later retrieval. /// /// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDBMut<'db> { - raw: TrieDBMut<'db>, +pub struct FatDBMut<'db, H: Hasher + 'db> { + raw: TrieDBMut<'db, H>, } -impl<'db> FatDBMut<'db> { +impl<'db, H: Hasher> FatDBMut<'db, H> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { FatDBMut { raw: TrieDBMut::new(db, root) } } /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> super::Result { Ok(FatDBMut { raw: TrieDBMut::from_existing(db, root)? }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { + pub fn db(&self) -> &HashDB { self.raw.db() } /// Get the backing database. - pub fn db_mut(&mut self) -> &mut HashDB { + pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } fn to_aux_key(key: &[u8]) -> H256 { keccak(key) - } + } // TODO } -impl<'db> TrieMut for FatDBMut<'db> { - fn root(&mut self) -> &H256 { +impl<'db, H: Hasher> TrieMut for FatDBMut<'db, H> { + type H = H; + fn root(&mut self) -> &::Out { self.raw.root() } @@ -73,11 +74,11 @@ impl<'db> TrieMut for FatDBMut<'db> { fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result> where 'a: 'key { - self.raw.get(&keccak(key)) + self.raw.get(&keccak(key)) // TODO } fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { - let hash = keccak(key); + let hash = keccak(key); // TODO let out = self.raw.insert(&hash, value)?; let db = self.raw.db_mut(); diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index d1563becff0..a744c8d6501 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -34,7 +34,8 @@ extern crate log; use std::{fmt, error}; use ethereum_types::H256; use keccak::KECCAK_NULL_RLP; -use hashdb::{HashDB, DBValue}; +use hashdb::{HashDB, DBValue, Hasher, KeccakHasher}; +use std::marker::PhantomData; pub mod node; pub mod triedb; @@ -108,6 +109,7 @@ pub type TrieItem<'a> = Result<(Vec, DBValue)>; /// a DBValue), any function taking raw bytes (where no recording will be made), /// or any tuple of (&mut Recorder, FnOnce(&[u8])) pub trait Query { + type H: Hasher; /// Output item. type Item; @@ -115,37 +117,40 @@ pub trait Query { fn decode(self, &[u8]) -> Self::Item; /// Record that a node has been passed through. - fn record(&mut self, &H256, &[u8], u32) { } + fn record(&mut self, hash: &::Out, &[u8], u32) { } } -impl<'a> Query for &'a mut Recorder { +impl<'a, H: Hasher> Query for &'a mut Recorder { + type H = H; type Item = DBValue; - fn decode(self, value: &[u8]) -> DBValue { DBValue::from_slice(value) } - fn record(&mut self, hash: &H256, data: &[u8], depth: u32) { + fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { (&mut **self).record(hash, data, depth); } } -impl Query for F where F: for<'a> FnOnce(&'a [u8]) -> T { - type Item = T; - - fn decode(self, value: &[u8]) -> T { (self)(value) } -} +// TODO: fix "the type parameter `H` is not constrained by the impl trait, self type, or predicates" +//impl Query for F where F: for<'a> FnOnce(&'a [u8]) -> T { +// type H = H; +// type Item = T; +// +// fn decode(self, value: &[u8]) -> T { (self)(value) } +//} -impl<'a, F, T> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T { +impl<'a, F, T, H: Hasher> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T { + type H = H; type Item = T; - fn decode(self, value: &[u8]) -> T { (self.1)(value) } - fn record(&mut self, hash: &H256, data: &[u8], depth: u32) { + fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { self.0.record(hash, data, depth) } } /// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait Trie { + type H: Hasher; /// Return the root of the trie. - fn root(&self) -> &H256; + fn root(&self) -> &::Out; /// Is the trie empty? fn is_empty(&self) -> bool { *self.root() == KECCAK_NULL_RLP } @@ -171,8 +176,9 @@ pub trait Trie { /// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait TrieMut { + type H: Hasher; /// Return the root of the trie. - fn root(&mut self) -> &H256; + fn root(&mut self) -> &::Out; /// Is the trie empty? fn is_empty(&self) -> bool; @@ -219,19 +225,20 @@ impl Default for TrieSpec { /// Trie factory. #[derive(Default, Clone)] -pub struct TrieFactory { +pub struct TrieFactory { spec: TrieSpec, + marker: PhantomData } /// All different kinds of tries. /// This is used to prevent a heap allocation for every created trie. -pub enum TrieKinds<'db> { +pub enum TrieKinds<'db, H: Hasher + 'db> { /// A generic trie db. - Generic(TrieDB<'db>), + Generic(TrieDB<'db, H>), /// A secure trie db. - Secure(SecTrieDB<'db>), + Secure(SecTrieDB<'db, H>), /// A fat trie db. - Fat(FatDB<'db>), + Fat(FatDB<'db, H>), } // wrapper macro for making the match easier to deal with. @@ -245,8 +252,9 @@ macro_rules! wrapper { } } -impl<'db> Trie for TrieKinds<'db> { - fn root(&self) -> &H256 { +impl<'db, H: Hasher> Trie for TrieKinds<'db, H> { + type H = H; + fn root(&self) -> &::Out { wrapper!(self, root,) } @@ -269,16 +277,14 @@ impl<'db> Trie for TrieKinds<'db> { } } -impl TrieFactory { +impl TrieFactory { /// Creates new factory. pub fn new(spec: TrieSpec) -> Self { - TrieFactory { - spec: spec, - } + TrieFactory { spec } } /// Create new immutable instance of Trie. - pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result> { + pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), @@ -287,7 +293,7 @@ impl TrieFactory { } /// Create new mutable instance of Trie. - pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Box { + pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Box + 'db> { match self.spec { TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), @@ -296,7 +302,7 @@ impl TrieFactory { } /// Create new mutable instance of trie and check for errors. - pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result> { + pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result + 'db>> { match self.spec { TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 2d63f7d00e1..5ea396acd20 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -16,7 +16,7 @@ //! Trie lookup via HashDB. -use hashdb::HashDB; +use hashdb::{HashDB, Hasher}; use nibbleslice::NibbleSlice; use ethereum_types::H256; @@ -24,16 +24,16 @@ use super::{TrieError, Query}; use super::node::Node; /// Trie lookup helper object. -pub struct Lookup<'a, Q: Query> { +pub struct Lookup<'a, H: Hasher + 'a, Q: Query> { /// database to query from. - pub db: &'a HashDB, + pub db: &'a HashDB, /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at - pub hash: H256, + pub hash: H::Out, // TODO } -impl<'a, Q: Query> Lookup<'a, Q> { +impl<'a, H: Hasher, Q: Query> Lookup<'a, H, Q> { /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result> { @@ -82,7 +82,7 @@ impl<'a, Q: Query> Lookup<'a, Q> { // check if new node data is inline or hash. if let Some(h) = Node::try_decode_hash(&node_data) { - hash = h; + hash = as Hasher>::H::Out(h); // REVIEW: this is pretty horrible. Better way? break } } diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 35a515b704d..f5c04ef2641 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -19,10 +19,12 @@ use keccak::keccak; use ethereum_types::H256; use bytes::Bytes; +use hashdb::Hasher; +use std::marker::PhantomData; /// A record of a visited node. #[derive(PartialEq, Eq, Debug, Clone)] -pub struct Record { +pub struct Record { /// The depth of this node. pub depth: u32, @@ -30,23 +32,25 @@ pub struct Record { pub data: Bytes, /// The hash of the data. - pub hash: H256, + pub hash: ::Out, + marker: PhantomData, } /// Records trie nodes as they pass it. #[derive(Debug)] -pub struct Recorder { - nodes: Vec, +pub struct Recorder { + nodes: Vec>, min_depth: u32, + marker: PhantomData, } -impl Default for Recorder { +impl Default for Recorder { fn default() -> Self { Recorder::new() } } -impl Recorder { +impl Recorder { /// Create a new `Recorder` which records all given nodes. #[inline] pub fn new() -> Self { @@ -58,11 +62,13 @@ impl Recorder { Recorder { nodes: Vec::new(), min_depth: depth, + marker: PhantomData } } /// Record a visited node, given its hash, data, and depth. - pub fn record(&mut self, hash: &H256, data: &[u8], depth: u32) { +// pub fn record(&mut self, hash: &H256, data: &[u8], depth: u32) { + pub fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { debug_assert_eq!(keccak(data), *hash); if depth >= self.min_depth { @@ -70,12 +76,13 @@ impl Recorder { depth: depth, data: data.into(), hash: *hash, + marker: PhantomData }) } } /// Drain all visited records. - pub fn drain(&mut self) -> Vec { + pub fn drain(&mut self) -> Vec> { ::std::mem::replace(&mut self.nodes, Vec::new()) } } diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index a9176d022ab..05c42058bdf 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -16,53 +16,55 @@ use ethereum_types::H256; use keccak::keccak; -use hashdb::HashDB; +use hashdb::{HashDB, Hasher}; use super::triedb::TrieDB; use super::{Trie, TrieItem, TrieIterator, Query}; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` trait object. You can use `raw()` to get the backing `TrieDB` object. -pub struct SecTrieDB<'db> { - raw: TrieDB<'db> +pub struct SecTrieDB<'db, H: Hasher + 'db> { + raw: TrieDB<'db, H> } -impl<'db> SecTrieDB<'db> { +impl<'db, H: Hasher> SecTrieDB<'db, H> { /// Create a new trie with the backing database `db` and empty `root` /// /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. /// Returns an error if root does not exist. - pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { Ok(SecTrieDB { raw: TrieDB::new(db, root)? }) } /// Get a reference to the underlying raw `TrieDB` struct. - pub fn raw(&self) -> &TrieDB { + pub fn raw(&self) -> &TrieDB { &self.raw } /// Get a mutable reference to the underlying raw `TrieDB` struct. - pub fn raw_mut(&mut self) -> &mut TrieDB<'db> { + pub fn raw_mut(&mut self) -> &mut TrieDB<'db, H> { &mut self.raw } } -impl<'db> Trie for SecTrieDB<'db> { - fn iter<'a>(&'a self) -> super::Result + 'a>> { - TrieDB::iter(&self.raw) - } +impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> { + type H = H; - fn root(&self) -> &H256 { self.raw.root() } + fn root(&self) -> &::Out{ self.raw.root() } fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(&keccak(key)) + self.raw.contains(&keccak(key)) // TODO } fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result> where 'a: 'key { - self.raw.get_with(&keccak(key), query) + self.raw.get_with(&keccak(key), query) // TODO + } + + fn iter<'a>(&'a self) -> super::Result + 'a>> { + TrieDB::iter(&self.raw) } } diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index b0436b271f6..4449395cce5 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -16,41 +16,42 @@ use ethereum_types::H256; use keccak::keccak; -use hashdb::{HashDB, DBValue}; +use hashdb::{HashDB, DBValue, Hasher}; use super::triedbmut::TrieDBMut; use super::TrieMut; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. -pub struct SecTrieDBMut<'db> { - raw: TrieDBMut<'db> +pub struct SecTrieDBMut<'db, H: Hasher + 'db> { + raw: TrieDBMut<'db, H> } -impl<'db> SecTrieDBMut<'db> { +impl<'db, H: Hasher> SecTrieDBMut<'db, H> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { SecTrieDBMut { raw: TrieDBMut::new(db, root) } } /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> super::Result { Ok(SecTrieDBMut { raw: TrieDBMut::from_existing(db, root)? }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { self.raw.db() } + pub fn db(&self) -> &HashDB { self.raw.db() } /// Get the backing database. - pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } + pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } -impl<'db> TrieMut for SecTrieDBMut<'db> { - fn root(&mut self) -> &H256 { +impl<'db, H: Hasher> TrieMut for SecTrieDBMut<'db, H> { + type H = H; + fn root(&mut self) -> &::Out { self.raw.root() } @@ -65,15 +66,15 @@ impl<'db> TrieMut for SecTrieDBMut<'db> { fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result> where 'a: 'key { - self.raw.get(&keccak(key)) + self.raw.get(&keccak(key)) // TODO } fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { - self.raw.insert(&keccak(key), value) + self.raw.insert(&keccak(key), value) // TODO } fn remove(&mut self, key: &[u8]) -> super::Result> { - self.raw.remove(&keccak(key)) + self.raw.remove(&keccak(key)) // TODO } } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 259525e2ee2..c34739d1c8a 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -49,30 +49,26 @@ use bytes::{ToPretty, Bytes}; /// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar")); /// } /// ``` -pub struct TrieDB<'db> { - db: &'db HashDB, - root: &'db H256, +pub struct TrieDB<'db, H: Hasher + 'db> { + db: &'db HashDB, + root: &'db H::Out, /// The number of hashes performed so far in operations on this trie. hash_count: usize, } -impl<'db> TrieDB<'db> { +impl<'db, H: Hasher> TrieDB<'db, H> { /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist - pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { if !db.contains(root) { Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { - Ok(TrieDB { - db: db, - root: root, - hash_count: 0 - }) + Ok(TrieDB {db, root, hash_count: 0}) } } /// Get the backing database. - pub fn db(&'db self) -> &'db HashDB { + pub fn db(&'db self) -> &'db HashDB { self.db } @@ -101,12 +97,9 @@ impl<'db> TrieDB<'db> { } } -impl<'db> Trie for TrieDB<'db> { - fn iter<'a>(&'a self) -> super::Result + 'a>> { - TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>) - } - - fn root(&self) -> &H256 { self.root } +impl<'db, H: Hasher> Trie for TrieDB<'db, H> { + type H = H; + fn root(&self) -> &::Out { self.root } fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result> where 'a: 'key @@ -117,15 +110,19 @@ impl<'db> Trie for TrieDB<'db> { hash: self.root.clone(), }.look_up(NibbleSlice::new(key)) } + + fn iter<'a>(&'a self) -> super::Result + 'a>> { + TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>) + } } // This is for pretty debug output only -struct TrieAwareDebugNode<'db, 'a> { - trie: &'db TrieDB<'db>, +struct TrieAwareDebugNode<'db, 'a, H: Hasher + 'db> { + trie: &'db TrieDB<'db, H>, key: &'a[u8] } -impl<'db, 'a> fmt::Debug for TrieAwareDebugNode<'db, 'a> { +impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Ok(node) = self.trie.get_raw_or_lookup(self.key) { match Node::decoded(&node) { @@ -161,7 +158,7 @@ impl<'db, 'a> fmt::Debug for TrieAwareDebugNode<'db, 'a> { } -impl<'db> fmt::Debug for TrieDB<'db> { +impl<'db, H: Hasher> fmt::Debug for TrieDB<'db, H> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let root_rlp = self.db.get(self.root).expect("Trie root not found!"); f.debug_struct("TrieDB") @@ -202,15 +199,15 @@ impl Crumb { } /// Iterator for going through all values in the trie. -pub struct TrieDBIterator<'a> { - db: &'a TrieDB<'a>, +pub struct TrieDBIterator<'a, H: Hasher + 'a> { + db: &'a TrieDB<'a, H>, trail: Vec, key_nibbles: Bytes, } -impl<'a> TrieDBIterator<'a> { +impl<'a, H: Hasher> TrieDBIterator<'a, H> { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> super::Result> { + pub fn new(db: &'a TrieDB) -> super::Result> { let mut r = TrieDBIterator { db: db, trail: vec![], @@ -319,7 +316,7 @@ impl<'a> TrieDBIterator<'a> { } } -impl<'a> TrieIterator for TrieDBIterator<'a> { +impl<'a, H: Hasher> TrieIterator for TrieDBIterator<'a, H> { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> super::Result<()> { self.trail.clear(); @@ -329,7 +326,7 @@ impl<'a> TrieIterator for TrieDBIterator<'a> { } } -impl<'a> Iterator for TrieDBIterator<'a> { +impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> { type Item = TrieItem<'a>; fn next(&mut self) -> Option { diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index b8d919deea2..d959b280ccc 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -25,7 +25,7 @@ use hashdb::HashDB; use bytes::ToPretty; use nibbleslice::NibbleSlice; use rlp::{Rlp, RlpStream}; -use hashdb::DBValue; +use hashdb::{Hasher, DBValue, KeccakHasher}; use std::collections::{HashSet, VecDeque}; use std::mem; @@ -87,7 +87,7 @@ enum Node { impl Node { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle { + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle { RlpNode::try_decode_hash(&node) .map(NodeHandle::Hash) .unwrap_or_else(|| { @@ -97,7 +97,7 @@ impl Node { } // decode a node from rlp without getting its children. - fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { + fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { match RlpNode::decoded(rlp).expect("rlp read from db; qed") { RlpNode::Empty => Node::Empty, RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)), @@ -292,22 +292,22 @@ impl<'a> Index<&'a StorageHandle> for NodeStorage { /// assert!(!t.contains(b"foo").unwrap()); /// } /// ``` -pub struct TrieDBMut<'a> { +pub struct TrieDBMut<'a, H: Hasher + 'a> { storage: NodeStorage, - db: &'a mut HashDB, - root: &'a mut H256, + db: &'a mut HashDB, + root: &'a mut H256, // TOOD root_handle: NodeHandle, - death_row: HashSet, + death_row: HashSet, // TODO /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, } -impl<'a> TrieDBMut<'a> { +impl<'a, H: Hasher> TrieDBMut<'a, H> { /// Create a new trie with backing database `db` and empty `root`. - pub fn new(db: &'a mut HashDB, root: &'a mut H256) -> Self { + pub fn new(db: &'a mut HashDB, root: &'a mut H256) -> Self { *root = KECCAK_NULL_RLP; - let root_handle = NodeHandle::Hash(KECCAK_NULL_RLP); + let root_handle = NodeHandle::Hash(KECCAK_NULL_RLP); // TODO TrieDBMut { storage: NodeStorage::empty(), @@ -321,7 +321,7 @@ impl<'a> TrieDBMut<'a> { /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. - pub fn from_existing(db: &'a mut HashDB, root: &'a mut H256) -> super::Result { + pub fn from_existing(db: &'a mut HashDB, root: &'a mut H256) -> super::Result { if !db.contains(root) { return Err(Box::new(TrieError::InvalidStateRoot(*root))); } @@ -337,12 +337,12 @@ impl<'a> TrieDBMut<'a> { }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { + pub fn db(&self) -> &HashDB { self.db } /// Get the backing database mutably. - pub fn db_mut(&mut self) -> &mut HashDB { + pub fn db_mut(&mut self) -> &mut HashDB { self.db } @@ -873,15 +873,17 @@ impl<'a> TrieDBMut<'a> { } } -impl<'a> TrieMut for TrieDBMut<'a> { - fn root(&mut self) -> &H256 { +impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> { + type H = H; + + fn root(&mut self) -> &::Out { self.commit(); self.root } fn is_empty(&self) -> bool { match self.root_handle { - NodeHandle::Hash(h) => h == KECCAK_NULL_RLP, + NodeHandle::Hash(h) => h == KECCAK_NULL_RLP, // TODO NodeHandle::InMemory(ref h) => match self.storage[h] { Node::Empty => true, _ => false, @@ -938,7 +940,7 @@ impl<'a> TrieMut for TrieDBMut<'a> { } } -impl<'a> Drop for TrieDBMut<'a> { +impl<'a, H: Hasher> Drop for TrieDBMut<'a, H> { fn drop(&mut self) { self.commit(); } From 3a7c490a463336319560c115b9d45d56d8370983 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 15:34:11 +0200 Subject: [PATCH 011/160] Use v0.2 of hashdb and memorydb --- util/patricia_trie/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 48b06b21463..fdb09f90348 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -12,10 +12,10 @@ rand = "0.4" ethcore-bytes = { version = "0.1.0", path = "../bytes" } ethereum-types = "0.3" keccak-hash = { version = "0.1.0", path = "../hash" } -hashdb = { version = "0.1.1", path = "../hashdb" } +hashdb = { version = "0.2", path = "../hashdb" } rlp = { version = "0.2.1", path = "../rlp" } triehash = { version = "0.1.0", path = "../triehash" } -memorydb = { version = "0.1.0", path = "../memorydb" } +memorydb = { version = "0.2", path = "../memorydb" } ethcore-logger = { version = "1.9.0", path = "../../logger" } [dev-dependencies] From 83342c0b51db285acfc34c27f906e1208a96d89e Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 15:39:47 +0200 Subject: [PATCH 012/160] Make journaldb use generic hashdb/memorydb Bump version to 0.2.0 **NOTE** Hard coded to use `KeccakHasher` --- util/journaldb/Cargo.toml | 6 +++--- util/journaldb/src/archivedb.rs | 7 ++++--- util/journaldb/src/earlymergedb.rs | 7 ++++--- util/journaldb/src/lib.rs | 3 ++- util/journaldb/src/overlaydb.rs | 3 ++- util/journaldb/src/overlayrecentdb.rs | 11 ++++++----- util/journaldb/src/refcounteddb.rs | 5 +++-- util/journaldb/src/traits.rs | 4 ++-- 8 files changed, 26 insertions(+), 20 deletions(-) diff --git a/util/journaldb/Cargo.toml b/util/journaldb/Cargo.toml index dea70bd6a43..9cd94d59b72 100644 --- a/util/journaldb/Cargo.toml +++ b/util/journaldb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "journaldb" -version = "0.1.0" +version = "0.2.0" authors = ["Parity Technologies "] description = "A `HashDB` which can manage a short-term journal potentially containing many forks of mutually exclusive actions" license = "GPL3" @@ -8,11 +8,11 @@ license = "GPL3" [dependencies] ethcore-bytes = { path = "../bytes" } ethereum-types = "0.3" -hashdb = { path = "../hashdb" } +hashdb = { version = "0.2.0", path = "../hashdb" } heapsize = "0.4" kvdb = { path = "../kvdb" } log = "0.3" -memorydb = { path = "../memorydb" } +memorydb = { version="0.2.0", path = "../memorydb" } parking_lot = "0.5" plain_hasher = { path = "../plain_hasher" } rlp = { path = "../rlp" } diff --git a/util/journaldb/src/archivedb.rs b/util/journaldb/src/archivedb.rs index b58558a332a..18fc9958764 100644 --- a/util/journaldb/src/archivedb.rs +++ b/util/journaldb/src/archivedb.rs @@ -37,7 +37,7 @@ use bytes::Bytes; /// immediately. As this is an "archive" database, nothing is ever removed. This means /// that the states of any block the node has ever processed will be accessible. pub struct ArchiveDB { - overlay: MemoryDB, + overlay: MemoryDB, backing: Arc, latest_era: Option, column: Option, @@ -63,6 +63,7 @@ impl ArchiveDB { } impl HashDB for ArchiveDB { + type H = KeccakHasher; fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| (H256::from_slice(&*key), 1)) @@ -108,7 +109,7 @@ impl HashDB for ArchiveDB { } impl JournalDB for ArchiveDB { - fn boxed_clone(&self) -> Box { + fn boxed_clone(&self) -> Box> { Box::new(ArchiveDB { overlay: self.overlay.clone(), backing: self.backing.clone(), @@ -191,7 +192,7 @@ impl JournalDB for ArchiveDB { &self.backing } - fn consolidate(&mut self, with: MemoryDB) { + fn consolidate(&mut self, with: MemoryDB) { self.overlay.consolidate(with); } } diff --git a/util/journaldb/src/earlymergedb.rs b/util/journaldb/src/earlymergedb.rs index c26a67e0ad2..98a526cca24 100644 --- a/util/journaldb/src/earlymergedb.rs +++ b/util/journaldb/src/earlymergedb.rs @@ -105,7 +105,7 @@ enum RemoveFrom { /// /// TODO: `store_reclaim_period` pub struct EarlyMergeDB { - overlay: MemoryDB, + overlay: MemoryDB, backing: Arc, refs: Option>>>, latest_era: Option, @@ -286,6 +286,7 @@ impl EarlyMergeDB { } impl HashDB for EarlyMergeDB { + type H = KeccakHasher; fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| (H256::from_slice(&*key), 1)) @@ -329,7 +330,7 @@ impl HashDB for EarlyMergeDB { } impl JournalDB for EarlyMergeDB { - fn boxed_clone(&self) -> Box { + fn boxed_clone(&self) -> Box> { Box::new(EarlyMergeDB { overlay: self.overlay.clone(), backing: self.backing.clone(), @@ -513,7 +514,7 @@ impl JournalDB for EarlyMergeDB { Ok(ops) } - fn consolidate(&mut self, with: MemoryDB) { + fn consolidate(&mut self, with: MemoryDB) { self.overlay.consolidate(with); } } diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index c1fb23b6cd1..a6471f39e60 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -39,6 +39,7 @@ extern crate kvdb_memorydb; use std::{fmt, str}; use std::sync::Arc; +use hashdb::KeccakHasher; /// Export the journaldb module. mod traits; @@ -140,7 +141,7 @@ impl fmt::Display for Algorithm { } /// Create a new `JournalDB` trait object over a generic key-value database. -pub fn new(backing: Arc<::kvdb::KeyValueDB>, algorithm: Algorithm, col: Option) -> Box { +pub fn new(backing: Arc<::kvdb::KeyValueDB>, algorithm: Algorithm, col: Option) -> Box> { match algorithm { Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(backing, col)), Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(backing, col)), diff --git a/util/journaldb/src/overlaydb.rs b/util/journaldb/src/overlaydb.rs index 54d0bb12d76..bd7a1a89a0d 100644 --- a/util/journaldb/src/overlaydb.rs +++ b/util/journaldb/src/overlaydb.rs @@ -36,7 +36,7 @@ use kvdb::{KeyValueDB, DBTransaction}; /// queries have an immediate effect in terms of these functions. #[derive(Clone)] pub struct OverlayDB { - overlay: MemoryDB, + overlay: MemoryDB, backing: Arc, column: Option, } @@ -153,6 +153,7 @@ impl OverlayDB { } impl HashDB for OverlayDB { + type H = KeccakHasher; fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| { diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index 2c9ce5cb1dd..b0dbbbc2ee6 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -65,7 +65,7 @@ use util::DatabaseKey; /// 7. Delete ancient record from memory and disk. pub struct OverlayRecentDB { - transaction_overlay: MemoryDB, + transaction_overlay: MemoryDB, backing: Arc, journal_overlay: Arc>, column: Option, @@ -119,8 +119,8 @@ impl<'a> Encodable for DatabaseValueRef<'a> { #[derive(PartialEq)] struct JournalOverlay { - backing_overlay: MemoryDB, // Nodes added in the history period - pending_overlay: H256FastMap, // Nodes being transfered from backing_overlay to backing db + backing_overlay: MemoryDB, // Nodes added in the history period + pending_overlay: H256FastMap, // Nodes being transfered from backing_overlay to backing db journal: HashMap>, latest_era: Option, earliest_era: Option, @@ -242,7 +242,7 @@ fn to_short_key(key: &H256) -> H256 { } impl JournalDB for OverlayRecentDB { - fn boxed_clone(&self) -> Box { + fn boxed_clone(&self) -> Box> { Box::new(self.clone()) } @@ -433,12 +433,13 @@ impl JournalDB for OverlayRecentDB { Ok(ops) } - fn consolidate(&mut self, with: MemoryDB) { + fn consolidate(&mut self, with: MemoryDB) { self.transaction_overlay.consolidate(with); } } impl HashDB for OverlayRecentDB { + type H = KeccakHasher; fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| (H256::from_slice(&*key), 1)) diff --git a/util/journaldb/src/refcounteddb.rs b/util/journaldb/src/refcounteddb.rs index d182d5cf803..b72854b22f7 100644 --- a/util/journaldb/src/refcounteddb.rs +++ b/util/journaldb/src/refcounteddb.rs @@ -79,6 +79,7 @@ impl RefCountedDB { } impl HashDB for RefCountedDB { + type H = KeccakHasher; fn keys(&self) -> HashMap { self.forward.keys() } fn get(&self, key: &H256) -> Option { self.forward.get(key) } fn contains(&self, key: &H256) -> bool { self.forward.contains(key) } @@ -88,7 +89,7 @@ impl HashDB for RefCountedDB { } impl JournalDB for RefCountedDB { - fn boxed_clone(&self) -> Box { + fn boxed_clone(&self) -> Box> { Box::new(RefCountedDB { forward: self.forward.clone(), backing: self.backing.clone(), @@ -197,7 +198,7 @@ impl JournalDB for RefCountedDB { self.forward.commit_to_batch(batch) } - fn consolidate(&mut self, mut with: MemoryDB) { + fn consolidate(&mut self, mut with: MemoryDB) { for (key, (value, rc)) in with.drain() { for _ in 0..rc { self.emplace(key, value.clone()); diff --git a/util/journaldb/src/traits.rs b/util/journaldb/src/traits.rs index aaf5b279701..bfed0e51211 100644 --- a/util/journaldb/src/traits.rs +++ b/util/journaldb/src/traits.rs @@ -27,7 +27,7 @@ use bytes::Bytes; /// exclusive actions. pub trait JournalDB: HashDB { /// Return a copy of ourself, in a box. - fn boxed_clone(&self) -> Box; + fn boxed_clone(&self) -> Box>; /// Returns heap memory size used fn mem_used(&self) -> usize; @@ -76,7 +76,7 @@ pub trait JournalDB: HashDB { fn flush(&self) {} /// Consolidate all the insertions and deletions in the given memory overlay. - fn consolidate(&mut self, overlay: ::memorydb::MemoryDB); + fn consolidate(&mut self, overlay: ::memorydb::MemoryDB); /// Commit all changes in a single batch #[cfg(test)] From aa268bd5f0ce6d2729dc4f35b067f4151531abd6 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 28 May 2018 16:21:42 +0200 Subject: [PATCH 013/160] Version bump: use newer hasdb --- util/plain_hasher/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/plain_hasher/Cargo.toml b/util/plain_hasher/Cargo.toml index f23c823dcf4..9b2cd55033f 100644 --- a/util/plain_hasher/Cargo.toml +++ b/util/plain_hasher/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "plain_hasher" description = "Hasher for 32-bit keys." -version = "0.1.0" +version = "0.2.0" authors = ["Parity Technologies "] license = "MIT" keywords = ["hash", "hasher"] @@ -10,4 +10,4 @@ homepage = "https://github.com/paritytech/plain_hasher" [dependencies] crunchy = "0.1.6" ethereum-types = "0.3" -hashdb = { version = "0.1.1", path = "../hashdb" } \ No newline at end of file +hashdb = { version = "0.2.0", path = "../hashdb" } \ No newline at end of file From bf01073c38d2e603194422e4e04b332372c59ed0 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 12:22:16 +0200 Subject: [PATCH 014/160] Additional bounds on Hasher output --- util/hashdb/Cargo.toml | 3 ++- util/hashdb/src/lib.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/util/hashdb/Cargo.toml b/util/hashdb/Cargo.toml index 15520c1921f..6ac3967ca50 100644 --- a/util/hashdb/Cargo.toml +++ b/util/hashdb/Cargo.toml @@ -9,4 +9,5 @@ license = "GPL-3.0" elastic-array = "0.10" ethereum-types = "0.3" heapsize = "0.4" -tiny-keccak = "1.4.2" \ No newline at end of file +tiny-keccak = "1.4.2" +rlp = "*" \ No newline at end of file diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index fd4da5ceb7a..3f8da276b79 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -19,6 +19,7 @@ extern crate elastic_array; extern crate ethereum_types; extern crate heapsize; extern crate tiny_keccak; +extern crate rlp; use elastic_array::ElasticArray128; use ethereum_types::H256; @@ -28,7 +29,7 @@ use std::{fmt::Debug, hash::Hash}; use tiny_keccak::Keccak; pub trait Hasher: Sync + Send { - type Out: Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? */ + HeapSizeOf; + type Out: AsRef<[u8]> + Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? --> */ + HeapSizeOf /* …and this? -> */ + rlp::Decodable; const HASHED_NULL_RLP: Self::Out; fn hash(x: &[u8]) -> Self::Out; } From 013666f3c5bd0e23d72338c4d76151aba3ca743b Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 12:23:15 +0200 Subject: [PATCH 015/160] TrieDB, TrieDBMut, SecTrieDB, FatDB typechecks --- util/patricia_trie/src/fatdb.rs | 52 ++++----- util/patricia_trie/src/fatdbmut.rs | 29 +++-- util/patricia_trie/src/lib.rs | 141 ++++++++++++------------ util/patricia_trie/src/lookup.rs | 13 +-- util/patricia_trie/src/node.rs | 9 +- util/patricia_trie/src/recorder.rs | 8 +- util/patricia_trie/src/sectriedb.rs | 23 ++-- util/patricia_trie/src/sectriedbmut.rs | 13 ++- util/patricia_trie/src/triedb.rs | 89 ++++++++------- util/patricia_trie/src/triedbmut.rs | 144 ++++++++++++------------- 10 files changed, 254 insertions(+), 267 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index 4cdd131c95a..3d96564c934 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -14,10 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethereum_types::H256; -use keccak::keccak; use hashdb::{HashDB, Hasher}; use super::{TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query}; +use rlp::{Decodable, Encodable}; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// Additionaly it stores inserted hash-key mappings for later retrieval. @@ -27,43 +26,35 @@ pub struct FatDB<'db, H: Hasher + 'db> { raw: TrieDB<'db, H>, } -impl<'db, H: Hasher> FatDB<'db, H> { +impl<'db, H: Hasher> FatDB<'db, H> where H::Out: Decodable { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { - let fatdb = FatDB { - raw: TrieDB::new(db, root)? - }; - - Ok(fatdb) + pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { + Ok(FatDB { raw: TrieDB::new(db, root)? }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { - self.raw.db() - } + pub fn db(&self) -> &HashDB { self.raw.db() } } -impl<'db, H: Hasher> Trie for FatDB<'db, H> { +impl<'db, H: Hasher> Trie for FatDB<'db, H> where H::Out: Decodable + Encodable { type H = H; - fn root(&self) -> &::Out { - self.raw.root() - } + fn root(&self) -> &::Out { self.raw.root() } - fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(&keccak(key)) // TODO + fn contains(&self, key: &[u8]) -> super::Result::Out> { + self.raw.contains(&Self::H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> where 'a: 'key { - self.raw.get_with(&keccak(key), query) // TODO + self.raw.get_with(&Self::H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> super::Result + 'a>> { - FatDBIterator::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) + fn iter<'a>(&'a self) -> super::Result> + 'a>, ::Out> { + FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) } } @@ -73,9 +64,9 @@ pub struct FatDBIterator<'db, H: Hasher + 'db> { trie: &'db TrieDB<'db, H>, } -impl<'db, H: Hasher> FatDBIterator<'db, H> { +impl<'db, H: Hasher> FatDBIterator<'db, H> where H::Out: Decodable { /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> super::Result { + pub fn new(trie: &'db TrieDB) -> super::Result { Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie: trie, @@ -83,20 +74,21 @@ impl<'db, H: Hasher> FatDBIterator<'db, H> { } } -impl<'db, H: Hasher> TrieIterator for FatDBIterator<'db, H> { - fn seek(&mut self, key: &[u8]) -> super::Result<()> { - self.trie_iterator.seek(&keccak(key)) +impl<'db, H: Hasher> TrieIterator for FatDBIterator<'db, H> where H::Out: Decodable { + fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { + let hashed_key = H::hash(key); + self.trie_iterator.seek(&hashed_key.as_ref()) } } -impl<'db, H: Hasher> Iterator for FatDBIterator<'db, H> { - type Item = TrieItem<'db>; +impl<'db, H: Hasher> Iterator for FatDBIterator<'db, H> where H::Out: Decodable { + type Item = TrieItem<'db, H>; fn next(&mut self) -> Option { self.trie_iterator.next() .map(|res| res.map(|(hash, value)| { - let aux_hash = keccak(hash); + let aux_hash = &H::hash(&hash); (self.trie.db().get(&aux_hash).expect("Missing fatdb hash").into_vec(), value) }) ) diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 1d007586365..57f691cf6dc 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethereum_types::H256; -use keccak::keccak; +//use ethereum_types::H256; +//use keccak::keccak; use hashdb::{HashDB, DBValue, Hasher}; use super::{TrieDBMut, TrieMut}; @@ -31,14 +31,14 @@ impl<'db, H: Hasher> FatDBMut<'db, H> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + pub fn new(db: &'db mut HashDB, root: &'db mut H::Out) -> Self { FatDBMut { raw: TrieDBMut::new(db, root) } } /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { Ok(FatDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -52,33 +52,27 @@ impl<'db, H: Hasher> FatDBMut<'db, H> { self.raw.db_mut() } - fn to_aux_key(key: &[u8]) -> H256 { - keccak(key) - } // TODO + fn to_aux_key(key: &[u8]) -> H::Out { H::hash(key) } } impl<'db, H: Hasher> TrieMut for FatDBMut<'db, H> { type H = H; - fn root(&mut self) -> &::Out { - self.raw.root() - } + fn root(&mut self) -> &::Out { self.raw.root() } - fn is_empty(&self) -> bool { - self.raw.is_empty() - } + fn is_empty(&self) -> bool { self.raw.is_empty() } fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(&keccak(key)) + self.raw.contains(Self::H::hash(key)) } fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result> where 'a: 'key { - self.raw.get(&keccak(key)) // TODO + self.raw.get(Self::H::hash(key)) } fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { - let hash = keccak(key); // TODO + let hash = Self::H::hash(key); let out = self.raw.insert(&hash, value)?; let db = self.raw.db_mut(); @@ -90,7 +84,8 @@ impl<'db, H: Hasher> TrieMut for FatDBMut<'db, H> { } fn remove(&mut self, key: &[u8]) -> super::Result> { - let hash = keccak(key); +// let hash = keccak(key); //TODO + let hash = Self::H::hash(key); let out = self.raw.remove(&hash)?; // don't remove if it already exists. diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index a744c8d6501..da353c2389f 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -32,30 +32,28 @@ extern crate trie_standardmap as standardmap; extern crate log; use std::{fmt, error}; -use ethereum_types::H256; -use keccak::KECCAK_NULL_RLP; -use hashdb::{HashDB, DBValue, Hasher, KeccakHasher}; +use hashdb::{HashDB, DBValue, Hasher}; use std::marker::PhantomData; pub mod node; pub mod triedb; pub mod triedbmut; pub mod sectriedb; -pub mod sectriedbmut; +//pub mod sectriedbmut; pub mod recorder; mod fatdb; -mod fatdbmut; +//mod fatdbmut; mod lookup; mod nibbleslice; mod nibblevec; -pub use self::triedbmut::TrieDBMut; pub use self::triedb::{TrieDB, TrieDBIterator}; -pub use self::sectriedbmut::SecTrieDBMut; +pub use self::triedbmut::TrieDBMut; +//pub use self::sectriedbmut::SecTrieDBMut; pub use self::sectriedb::SecTrieDB; pub use self::fatdb::{FatDB, FatDBIterator}; -pub use self::fatdbmut::FatDBMut; +//pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; /// Trie Errors. @@ -63,27 +61,26 @@ pub use self::recorder::Recorder; /// These borrow the data within them to avoid excessive copying on every /// trie operation. #[derive(Debug, PartialEq, Eq, Clone)] -pub enum TrieError { +pub enum TrieError { /// Attempted to create a trie with a state root not in the DB. - InvalidStateRoot(H256), + InvalidStateRoot(T), /// Trie item not found in the database, - IncompleteDatabase(H256), + IncompleteDatabase(T), /// Corrupt Trie item DecoderError(rlp::DecoderError), } -impl fmt::Display for TrieError { +impl fmt::Display for TrieError where T: std::fmt::Debug { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { - TrieError::InvalidStateRoot(ref root) => write!(f, "Invalid state root: {}", root), - TrieError::IncompleteDatabase(ref missing) => - write!(f, "Database missing expected key: {}", missing), + TrieError::InvalidStateRoot(ref root) => write!(f, "Invalid state root: {:?}", root), + TrieError::IncompleteDatabase(ref missing) => write!(f, "Database missing expected key: {:?}", missing), TrieError::DecoderError(ref err) => write!(f, "Decoding failed with {}", err), } } } -impl error::Error for TrieError { +impl error::Error for TrieError where T: std::fmt::Debug { fn description(&self) -> &str { match *self { TrieError::InvalidStateRoot(_) => "Invalid state root", @@ -93,55 +90,50 @@ impl error::Error for TrieError { } } -impl From for Box { +impl From for Box> { fn from(e: rlp::DecoderError) -> Self { Box::new(TrieError::DecoderError(e)) } } /// Trie result type. Boxed to avoid copying around extra space for `H256`s on successful queries. -pub type Result = ::std::result::Result>; +pub type Result = ::std::result::Result>>; /// Trie-Item type. -pub type TrieItem<'a> = Result<(Vec, DBValue)>; +pub type TrieItem<'a, U> = Result<(Vec, DBValue), U>; /// Description of what kind of query will be made to the trie. /// /// This is implemented for any &mut recorder (where the query will return /// a DBValue), any function taking raw bytes (where no recording will be made), /// or any tuple of (&mut Recorder, FnOnce(&[u8])) -pub trait Query { - type H: Hasher; +pub trait Query { /// Output item. type Item; /// Decode a byte-slice into the desired item. - fn decode(self, &[u8]) -> Self::Item; + fn decode(self, data: &[u8]) -> Self::Item; /// Record that a node has been passed through. - fn record(&mut self, hash: &::Out, &[u8], u32) { } + #[allow(unused_variables)] + fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) {} } -impl<'a, H: Hasher> Query for &'a mut Recorder { - type H = H; +impl<'a, H: Hasher> Query for &'a mut Recorder { type Item = DBValue; fn decode(self, value: &[u8]) -> DBValue { DBValue::from_slice(value) } - fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { + fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) { (&mut **self).record(hash, data, depth); } } -// TODO: fix "the type parameter `H` is not constrained by the impl trait, self type, or predicates" -//impl Query for F where F: for<'a> FnOnce(&'a [u8]) -> T { -// type H = H; -// type Item = T; -// -// fn decode(self, value: &[u8]) -> T { (self)(value) } -//} +impl Query for F where F: for<'a> FnOnce(&'a [u8]) -> T { + type Item = T; + fn decode(self, value: &[u8]) -> T { (self)(value) } +} -impl<'a, F, T, H: Hasher> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T { - type H = H; +impl<'a, F, T, H: Hasher> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T { type Item = T; fn decode(self, value: &[u8]) -> T { (self.1)(value) } - fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { + fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) { self.0.record(hash, data, depth) } } @@ -153,25 +145,24 @@ pub trait Trie { fn root(&self) -> &::Out; /// Is the trie empty? - fn is_empty(&self) -> bool { *self.root() == KECCAK_NULL_RLP } + fn is_empty(&self) -> bool { *self.root() == Self::H::HASHED_NULL_RLP } /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result { - self.get(key).map(|x| x.is_some()) + fn contains(&self, key: &[u8]) -> Result::Out> { + self.get(key).map(|x|x.is_some() ) } /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result> where 'a: 'key { + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> where 'a: 'key { self.get_with(key, DBValue::from_slice) } /// Search for the key with the given query parameter. See the docs of the `Query` /// trait for more details. - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) - -> Result> where 'a: 'key; + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> where 'a: 'key; /// Returns a depth-first iterator over the elements of trie. - fn iter<'a>(&'a self) -> Result + 'a>>; + fn iter<'a>(&'a self) -> Result> + 'a>, ::Out>; } /// A key-value datastore implemented as a database-backed modified Merkle tree. @@ -184,26 +175,26 @@ pub trait TrieMut { fn is_empty(&self) -> bool; /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result::Out> { self.get(key).map(|x| x.is_some()) } /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result> where 'a: 'key; + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> where 'a: 'key; /// Insert a `key`/`value` pair into the trie. An empty value is equivalent to removing /// `key` from the trie. Returns the old value associated with this key, if it existed. - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result>; + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out>; /// Remove a `key` from the trie. Equivalent to making it equal to the empty /// value. Returns the old value associated with this key, if it existed. - fn remove(&mut self, key: &[u8]) -> Result>; + fn remove(&mut self, key: &[u8]) -> Result, ::Out>; } /// A trie iterator that also supports random access. -pub trait TrieIterator : Iterator { +pub trait TrieIterator: Iterator { /// Position the iterator on the first element with key > `key` - fn seek(&mut self, key: &[u8]) -> Result<()>; + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out>; } /// Trie types @@ -236,6 +227,7 @@ pub enum TrieKinds<'db, H: Hasher + 'db> { /// A generic trie db. Generic(TrieDB<'db, H>), /// A secure trie db. +// Secure(TrieDB<'db, H>), Secure(SecTrieDB<'db, H>), /// A fat trie db. Fat(FatDB<'db, H>), @@ -251,8 +243,7 @@ macro_rules! wrapper { } } } - -impl<'db, H: Hasher> Trie for TrieKinds<'db, H> { +impl<'db, H: Hasher> Trie for TrieKinds<'db, H> where H::Out: rlp::Decodable + rlp::Encodable { type H = H; fn root(&self) -> &::Out { wrapper!(self, root,) @@ -262,29 +253,31 @@ impl<'db, H: Hasher> Trie for TrieKinds<'db, H> { wrapper!(self, is_empty,) } - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result::Out> { wrapper!(self, contains, key) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> where 'a: 'key { wrapper!(self, get_with, key, query) } - fn iter<'a>(&'a self) -> Result + 'a>> { + fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { wrapper!(self, iter,) } } -impl TrieFactory { +impl TrieFactory +where H::Out: rlp::Decodable + rlp::Encodable +{ /// Creates new factory. pub fn new(spec: TrieSpec) -> Self { - TrieFactory { spec } + TrieFactory { spec, marker: PhantomData } } /// Create new immutable instance of Trie. - pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H256) -> Result> { + pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), @@ -293,23 +286,29 @@ impl TrieFactory { } /// Create new mutable instance of Trie. - pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Box + 'db> { - match self.spec { - TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), - TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), - TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), - } - } +// pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { +// match self.spec { +// TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), +// TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), +// TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), +// } +// } /// Create new mutable instance of trie and check for errors. - pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H256) -> Result + 'db>> { - match self.spec { - TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), - TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), - TrieSpec::Fat => Ok(Box::new(FatDBMut::from_existing(db, root)?)), - } - } +// pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { +// match self.spec { +// TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), +// TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), +// TrieSpec::Fat => Ok(Box::new(FatDBMut::from_existing(db, root)?)), +// } +// } /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } } + +/* +fn decode_node(node: &'db [u8]) -> Node { + Node::decoded(node).expect("rlp read from db; qed") + } +*/ \ No newline at end of file diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 5ea396acd20..d66cb3e20f8 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -18,13 +18,14 @@ use hashdb::{HashDB, Hasher}; use nibbleslice::NibbleSlice; -use ethereum_types::H256; +//use ethereum_types::H256; use super::{TrieError, Query}; use super::node::Node; +use rlp::Decodable; /// Trie lookup helper object. -pub struct Lookup<'a, H: Hasher + 'a, Q: Query> { +pub struct Lookup<'a, H: Hasher + 'a, Q: Query> { /// database to query from. pub db: &'a HashDB, /// Query object to record nodes and transform data. @@ -33,10 +34,10 @@ pub struct Lookup<'a, H: Hasher + 'a, Q: Query> { pub hash: H::Out, // TODO } -impl<'a, H: Hasher, Q: Query> Lookup<'a, H, Q> { +impl<'a, H: Hasher + 'a, Q: Query> Lookup<'a, H, Q> where H::Out: Decodable { /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. - pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result> { + pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result, H::Out> { let mut hash = self.hash; // this loop iterates through non-inline nodes. @@ -81,8 +82,8 @@ impl<'a, H: Hasher, Q: Query> Lookup<'a, H, Q> { } // check if new node data is inline or hash. - if let Some(h) = Node::try_decode_hash(&node_data) { - hash = as Hasher>::H::Out(h); // REVIEW: this is pretty horrible. Better way? + if let Some(h) = Node::try_decode_hash::(&node_data) { + hash = h; break } } diff --git a/util/patricia_trie/src/node.rs b/util/patricia_trie/src/node.rs index 0b99acded34..9bbea4e2d1a 100644 --- a/util/patricia_trie/src/node.rs +++ b/util/patricia_trie/src/node.rs @@ -14,12 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethereum_types::H256; use elastic_array::ElasticArray36; use nibbleslice::NibbleSlice; use nibblevec::NibbleVec; use bytes::*; -use rlp::{Rlp, RlpStream, Prototype, DecoderError}; +use rlp::{Rlp, RlpStream, Prototype, DecoderError, self}; use hashdb::DBValue; /// Partial node key type. @@ -35,7 +34,7 @@ pub enum Node<'a> { /// Extension node; has key slice and node data. Data may not be null. Extension(NibbleSlice<'a>, &'a [u8]), /// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data. - Branch([&'a [u8]; 16], Option<&'a [u8]>) + Branch([&'a [u8]; 16], Option<&'a [u8]>), } impl<'a> Node<'a> { @@ -104,7 +103,9 @@ impl<'a> Node<'a> { } } - pub fn try_decode_hash(node_data: &[u8]) -> Option { + pub fn try_decode_hash(node_data: &[u8]) -> Option + where O: rlp::Decodable // REVIEW: this is not necessary but is perhaps useful when reading the code? Keep or leave out? + { let r = Rlp::new(node_data); if r.is_data() && r.size() == 32 { Some(r.as_val().expect("Hash is the correct size of 32 bytes; qed")) diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index f5c04ef2641..7add50152f6 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -16,8 +16,8 @@ //! Trie query recorder. -use keccak::keccak; -use ethereum_types::H256; +//use keccak::keccak; +//use ethereum_types::H256; use bytes::Bytes; use hashdb::Hasher; use std::marker::PhantomData; @@ -32,7 +32,7 @@ pub struct Record { pub data: Bytes, /// The hash of the data. - pub hash: ::Out, + pub hash: H::Out, marker: PhantomData, } @@ -69,7 +69,7 @@ impl Recorder { /// Record a visited node, given its hash, data, and depth. // pub fn record(&mut self, hash: &H256, data: &[u8], depth: u32) { pub fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { - debug_assert_eq!(keccak(data), *hash); + debug_assert_eq!(H::hash(data), *hash); if depth >= self.min_depth { self.nodes.push(Record { diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index 05c42058bdf..fde26792bfd 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -14,11 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethereum_types::H256; -use keccak::keccak; +//use ethereum_types::H256; +//use keccak::keccak; use hashdb::{HashDB, Hasher}; use super::triedb::TrieDB; use super::{Trie, TrieItem, TrieIterator, Query}; +use rlp::{Decodable, Encodable}; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// @@ -27,13 +28,13 @@ pub struct SecTrieDB<'db, H: Hasher + 'db> { raw: TrieDB<'db, H> } -impl<'db, H: Hasher> SecTrieDB<'db, H> { +impl<'db, H: Hasher> SecTrieDB<'db, H> where H::Out: Decodable { /// Create a new trie with the backing database `db` and empty `root` /// /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. /// Returns an error if root does not exist. - pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { Ok(SecTrieDB { raw: TrieDB::new(db, root)? }) } @@ -48,22 +49,22 @@ impl<'db, H: Hasher> SecTrieDB<'db, H> { } } -impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> { +impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> where H::Out: Decodable + Encodable { type H = H; - fn root(&self) -> &::Out{ self.raw.root() } + fn root(&self) -> &::Out { self.raw.root() } - fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(&keccak(key)) // TODO + fn contains(&self, key: &[u8]) -> super::Result::Out> { + self.raw.contains(&Self::H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> where 'a: 'key { - self.raw.get_with(&keccak(key), query) // TODO + self.raw.get_with(&Self::H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> super::Result + 'a>> { + fn iter<'a>(&'a self) -> super::Result> + 'a>, ::Out> { TrieDB::iter(&self.raw) } } diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 4449395cce5..c981d9cc4f3 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethereum_types::H256; use keccak::keccak; use hashdb::{HashDB, DBValue, Hasher}; use super::triedbmut::TrieDBMut; @@ -31,14 +30,14 @@ impl<'db, H: Hasher> SecTrieDBMut<'db, H> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + pub fn new(db: &'db mut HashDB, root: &'db mut H::Out) -> Self { SecTrieDBMut { raw: TrieDBMut::new(db, root) } } /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { Ok(SecTrieDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -60,21 +59,21 @@ impl<'db, H: Hasher> TrieMut for SecTrieDBMut<'db, H> { } fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(&keccak(key)) + self.raw.contains(&Self::H::hash(key)) } fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result> where 'a: 'key { - self.raw.get(&keccak(key)) // TODO + self.raw.get(&Self::H::hash(key)) } fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { - self.raw.insert(&keccak(key), value) // TODO + self.raw.insert(&Self::H::hash(key), value) } fn remove(&mut self, key: &[u8]) -> super::Result> { - self.raw.remove(&keccak(key)) // TODO + self.raw.remove(&Self::H::hash(key)) } } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index c34739d1c8a..8418fabfc41 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -20,8 +20,8 @@ use nibbleslice::NibbleSlice; use super::node::{Node, OwnedNode}; use super::lookup::Lookup; use super::{Trie, TrieItem, TrieError, TrieIterator, Query}; -use ethereum_types::H256; -use bytes::{ToPretty, Bytes}; +use bytes::Bytes; +use rlp::{Decodable, Encodable}; /// A `Trie` implementation using a generic `HashDB` backing database. /// @@ -56,10 +56,10 @@ pub struct TrieDB<'db, H: Hasher + 'db> { hash_count: usize, } -impl<'db, H: Hasher> TrieDB<'db, H> { +impl<'db, H: Hasher> TrieDB<'db, H> where H::Out: Decodable { /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist - pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { if !db.contains(root) { Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { @@ -68,12 +68,10 @@ impl<'db, H: Hasher> TrieDB<'db, H> { } /// Get the backing database. - pub fn db(&'db self) -> &'db HashDB { - self.db - } + pub fn db(&'db self) -> &'db HashDB { self.db } /// Get the data of the root node. - fn root_data(&self) -> super::Result { + fn root_data(&self) -> super::Result { self.db .get(self.root) .ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) @@ -82,26 +80,22 @@ impl<'db, H: Hasher> TrieDB<'db, H> { /// Given some node-describing data `node`, return the actual node RLP. /// This could be a simple identity operation in the case that the node is sufficiently small, but /// may require a database lookup. - fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> super::Result { - match Node::try_decode_hash(node) { + fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> super::Result { + // REVIEW: is it better to parametrize on `H` or like this with `H::Out` (saves us from adding `use hashdb::Hasher`) + match Node::try_decode_hash::(node) { Some(key) => { self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) } None => Ok(DBValue::from_slice(node)) } } - - /// Create a node from raw rlp bytes, assumes valid rlp because encoded locally - fn decode_node(node: &'db [u8]) -> Node { - Node::decoded(node).expect("rlp read from db; qed") - } } -impl<'db, H: Hasher> Trie for TrieDB<'db, H> { +impl<'db, H: Hasher> Trie for TrieDB<'db, H> where H::Out: Decodable + Encodable { type H = H; fn root(&self) -> &::Out { self.root } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, H::Out> where 'a: 'key { Lookup { @@ -111,7 +105,7 @@ impl<'db, H: Hasher> Trie for TrieDB<'db, H> { }.look_up(NibbleSlice::new(key)) } - fn iter<'a>(&'a self) -> super::Result + 'a>> { + fn iter<'a>(&'a self) -> super::Result> + 'a>, H::Out> { TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>) } } @@ -122,7 +116,7 @@ struct TrieAwareDebugNode<'db, 'a, H: Hasher + 'db> { key: &'a[u8] } -impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> { +impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> where H::Out: Decodable { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Ok(node) = self.trie.get_raw_or_lookup(self.key) { match Node::decoded(&node) { @@ -135,7 +129,7 @@ impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> { .field("item", &TrieAwareDebugNode{trie: self.trie, key: item}) .finish(), Ok(Node::Branch(ref nodes, ref value)) => { - let nodes: Vec = nodes.into_iter().map(|n| TrieAwareDebugNode{trie: self.trie, key: n} ).collect(); + let nodes: Vec> = nodes.into_iter().map(|n| TrieAwareDebugNode{trie: self.trie, key: n} ).collect(); f.debug_struct("Node::Branch") .field("nodes", &nodes) .field("value", &value) @@ -158,7 +152,7 @@ impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> { } -impl<'db, H: Hasher> fmt::Debug for TrieDB<'db, H> { +impl<'db, H: Hasher> fmt::Debug for TrieDB<'db, H> where H::Out: Decodable { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let root_rlp = self.db.get(self.root).expect("Trie root not found!"); f.debug_struct("TrieDB") @@ -205,9 +199,9 @@ pub struct TrieDBIterator<'a, H: Hasher + 'a> { key_nibbles: Bytes, } -impl<'a, H: Hasher> TrieDBIterator<'a, H> { +impl<'a, H: Hasher> TrieDBIterator<'a, H> where H::Out: Decodable { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> super::Result> { + pub fn new(db: &'a TrieDB) -> super::Result, H::Out> { let mut r = TrieDBIterator { db: db, trail: vec![], @@ -218,10 +212,10 @@ impl<'a, H: Hasher> TrieDBIterator<'a, H> { Ok(r) } - fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> super::Result<()> { + fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> super::Result<(), H::Out> { loop { let (data, mid) = { - let node = TrieDB::decode_node(&node_data); + let node = Node::decoded(&node_data).expect("rlp read from db; qed"); match node { Node::Leaf(slice, _) => { if slice == key { @@ -282,9 +276,10 @@ impl<'a, H: Hasher> TrieDBIterator<'a, H> { } /// Descend into a payload. - fn descend(&mut self, d: &[u8]) -> super::Result<()> { - let node = TrieDB::decode_node(&self.db.get_raw_or_lookup(d)?).into(); - Ok(self.descend_into_node(node)) + fn descend(&mut self, d: &[u8]) -> super::Result<(), H::Out> { + let node_data = &self.db.get_raw_or_lookup(d)?; + let node = Node::decoded(&node_data).expect("rlp read from db; qed"); + Ok(self.descend_into_node(node.into())) } /// Descend into a payload. @@ -316,9 +311,9 @@ impl<'a, H: Hasher> TrieDBIterator<'a, H> { } } -impl<'a, H: Hasher> TrieIterator for TrieDBIterator<'a, H> { +impl<'a, H: Hasher> TrieIterator for TrieDBIterator<'a, H> where H::Out: Decodable { /// Position the iterator on the first element with key >= `key` - fn seek(&mut self, key: &[u8]) -> super::Result<()> { + fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { self.trail.clear(); self.key_nibbles.clear(); let root_rlp = self.db.root_data()?; @@ -326,16 +321,16 @@ impl<'a, H: Hasher> TrieIterator for TrieDBIterator<'a, H> { } } -impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> { - type Item = TrieItem<'a>; + +impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> where H::Out: Decodable { + type Item = TrieItem<'a, H>; fn next(&mut self) -> Option { - enum IterStep { + enum IterStep { Continue, PopTrail, - Descend(super::Result), + Descend(super::Result), } - loop { let iter_step = { self.trail.last_mut()?.increment(); @@ -356,7 +351,7 @@ impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> { (Status::At, &OwnedNode::Leaf(_, ref v)) | (Status::At, &OwnedNode::Branch(_, Some(ref v))) => { return Some(Ok((self.key(), v.clone()))); }, - (Status::At, &OwnedNode::Extension(_, ref d)) => IterStep::Descend(self.db.get_raw_or_lookup(&*d)), + (Status::At, &OwnedNode::Extension(_, ref d)) => IterStep::Descend::(self.db.get_raw_or_lookup(&*d)), (Status::At, &OwnedNode::Branch(_, _)) => IterStep::Continue, (Status::AtChild(i), &OwnedNode::Branch(ref children, _)) if children[i].len() > 0 => { match i { @@ -364,7 +359,7 @@ impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> { i => *self.key_nibbles.last_mut() .expect("pushed as 0; moves sequentially; removed afterwards; qed") = i as u8, } - IterStep::Descend(self.db.get_raw_or_lookup(&*children[i])) + IterStep::Descend::(self.db.get_raw_or_lookup(&*children[i])) }, (Status::AtChild(i), &OwnedNode::Branch(_, _)) => { if i == 0 { @@ -380,11 +375,23 @@ impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> { IterStep::PopTrail => { self.trail.pop(); }, - IterStep::Descend(Ok(d)) => { - self.descend_into_node(TrieDB::decode_node(&d).into()) + IterStep::Descend::(Ok(d)) => { + let node = Node::decoded(&d).expect("rlp read from db; qed"); + self.descend_into_node(node.into()) }, - IterStep::Descend(Err(e)) => { - return Some(Err(e)) + IterStep::Descend::(Err(e)) => { +// return Some(Err(e)) // <–– REVIEW: This causes a compiler error I can't figure out: + /* + error[E0308]: mismatched types + --> util/patricia_trie/src/triedb.rs:404:22 + | + 404 | return Some(Err(e)) + | ^ expected type parameter, found associated type + | + = note: expected type `std::boxed::Box>` + found type `std::boxed::Box::Out>>` + */ + panic!("FIXME: this causes `expected type parameter, found associated type`") } IterStep::Continue => {}, } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index d959b280ccc..4ee3f50a69d 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -24,15 +24,13 @@ use super::node::NodeKey; use hashdb::HashDB; use bytes::ToPretty; use nibbleslice::NibbleSlice; -use rlp::{Rlp, RlpStream}; -use hashdb::{Hasher, DBValue, KeccakHasher}; +use rlp::{Rlp, RlpStream, Encodable, Decodable}; +use hashdb::{Hasher, DBValue}; use std::collections::{HashSet, VecDeque}; use std::mem; use std::ops::Index; -use ethereum_types::H256; use elastic_array::ElasticArray1024; -use keccak::{KECCAK_NULL_RLP}; // For lookups into the Node storage buffer. // This is deliberately non-copyable. @@ -41,26 +39,26 @@ struct StorageHandle(usize); // Handles to nodes in the trie. #[derive(Debug)] -enum NodeHandle { +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node - Hash(H256), + Hash(H::Out), } -impl From for NodeHandle { +impl From for NodeHandle { fn from(handle: StorageHandle) -> Self { NodeHandle::InMemory(handle) } } -impl From for NodeHandle { - fn from(hash: H256) -> Self { - NodeHandle::Hash(hash) - } -} +//impl From for NodeHandle { +// fn from(hash: H256) -> Self { +// NodeHandle::Hash(hash) +// } +//} -fn empty_children() -> Box<[Option; 16]> { +fn empty_children() -> Box<[Option>; 16]> { Box::new([ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, @@ -69,7 +67,7 @@ fn empty_children() -> Box<[Option; 16]> { /// Node types in the Trie. #[derive(Debug)] -enum Node { +enum Node { /// Empty node. Empty, /// A leaf node contains the end of a key and a value. @@ -80,15 +78,15 @@ enum Node { /// The shared portion is encoded from a `NibbleSlice` meaning it contains /// a flag indicating it is an extension. /// The child node is always a branch. - Extension(NodeKey, NodeHandle), + Extension(NodeKey, NodeHandle), /// A branch has up to 16 children and an optional value. - Branch(Box<[Option; 16]>, Option) + Branch(Box<[Option>; 16]>, Option) } -impl Node { +impl Node where H::Out: Decodable { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle { - RlpNode::try_decode_hash(&node) + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle { + RlpNode::try_decode_hash::(&node) .map(NodeHandle::Hash) .unwrap_or_else(|| { let child = Node::from_rlp(node, db, storage); @@ -97,7 +95,7 @@ impl Node { } // decode a node from rlp without getting its children. - fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { + fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { match RlpNode::decoded(rlp).expect("rlp read from db; qed") { RlpNode::Empty => Node::Empty, RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)), @@ -130,7 +128,7 @@ impl Node { // encode a node to RLP // TODO: parallelize fn into_rlp(self, mut child_cb: F) -> ElasticArray1024 - where F: FnMut(NodeHandle, &mut RlpStream) + where F: FnMut(NodeHandle, &mut RlpStream) { match self { Node::Empty => { @@ -172,25 +170,25 @@ impl Node { } // post-inspect action. -enum Action { +enum Action { // Replace a node with a new one. - Replace(Node), + Replace(Node), // Restore the original node. This trusts that the node is actually the original. - Restore(Node), + Restore(Node), // if it is a new node, just clears the storage. Delete, } // post-insert action. Same as action without delete -enum InsertAction { +enum InsertAction { // Replace a node with a new one. - Replace(Node), + Replace(Node), // Restore the original node. - Restore(Node), + Restore(Node), } -impl InsertAction { - fn into_action(self) -> Action { +impl InsertAction { + fn into_action(self) -> Action { match self { InsertAction::Replace(n) => Action::Replace(n), InsertAction::Restore(n) => Action::Restore(n), @@ -198,7 +196,7 @@ impl InsertAction { } // unwrap the node, disregarding replace or restore state. - fn unwrap_node(self) -> Node { + fn unwrap_node(self) -> Node { match self { InsertAction::Replace(n) | InsertAction::Restore(n) => n, } @@ -206,20 +204,20 @@ impl InsertAction { } // What kind of node is stored here. -enum Stored { +enum Stored { // A new node. - New(Node), + New(Node), // A cached node, loaded from the DB. - Cached(Node, H256), + Cached(Node, H::Out), } /// Compact and cache-friendly storage for Trie nodes. -struct NodeStorage { - nodes: Vec, +struct NodeStorage { + nodes: Vec>, free_indices: VecDeque, } -impl NodeStorage { +impl NodeStorage { /// Create a new storage. fn empty() -> Self { NodeStorage { @@ -229,7 +227,7 @@ impl NodeStorage { } /// Allocate a new node in the storage. - fn alloc(&mut self, stored: Stored) -> StorageHandle { + fn alloc(&mut self, stored: Stored) -> StorageHandle { if let Some(idx) = self.free_indices.pop_front() { self.nodes[idx] = stored; StorageHandle(idx) @@ -240,7 +238,7 @@ impl NodeStorage { } /// Remove a node from the storage, consuming the handle and returning the node. - fn destroy(&mut self, handle: StorageHandle) -> Stored { + fn destroy(&mut self, handle: StorageHandle) -> Stored { let idx = handle.0; self.free_indices.push_back(idx); @@ -248,10 +246,10 @@ impl NodeStorage { } } -impl<'a> Index<&'a StorageHandle> for NodeStorage { - type Output = Node; +impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { + type Output = Node; - fn index(&self, handle: &'a StorageHandle) -> &Node { + fn index(&self, handle: &'a StorageHandle) -> &Node { match self.nodes[handle.0] { Stored::New(ref node) => node, Stored::Cached(ref node, _) => node, @@ -292,22 +290,22 @@ impl<'a> Index<&'a StorageHandle> for NodeStorage { /// assert!(!t.contains(b"foo").unwrap()); /// } /// ``` -pub struct TrieDBMut<'a, H: Hasher + 'a> { - storage: NodeStorage, +pub struct TrieDBMut<'a, H: Hasher + 'a> where H::Out: Decodable + Encodable { + storage: NodeStorage, db: &'a mut HashDB, - root: &'a mut H256, // TOOD - root_handle: NodeHandle, - death_row: HashSet, // TODO + root: &'a mut H::Out, + root_handle: NodeHandle, + death_row: HashSet, /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, } -impl<'a, H: Hasher> TrieDBMut<'a, H> { +impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { /// Create a new trie with backing database `db` and empty `root`. - pub fn new(db: &'a mut HashDB, root: &'a mut H256) -> Self { - *root = KECCAK_NULL_RLP; - let root_handle = NodeHandle::Hash(KECCAK_NULL_RLP); // TODO + pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { + *root = H::HASHED_NULL_RLP; + let root_handle = NodeHandle::Hash(H::HASHED_NULL_RLP); TrieDBMut { storage: NodeStorage::empty(), @@ -321,7 +319,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. - pub fn from_existing(db: &'a mut HashDB, root: &'a mut H256) -> super::Result { + pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> super::Result { if !db.contains(root) { return Err(Box::new(TrieError::InvalidStateRoot(*root))); } @@ -347,7 +345,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } // cache a node by hash - fn cache(&mut self, hash: H256) -> super::Result { + fn cache(&mut self, hash: H::Out) -> super::Result { let node_rlp = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; let node = Node::from_rlp(&node_rlp, &*self.db, &mut self.storage); Ok(self.storage.alloc(Stored::Cached(node, hash))) @@ -355,8 +353,8 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { // inspect a node, choosing either to replace, restore, or delete it. // if restored or replaced, returns the new node along with a flag of whether it was changed. - fn inspect(&mut self, stored: Stored, inspector: F) -> super::Result> - where F: FnOnce(&mut Self, Node) -> super::Result { + fn inspect(&mut self, stored: Stored, inspector: F) -> super::Result, bool)>, H::Out> + where F: FnOnce(&mut Self, Node) -> super::Result, H::Out> { Ok(match stored { Stored::New(node) => match inspector(self, node)? { Action::Restore(node) => Some((Stored::New(node), false)), @@ -378,7 +376,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } // walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> super::Result> + fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> super::Result, H::Out> where 'x: 'key { let mut handle = handle; @@ -426,9 +424,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } /// insert a key, value pair into the trie, creating new nodes if necessary. - fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) - -> super::Result<(StorageHandle, bool)> - { + fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> super::Result<(StorageHandle, bool), H::Out> { let h = match handle { NodeHandle::InMemory(h) => h, NodeHandle::Hash(h) => self.cache(h)?, @@ -442,9 +438,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } /// the insertion inspector. - fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) - -> super::Result - { + fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> super::Result, H::Out> { trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty()); Ok(match node { @@ -605,9 +599,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } /// Remove a node from the trie based on key. - fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) - -> super::Result> - { + fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> super::Result, H::Out> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), NodeHandle::Hash(h) => { @@ -622,7 +614,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } /// the removal inspector - fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> super::Result { + fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> super::Result, H::Out> { Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete, (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), @@ -708,7 +700,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { /// _invalid state_ means: /// - Branch node where there is only a single entry; /// - Extension node followed by anything other than a Branch node. - fn fix(&mut self, node: Node) -> super::Result { + fn fix(&mut self, node: Node) -> super::Result, H::Out> { match node { Node::Branch(mut children, value) => { // if only a single value, transmute to leaf/extension and feed through fixed. @@ -845,7 +837,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { /// commit a node, hashing it, committing it to the db, /// and writing it to the rlp stream as necessary. - fn commit_node(&mut self, handle: NodeHandle, stream: &mut RlpStream) { + fn commit_node(&mut self, handle: NodeHandle, stream: &mut RlpStream) { match handle { NodeHandle::Hash(h) => stream.append(&h), NodeHandle::InMemory(h) => match self.storage.destroy(h) { @@ -865,7 +857,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } // a hack to get the root node's handle - fn root_handle(&self) -> NodeHandle { + fn root_handle(&self) -> NodeHandle { match self.root_handle { NodeHandle::Hash(h) => NodeHandle::Hash(h), NodeHandle::InMemory(StorageHandle(x)) => NodeHandle::InMemory(StorageHandle(x)), @@ -873,7 +865,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> { } } -impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> { +impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> where H::Out: Decodable + Encodable { type H = H; fn root(&mut self) -> &::Out { @@ -883,7 +875,7 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> { fn is_empty(&self) -> bool { match self.root_handle { - NodeHandle::Hash(h) => h == KECCAK_NULL_RLP, // TODO + NodeHandle::Hash(h) => h == Self::H::HASHED_NULL_RLP, // TODO NodeHandle::InMemory(ref h) => match self.storage[h] { Node::Empty => true, _ => false, @@ -891,12 +883,12 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> { } } - fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result> where 'x: 'key { + fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result, ::Out> where 'x: 'key { self.lookup(NibbleSlice::new(key), &self.root_handle) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result, ::Out> { if value.is_empty() { return self.remove(key) } let mut old_val = None; @@ -917,7 +909,7 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> { Ok(old_val) } - fn remove(&mut self, key: &[u8]) -> super::Result> { + fn remove(&mut self, key: &[u8]) -> super::Result, ::Out> { trace!(target: "trie", "remove: key={:?}", key.pretty()); let root_handle = self.root_handle(); @@ -931,8 +923,8 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> { } None => { trace!(target: "trie", "remove: obliterated trie"); - self.root_handle = NodeHandle::Hash(KECCAK_NULL_RLP); - *self.root = KECCAK_NULL_RLP; + self.root_handle = NodeHandle::Hash(Self::H::HASHED_NULL_RLP); + *self.root = Self::H::HASHED_NULL_RLP; } } @@ -940,7 +932,7 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> { } } -impl<'a, H: Hasher> Drop for TrieDBMut<'a, H> { +impl<'a, H: Hasher> Drop for TrieDBMut<'a, H> where H::Out: Decodable + Encodable { fn drop(&mut self) { self.commit(); } From 1f892fe8b3ace64345b322c59d695d800348bfd8 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 12:56:51 +0200 Subject: [PATCH 016/160] Fix tests --- util/patricia_trie/src/triedb.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 47449bdcda4..49ae4756a9e 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -41,7 +41,7 @@ use rlp::{Decodable, Encodable}; /// use ethereum_types::H256; /// /// fn main() { -/// let mut memdb = MemoryDB::new(); +/// let mut memdb = MemoryDB::::new(); /// let mut root = H256::new(); /// TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap(); /// let t = TrieDB::new(&memdb, &root).unwrap(); @@ -404,11 +404,12 @@ fn iterator() { use memorydb::*; use super::TrieMut; use super::triedbmut::*; + use hashdb::KeccakHasher; let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { @@ -429,8 +430,8 @@ fn iterator_seek() { let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { @@ -472,8 +473,8 @@ fn get_len() { use super::TrieMut; use super::triedbmut::*; - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); @@ -495,8 +496,8 @@ fn debug_output_supports_pretty_print() { let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); let root = { let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { @@ -606,8 +607,8 @@ fn test_lookup_with_corrupt_data_returns_decoder_error() { use rlp; use ethereum_types::H512; - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); From f98e905f1350d043b28e41734efb9d8a2518e66b Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 15:01:57 +0200 Subject: [PATCH 017/160] FatDBMut and test fixes --- util/patricia_trie/src/fatdb.rs | 21 +- util/patricia_trie/src/fatdbmut.rs | 63 ++- util/patricia_trie/src/lib.rs | 11 +- util/patricia_trie/src/recorder.rs | 250 ++++----- util/patricia_trie/src/sectriedb.rs | 36 +- util/patricia_trie/src/sectriedbmut.rs | 30 +- util/patricia_trie/src/triedb.rs | 3 +- util/patricia_trie/src/triedbmut.rs | 732 ++++++++++++------------- 8 files changed, 571 insertions(+), 575 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index 3d96564c934..b818b2d3f85 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -44,13 +44,13 @@ impl<'db, H: Hasher> Trie for FatDB<'db, H> where H::Out: Decodable + Encodable fn root(&self) -> &::Out { self.raw.root() } fn contains(&self, key: &[u8]) -> super::Result::Out> { - self.raw.contains(&Self::H::hash(key).as_ref()) + self.raw.contains(Self::H::hash(key).as_ref()) } fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> where 'a: 'key { - self.raw.get_with(&Self::H::hash(key).as_ref(), query) + self.raw.get_with(Self::H::hash(key).as_ref(), query) } fn iter<'a>(&'a self) -> super::Result> + 'a>, ::Out> { @@ -77,7 +77,7 @@ impl<'db, H: Hasher> FatDBIterator<'db, H> where H::Out: Decodable { impl<'db, H: Hasher> TrieIterator for FatDBIterator<'db, H> where H::Out: Decodable { fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { let hashed_key = H::hash(key); - self.trie_iterator.seek(&hashed_key.as_ref()) + self.trie_iterator.seek(hashed_key.as_ref()) } } @@ -86,12 +86,12 @@ impl<'db, H: Hasher> Iterator for FatDBIterator<'db, H> where H::Out: Decodable fn next(&mut self) -> Option { self.trie_iterator.next() - .map(|res| + .map(|res| { res.map(|(hash, value)| { - let aux_hash = &H::hash(&hash); + let aux_hash = H::hash(&hash); (self.trie.db().get(&aux_hash).expect("Missing fatdb hash").into_vec(), value) }) - ) + }) } } @@ -101,14 +101,17 @@ fn fatdb_to_trie() { use hashdb::DBValue; use super::fatdbmut::FatDBMut; use super::TrieMut; + use hashdb::KeccakHasher; - let mut memdb = MemoryDB::new(); - let mut root = H256::default(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); { let mut t = FatDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } let t = FatDB::new(&memdb, &root).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); - assert_eq!(t.iter().unwrap().map(Result::unwrap).collect::>(), vec![(vec![0x01u8, 0x23], DBValue::from_slice(&[0x01u8, 0x23] as &[u8]))]); + assert_eq!( + t.iter().unwrap().map(Result::unwrap).collect::>(), + vec![(vec![0x01u8, 0x23], DBValue::from_slice(&[0x01u8, 0x23] as &[u8]))]); } diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 57f691cf6dc..083dcad6fcd 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -18,16 +18,17 @@ //use keccak::keccak; use hashdb::{HashDB, DBValue, Hasher}; use super::{TrieDBMut, TrieMut}; +use rlp::{Encodable, Decodable}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// Additionaly it stores inserted hash-key mappings for later retrieval. /// /// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDBMut<'db, H: Hasher + 'db> { +pub struct FatDBMut<'db, H: Hasher + 'db> where H::Out: Decodable + Encodable { raw: TrieDBMut<'db, H>, } -impl<'db, H: Hasher> FatDBMut<'db, H> { +impl<'db, H: Hasher> FatDBMut<'db, H> where H::Out: Decodable + Encodable { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. @@ -38,7 +39,7 @@ impl<'db, H: Hasher> FatDBMut<'db, H> { /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { Ok(FatDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -51,64 +52,62 @@ impl<'db, H: Hasher> FatDBMut<'db, H> { pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } - - fn to_aux_key(key: &[u8]) -> H::Out { H::hash(key) } } -impl<'db, H: Hasher> TrieMut for FatDBMut<'db, H> { +impl<'db, H: Hasher> TrieMut for FatDBMut<'db, H> where H::Out: Decodable + Encodable { type H = H; fn root(&mut self) -> &::Out { self.raw.root() } fn is_empty(&self) -> bool { self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(Self::H::hash(key)) + fn contains(&self, key: &[u8]) -> super::Result::Out> { + self.raw.contains(Self::H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result, ::Out> where 'a: 'key { - self.raw.get(Self::H::hash(key)) + self.raw.get(Self::H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result, ::Out> { let hash = Self::H::hash(key); - let out = self.raw.insert(&hash, value)?; + let out = self.raw.insert(hash.as_ref(), value)?; let db = self.raw.db_mut(); // don't insert if it doesn't exist. if out.is_none() { - db.emplace(Self::to_aux_key(&hash), DBValue::from_slice(key)); + let aux_hash = Self::H::hash(hash.as_ref()); + db.emplace(aux_hash, DBValue::from_slice(key)); } Ok(out) } - fn remove(&mut self, key: &[u8]) -> super::Result> { -// let hash = keccak(key); //TODO + fn remove(&mut self, key: &[u8]) -> super::Result, ::Out> { let hash = Self::H::hash(key); - let out = self.raw.remove(&hash)?; + let out = self.raw.remove(hash.as_ref())?; // don't remove if it already exists. if out.is_some() { - self.raw.db_mut().remove(&Self::to_aux_key(&hash)); + self.raw.db_mut().remove(&hash); } Ok(out) } } -#[test] -fn fatdb_to_trie() { - use memorydb::MemoryDB; - use super::TrieDB; - use super::Trie; - - let mut memdb = MemoryDB::new(); - let mut root = H256::default(); - { - let mut t = FatDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - let t = TrieDB::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -} +//#[test] +//fn fatdb_to_trie() { +// use memorydb::MemoryDB; +// use super::TrieDB; +// use super::Trie; +// +// let mut memdb = MemoryDB::new(); +// let mut root = H256::default(); +// { +// let mut t = FatDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// } +// let t = TrieDB::new(&memdb, &root).unwrap(); +// assert_eq!(t.get(&keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +//} diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index da353c2389f..547bc57e3d3 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -43,7 +43,7 @@ pub mod sectriedb; pub mod recorder; mod fatdb; -//mod fatdbmut; +mod fatdbmut; mod lookup; mod nibbleslice; mod nibblevec; @@ -53,7 +53,7 @@ pub use self::triedbmut::TrieDBMut; //pub use self::sectriedbmut::SecTrieDBMut; pub use self::sectriedb::SecTrieDB; pub use self::fatdb::{FatDB, FatDBIterator}; -//pub use self::fatdbmut::FatDBMut; +pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; /// Trie Errors. @@ -195,6 +195,7 @@ pub trait TrieMut { pub trait TrieIterator: Iterator { /// Position the iterator on the first element with key > `key` fn seek(&mut self, key: &[u8]) -> Result<(), H::Out>; +// fn seek(&mut self, key: &H::Out) -> Result<(), H::Out>; } /// Trie types @@ -306,9 +307,3 @@ where H::Out: rlp::Decodable + rlp::Encodable /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } } - -/* -fn decode_node(node: &'db [u8]) -> Node { - Node::decoded(node).expect("rlp read from db; qed") - } -*/ \ No newline at end of file diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 7add50152f6..975dcd7da82 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -87,128 +87,128 @@ impl Recorder { } } -#[cfg(test)] -mod tests { - use super::*; - use ethereum_types::H256; - - #[test] - fn basic_recorder() { - let mut basic = Recorder::new(); - - let node1 = vec![1, 2, 3, 4]; - let node2 = vec![4, 5, 6, 7, 8, 9, 10]; - - let (hash1, hash2) = (keccak(&node1), keccak(&node2)); - basic.record(&hash1, &node1, 0); - basic.record(&hash2, &node2, 456); - - let record1 = Record { - data: node1, - hash: hash1, - depth: 0, - }; - - let record2 = Record { - data: node2, - hash: hash2, - depth: 456 - }; - - assert_eq!(basic.drain(), vec![record1, record2]); - } - - #[test] - fn basic_recorder_min_depth() { - let mut basic = Recorder::with_depth(400); - - let node1 = vec![1, 2, 3, 4]; - let node2 = vec![4, 5, 6, 7, 8, 9, 10]; - - let hash1 = keccak(&node1); - let hash2 = keccak(&node2); - basic.record(&hash1, &node1, 0); - basic.record(&hash2, &node2, 456); - - let records = basic.drain(); - - assert_eq!(records.len(), 1); - - assert_eq!(records[0].clone(), Record { - data: node2, - hash: hash2, - depth: 456, - }); - } - - #[test] - fn trie_record() { - use super::super::{TrieDB, TrieDBMut, Trie, TrieMut}; - use memorydb::MemoryDB; - - let mut db = MemoryDB::new(); - - let mut root = H256::default(); - - { - let mut x = TrieDBMut::new(&mut db, &mut root); - - x.insert(b"dog", b"cat").unwrap(); - x.insert(b"lunch", b"time").unwrap(); - x.insert(b"notdog", b"notcat").unwrap(); - x.insert(b"hotdog", b"hotcat").unwrap(); - x.insert(b"letter", b"confusion").unwrap(); - x.insert(b"insert", b"remove").unwrap(); - x.insert(b"pirate", b"aargh!").unwrap(); - x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); - } - - let trie = TrieDB::new(&db, &root).unwrap(); - let mut recorder = Recorder::new(); - - trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); - - let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); - assert_eq!(nodes, vec![ - vec![ - 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, - 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, - 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, 59, - 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, 0, 236, - 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 - ], - vec![ - 248, 60, 206, 134, 32, 105, 114, 97, 116, 101, 134, 97, 97, 114, 103, 104, 33, - 128, 128, 128, 128, 128, 128, 128, 128, 221, 136, 32, 111, 32, 104, 111, 32, 104, - 111, 147, 97, 110, 100, 32, 97, 32, 98, 111, 116, 116, 108, 101, 32, 111, 102, - 32, 114, 117, 109, 128, 128, 128, 128, 128, 128, 128 - ] - ]); - - trie.get_with(b"letter", &mut recorder).unwrap().unwrap(); - - let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); - assert_eq!(nodes, vec![ - vec![ - 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, - 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, - 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, - 59, 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, - 0, 236, 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 - ], - vec![ - 248, 99, 128, 128, 128, 128, 200, 131, 32, 111, 103, 131, 99, 97, 116, 128, 128, - 128, 206, 134, 32, 111, 116, 100, 111, 103, 134, 104, 111, 116, 99, 97, 116, 206, - 134, 32, 110, 115, 101, 114, 116, 134, 114, 101, 109, 111, 118, 101, 128, 128, - 160, 202, 250, 252, 153, 229, 63, 255, 13, 100, 197, 80, 120, 190, 186, 92, 5, - 255, 135, 245, 205, 180, 213, 161, 8, 47, 107, 13, 105, 218, 1, 9, 5, 128, - 206, 134, 32, 111, 116, 100, 111, 103, 134, 110, 111, 116, 99, 97, 116, 128, 128 - ], - vec![ - 235, 128, 128, 128, 128, 128, 128, 208, 133, 53, 116, 116, 101, 114, 137, 99, - 111, 110, 102, 117, 115, 105, 111, 110, 202, 132, 53, 110, 99, 104, 132, 116, - 105, 109, 101, 128, 128, 128, 128, 128, 128, 128, 128, 128 - ] - ]); - } -} +//#[cfg(test)] +//mod tests { +// use super::*; +// use ethereum_types::H256; +// +// #[test] +// fn basic_recorder() { +// let mut basic = Recorder::new(); +// +// let node1 = vec![1, 2, 3, 4]; +// let node2 = vec![4, 5, 6, 7, 8, 9, 10]; +// +// let (hash1, hash2) = (keccak(&node1), keccak(&node2)); +// basic.record(&hash1, &node1, 0); +// basic.record(&hash2, &node2, 456); +// +// let record1 = Record { +// data: node1, +// hash: hash1, +// depth: 0, +// }; +// +// let record2 = Record { +// data: node2, +// hash: hash2, +// depth: 456 +// }; +// +// assert_eq!(basic.drain(), vec![record1, record2]); +// } +// +// #[test] +// fn basic_recorder_min_depth() { +// let mut basic = Recorder::with_depth(400); +// +// let node1 = vec![1, 2, 3, 4]; +// let node2 = vec![4, 5, 6, 7, 8, 9, 10]; +// +// let hash1 = keccak(&node1); +// let hash2 = keccak(&node2); +// basic.record(&hash1, &node1, 0); +// basic.record(&hash2, &node2, 456); +// +// let records = basic.drain(); +// +// assert_eq!(records.len(), 1); +// +// assert_eq!(records[0].clone(), Record { +// data: node2, +// hash: hash2, +// depth: 456, +// }); +// } +// +// #[test] +// fn trie_record() { +// use super::super::{TrieDB, TrieDBMut, Trie, TrieMut}; +// use memorydb::MemoryDB; +// +// let mut db = MemoryDB::new(); +// +// let mut root = H256::default(); +// +// { +// let mut x = TrieDBMut::new(&mut db, &mut root); +// +// x.insert(b"dog", b"cat").unwrap(); +// x.insert(b"lunch", b"time").unwrap(); +// x.insert(b"notdog", b"notcat").unwrap(); +// x.insert(b"hotdog", b"hotcat").unwrap(); +// x.insert(b"letter", b"confusion").unwrap(); +// x.insert(b"insert", b"remove").unwrap(); +// x.insert(b"pirate", b"aargh!").unwrap(); +// x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); +// } +// +// let trie = TrieDB::new(&db, &root).unwrap(); +// let mut recorder = Recorder::new(); +// +// trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); +// +// let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); +// assert_eq!(nodes, vec![ +// vec![ +// 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, +// 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, +// 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, 59, +// 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, 0, 236, +// 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 +// ], +// vec![ +// 248, 60, 206, 134, 32, 105, 114, 97, 116, 101, 134, 97, 97, 114, 103, 104, 33, +// 128, 128, 128, 128, 128, 128, 128, 128, 221, 136, 32, 111, 32, 104, 111, 32, 104, +// 111, 147, 97, 110, 100, 32, 97, 32, 98, 111, 116, 116, 108, 101, 32, 111, 102, +// 32, 114, 117, 109, 128, 128, 128, 128, 128, 128, 128 +// ] +// ]); +// +// trie.get_with(b"letter", &mut recorder).unwrap().unwrap(); +// +// let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); +// assert_eq!(nodes, vec![ +// vec![ +// 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, +// 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, +// 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, +// 59, 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, +// 0, 236, 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 +// ], +// vec![ +// 248, 99, 128, 128, 128, 128, 200, 131, 32, 111, 103, 131, 99, 97, 116, 128, 128, +// 128, 206, 134, 32, 111, 116, 100, 111, 103, 134, 104, 111, 116, 99, 97, 116, 206, +// 134, 32, 110, 115, 101, 114, 116, 134, 114, 101, 109, 111, 118, 101, 128, 128, +// 160, 202, 250, 252, 153, 229, 63, 255, 13, 100, 197, 80, 120, 190, 186, 92, 5, +// 255, 135, 245, 205, 180, 213, 161, 8, 47, 107, 13, 105, 218, 1, 9, 5, 128, +// 206, 134, 32, 111, 116, 100, 111, 103, 134, 110, 111, 116, 99, 97, 116, 128, 128 +// ], +// vec![ +// 235, 128, 128, 128, 128, 128, 128, 208, 133, 53, 116, 116, 101, 114, 137, 99, +// 111, 110, 102, 117, 115, 105, 111, 110, 202, 132, 53, 110, 99, 104, 132, 116, +// 105, 109, 101, 128, 128, 128, 128, 128, 128, 128, 128, 128 +// ] +// ]); +// } +//} diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index fde26792bfd..d0a8ea78271 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -55,13 +55,13 @@ impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> where H::Out: Decodable + Encoda fn root(&self) -> &::Out { self.raw.root() } fn contains(&self, key: &[u8]) -> super::Result::Out> { - self.raw.contains(&Self::H::hash(key).as_ref()) + self.raw.contains(Self::H::hash(key).as_ref()) } fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> where 'a: 'key { - self.raw.get_with(&Self::H::hash(key).as_ref(), query) + self.raw.get_with(Self::H::hash(key).as_ref(), query) } fn iter<'a>(&'a self) -> super::Result> + 'a>, ::Out> { @@ -69,19 +69,19 @@ impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> where H::Out: Decodable + Encoda } } -#[test] -fn trie_to_sectrie() { - use memorydb::MemoryDB; - use hashdb::DBValue; - use super::triedbmut::TrieDBMut; - use super::TrieMut; - - let mut memdb = MemoryDB::new(); - let mut root = H256::default(); - { - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); - } - let t = SecTrieDB::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -} +//#[test] +//fn trie_to_sectrie() { +// use memorydb::MemoryDB; +// use hashdb::DBValue; +// use super::triedbmut::TrieDBMut; +// use super::TrieMut; +// +// let mut memdb = MemoryDB::new(); +// let mut root = H256::default(); +// { +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); +// } +// let t = SecTrieDB::new(&memdb, &root).unwrap(); +// assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +//} diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index c981d9cc4f3..3e5449f01ca 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -77,18 +77,18 @@ impl<'db, H: Hasher> TrieMut for SecTrieDBMut<'db, H> { } } -#[test] -fn sectrie_to_trie() { - use memorydb::*; - use super::triedb::*; - use super::Trie; - - let mut memdb = MemoryDB::new(); - let mut root = H256::default(); - { - let mut t = SecTrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - let t = TrieDB::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -} +//#[test] +//fn sectrie_to_trie() { +// use memorydb::*; +// use super::triedb::*; +// use super::Trie; +// +// let mut memdb = MemoryDB::new(); +// let mut root = H256::default(); +// { +// let mut t = SecTrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// } +// let t = TrieDB::new(&memdb, &root).unwrap(); +// assert_eq!(t.get(&keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +//} diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 49ae4756a9e..d4639d5dc75 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -317,7 +317,7 @@ impl<'a, H: Hasher> TrieIterator for TrieDBIterator<'a, H> where H::Out: Deco self.trail.clear(); self.key_nibbles.clear(); let root_rlp = self.db.root_data()?; - self.seek(root_rlp, NibbleSlice::new(key)) + self.seek(root_rlp, NibbleSlice::new(key.as_ref())) } } @@ -404,7 +404,6 @@ fn iterator() { use memorydb::*; use super::TrieMut; use super::triedbmut::*; - use hashdb::KeccakHasher; let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 4ee3f50a69d..74580e21887 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -278,7 +278,7 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// use ethereum_types::H256; /// /// fn main() { -/// let mut memdb = MemoryDB::new(); +/// let mut memdb = MemoryDB::::new(); /// let mut root = H256::new(); /// let mut t = TrieDBMut::new(&mut memdb, &mut root); /// assert!(t.is_empty()); @@ -290,7 +290,7 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// assert!(!t.contains(b"foo").unwrap()); /// } /// ``` -pub struct TrieDBMut<'a, H: Hasher + 'a> where H::Out: Decodable + Encodable { +pub struct TrieDBMut<'a, H: Hasher + 'a> where H::Out: Decodable + Encodable { storage: NodeStorage, db: &'a mut HashDB, root: &'a mut H::Out, @@ -938,367 +938,367 @@ impl<'a, H: Hasher> Drop for TrieDBMut<'a, H> where H::Out: Decodable + Encodabl } } -#[cfg(test)] -mod tests { - extern crate triehash; - - use self::triehash::trie_root; - use hashdb::*; - use memorydb::*; - use super::*; - use bytes::ToPretty; - use keccak::KECCAK_NULL_RLP; - use super::super::TrieMut; - use standardmap::*; - - fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec, Vec)]) -> TrieDBMut<'db> { - let mut t = TrieDBMut::new(db, root); - for i in 0..v.len() { - let key: &[u8]= &v[i].0; - let val: &[u8] = &v[i].1; - t.insert(key, val).unwrap(); - } - t - } - - fn unpopulate_trie<'db>(t: &mut TrieDBMut<'db>, v: &[(Vec, Vec)]) { - for i in v { - let key: &[u8]= &i.0; - t.remove(key).unwrap(); - } - } - - #[test] - fn playpen() { - ::ethcore_logger::init_log(); - - let mut seed = H256::new(); - for test_i in 0..10 { - if test_i % 50 == 0 { - debug!("{:?} of 10000 stress tests done", test_i); - } - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 100, - }.make_with(&mut seed); - - let real = trie_root(x.clone()); - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut memtrie = populate_trie(&mut memdb, &mut root, &x); - - memtrie.commit(); - if *memtrie.root() != real { - println!("TRIE MISMATCH"); - println!(""); - println!("{:?} vs {:?}", memtrie.root(), real); - for i in &x { - println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); - } - } - assert_eq!(*memtrie.root(), real); - unpopulate_trie(&mut memtrie, &x); - memtrie.commit(); - if *memtrie.root() != KECCAK_NULL_RLP { - println!("- TRIE MISMATCH"); - println!(""); - println!("{:?} vs {:?}", memtrie.root(), real); - for i in &x { - println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); - } - } - assert_eq!(*memtrie.root(), KECCAK_NULL_RLP); - } - } - - #[test] - fn init() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - assert_eq!(*t.root(), KECCAK_NULL_RLP); - } - - #[test] - fn insert_on_empty() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); - } - - #[test] - fn remove_to_empty() { - let big_value = b"00000000000000000000000000000000"; - - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t1 = TrieDBMut::new(&mut memdb, &mut root); - t1.insert(&[0x01, 0x23], big_value).unwrap(); - t1.insert(&[0x01, 0x34], big_value).unwrap(); - let mut memdb2 = MemoryDB::new(); - let mut root2 = H256::new(); - let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); - t2.insert(&[0x01], big_value).unwrap(); - t2.insert(&[0x01, 0x23], big_value).unwrap(); - t2.insert(&[0x01, 0x34], big_value).unwrap(); - t2.remove(&[0x01]).unwrap(); - } - - #[test] - fn insert_replace_root() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); - } - - #[test] - fn insert_make_branch_root() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) - ])); - } - - #[test] - fn insert_into_branch_root() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); - t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), - (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), - ])); - } - - #[test] - fn insert_value_into_branch_root() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[], &[0x0]).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ - (vec![], vec![0x0]), - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - ])); - } - - #[test] - fn insert_split_leaf() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), - ])); - } - - #[test] - fn insert_split_extenstion() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); - t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); - t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01, 0x23, 0x45], vec![0x01]), - (vec![0x01, 0xf3, 0x45], vec![0x02]), - (vec![0x01, 0xf3, 0xf5], vec![0x03]), - ])); - } - - #[test] - fn insert_big_value() { - let big_value0 = b"00000000000000000000000000000000"; - let big_value1 = b"11111111111111111111111111111111"; - - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], big_value0).unwrap(); - t.insert(&[0x11u8, 0x23], big_value1).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], big_value0.to_vec()), - (vec![0x11u8, 0x23], big_value1.to_vec()) - ])); - } - - #[test] - fn insert_duplicate_value() { - let big_value = b"00000000000000000000000000000000"; - - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], big_value).unwrap(); - t.insert(&[0x11u8, 0x23], big_value).unwrap(); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], big_value.to_vec()), - (vec![0x11u8, 0x23], big_value.to_vec()) - ])); - } - - #[test] - fn test_at_empty() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let t = TrieDBMut::new(&mut memdb, &mut root); - assert_eq!(t.get(&[0x5]), Ok(None)); - } - - #[test] - fn test_at_one() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); - t.commit(); - assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); - } - - #[test] - fn test_at_three() { - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); - t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); - assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); - assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); - assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); - assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); - t.commit(); - assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); - assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); - assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); - assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); - } - - #[test] - fn stress() { - let mut seed = H256::new(); - for _ in 0..50 { - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 4, - }.make_with(&mut seed); - - let real = trie_root(x.clone()); - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut memtrie = populate_trie(&mut memdb, &mut root, &x); - let mut y = x.clone(); - y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let mut memdb2 = MemoryDB::new(); - let mut root2 = H256::new(); - let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); - if *memtrie.root() != real || *memtrie_sorted.root() != real { - println!("TRIE MISMATCH"); - println!(""); - println!("ORIGINAL... {:?}", memtrie.root()); - for i in &x { - println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); - } - println!("SORTED... {:?}", memtrie_sorted.root()); - for i in &y { - println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); - } - } - assert_eq!(*memtrie.root(), real); - assert_eq!(*memtrie_sorted.root(), real); - } - } - - #[test] - fn test_trie_existing() { - let mut root = H256::new(); - let mut db = MemoryDB::new(); - { - let mut t = TrieDBMut::new(&mut db, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - } - - { - let _ = TrieDBMut::from_existing(&mut db, &mut root); - } - } - - #[test] - fn insert_empty() { - let mut seed = H256::new(); - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 4, - }.make_with(&mut seed); - - let mut db = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut db, &mut root); - for &(ref key, ref value) in &x { - t.insert(key, value).unwrap(); - } - - assert_eq!(*t.root(), trie_root(x.clone())); - - for &(ref key, _) in &x { - t.insert(key, &[]).unwrap(); - } - - assert!(t.is_empty()); - assert_eq!(*t.root(), KECCAK_NULL_RLP); - } - - #[test] - fn return_old_values() { - let mut seed = H256::new(); - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 4, - }.make_with(&mut seed); - - let mut db = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut db, &mut root); - for &(ref key, ref value) in &x { - assert!(t.insert(key, value).unwrap().is_none()); - assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); - } - - for (key, value) in x { - assert_eq!(t.remove(&key).unwrap(), Some(DBValue::from_slice(&value))); - assert!(t.remove(&key).unwrap().is_none()); - } - } -} +//#[cfg(test)] +//mod tests { +// extern crate triehash; +// +// use self::triehash::trie_root; +// use hashdb::*; +// use memorydb::*; +// use super::*; +// use bytes::ToPretty; +// use keccak::KECCAK_NULL_RLP; +// use super::super::TrieMut; +// use standardmap::*; +// +// fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec, Vec)]) -> TrieDBMut<'db> { +// let mut t = TrieDBMut::new(db, root); +// for i in 0..v.len() { +// let key: &[u8]= &v[i].0; +// let val: &[u8] = &v[i].1; +// t.insert(key, val).unwrap(); +// } +// t +// } +// +// fn unpopulate_trie<'db>(t: &mut TrieDBMut<'db>, v: &[(Vec, Vec)]) { +// for i in v { +// let key: &[u8]= &i.0; +// t.remove(key).unwrap(); +// } +// } +// +// #[test] +// fn playpen() { +// ::ethcore_logger::init_log(); +// +// let mut seed = H256::new(); +// for test_i in 0..10 { +// if test_i % 50 == 0 { +// debug!("{:?} of 10000 stress tests done", test_i); +// } +// let x = StandardMap { +// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), +// min_key: 5, +// journal_key: 0, +// value_mode: ValueMode::Index, +// count: 100, +// }.make_with(&mut seed); +// +// let real = trie_root(x.clone()); +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut memtrie = populate_trie(&mut memdb, &mut root, &x); +// +// memtrie.commit(); +// if *memtrie.root() != real { +// println!("TRIE MISMATCH"); +// println!(""); +// println!("{:?} vs {:?}", memtrie.root(), real); +// for i in &x { +// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); +// } +// } +// assert_eq!(*memtrie.root(), real); +// unpopulate_trie(&mut memtrie, &x); +// memtrie.commit(); +// if *memtrie.root() != KECCAK_NULL_RLP { +// println!("- TRIE MISMATCH"); +// println!(""); +// println!("{:?} vs {:?}", memtrie.root(), real); +// for i in &x { +// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); +// } +// } +// assert_eq!(*memtrie.root(), KECCAK_NULL_RLP); +// } +// } +// +// #[test] +// fn init() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// assert_eq!(*t.root(), KECCAK_NULL_RLP); +// } +// +// #[test] +// fn insert_on_empty() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); +// } +// +// #[test] +// fn remove_to_empty() { +// let big_value = b"00000000000000000000000000000000"; +// +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t1 = TrieDBMut::new(&mut memdb, &mut root); +// t1.insert(&[0x01, 0x23], big_value).unwrap(); +// t1.insert(&[0x01, 0x34], big_value).unwrap(); +// let mut memdb2 = MemoryDB::new(); +// let mut root2 = H256::new(); +// let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); +// t2.insert(&[0x01], big_value).unwrap(); +// t2.insert(&[0x01, 0x23], big_value).unwrap(); +// t2.insert(&[0x01, 0x34], big_value).unwrap(); +// t2.remove(&[0x01]).unwrap(); +// } +// +// #[test] +// fn insert_replace_root() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); +// } +// +// #[test] +// fn insert_make_branch_root() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ +// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), +// (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) +// ])); +// } +// +// #[test] +// fn insert_into_branch_root() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); +// t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ +// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), +// (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), +// (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), +// ])); +// } +// +// #[test] +// fn insert_value_into_branch_root() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// t.insert(&[], &[0x0]).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ +// (vec![], vec![0x0]), +// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), +// ])); +// } +// +// #[test] +// fn insert_split_leaf() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ +// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), +// (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), +// ])); +// } +// +// #[test] +// fn insert_split_extenstion() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); +// t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); +// t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ +// (vec![0x01, 0x23, 0x45], vec![0x01]), +// (vec![0x01, 0xf3, 0x45], vec![0x02]), +// (vec![0x01, 0xf3, 0xf5], vec![0x03]), +// ])); +// } +// +// #[test] +// fn insert_big_value() { +// let big_value0 = b"00000000000000000000000000000000"; +// let big_value1 = b"11111111111111111111111111111111"; +// +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], big_value0).unwrap(); +// t.insert(&[0x11u8, 0x23], big_value1).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ +// (vec![0x01u8, 0x23], big_value0.to_vec()), +// (vec![0x11u8, 0x23], big_value1.to_vec()) +// ])); +// } +// +// #[test] +// fn insert_duplicate_value() { +// let big_value = b"00000000000000000000000000000000"; +// +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], big_value).unwrap(); +// t.insert(&[0x11u8, 0x23], big_value).unwrap(); +// assert_eq!(*t.root(), trie_root(vec![ +// (vec![0x01u8, 0x23], big_value.to_vec()), +// (vec![0x11u8, 0x23], big_value.to_vec()) +// ])); +// } +// +// #[test] +// fn test_at_empty() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let t = TrieDBMut::new(&mut memdb, &mut root); +// assert_eq!(t.get(&[0x5]), Ok(None)); +// } +// +// #[test] +// fn test_at_one() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); +// t.commit(); +// assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); +// } +// +// #[test] +// fn test_at_three() { +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut memdb, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); +// t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); +// assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +// assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); +// assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); +// assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); +// t.commit(); +// assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +// assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); +// assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); +// assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); +// } +// +// #[test] +// fn stress() { +// let mut seed = H256::new(); +// for _ in 0..50 { +// let x = StandardMap { +// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), +// min_key: 5, +// journal_key: 0, +// value_mode: ValueMode::Index, +// count: 4, +// }.make_with(&mut seed); +// +// let real = trie_root(x.clone()); +// let mut memdb = MemoryDB::new(); +// let mut root = H256::new(); +// let mut memtrie = populate_trie(&mut memdb, &mut root, &x); +// let mut y = x.clone(); +// y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); +// let mut memdb2 = MemoryDB::new(); +// let mut root2 = H256::new(); +// let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); +// if *memtrie.root() != real || *memtrie_sorted.root() != real { +// println!("TRIE MISMATCH"); +// println!(""); +// println!("ORIGINAL... {:?}", memtrie.root()); +// for i in &x { +// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); +// } +// println!("SORTED... {:?}", memtrie_sorted.root()); +// for i in &y { +// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); +// } +// } +// assert_eq!(*memtrie.root(), real); +// assert_eq!(*memtrie_sorted.root(), real); +// } +// } +// +// #[test] +// fn test_trie_existing() { +// let mut root = H256::new(); +// let mut db = MemoryDB::new(); +// { +// let mut t = TrieDBMut::new(&mut db, &mut root); +// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); +// } +// +// { +// let _ = TrieDBMut::from_existing(&mut db, &mut root); +// } +// } +// +// #[test] +// fn insert_empty() { +// let mut seed = H256::new(); +// let x = StandardMap { +// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), +// min_key: 5, +// journal_key: 0, +// value_mode: ValueMode::Index, +// count: 4, +// }.make_with(&mut seed); +// +// let mut db = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut db, &mut root); +// for &(ref key, ref value) in &x { +// t.insert(key, value).unwrap(); +// } +// +// assert_eq!(*t.root(), trie_root(x.clone())); +// +// for &(ref key, _) in &x { +// t.insert(key, &[]).unwrap(); +// } +// +// assert!(t.is_empty()); +// assert_eq!(*t.root(), KECCAK_NULL_RLP); +// } +// +// #[test] +// fn return_old_values() { +// let mut seed = H256::new(); +// let x = StandardMap { +// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), +// min_key: 5, +// journal_key: 0, +// value_mode: ValueMode::Index, +// count: 4, +// }.make_with(&mut seed); +// +// let mut db = MemoryDB::new(); +// let mut root = H256::new(); +// let mut t = TrieDBMut::new(&mut db, &mut root); +// for &(ref key, ref value) in &x { +// assert!(t.insert(key, value).unwrap().is_none()); +// assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); +// } +// +// for (key, value) in x { +// assert_eq!(t.remove(&key).unwrap(), Some(DBValue::from_slice(&value))); +// assert!(t.remove(&key).unwrap().is_none()); +// } +// } +//} From b42f92e7eddfb5594a0d3d2b98319ed34b752d84 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 15:08:06 +0200 Subject: [PATCH 018/160] FatDBMut test --- util/patricia_trie/src/fatdbmut.rs | 32 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 083dcad6fcd..41cd53c366f 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -96,18 +96,20 @@ impl<'db, H: Hasher> TrieMut for FatDBMut<'db, H> where H::Out: Decodable + Enco } } -//#[test] -//fn fatdb_to_trie() { -// use memorydb::MemoryDB; -// use super::TrieDB; -// use super::Trie; -// -// let mut memdb = MemoryDB::new(); -// let mut root = H256::default(); -// { -// let mut t = FatDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// } -// let t = TrieDB::new(&memdb, &root).unwrap(); -// assert_eq!(t.get(&keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -//} +#[test] +fn fatdbmut_to_trie() { + use memorydb::MemoryDB; + use super::TrieDB; + use super::Trie; + use hashdb::KeccakHasher; + use keccak; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = FatDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + } + let t = TrieDB::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +} From 5e9fda0ccb93e539c3f47dff56501e4f0c54780d Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 15:10:31 +0200 Subject: [PATCH 019/160] SecTrieDB tests --- util/patricia_trie/src/sectriedb.rs | 34 +++++++++++++++-------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index d0a8ea78271..37c3486e8a5 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -69,19 +69,21 @@ impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> where H::Out: Decodable + Encoda } } -//#[test] -//fn trie_to_sectrie() { -// use memorydb::MemoryDB; -// use hashdb::DBValue; -// use super::triedbmut::TrieDBMut; -// use super::TrieMut; -// -// let mut memdb = MemoryDB::new(); -// let mut root = H256::default(); -// { -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); -// } -// let t = SecTrieDB::new(&memdb, &root).unwrap(); -// assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -//} +#[test] +fn trie_to_sectrie() { + use memorydb::MemoryDB; + use hashdb::DBValue; + use super::triedbmut::TrieDBMut; + use super::TrieMut; + use hashdb::KeccakHasher; + use keccak; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&keccak::keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); + } + let t = SecTrieDB::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +} From 841abc46ba7e9f33432bb61ceeb505f832b85ae5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 15:30:42 +0200 Subject: [PATCH 020/160] Tests for TrieDBMut --- util/patricia_trie/src/triedbmut.rs | 732 ++++++++++++++-------------- 1 file changed, 368 insertions(+), 364 deletions(-) diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 74580e21887..4759652bb3c 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -938,367 +938,371 @@ impl<'a, H: Hasher> Drop for TrieDBMut<'a, H> where H::Out: Decodable + Encodabl } } -//#[cfg(test)] -//mod tests { -// extern crate triehash; -// -// use self::triehash::trie_root; -// use hashdb::*; -// use memorydb::*; -// use super::*; -// use bytes::ToPretty; -// use keccak::KECCAK_NULL_RLP; -// use super::super::TrieMut; -// use standardmap::*; -// -// fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec, Vec)]) -> TrieDBMut<'db> { -// let mut t = TrieDBMut::new(db, root); -// for i in 0..v.len() { -// let key: &[u8]= &v[i].0; -// let val: &[u8] = &v[i].1; -// t.insert(key, val).unwrap(); -// } -// t -// } -// -// fn unpopulate_trie<'db>(t: &mut TrieDBMut<'db>, v: &[(Vec, Vec)]) { -// for i in v { -// let key: &[u8]= &i.0; -// t.remove(key).unwrap(); -// } -// } -// -// #[test] -// fn playpen() { -// ::ethcore_logger::init_log(); -// -// let mut seed = H256::new(); -// for test_i in 0..10 { -// if test_i % 50 == 0 { -// debug!("{:?} of 10000 stress tests done", test_i); -// } -// let x = StandardMap { -// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), -// min_key: 5, -// journal_key: 0, -// value_mode: ValueMode::Index, -// count: 100, -// }.make_with(&mut seed); -// -// let real = trie_root(x.clone()); -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut memtrie = populate_trie(&mut memdb, &mut root, &x); -// -// memtrie.commit(); -// if *memtrie.root() != real { -// println!("TRIE MISMATCH"); -// println!(""); -// println!("{:?} vs {:?}", memtrie.root(), real); -// for i in &x { -// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); -// } -// } -// assert_eq!(*memtrie.root(), real); -// unpopulate_trie(&mut memtrie, &x); -// memtrie.commit(); -// if *memtrie.root() != KECCAK_NULL_RLP { -// println!("- TRIE MISMATCH"); -// println!(""); -// println!("{:?} vs {:?}", memtrie.root(), real); -// for i in &x { -// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); -// } -// } -// assert_eq!(*memtrie.root(), KECCAK_NULL_RLP); -// } -// } -// -// #[test] -// fn init() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// assert_eq!(*t.root(), KECCAK_NULL_RLP); -// } -// -// #[test] -// fn insert_on_empty() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); -// } -// -// #[test] -// fn remove_to_empty() { -// let big_value = b"00000000000000000000000000000000"; -// -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t1 = TrieDBMut::new(&mut memdb, &mut root); -// t1.insert(&[0x01, 0x23], big_value).unwrap(); -// t1.insert(&[0x01, 0x34], big_value).unwrap(); -// let mut memdb2 = MemoryDB::new(); -// let mut root2 = H256::new(); -// let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); -// t2.insert(&[0x01], big_value).unwrap(); -// t2.insert(&[0x01, 0x23], big_value).unwrap(); -// t2.insert(&[0x01, 0x34], big_value).unwrap(); -// t2.remove(&[0x01]).unwrap(); -// } -// -// #[test] -// fn insert_replace_root() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); -// } -// -// #[test] -// fn insert_make_branch_root() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ -// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), -// (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) -// ])); -// } -// -// #[test] -// fn insert_into_branch_root() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); -// t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ -// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), -// (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), -// (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), -// ])); -// } -// -// #[test] -// fn insert_value_into_branch_root() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// t.insert(&[], &[0x0]).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ -// (vec![], vec![0x0]), -// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), -// ])); -// } -// -// #[test] -// fn insert_split_leaf() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ -// (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), -// (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), -// ])); -// } -// -// #[test] -// fn insert_split_extenstion() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); -// t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); -// t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ -// (vec![0x01, 0x23, 0x45], vec![0x01]), -// (vec![0x01, 0xf3, 0x45], vec![0x02]), -// (vec![0x01, 0xf3, 0xf5], vec![0x03]), -// ])); -// } -// -// #[test] -// fn insert_big_value() { -// let big_value0 = b"00000000000000000000000000000000"; -// let big_value1 = b"11111111111111111111111111111111"; -// -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], big_value0).unwrap(); -// t.insert(&[0x11u8, 0x23], big_value1).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ -// (vec![0x01u8, 0x23], big_value0.to_vec()), -// (vec![0x11u8, 0x23], big_value1.to_vec()) -// ])); -// } -// -// #[test] -// fn insert_duplicate_value() { -// let big_value = b"00000000000000000000000000000000"; -// -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], big_value).unwrap(); -// t.insert(&[0x11u8, 0x23], big_value).unwrap(); -// assert_eq!(*t.root(), trie_root(vec![ -// (vec![0x01u8, 0x23], big_value.to_vec()), -// (vec![0x11u8, 0x23], big_value.to_vec()) -// ])); -// } -// -// #[test] -// fn test_at_empty() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let t = TrieDBMut::new(&mut memdb, &mut root); -// assert_eq!(t.get(&[0x5]), Ok(None)); -// } -// -// #[test] -// fn test_at_one() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); -// t.commit(); -// assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); -// } -// -// #[test] -// fn test_at_three() { -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); -// t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); -// assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -// assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); -// assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); -// assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); -// t.commit(); -// assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -// assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); -// assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); -// assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); -// } -// -// #[test] -// fn stress() { -// let mut seed = H256::new(); -// for _ in 0..50 { -// let x = StandardMap { -// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), -// min_key: 5, -// journal_key: 0, -// value_mode: ValueMode::Index, -// count: 4, -// }.make_with(&mut seed); -// -// let real = trie_root(x.clone()); -// let mut memdb = MemoryDB::new(); -// let mut root = H256::new(); -// let mut memtrie = populate_trie(&mut memdb, &mut root, &x); -// let mut y = x.clone(); -// y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); -// let mut memdb2 = MemoryDB::new(); -// let mut root2 = H256::new(); -// let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); -// if *memtrie.root() != real || *memtrie_sorted.root() != real { -// println!("TRIE MISMATCH"); -// println!(""); -// println!("ORIGINAL... {:?}", memtrie.root()); -// for i in &x { -// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); -// } -// println!("SORTED... {:?}", memtrie_sorted.root()); -// for i in &y { -// println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); -// } -// } -// assert_eq!(*memtrie.root(), real); -// assert_eq!(*memtrie_sorted.root(), real); -// } -// } -// -// #[test] -// fn test_trie_existing() { -// let mut root = H256::new(); -// let mut db = MemoryDB::new(); -// { -// let mut t = TrieDBMut::new(&mut db, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// } -// -// { -// let _ = TrieDBMut::from_existing(&mut db, &mut root); -// } -// } -// -// #[test] -// fn insert_empty() { -// let mut seed = H256::new(); -// let x = StandardMap { -// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), -// min_key: 5, -// journal_key: 0, -// value_mode: ValueMode::Index, -// count: 4, -// }.make_with(&mut seed); -// -// let mut db = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut db, &mut root); -// for &(ref key, ref value) in &x { -// t.insert(key, value).unwrap(); -// } -// -// assert_eq!(*t.root(), trie_root(x.clone())); -// -// for &(ref key, _) in &x { -// t.insert(key, &[]).unwrap(); -// } -// -// assert!(t.is_empty()); -// assert_eq!(*t.root(), KECCAK_NULL_RLP); -// } -// -// #[test] -// fn return_old_values() { -// let mut seed = H256::new(); -// let x = StandardMap { -// alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), -// min_key: 5, -// journal_key: 0, -// value_mode: ValueMode::Index, -// count: 4, -// }.make_with(&mut seed); -// -// let mut db = MemoryDB::new(); -// let mut root = H256::new(); -// let mut t = TrieDBMut::new(&mut db, &mut root); -// for &(ref key, ref value) in &x { -// assert!(t.insert(key, value).unwrap().is_none()); -// assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); -// } -// -// for (key, value) in x { -// assert_eq!(t.remove(&key).unwrap(), Some(DBValue::from_slice(&value))); -// assert!(t.remove(&key).unwrap().is_none()); -// } -// } -//} +#[cfg(test)] +mod tests { + extern crate triehash; + + use self::triehash::trie_root; + use hashdb::*; + use memorydb::*; + use super::*; + use bytes::ToPretty; + use super::super::TrieMut; + use standardmap::*; + use rlp::{Decodable, Encodable}; + + fn populate_trie<'db, H: Hasher>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H> + where H::Out: Decodable + Encodable + { + let mut t = TrieDBMut::new(db, root); + for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(key, val).unwrap(); + } + t + } + + fn unpopulate_trie<'db, H: Hasher>(t: &mut TrieDBMut<'db, H>, v: &[(Vec, Vec)]) + where H::Out: Decodable + Encodable + { + for i in v { + let key: &[u8]= &i.0; + t.remove(key).unwrap(); + } + } + + #[test] + fn playpen() { + ::ethcore_logger::init_log(); + + let mut seed = ::Out::new(); + for test_i in 0..10 { + if test_i % 50 == 0 { + debug!("{:?} of 10000 stress tests done", test_i); + } + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 100, + }.make_with(&mut seed); + + let real = trie_root(x.clone()); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + + memtrie.commit(); + if *memtrie.root() != real { + println!("TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in &x { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + } + assert_eq!(*memtrie.root(), real); + unpopulate_trie(&mut memtrie, &x); + memtrie.commit(); + if *memtrie.root() != KeccakHasher::HASHED_NULL_RLP { + println!("- TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in &x { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + } + assert_eq!(*memtrie.root(), KeccakHasher::HASHED_NULL_RLP); + } + } + + #[test] + fn init() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + assert_eq!(*t.root(), KeccakHasher::HASHED_NULL_RLP); + } + + #[test] + fn insert_on_empty() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); + } + + #[test] + fn remove_to_empty() { + let big_value = b"00000000000000000000000000000000"; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t1 = TrieDBMut::new(&mut memdb, &mut root); + t1.insert(&[0x01, 0x23], big_value).unwrap(); + t1.insert(&[0x01, 0x34], big_value).unwrap(); + let mut memdb2 = MemoryDB::::new(); + let mut root2 = ::Out::default(); + let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); + t2.insert(&[0x01], big_value).unwrap(); + t2.insert(&[0x01, 0x23], big_value).unwrap(); + t2.insert(&[0x01, 0x34], big_value).unwrap(); + t2.remove(&[0x01]).unwrap(); + } + + #[test] + fn insert_replace_root() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); + } + + #[test] + fn insert_make_branch_root() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) + ])); + } + + #[test] + fn insert_into_branch_root() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ])); + } + + #[test] + fn insert_value_into_branch_root() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[], &[0x0]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ + (vec![], vec![0x0]), + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + ])); + } + + #[test] + fn insert_split_leaf() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), + ])); + } + + #[test] + fn insert_split_extenstion() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); + t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); + t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01, 0x23, 0x45], vec![0x01]), + (vec![0x01, 0xf3, 0x45], vec![0x02]), + (vec![0x01, 0xf3, 0xf5], vec![0x03]), + ])); + } + + #[test] + fn insert_big_value() { + let big_value0 = b"00000000000000000000000000000000"; + let big_value1 = b"11111111111111111111111111111111"; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], big_value0).unwrap(); + t.insert(&[0x11u8, 0x23], big_value1).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value0.to_vec()), + (vec![0x11u8, 0x23], big_value1.to_vec()) + ])); + } + + #[test] + fn insert_duplicate_value() { + let big_value = b"00000000000000000000000000000000"; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], big_value).unwrap(); + t.insert(&[0x11u8, 0x23], big_value).unwrap(); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value.to_vec()), + (vec![0x11u8, 0x23], big_value.to_vec()) + ])); + } + + #[test] + fn test_at_empty() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let t = TrieDBMut::new(&mut memdb, &mut root); + assert_eq!(t.get(&[0x5]), Ok(None)); + } + + #[test] + fn test_at_one() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); + t.commit(); + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); + } + + #[test] + fn test_at_three() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); + assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); + assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); + assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); + t.commit(); + assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); + assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); + assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); + assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); + } + + #[test] + fn stress() { + let mut seed = ::Out::new(); + for _ in 0..50 { + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + }.make_with(&mut seed); + + let real = trie_root(x.clone()); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + let mut y = x.clone(); + y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); + let mut memdb2 = MemoryDB::::new(); + let mut root2 = ::Out::default(); + let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); + if *memtrie.root() != real || *memtrie_sorted.root() != real { + println!("TRIE MISMATCH"); + println!(""); + println!("ORIGINAL... {:?}", memtrie.root()); + for i in &x { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("SORTED... {:?}", memtrie_sorted.root()); + for i in &y { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + } + assert_eq!(*memtrie.root(), real); + assert_eq!(*memtrie_sorted.root(), real); + } + } + + #[test] + fn test_trie_existing() { + let mut db = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = TrieDBMut::new(&mut db, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + } + + { + let _ = TrieDBMut::from_existing(&mut db, &mut root); + } + } + + #[test] + fn insert_empty() { + let mut seed = ::Out::new(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + }.make_with(&mut seed); + + let mut db = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::new(&mut db, &mut root); + for &(ref key, ref value) in &x { + t.insert(key, value).unwrap(); + } + + assert_eq!(*t.root(), trie_root(x.clone())); + + for &(ref key, _) in &x { + t.insert(key, &[]).unwrap(); + } + + assert!(t.is_empty()); + assert_eq!(*t.root(), KeccakHasher::HASHED_NULL_RLP); + } + + #[test] + fn return_old_values() { + let mut seed = ::Out::new(); + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + }.make_with(&mut seed); + + let mut db = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::new(&mut db, &mut root); + for &(ref key, ref value) in &x { + assert!(t.insert(key, value).unwrap().is_none()); + assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); + } + + for (key, value) in x { + assert_eq!(t.remove(&key).unwrap(), Some(DBValue::from_slice(&value))); + assert!(t.remove(&key).unwrap().is_none()); + } + } +} From f29132ce552bf91439d1fce174db0df0f25db0d7 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 15:39:27 +0200 Subject: [PATCH 021/160] Recorder tests --- util/patricia_trie/src/recorder.rs | 255 +++++++++++++++-------------- 1 file changed, 130 insertions(+), 125 deletions(-) diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 975dcd7da82..27738dbb3a8 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -87,128 +87,133 @@ impl Recorder { } } -//#[cfg(test)] -//mod tests { -// use super::*; -// use ethereum_types::H256; -// -// #[test] -// fn basic_recorder() { -// let mut basic = Recorder::new(); -// -// let node1 = vec![1, 2, 3, 4]; -// let node2 = vec![4, 5, 6, 7, 8, 9, 10]; -// -// let (hash1, hash2) = (keccak(&node1), keccak(&node2)); -// basic.record(&hash1, &node1, 0); -// basic.record(&hash2, &node2, 456); -// -// let record1 = Record { -// data: node1, -// hash: hash1, -// depth: 0, -// }; -// -// let record2 = Record { -// data: node2, -// hash: hash2, -// depth: 456 -// }; -// -// assert_eq!(basic.drain(), vec![record1, record2]); -// } -// -// #[test] -// fn basic_recorder_min_depth() { -// let mut basic = Recorder::with_depth(400); -// -// let node1 = vec![1, 2, 3, 4]; -// let node2 = vec![4, 5, 6, 7, 8, 9, 10]; -// -// let hash1 = keccak(&node1); -// let hash2 = keccak(&node2); -// basic.record(&hash1, &node1, 0); -// basic.record(&hash2, &node2, 456); -// -// let records = basic.drain(); -// -// assert_eq!(records.len(), 1); -// -// assert_eq!(records[0].clone(), Record { -// data: node2, -// hash: hash2, -// depth: 456, -// }); -// } -// -// #[test] -// fn trie_record() { -// use super::super::{TrieDB, TrieDBMut, Trie, TrieMut}; -// use memorydb::MemoryDB; -// -// let mut db = MemoryDB::new(); -// -// let mut root = H256::default(); -// -// { -// let mut x = TrieDBMut::new(&mut db, &mut root); -// -// x.insert(b"dog", b"cat").unwrap(); -// x.insert(b"lunch", b"time").unwrap(); -// x.insert(b"notdog", b"notcat").unwrap(); -// x.insert(b"hotdog", b"hotcat").unwrap(); -// x.insert(b"letter", b"confusion").unwrap(); -// x.insert(b"insert", b"remove").unwrap(); -// x.insert(b"pirate", b"aargh!").unwrap(); -// x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); -// } -// -// let trie = TrieDB::new(&db, &root).unwrap(); -// let mut recorder = Recorder::new(); -// -// trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); -// -// let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); -// assert_eq!(nodes, vec![ -// vec![ -// 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, -// 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, -// 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, 59, -// 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, 0, 236, -// 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 -// ], -// vec![ -// 248, 60, 206, 134, 32, 105, 114, 97, 116, 101, 134, 97, 97, 114, 103, 104, 33, -// 128, 128, 128, 128, 128, 128, 128, 128, 221, 136, 32, 111, 32, 104, 111, 32, 104, -// 111, 147, 97, 110, 100, 32, 97, 32, 98, 111, 116, 116, 108, 101, 32, 111, 102, -// 32, 114, 117, 109, 128, 128, 128, 128, 128, 128, 128 -// ] -// ]); -// -// trie.get_with(b"letter", &mut recorder).unwrap().unwrap(); -// -// let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); -// assert_eq!(nodes, vec![ -// vec![ -// 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, -// 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, -// 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, -// 59, 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, -// 0, 236, 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 -// ], -// vec![ -// 248, 99, 128, 128, 128, 128, 200, 131, 32, 111, 103, 131, 99, 97, 116, 128, 128, -// 128, 206, 134, 32, 111, 116, 100, 111, 103, 134, 104, 111, 116, 99, 97, 116, 206, -// 134, 32, 110, 115, 101, 114, 116, 134, 114, 101, 109, 111, 118, 101, 128, 128, -// 160, 202, 250, 252, 153, 229, 63, 255, 13, 100, 197, 80, 120, 190, 186, 92, 5, -// 255, 135, 245, 205, 180, 213, 161, 8, 47, 107, 13, 105, 218, 1, 9, 5, 128, -// 206, 134, 32, 111, 116, 100, 111, 103, 134, 110, 111, 116, 99, 97, 116, 128, 128 -// ], -// vec![ -// 235, 128, 128, 128, 128, 128, 128, 208, 133, 53, 116, 116, 101, 114, 137, 99, -// 111, 110, 102, 117, 115, 105, 111, 110, 202, 132, 53, 110, 99, 104, 132, 116, -// 105, 109, 101, 128, 128, 128, 128, 128, 128, 128, 128, 128 -// ] -// ]); -// } -//} +#[cfg(test)] +mod tests { + use super::*; + use keccak::keccak; + use hashdb::KeccakHasher; + + #[test] + fn basic_recorder() { + let mut basic = Recorder::::new(); + + let node1 = vec![1, 2, 3, 4]; + let node2 = vec![4, 5, 6, 7, 8, 9, 10]; + + let (hash1, hash2) = (keccak(&node1), keccak(&node2)); + basic.record(&hash1, &node1, 0); + basic.record(&hash2, &node2, 456); + + let record1 = Record { + data: node1, + hash: hash1, + depth: 0, + marker: PhantomData, + }; + + let record2 = Record { + data: node2, + hash: hash2, + depth: 456, + marker: PhantomData, + }; + + + assert_eq!(basic.drain(), vec![record1, record2]); + } + + #[test] + fn basic_recorder_min_depth() { + let mut basic = Recorder::::with_depth(400); + + let node1 = vec![1, 2, 3, 4]; + let node2 = vec![4, 5, 6, 7, 8, 9, 10]; + + let hash1 = keccak(&node1); + let hash2 = keccak(&node2); + basic.record(&hash1, &node1, 0); + basic.record(&hash2, &node2, 456); + + let records = basic.drain(); + + assert_eq!(records.len(), 1); + + assert_eq!(records[0].clone(), Record { + data: node2, + hash: hash2, + depth: 456, + marker: PhantomData, + }); + } + + #[test] + fn trie_record() { + use super::super::{TrieDB, TrieDBMut, Trie, TrieMut}; + use memorydb::MemoryDB; + + let mut db = MemoryDB::::new(); + + let mut root = ::Out::default(); + + { + let mut x = TrieDBMut::new(&mut db, &mut root); + + x.insert(b"dog", b"cat").unwrap(); + x.insert(b"lunch", b"time").unwrap(); + x.insert(b"notdog", b"notcat").unwrap(); + x.insert(b"hotdog", b"hotcat").unwrap(); + x.insert(b"letter", b"confusion").unwrap(); + x.insert(b"insert", b"remove").unwrap(); + x.insert(b"pirate", b"aargh!").unwrap(); + x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); + } + + let trie = TrieDB::new(&db, &root).unwrap(); + let mut recorder = Recorder::::new(); + + trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); + + let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); + assert_eq!(nodes, vec![ + vec![ + 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, + 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, + 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, 59, + 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, 0, 236, + 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 + ], + vec![ + 248, 60, 206, 134, 32, 105, 114, 97, 116, 101, 134, 97, 97, 114, 103, 104, 33, + 128, 128, 128, 128, 128, 128, 128, 128, 221, 136, 32, 111, 32, 104, 111, 32, 104, + 111, 147, 97, 110, 100, 32, 97, 32, 98, 111, 116, 116, 108, 101, 32, 111, 102, + 32, 114, 117, 109, 128, 128, 128, 128, 128, 128, 128 + ] + ]); + + trie.get_with(b"letter", &mut recorder).unwrap().unwrap(); + + let nodes: Vec<_> = recorder.drain().into_iter().map(|r| r.data).collect(); + assert_eq!(nodes, vec![ + vec![ + 248, 81, 128, 128, 128, 128, 128, 128, 160, 50, 19, 71, 57, 213, 63, 125, 149, + 92, 119, 88, 96, 80, 126, 59, 11, 160, 142, 98, 229, 237, 200, 231, 224, 79, 118, + 215, 93, 144, 246, 179, 176, 160, 118, 211, 171, 199, 172, 136, 136, 240, 221, + 59, 110, 82, 86, 54, 23, 95, 48, 108, 71, 125, 59, 51, 253, 210, 18, 116, 79, + 0, 236, 102, 142, 48, 128, 128, 128, 128, 128, 128, 128, 128, 128 + ], + vec![ + 248, 99, 128, 128, 128, 128, 200, 131, 32, 111, 103, 131, 99, 97, 116, 128, 128, + 128, 206, 134, 32, 111, 116, 100, 111, 103, 134, 104, 111, 116, 99, 97, 116, 206, + 134, 32, 110, 115, 101, 114, 116, 134, 114, 101, 109, 111, 118, 101, 128, 128, + 160, 202, 250, 252, 153, 229, 63, 255, 13, 100, 197, 80, 120, 190, 186, 92, 5, + 255, 135, 245, 205, 180, 213, 161, 8, 47, 107, 13, 105, 218, 1, 9, 5, 128, + 206, 134, 32, 111, 116, 100, 111, 103, 134, 110, 111, 116, 99, 97, 116, 128, 128 + ], + vec![ + 235, 128, 128, 128, 128, 128, 128, 208, 133, 53, 116, 116, 101, 114, 137, 99, + 111, 110, 102, 117, 115, 105, 111, 110, 202, 132, 53, 110, 99, 104, 132, 116, + 105, 109, 101, 128, 128, 128, 128, 128, 128, 128, 128, 128 + ] + ]); + } +} From 201cdcbd5e3a25ff0ab57ad4ddc4be0b110c58b1 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 15:46:14 +0200 Subject: [PATCH 022/160] SecTrieDBMut + test --- util/patricia_trie/src/lib.rs | 32 +++++++------- util/patricia_trie/src/sectriedbmut.rs | 58 +++++++++++++------------- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 547bc57e3d3..790f75532eb 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -39,7 +39,7 @@ pub mod node; pub mod triedb; pub mod triedbmut; pub mod sectriedb; -//pub mod sectriedbmut; +pub mod sectriedbmut; pub mod recorder; mod fatdb; @@ -50,7 +50,7 @@ mod nibblevec; pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::triedbmut::TrieDBMut; -//pub use self::sectriedbmut::SecTrieDBMut; +pub use self::sectriedbmut::SecTrieDBMut; pub use self::sectriedb::SecTrieDB; pub use self::fatdb::{FatDB, FatDBIterator}; pub use self::fatdbmut::FatDBMut; @@ -287,22 +287,22 @@ where H::Out: rlp::Decodable + rlp::Encodable } /// Create new mutable instance of Trie. -// pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { -// match self.spec { -// TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), -// TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), -// TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), -// } -// } + pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { + match self.spec { + TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), + TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), + TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), + } + } /// Create new mutable instance of trie and check for errors. -// pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { -// match self.spec { -// TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), -// TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), -// TrieSpec::Fat => Ok(Box::new(FatDBMut::from_existing(db, root)?)), -// } -// } + pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { + match self.spec { + TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), + TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), + TrieSpec::Fat => Ok(Box::new(FatDBMut::from_existing(db, root)?)), + } + } /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 3e5449f01ca..425ae1e0c8d 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -14,19 +14,19 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use keccak::keccak; use hashdb::{HashDB, DBValue, Hasher}; use super::triedbmut::TrieDBMut; use super::TrieMut; +use rlp::{Encodable, Decodable}; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. -pub struct SecTrieDBMut<'db, H: Hasher + 'db> { +pub struct SecTrieDBMut<'db, H: Hasher + 'db> where H::Out: Decodable + Encodable { raw: TrieDBMut<'db, H> } -impl<'db, H: Hasher> SecTrieDBMut<'db, H> { +impl<'db, H: Hasher> SecTrieDBMut<'db, H> where H::Out: Decodable + Encodable { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. @@ -37,7 +37,7 @@ impl<'db, H: Hasher> SecTrieDBMut<'db, H> { /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { Ok(SecTrieDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -48,7 +48,7 @@ impl<'db, H: Hasher> SecTrieDBMut<'db, H> { pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } -impl<'db, H: Hasher> TrieMut for SecTrieDBMut<'db, H> { +impl<'db, H: Hasher> TrieMut for SecTrieDBMut<'db, H> where H::Out: Decodable + Encodable { type H = H; fn root(&mut self) -> &::Out { self.raw.root() @@ -58,37 +58,39 @@ impl<'db, H: Hasher> TrieMut for SecTrieDBMut<'db, H> { self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> super::Result { - self.raw.contains(&Self::H::hash(key)) + fn contains(&self, key: &[u8]) -> super::Result::Out> { + self.raw.contains(&Self::H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result, ::Out> where 'a: 'key { - self.raw.get(&Self::H::hash(key)) + self.raw.get(&Self::H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { - self.raw.insert(&Self::H::hash(key), value) + fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result, ::Out> { + self.raw.insert(&Self::H::hash(key).as_ref(), value) } - fn remove(&mut self, key: &[u8]) -> super::Result> { - self.raw.remove(&Self::H::hash(key)) + fn remove(&mut self, key: &[u8]) -> super::Result, ::Out> { + self.raw.remove(&Self::H::hash(key).as_ref()) } } -//#[test] -//fn sectrie_to_trie() { -// use memorydb::*; -// use super::triedb::*; -// use super::Trie; -// -// let mut memdb = MemoryDB::new(); -// let mut root = H256::default(); -// { -// let mut t = SecTrieDBMut::new(&mut memdb, &mut root); -// t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); -// } -// let t = TrieDB::new(&memdb, &root).unwrap(); -// assert_eq!(t.get(&keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -//} +#[test] +fn sectrie_to_trie() { + use memorydb::*; + use super::triedb::*; + use super::Trie; + use hashdb::KeccakHasher; + use keccak; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = SecTrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + } + let t = TrieDB::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); +} From 6bf7e5bdce1b26928ec726a4339927ef811f0cf1 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 15:53:51 +0200 Subject: [PATCH 023/160] Remove scratch pad files --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 82506652961..de1a78bf4fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,7 +136,6 @@ members = [ "transaction-pool", "whisper", "whisper/cli", - "util/learn-generics", ] [patch.crates-io] From fc058e6b16348bfed55234fc8e2936428707ba00 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 16:02:40 +0200 Subject: [PATCH 024/160] Remove more traces of scratch pad files --- Cargo.lock | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c44401b301..06a6658bf9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1526,13 +1526,6 @@ name = "lazycell" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "learn-generics" -version = "0.1.0" -dependencies = [ - "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "libc" version = "0.2.36" From 03ff20eb943474b3257307979b983d1e24a3846d Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 16:15:16 +0200 Subject: [PATCH 025/160] Add a test for using MemoryDB with a different hasher (TODO) --- util/memorydb/src/lib.rs | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 2604dcc387d..3da8e8bc454 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -329,26 +329,19 @@ mod tests { assert_eq!(overlay.get(&negative_remove_key).unwrap(), &(DBValue::from_slice(b"negative"), -2)); } -// #[test] -// fn other_hashers() { -// struct DummyHasher; -// impl Hasher for DummyHasher { -// // TODO: Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) -// // It looks like this work must touch H256FastMap as well. -// type Out = ethereum_types::H264; -// const HASHED_NULL_RLP: ethereum_types::H264= ethereum_types::H264([0; 33]); -// fn hash(_x: &[u8]) -> Self::Out { -// ethereum_types::H264(*b"010102020101020201010202010102025") -// -// } -// } -// -// impl HeapSizeOf for DummyHasher { -// fn heap_size_of_children(&self) -> usize { 0 } -// } -// let mut db = MemoryDB::::new(); -//// let key = db.insert(b"32103210321032103210321032103210"); -// // Fails -//// assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); -// } + #[test] + fn other_hashers() { + struct DummyHasher; + impl Hasher for DummyHasher { + type Out = ethereum_types::H160; + const HASHED_NULL_RLP: ethereum_types::H160= ethereum_types::H160([0; 20]); + fn hash(_x: &[u8]) -> Self::Out { ethereum_types::H160(*b"01010202010102020101") } + } + impl HeapSizeOf for DummyHasher { fn heap_size_of_children(&self) -> usize { 0 } } + + let mut db = MemoryDB::::new(); + // TODO: Fails. Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) + // let key = db.insert(b"32103210321032103210321032103210"); + // assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); + } } From 8082d40e080fadbd84c2e3d254ed1f2bad58e46c Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 30 May 2018 16:22:05 +0200 Subject: [PATCH 026/160] cleanup --- util/patricia_trie/src/recorder.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 27738dbb3a8..e3a00ef0d3f 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -16,8 +16,6 @@ //! Trie query recorder. -//use keccak::keccak; -//use ethereum_types::H256; use bytes::Bytes; use hashdb::Hasher; use std::marker::PhantomData; @@ -67,7 +65,6 @@ impl Recorder { } /// Record a visited node, given its hash, data, and depth. -// pub fn record(&mut self, hash: &H256, data: &[u8], depth: u32) { pub fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { debug_assert_eq!(H::hash(data), *hash); From c9fc6a91061c1bf23641d52c85a004d0b4647bd6 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 31 May 2018 08:01:00 +0200 Subject: [PATCH 027/160] cleanup --- util/hashdb/src/lib.rs | 4 ++-- util/patricia_trie/src/lookup.rs | 8 +++----- util/patricia_trie/src/triedbmut.rs | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 3f8da276b79..322b18b69ca 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -34,8 +34,8 @@ pub trait Hasher: Sync + Send { fn hash(x: &[u8]) -> Self::Out; } -#[derive(Debug, Clone, PartialEq)] // REVIEW: Where do the concrete Hasher implementations go? Own crate? +#[derive(Debug, Clone, PartialEq)] pub struct KeccakHasher; impl Hasher for KeccakHasher { type Out = H256; @@ -69,7 +69,7 @@ pub trait HashDB: Send + Sync { /// is considered dead. fn insert(&mut self, value: &[u8]) -> ::Out; - /// Like `insert()` , except you provide the key and the data is all moved. + /// Like `insert()`, except you provide the key and the data is all moved. fn emplace(&mut self, key: ::Out, value: DBValue); /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index d66cb3e20f8..e0fabe298fd 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -18,11 +18,9 @@ use hashdb::{HashDB, Hasher}; use nibbleslice::NibbleSlice; -//use ethereum_types::H256; - -use super::{TrieError, Query}; -use super::node::Node; use rlp::Decodable; +use super::node::Node; +use super::{TrieError, Query}; /// Trie lookup helper object. pub struct Lookup<'a, H: Hasher + 'a, Q: Query> { @@ -31,7 +29,7 @@ pub struct Lookup<'a, H: Hasher + 'a, Q: Query> { /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at - pub hash: H::Out, // TODO + pub hash: H::Out, } impl<'a, H: Hasher + 'a, Q: Query> Lookup<'a, H, Q> where H::Out: Decodable { diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 4759652bb3c..c56de9653e2 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -875,7 +875,7 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> where H::Out: Decodable + Encod fn is_empty(&self) -> bool { match self.root_handle { - NodeHandle::Hash(h) => h == Self::H::HASHED_NULL_RLP, // TODO + NodeHandle::Hash(h) => h == Self::H::HASHED_NULL_RLP, NodeHandle::InMemory(ref h) => match self.storage[h] { Node::Empty => true, _ => false, From 13d1de3e40ba1dbe36005da10be246adc298fb8a Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 31 May 2018 08:13:39 +0200 Subject: [PATCH 028/160] Comment out WIP test --- util/memorydb/src/lib.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 3da8e8bc454..2439819a498 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -329,19 +329,19 @@ mod tests { assert_eq!(overlay.get(&negative_remove_key).unwrap(), &(DBValue::from_slice(b"negative"), -2)); } - #[test] - fn other_hashers() { - struct DummyHasher; - impl Hasher for DummyHasher { - type Out = ethereum_types::H160; - const HASHED_NULL_RLP: ethereum_types::H160= ethereum_types::H160([0; 20]); - fn hash(_x: &[u8]) -> Self::Out { ethereum_types::H160(*b"01010202010102020101") } - } - impl HeapSizeOf for DummyHasher { fn heap_size_of_children(&self) -> usize { 0 } } - - let mut db = MemoryDB::::new(); - // TODO: Fails. Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) - // let key = db.insert(b"32103210321032103210321032103210"); - // assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); - } +// #[test] +// fn other_hashers() { +// struct DummyHasher; +// impl Hasher for DummyHasher { +// type Out = ethereum_types::H160; +// const HASHED_NULL_RLP: ethereum_types::H160= ethereum_types::H160([0; 20]); +// fn hash(_x: &[u8]) -> Self::Out { ethereum_types::H160(*b"01010202010102020101") } +// } +// impl HeapSizeOf for DummyHasher { fn heap_size_of_children(&self) -> usize { 0 } } +// +// let mut db = MemoryDB::::new(); +// // TODO: Fails. Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) +// let key = db.insert(b"32103210321032103210321032103210"); +// assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); +// } } From 33e05317676a10878f108197da12a88782e03b25 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 31 May 2018 08:15:26 +0200 Subject: [PATCH 029/160] cleanup --- util/patricia_trie/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 790f75532eb..fb16d46ce2d 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -195,7 +195,6 @@ pub trait TrieMut { pub trait TrieIterator: Iterator { /// Position the iterator on the first element with key > `key` fn seek(&mut self, key: &[u8]) -> Result<(), H::Out>; -// fn seek(&mut self, key: &H::Out) -> Result<(), H::Out>; } /// Trie types From ef707b085bf29e735d41345de6119efb9c1e9cf0 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 31 May 2018 09:34:15 +0200 Subject: [PATCH 030/160] TrieError needs T: Debug --- ethcore/vm/src/error.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/vm/src/error.rs b/ethcore/vm/src/error.rs index fe8d7054cfe..3d5f867addc 100644 --- a/ethcore/vm/src/error.rs +++ b/ethcore/vm/src/error.rs @@ -72,8 +72,8 @@ pub enum Error { } -impl From> for Error { - fn from(err: Box) -> Self { +impl From>> for Error where T: fmt::Debug { + fn from(err: Box>) -> Self { Error::Internal(format!("Internal error: {}", err)) } } From ada0b7e45c7eece2c5ebbbb34ca08c3de3ed2e7a Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 31 May 2018 11:50:28 +0200 Subject: [PATCH 031/160] Cleanup --- util/hashdb/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 322b18b69ca..01461689737 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -85,21 +85,21 @@ pub trait AsHashDB { fn as_hashdb_mut(&mut self) -> &mut HashDB; } -impl> AsHashDB for T { - fn as_hashdb(&self) -> &HashDB { +impl> AsHashDB for T { + fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } } -impl<'a, HF: Hasher> AsHashDB for &'a mut HashDB { - fn as_hashdb(&self) -> &HashDB { +impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { + fn as_hashdb(&self) -> &HashDB { &**self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { &mut **self } } From 56af2c47d53df77a8a234d1ed65960ed1f631a20 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 31 May 2018 11:50:36 +0200 Subject: [PATCH 032/160] Don't compare rc against i32::MIN --- util/memorydb/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 2439819a498..1318b445d35 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -176,7 +176,7 @@ impl MemoryDB { } impl HashDB for MemoryDB { - type H = H; // REVIEW this is a bit confusing, but not sure what is better (e.g. using `impl … … type H = HH;`) + type H = H; // REVIEW: this method is what made it necessary to add a type param to H256FastMap, which I'd rather have avoided. // The problem is that the keys returned are `H256` and type inference fails on the `collect()` call. @@ -222,7 +222,7 @@ impl HashDB for MemoryDB { match self.data.entry(key) { Entry::Occupied(mut entry) => { let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); - if *rc >= -0x80000000i32 && *rc <= 0 { + if *rc <= 0 { *old_value = DBValue::from_slice(value); } *rc += 1; @@ -242,7 +242,7 @@ impl HashDB for MemoryDB { match self.data.entry(key) { Entry::Occupied(mut entry) => { let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); - if *rc >= -0x80000000i32 && *rc <= 0 { + if *rc <= 0 { *old_value = value; } *rc += 1; From 3f1cfc535cd83f9bc8d6293ff6f0bfc08e414cc7 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 1 Jun 2018 08:35:29 +0200 Subject: [PATCH 033/160] missing use's --- util/plain_hasher/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/plain_hasher/src/lib.rs b/util/plain_hasher/src/lib.rs index 8cdec9df72a..20bb08075f7 100644 --- a/util/plain_hasher/src/lib.rs +++ b/util/plain_hasher/src/lib.rs @@ -5,7 +5,8 @@ extern crate hashdb; use ethereum_types::H256; use hashdb::Hasher; - +use std::hash; +use std::collections::{HashMap, HashSet}; /// Specialized version of `HashMap` with H256 keys and fast hashing function. pub type H256FastMap = HashMap<::Out, T, hash::BuildHasherDefault>; /// Specialized version of `HashSet` with H256 keys and fast hashing function. From 4e2e81b9c67587d1cc70ea1be205debaec9f1094 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 4 Jun 2018 09:46:09 +0200 Subject: [PATCH 034/160] Remove Decodable bound --- util/hashdb/Cargo.toml | 6 +++--- util/hashdb/src/lib.rs | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/util/hashdb/Cargo.toml b/util/hashdb/Cargo.toml index 6ac3967ca50..962f0e7f694 100644 --- a/util/hashdb/Cargo.toml +++ b/util/hashdb/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" -ethereum-types = "0.3" heapsize = "0.4" -tiny-keccak = "1.4.2" -rlp = "*" \ No newline at end of file +# TODO: Used by the `KeccakHasher` impl which I think should move to own crate? +ethereum-types = "0.3" +tiny-keccak = "1.4.2" \ No newline at end of file diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 01461689737..0a9a9fc588f 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -19,7 +19,6 @@ extern crate elastic_array; extern crate ethereum_types; extern crate heapsize; extern crate tiny_keccak; -extern crate rlp; use elastic_array::ElasticArray128; use ethereum_types::H256; @@ -29,7 +28,7 @@ use std::{fmt::Debug, hash::Hash}; use tiny_keccak::Keccak; pub trait Hasher: Sync + Send { - type Out: AsRef<[u8]> + Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? --> */ + HeapSizeOf /* …and this? -> */ + rlp::Decodable; + type Out: AsRef<[u8]> + Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? --> */ + HeapSizeOf; const HASHED_NULL_RLP: Self::Out; fn hash(x: &[u8]) -> Self::Out; } From c149786300cf062b280edacb1521af6cfe297740 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 4 Jun 2018 12:12:50 +0200 Subject: [PATCH 035/160] Comment out fatdb/sectriedb --- util/patricia_trie/src/lib.rs | 59 ++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index fb16d46ce2d..93614c3faf1 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -38,22 +38,22 @@ use std::marker::PhantomData; pub mod node; pub mod triedb; pub mod triedbmut; -pub mod sectriedb; -pub mod sectriedbmut; +//pub mod sectriedb; +//pub mod sectriedbmut; pub mod recorder; -mod fatdb; -mod fatdbmut; +//mod fatdb; +//mod fatdbmut; mod lookup; mod nibbleslice; mod nibblevec; pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::triedbmut::TrieDBMut; -pub use self::sectriedbmut::SecTrieDBMut; -pub use self::sectriedb::SecTrieDB; -pub use self::fatdb::{FatDB, FatDBIterator}; -pub use self::fatdbmut::FatDBMut; +//pub use self::sectriedbmut::SecTrieDBMut; +//pub use self::sectriedb::SecTrieDB; +//pub use self::fatdb::{FatDB, FatDBIterator}; +//pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; /// Trie Errors. @@ -227,10 +227,13 @@ pub enum TrieKinds<'db, H: Hasher + 'db> { /// A generic trie db. Generic(TrieDB<'db, H>), /// A secure trie db. -// Secure(TrieDB<'db, H>), - Secure(SecTrieDB<'db, H>), + Secure(TrieDB<'db, H>), /// A fat trie db. - Fat(FatDB<'db, H>), + Fat(TrieDB<'db, H>), +// /// A secure trie db. +// Secure(SecTrieDB<'db, H>), +// /// A fat trie db. +// Fat(FatDB<'db, H>), } // wrapper macro for making the match easier to deal with. @@ -280,8 +283,10 @@ where H::Out: rlp::Decodable + rlp::Encodable pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), - TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), - TrieSpec::Fat => Ok(TrieKinds::Fat(FatDB::new(db, root)?)), + TrieSpec::Secure => Ok(TrieKinds::Secure(TrieDB::new(db, root)?)), + TrieSpec::Fat => Ok(TrieKinds::Fat(TrieDB::new(db, root)?)), +// TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), +// TrieSpec::Fat => Ok(TrieKinds::Fat(FatDB::new(db, root)?)), } } @@ -289,20 +294,22 @@ where H::Out: rlp::Decodable + rlp::Encodable pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { match self.spec { TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), - TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), - TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), + TrieSpec::Secure => Box::new(TrieDBMut::new(db, root)), + TrieSpec::Fat => Box::new(TrieDBMut::new(db, root)), +// TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), +// TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), } } - /// Create new mutable instance of trie and check for errors. - pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { - match self.spec { - TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), - TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), - TrieSpec::Fat => Ok(Box::new(FatDBMut::from_existing(db, root)?)), - } - } - - /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). - pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } +// /// Create new mutable instance of trie and check for errors. +// pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { +// match self.spec { +// TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), +// TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), +// TrieSpec::Fat => Ok(Box::new(FatDBMut::from_existing(db, root)?)), +// } +// } + +// /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). +// pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } } From 68fa728b99e3be573ca587fbfb8a09a5d7b994a7 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 4 Jun 2018 12:47:11 +0200 Subject: [PATCH 036/160] Add `NodeCodec` with impl for `Node` (the Rlp-kind) --- util/patricia_trie/src/lib.rs | 6 +- util/patricia_trie/src/lookup.rs | 1 + util/patricia_trie/src/node.rs | 97 +++++++++++++--------------- util/patricia_trie/src/node_codec.rs | 17 +++++ util/patricia_trie/src/triedb.rs | 3 +- util/patricia_trie/src/triedbmut.rs | 11 ++-- 6 files changed, 73 insertions(+), 62 deletions(-) create mode 100644 util/patricia_trie/src/node_codec.rs diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 93614c3faf1..157fedf11bf 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -36,6 +36,7 @@ use hashdb::{HashDB, DBValue, Hasher}; use std::marker::PhantomData; pub mod node; +pub mod node_codec; pub mod triedb; pub mod triedbmut; //pub mod sectriedb; @@ -246,6 +247,7 @@ macro_rules! wrapper { } } } + impl<'db, H: Hasher> Trie for TrieKinds<'db, H> where H::Out: rlp::Decodable + rlp::Encodable { type H = H; fn root(&self) -> &::Out { @@ -271,9 +273,7 @@ impl<'db, H: Hasher> Trie for TrieKinds<'db, H> where H::Out: rlp::Decodable + r } } -impl TrieFactory -where H::Out: rlp::Decodable + rlp::Encodable -{ +impl TrieFactory where H::Out: rlp::Decodable + rlp::Encodable { /// Creates new factory. pub fn new(spec: TrieSpec) -> Self { TrieFactory { spec, marker: PhantomData } diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index e0fabe298fd..854a1c3d3b0 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -20,6 +20,7 @@ use hashdb::{HashDB, Hasher}; use nibbleslice::NibbleSlice; use rlp::Decodable; use super::node::Node; +use node_codec::NodeCodec; use super::{TrieError, Query}; /// Trie lookup helper object. diff --git a/util/patricia_trie/src/node.rs b/util/patricia_trie/src/node.rs index 9bbea4e2d1a..e9b0b061c3c 100644 --- a/util/patricia_trie/src/node.rs +++ b/util/patricia_trie/src/node.rs @@ -20,57 +20,13 @@ use nibblevec::NibbleVec; use bytes::*; use rlp::{Rlp, RlpStream, Prototype, DecoderError, self}; use hashdb::DBValue; +use node_codec::NodeCodec; /// Partial node key type. pub type NodeKey = ElasticArray36; -/// Type of node in the trie and essential information thereof. -#[derive(Eq, PartialEq, Debug, Clone)] -pub enum Node<'a> { - /// Null trie node; could be an empty root or an empty branch entry. - Empty, - /// Leaf node; has key slice and value. Value may not be empty. - Leaf(NibbleSlice<'a>, &'a [u8]), - /// Extension node; has key slice and node data. Data may not be null. - Extension(NibbleSlice<'a>, &'a [u8]), - /// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data. - Branch([&'a [u8]; 16], Option<&'a [u8]>), -} - -impl<'a> Node<'a> { - /// Decode the `node_rlp` and return the Node. - pub fn decoded(node_rlp: &'a [u8]) -> Result { - let r = Rlp::new(node_rlp); - match r.prototype()? { - // either leaf or extension - decode first item with NibbleSlice::??? - // and use is_leaf return to figure out which. - // if leaf, second item is a value (is_data()) - // if extension, second item is a node (either SHA3 to be looked up and - // fed back into this function or inline RLP which can be fed back into this function). - Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { - (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), - (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), - }, - // branch - first 16 are nodes, 17th is a value (or empty). - Prototype::List(17) => { - let mut nodes = [&[] as &[u8]; 16]; - for i in 0..16 { - nodes[i] = r.at(i)?.as_raw(); - } - Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) - }, - // an empty branch index. - Prototype::Data(0) => Ok(Node::Empty), - // something went wrong. - _ => Err(DecoderError::Custom("Rlp is not valid.")) - } - } - - /// Encode the node into RLP. - /// - /// Will always return the direct node RLP even if it's 32 or more bytes. To get the - /// RLP which would be valid for using in another node, use `encoded_and_added()`. - pub fn encoded(&self) -> Bytes { +impl<'a> NodeCodec<'a> for Node<'a> { + fn encoded(&self) -> Bytes { match *self { Node::Leaf(ref slice, ref value) => { let mut stream = RlpStream::new_list(2); @@ -101,12 +57,36 @@ impl<'a> Node<'a> { stream.out() } } - } - pub fn try_decode_hash(node_data: &[u8]) -> Option - where O: rlp::Decodable // REVIEW: this is not necessary but is perhaps useful when reading the code? Keep or leave out? - { - let r = Rlp::new(node_data); + } + fn decoded(data: &'a [u8]) -> Result { + let r = Rlp::new(data); + match r.prototype()? { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { + (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), + (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes = [&[] as &[u8]; 16]; + for i in 0..16 { + nodes[i] = r.at(i)?.as_raw(); + } + Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) + }, + // an empty branch index. + Prototype::Data(0) => Ok(Node::Empty), + // something went wrong. + _ => Err(DecoderError::Custom("Rlp is not valid.")) + } + } + fn try_decode_hash(data: &[u8]) -> Option where O: rlp::Decodable { + let r = Rlp::new(data); if r.is_data() && r.size() == 32 { Some(r.as_val().expect("Hash is the correct size of 32 bytes; qed")) } else { @@ -115,6 +95,19 @@ impl<'a> Node<'a> { } } +/// Type of node in the trie and essential information thereof. +#[derive(Eq, PartialEq, Debug, Clone)] +pub enum Node<'a> { + /// Null trie node; could be an empty root or an empty branch entry. + Empty, + /// Leaf node; has key slice and value. Value may not be empty. + Leaf(NibbleSlice<'a>, &'a [u8]), + /// Extension node; has key slice and node data. Data may not be null. + Extension(NibbleSlice<'a>, &'a [u8]), + /// Branch node; has array of 16 child nodes (each possibly null) and an optional immediate node data. + Branch([&'a [u8]; 16], Option<&'a [u8]>), +} + /// An owning node type. Useful for trie iterators. #[derive(Debug, PartialEq, Eq)] pub enum OwnedNode { diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs new file mode 100644 index 00000000000..f354a10a618 --- /dev/null +++ b/util/patricia_trie/src/node_codec.rs @@ -0,0 +1,17 @@ +//pub trait NodeCodec { +// type N; +// type O: Decodable + Debug; +// fn encoded(&self) -> Vec; +// fn decoded(data: &[u8]) -> Result; +// fn try_decode_hash(data: &[u8]) -> Option; +//} + +use bytes::*; +use rlp::{DecoderError, Decodable}; + +pub trait NodeCodec<'a>: Sized { + fn encoded(&self) -> Bytes; + fn decoded(data: &'a [u8]) -> Result; + fn try_decode_hash(data: &[u8]) -> Option where O: Decodable; +} + diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index d4639d5dc75..c2a52131374 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -18,6 +18,7 @@ use std::fmt; use hashdb::*; use nibbleslice::NibbleSlice; use super::node::{Node, OwnedNode}; +use node_codec::NodeCodec; use super::lookup::Lookup; use super::{Trie, TrieItem, TrieError, TrieIterator, Query}; use bytes::Bytes; @@ -151,7 +152,6 @@ impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> where H:: } } - impl<'db, H: Hasher> fmt::Debug for TrieDB<'db, H> where H::Out: Decodable { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let root_rlp = self.db.get(self.root).expect("Trie root not found!"); @@ -486,7 +486,6 @@ fn get_len() { assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()), Ok(None)); } - #[test] fn debug_output_supports_pretty_print() { use memorydb::*; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index c56de9653e2..cc48316e062 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -19,6 +19,7 @@ use super::{TrieError, TrieMut}; use super::lookup::Lookup; use super::node::Node as RlpNode; +use node_codec::NodeCodec; use super::node::NodeKey; use hashdb::HashDB; @@ -89,14 +90,14 @@ impl Node where H::Out: Decodable { RlpNode::try_decode_hash::(&node) .map(NodeHandle::Hash) .unwrap_or_else(|| { - let child = Node::from_rlp(node, db, storage); + let child = Node::from_encoded(node, db, storage); NodeHandle::InMemory(storage.alloc(Stored::New(child))) }) } // decode a node from rlp without getting its children. - fn from_rlp(rlp: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { - match RlpNode::decoded(rlp).expect("rlp read from db; qed") { + fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { + match RlpNode::decoded(data).expect("encoded bytes read from db; qed") { RlpNode::Empty => Node::Empty, RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)), RlpNode::Extension(key, cb) => { @@ -346,8 +347,8 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { // cache a node by hash fn cache(&mut self, hash: H::Out) -> super::Result { - let node_rlp = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; - let node = Node::from_rlp(&node_rlp, &*self.db, &mut self.storage); + let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; + let node = Node::from_encoded(&node_encoded, &*self.db, &mut self.storage); Ok(self.storage.alloc(Stored::Cached(node, hash))) } From c198d0f2a2e80386b8265cf2b6b6156869a59d75 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 4 Jun 2018 14:45:34 +0200 Subject: [PATCH 037/160] WIP try to extract more rlp-dependent code into `NodeCodec` trait --- Cargo.lock | 13 ------------- util/patricia_trie/src/node.rs | 17 +++++++++++++++++ util/patricia_trie/src/node_codec.rs | 15 +++++++-------- util/patricia_trie/src/triedbmut.rs | 26 ++++++++++++++++---------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc64f98d304..a6170c58fa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1203,7 +1203,6 @@ dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rlp 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2798,17 +2797,6 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "rlp" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "rlp_compress" version = "0.1.0" @@ -4025,7 +4013,6 @@ dependencies = [ "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" "checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" "checksum ring 0.12.1 (git+https://github.com/paritytech/ring)" = "" -"checksum rlp 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "89db7f8dfdd5eb7ab3ac3ece7a07fd273a680b4b224cb231181280e8996f9f0b" "checksum rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb)" = "" "checksum rocksdb-sys 0.3.0 (git+https://github.com/paritytech/rust-rocksdb)" = "" "checksum rpassword 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b273c91bd242ca03ad6d71c143b6f17a48790e61f21a6c78568fa2b6774a24a4" diff --git a/util/patricia_trie/src/node.rs b/util/patricia_trie/src/node.rs index e9b0b061c3c..2270af7e8eb 100644 --- a/util/patricia_trie/src/node.rs +++ b/util/patricia_trie/src/node.rs @@ -26,6 +26,8 @@ use node_codec::NodeCodec; pub type NodeKey = ElasticArray36; impl<'a> NodeCodec<'a> for Node<'a> { + type Encoding = Rlp<'a>; + type StreamEncoding = RlpStream; fn encoded(&self) -> Bytes { match *self { Node::Leaf(ref slice, ref value) => { @@ -93,6 +95,21 @@ impl<'a> NodeCodec<'a> for Node<'a> { None } } + + fn new_encoded(data: &'a [u8]) -> Self::Encoding { + Rlp::new(data) + } + + fn encoded_stream() -> Self::StreamEncoding { + RlpStream::new() + } + + fn encoded_list(size: usize) -> Self::StreamEncoding { + RlpStream::new_list(size) + } + + + } /// Type of node in the trie and essential information thereof. diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index f354a10a618..89c690fbf6b 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,17 +1,16 @@ -//pub trait NodeCodec { -// type N; -// type O: Decodable + Debug; -// fn encoded(&self) -> Vec; -// fn decoded(data: &[u8]) -> Result; -// fn try_decode_hash(data: &[u8]) -> Option; -//} - use bytes::*; use rlp::{DecoderError, Decodable}; pub trait NodeCodec<'a>: Sized { + type Encoding; + type StreamEncoding; + fn encoded(&self) -> Bytes; fn decoded(data: &'a [u8]) -> Result; fn try_decode_hash(data: &[u8]) -> Option where O: Decodable; + + fn new_encoded(data: &'a[u8]) -> Self::Encoding; + fn encoded_stream() -> Self::StreamEncoding; + fn encoded_list(size: usize) -> Self::StreamEncoding; } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index cc48316e062..066ae3caf59 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -103,11 +103,11 @@ impl Node where H::Out: Decodable { RlpNode::Extension(key, cb) => { Node::Extension(key.encoded(false), Self::inline_or_hash(cb, db, storage)) } - RlpNode::Branch(ref children_rlp, val) => { - let mut child = |i| { - let raw = children_rlp[i]; - let child_rlp = Rlp::new(raw); - if !child_rlp.is_empty() { + RlpNode::Branch(ref encoded_children, val) => { + let mut child = |i:usize| { + let raw = encoded_children[i]; + let encoded_child = RlpNode::new_encoded(raw); + if !encoded_child.is_empty() { Some(Self::inline_or_hash(raw, db, storage)) } else { None @@ -129,28 +129,29 @@ impl Node where H::Out: Decodable { // encode a node to RLP // TODO: parallelize fn into_rlp(self, mut child_cb: F) -> ElasticArray1024 - where F: FnMut(NodeHandle, &mut RlpStream) + where F: FnMut(NodeHandle, &mut RlpStream) // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` +// where F: FnMut(NodeHandle, &mut ::StreamEncoding) { match self { Node::Empty => { - let mut stream = RlpStream::new(); + let mut stream = RlpNode::encoded_stream(); stream.append_empty_data(); stream.drain() } Node::Leaf(partial, value) => { - let mut stream = RlpStream::new_list(2); + let mut stream = RlpNode::encoded_list(2); stream.append(&&*partial); stream.append(&&*value); stream.drain() } Node::Extension(partial, child) => { - let mut stream = RlpStream::new_list(2); + let mut stream = RlpNode::encoded_list(2); stream.append(&&*partial); child_cb(child, &mut stream); stream.drain() } Node::Branch(mut children, value) => { - let mut stream = RlpStream::new_list(17); + let mut stream = RlpNode::encoded_list(17); for child in children.iter_mut().map(Option::take) { if let Some(handle) = child { child_cb(handle, &mut stream); @@ -839,6 +840,11 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { /// commit a node, hashing it, committing it to the db, /// and writing it to the rlp stream as necessary. fn commit_node(&mut self, handle: NodeHandle, stream: &mut RlpStream) { +// fn commit_node( +// &mut self, handle: NodeHandle, +// stream: &mut ::StreamEncoding +// ) +// { match handle { NodeHandle::Hash(h) => stream.append(&h), NodeHandle::InMemory(h) => match self.storage.destroy(h) { From a7e9cbfb1f3365ce6e2aea791dafa6efe391e226 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 5 Jun 2018 18:09:02 +0200 Subject: [PATCH 038/160] WIP Add NodeCodec in the proper place --- util/patricia_trie/src/lib.rs | 25 ++- util/patricia_trie/src/lookup.rs | 11 +- util/patricia_trie/src/node.rs | 87 -------- util/patricia_trie/src/node_codec.rs | 119 ++++++++++- util/patricia_trie/src/recorder.rs | 3 + util/patricia_trie/src/triedb.rs | 305 ++++++++++++++------------- util/patricia_trie/src/triedbmut.rs | 155 +++++++++----- 7 files changed, 385 insertions(+), 320 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 157fedf11bf..17c9938a20b 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -56,6 +56,7 @@ pub use self::triedbmut::TrieDBMut; //pub use self::fatdb::{FatDB, FatDBIterator}; //pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; +use node_codec::NodeCodec; /// Trie Errors. /// @@ -217,20 +218,21 @@ impl Default for TrieSpec { /// Trie factory. #[derive(Default, Clone)] -pub struct TrieFactory { +pub struct TrieFactory> { spec: TrieSpec, - marker: PhantomData + mark_hash: PhantomData, + mark_codec: PhantomData, } /// All different kinds of tries. /// This is used to prevent a heap allocation for every created trie. -pub enum TrieKinds<'db, H: Hasher + 'db> { +pub enum TrieKinds<'db, H: Hasher + 'db, C: NodeCodec> { /// A generic trie db. - Generic(TrieDB<'db, H>), + Generic(TrieDB<'db, H, C>), /// A secure trie db. - Secure(TrieDB<'db, H>), + Secure(TrieDB<'db, H, C>), /// A fat trie db. - Fat(TrieDB<'db, H>), + Fat(TrieDB<'db, H, C>), // /// A secure trie db. // Secure(SecTrieDB<'db, H>), // /// A fat trie db. @@ -248,7 +250,7 @@ macro_rules! wrapper { } } -impl<'db, H: Hasher> Trie for TrieKinds<'db, H> where H::Out: rlp::Decodable + rlp::Encodable { +impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> where H::Out: rlp::Decodable + rlp::Encodable { type H = H; fn root(&self) -> &::Out { wrapper!(self, root,) @@ -273,14 +275,14 @@ impl<'db, H: Hasher> Trie for TrieKinds<'db, H> where H::Out: rlp::Decodable + r } } -impl TrieFactory where H::Out: rlp::Decodable + rlp::Encodable { +impl> TrieFactory where H::Out: rlp::Decodable + rlp::Encodable { /// Creates new factory. pub fn new(spec: TrieSpec) -> Self { - TrieFactory { spec, marker: PhantomData } + TrieFactory { spec, mark_hash: PhantomData, mark_codec: PhantomData } } /// Create new immutable instance of Trie. - pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { + pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), TrieSpec::Secure => Ok(TrieKinds::Secure(TrieDB::new(db, root)?)), @@ -290,6 +292,7 @@ impl TrieFactory where H::Out: rlp::Decodable + rlp::Encodable { } } +/* /// Create new mutable instance of Trie. pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { match self.spec { @@ -300,7 +303,7 @@ impl TrieFactory where H::Out: rlp::Decodable + rlp::Encodable { // TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), } } - +*/ // /// Create new mutable instance of trie and check for errors. // pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { // match self.spec { diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 854a1c3d3b0..458067f695a 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -22,18 +22,20 @@ use rlp::Decodable; use super::node::Node; use node_codec::NodeCodec; use super::{TrieError, Query}; +use std::marker::PhantomData; /// Trie lookup helper object. -pub struct Lookup<'a, H: Hasher + 'a, Q: Query> { +pub struct Lookup<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> { /// database to query from. pub db: &'a HashDB, /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at pub hash: H::Out, + pub marker: PhantomData, // TODO: probably not needed when all is said and done? When Query is made generic? } -impl<'a, H: Hasher + 'a, Q: Query> Lookup<'a, H, Q> where H::Out: Decodable { +impl<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> Lookup<'a, H, C, Q> where H::Out: Decodable { /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result, H::Out> { @@ -55,7 +57,7 @@ impl<'a, H: Hasher + 'a, Q: Query> Lookup<'a, H, Q> where H::Out: Decodable { // without incrementing the depth. let mut node_data = &node_data[..]; loop { - match Node::decoded(node_data)? { + match C::decode(node_data)? { Node::Leaf(slice, value) => { return Ok(match slice == key { true => Some(self.query.decode(value)), @@ -81,7 +83,8 @@ impl<'a, H: Hasher + 'a, Q: Query> Lookup<'a, H, Q> where H::Out: Decodable { } // check if new node data is inline or hash. - if let Some(h) = Node::try_decode_hash::(&node_data) { +// if let Some(h) = Node::try_decode_hash::(&node_data) { + if let Some(h) = C::try_decode_hash(&node_data) { hash = h; break } diff --git a/util/patricia_trie/src/node.rs b/util/patricia_trie/src/node.rs index 2270af7e8eb..d055e82f2ef 100644 --- a/util/patricia_trie/src/node.rs +++ b/util/patricia_trie/src/node.rs @@ -25,93 +25,6 @@ use node_codec::NodeCodec; /// Partial node key type. pub type NodeKey = ElasticArray36; -impl<'a> NodeCodec<'a> for Node<'a> { - type Encoding = Rlp<'a>; - type StreamEncoding = RlpStream; - fn encoded(&self) -> Bytes { - match *self { - Node::Leaf(ref slice, ref value) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(true)); - stream.append(value); - stream.out() - }, - Node::Extension(ref slice, ref raw_rlp) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(false)); - stream.append_raw(raw_rlp, 1); - stream.out() - }, - Node::Branch(ref nodes, ref value) => { - let mut stream = RlpStream::new_list(17); - for i in 0..16 { - stream.append_raw(nodes[i], 1); - } - match *value { - Some(ref n) => { stream.append(n); }, - None => { stream.append_empty_data(); }, - } - stream.out() - }, - Node::Empty => { - let mut stream = RlpStream::new(); - stream.append_empty_data(); - stream.out() - } - } - - } - fn decoded(data: &'a [u8]) -> Result { - let r = Rlp::new(data); - match r.prototype()? { - // either leaf or extension - decode first item with NibbleSlice::??? - // and use is_leaf return to figure out which. - // if leaf, second item is a value (is_data()) - // if extension, second item is a node (either SHA3 to be looked up and - // fed back into this function or inline RLP which can be fed back into this function). - Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { - (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), - (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), - }, - // branch - first 16 are nodes, 17th is a value (or empty). - Prototype::List(17) => { - let mut nodes = [&[] as &[u8]; 16]; - for i in 0..16 { - nodes[i] = r.at(i)?.as_raw(); - } - Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) - }, - // an empty branch index. - Prototype::Data(0) => Ok(Node::Empty), - // something went wrong. - _ => Err(DecoderError::Custom("Rlp is not valid.")) - } - } - fn try_decode_hash(data: &[u8]) -> Option where O: rlp::Decodable { - let r = Rlp::new(data); - if r.is_data() && r.size() == 32 { - Some(r.as_val().expect("Hash is the correct size of 32 bytes; qed")) - } else { - None - } - } - - fn new_encoded(data: &'a [u8]) -> Self::Encoding { - Rlp::new(data) - } - - fn encoded_stream() -> Self::StreamEncoding { - RlpStream::new() - } - - fn encoded_list(size: usize) -> Self::StreamEncoding { - RlpStream::new_list(size) - } - - - -} - /// Type of node in the trie and essential information thereof. #[derive(Eq, PartialEq, Debug, Clone)] pub enum Node<'a> { diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 89c690fbf6b..8d678c3ec79 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,16 +1,115 @@ use bytes::*; -use rlp::{DecoderError, Decodable}; +use nibbleslice::NibbleSlice; +//use nibblevec::NibbleVec; +use rlp::{self, Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; +use hashdb::Hasher; +use node::Node; +use std::marker::PhantomData; +//pub trait OldNodeCodec<'a>: Sized { +// type Encoding; +// type StreamEncoding; +// +// fn encoded(&self) -> Bytes; +// fn decoded(data: &'a [u8]) -> Result; +// fn try_decode_hash(data: &[u8]) -> Option where O: Decodable; +// +// fn new_encoded(data: &'a[u8]) -> Self::Encoding; +// fn encoded_stream() -> Self::StreamEncoding; +// fn encoded_list(size: usize) -> Self::StreamEncoding; +//} -pub trait NodeCodec<'a>: Sized { - type Encoding; - type StreamEncoding; - fn encoded(&self) -> Bytes; - fn decoded(data: &'a [u8]) -> Result; - fn try_decode_hash(data: &[u8]) -> Option where O: Decodable; +pub trait NodeCodec: Sized { + fn encode(&Node) -> Bytes; + fn decode(data: &[u8]) -> Result; // TODO: make the error generic here, perhaps an associated type on the trait + fn try_decode_hash(data: &[u8]) -> Option; + + // TODO: We don't want these here, but where do they go? Helper trait? + fn new_encoded<'a>(data: &'a [u8]) -> Rlp<'a>; + fn encoded_stream() -> RlpStream; + fn encoded_list(size: usize) -> RlpStream; - fn new_encoded(data: &'a[u8]) -> Self::Encoding; - fn encoded_stream() -> Self::StreamEncoding; - fn encoded_list(size: usize) -> Self::StreamEncoding; } +pub struct RlpNodeCodec {mark: PhantomData} + +impl NodeCodec for RlpNodeCodec where H::Out: Encodable + Decodable { + fn encode(node: &Node) -> Bytes { + match *node { + Node::Leaf(ref slice, ref value) => { + let mut stream = RlpStream::new_list(2); + stream.append(&&*slice.encoded(true)); + stream.append(value); + stream.out() + }, + Node::Extension(ref slice, ref raw_rlp) => { + let mut stream = RlpStream::new_list(2); + stream.append(&&*slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + stream.out() + }, + Node::Branch(ref nodes, ref value) => { + let mut stream = RlpStream::new_list(17); + for i in 0..16 { + stream.append_raw(nodes[i], 1); + } + match *value { + Some(ref n) => { stream.append(n); }, + None => { stream.append_empty_data(); }, + } + stream.out() + }, + Node::Empty => { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.out() + } + } + } + fn decode(data: &[u8]) -> Result { + let r = Rlp::new(data); + match r.prototype()? { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { + (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), + (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes = [&[] as &[u8]; 16]; + for i in 0..16 { + nodes[i] = r.at(i)?.as_raw(); + } + Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) + }, + // an empty branch index. + Prototype::Data(0) => Ok(Node::Empty), + // something went wrong. + _ => Err(DecoderError::Custom("Rlp is not valid.")) + } + } + fn try_decode_hash(data: &[u8]) -> Option { + let r = Rlp::new(data); + if r.is_data() && r.size() == 32 { + Some(r.as_val().expect("Hash is the correct size of 32 bytes; qed")) + } else { + None + } + } + + fn new_encoded<'a>(data: &'a [u8]) -> Rlp<'a> { + Rlp::new(data) + } + + fn encoded_stream() -> RlpStream { + RlpStream::new() + } + + fn encoded_list(size: usize) -> RlpStream{ + RlpStream::new_list(size) + } +} \ No newline at end of file diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index e3a00ef0d3f..66cf7bb250f 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -84,6 +84,8 @@ impl Recorder { } } +/* + #[cfg(test)] mod tests { use super::*; @@ -214,3 +216,4 @@ mod tests { ]); } } +*/ diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index c2a52131374..ce69f652e17 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -23,6 +23,7 @@ use super::lookup::Lookup; use super::{Trie, TrieItem, TrieError, TrieIterator, Query}; use bytes::Bytes; use rlp::{Decodable, Encodable}; +use std::marker::PhantomData; /// A `Trie` implementation using a generic `HashDB` backing database. /// @@ -40,31 +41,39 @@ use rlp::{Decodable, Encodable}; /// use hashdb::*; /// use memorydb::*; /// use ethereum_types::H256; +/// use node_codec::RlpNodeCodec; +/// +/// type RlpCodec = RlpNodeCodec; /// /// fn main() { /// let mut memdb = MemoryDB::::new(); /// let mut root = H256::new(); -/// TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap(); -/// let t = TrieDB::new(&memdb, &root).unwrap(); +/// TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap(); +/// let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); /// assert!(t.contains(b"foo").unwrap()); /// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar")); /// } /// ``` -pub struct TrieDB<'db, H: Hasher + 'db> { +pub struct TrieDB<'db, H, C> + where H: Hasher + 'db, C: NodeCodec +{ db: &'db HashDB, root: &'db H::Out, /// The number of hashes performed so far in operations on this trie. hash_count: usize, + codec_marker: PhantomData, } -impl<'db, H: Hasher> TrieDB<'db, H> where H::Out: Decodable { +impl<'db, H, C> TrieDB<'db, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { if !db.contains(root) { Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { - Ok(TrieDB {db, root, hash_count: 0}) + Ok(TrieDB {db, root, hash_count: 0, codec_marker: PhantomData}) } } @@ -82,8 +91,7 @@ impl<'db, H: Hasher> TrieDB<'db, H> where H::Out: Decodable { /// This could be a simple identity operation in the case that the node is sufficiently small, but /// may require a database lookup. fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> super::Result { - // REVIEW: is it better to parametrize on `H` or like this with `H::Out` (saves us from adding `use hashdb::Hasher`) - match Node::try_decode_hash::(node) { + match C::try_decode_hash(node) { Some(key) => { self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) } @@ -92,7 +100,9 @@ impl<'db, H: Hasher> TrieDB<'db, H> where H::Out: Decodable { } } -impl<'db, H: Hasher> Trie for TrieDB<'db, H> where H::Out: Decodable + Encodable { +impl<'db, H, C> Trie for TrieDB<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +{ type H = H; fn root(&self) -> &::Out { self.root } @@ -103,6 +113,7 @@ impl<'db, H: Hasher> Trie for TrieDB<'db, H> where H::Out: Decodable + Encodable db: self.db, query: query, hash: self.root.clone(), + marker: PhantomData::, }.look_up(NibbleSlice::new(key)) } @@ -112,15 +123,19 @@ impl<'db, H: Hasher> Trie for TrieDB<'db, H> where H::Out: Decodable + Encodable } // This is for pretty debug output only -struct TrieAwareDebugNode<'db, 'a, H: Hasher + 'db> { - trie: &'db TrieDB<'db, H>, +struct TrieAwareDebugNode<'db, 'a, H, C> +where H: Hasher + 'db, C: NodeCodec + 'db +{ + trie: &'db TrieDB<'db, H, C>, key: &'a[u8] } -impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> where H::Out: Decodable { +impl<'db, 'a, H, C> fmt::Debug for TrieAwareDebugNode<'db, 'a, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Ok(node) = self.trie.get_raw_or_lookup(self.key) { - match Node::decoded(&node) { + match C::decode(&node) { Ok(Node::Leaf(slice, value)) => f.debug_struct("Node::Leaf") .field("slice", &slice) .field("value", &value) @@ -130,7 +145,7 @@ impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> where H:: .field("item", &TrieAwareDebugNode{trie: self.trie, key: item}) .finish(), Ok(Node::Branch(ref nodes, ref value)) => { - let nodes: Vec> = nodes.into_iter().map(|n| TrieAwareDebugNode{trie: self.trie, key: n} ).collect(); + let nodes: Vec> = nodes.into_iter().map(|n| TrieAwareDebugNode{trie: self.trie, key: n} ).collect(); f.debug_struct("Node::Branch") .field("nodes", &nodes) .field("value", &value) @@ -152,7 +167,9 @@ impl<'db, 'a, H: Hasher> fmt::Debug for TrieAwareDebugNode<'db, 'a, H> where H:: } } -impl<'db, H: Hasher> fmt::Debug for TrieDB<'db, H> where H::Out: Decodable { +impl<'db, H, C> fmt::Debug for TrieDB<'db, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let root_rlp = self.db.get(self.root).expect("Trie root not found!"); f.debug_struct("TrieDB") @@ -193,21 +210,16 @@ impl Crumb { } /// Iterator for going through all values in the trie. -pub struct TrieDBIterator<'a, H: Hasher + 'a> { - db: &'a TrieDB<'a, H>, +pub struct TrieDBIterator<'a, H: Hasher + 'a, C: NodeCodec + 'a> { + db: &'a TrieDB<'a, H, C>, trail: Vec, key_nibbles: Bytes, } -impl<'a, H: Hasher> TrieDBIterator<'a, H> where H::Out: Decodable { +impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> where H::Out: Decodable { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> super::Result, H::Out> { - let mut r = TrieDBIterator { - db: db, - trail: vec![], - key_nibbles: Vec::new(), - }; - + pub fn new(db: &'a TrieDB) -> super::Result, H::Out> { + let mut r = TrieDBIterator { db, trail: vec![], key_nibbles: Vec::new() }; db.root_data().and_then(|root| r.descend(&root))?; Ok(r) } @@ -215,7 +227,8 @@ impl<'a, H: Hasher> TrieDBIterator<'a, H> where H::Out: Decodable { fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> super::Result<(), H::Out> { loop { let (data, mid) = { - let node = Node::decoded(&node_data).expect("rlp read from db; qed"); +// let node = Node::decoded(&node_data).expect("rlp read from db; qed"); + let node = C::decode(&node_data).expect("rlp read from db; qed"); match node { Node::Leaf(slice, _) => { if slice == key { @@ -278,16 +291,14 @@ impl<'a, H: Hasher> TrieDBIterator<'a, H> where H::Out: Decodable { /// Descend into a payload. fn descend(&mut self, d: &[u8]) -> super::Result<(), H::Out> { let node_data = &self.db.get_raw_or_lookup(d)?; - let node = Node::decoded(&node_data).expect("rlp read from db; qed"); +// let node = Node::decoded(&node_data).expect("rlp read from db; qed"); + let node = C::decode(&node_data).expect("rlp read from db; qed"); Ok(self.descend_into_node(node.into())) } /// Descend into a payload. fn descend_into_node(&mut self, node: OwnedNode) { - self.trail.push(Crumb { - status: Status::Entering, - node: node, - }); + self.trail.push(Crumb { status: Status::Entering, node }); match &self.trail.last().expect("just pushed item; qed").node { &OwnedNode::Leaf(ref n, _) | &OwnedNode::Extension(ref n, _) => { self.key_nibbles.extend((0..n.len()).map(|i| n.at(i))); @@ -311,7 +322,7 @@ impl<'a, H: Hasher> TrieDBIterator<'a, H> where H::Out: Decodable { } } -impl<'a, H: Hasher> TrieIterator for TrieDBIterator<'a, H> where H::Out: Decodable { +impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C> where H::Out: Decodable { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { self.trail.clear(); @@ -321,8 +332,7 @@ impl<'a, H: Hasher> TrieIterator for TrieDBIterator<'a, H> where H::Out: Deco } } - -impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> where H::Out: Decodable { +impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> where H::Out: Decodable { type Item = TrieItem<'a, H>; fn next(&mut self) -> Option { @@ -376,7 +386,7 @@ impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> where H::Out: Decodable { self.trail.pop(); }, IterStep::Descend::(Ok(d)) => { - let node = Node::decoded(&d).expect("rlp read from db; qed"); + let node = C::decode(&d).expect("rlp read from db; qed"); self.descend_into_node(node.into()) }, IterStep::Descend::(Err(e)) => { @@ -399,114 +409,108 @@ impl<'a, H: Hasher> Iterator for TrieDBIterator<'a, H> where H::Out: Decodable { } } -#[test] -fn iterator() { +#[cfg(test)] +mod tests { + use node_codec::RlpNodeCodec; + use hashdb::{Hasher, KeccakHasher, DBValue}; use memorydb::*; - use super::TrieMut; - use super::triedbmut::*; + use super::{TrieDB, Trie, Lookup, NibbleSlice}; + use super::super::{TrieMut, triedbmut::*}; - let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; + type RlpCodec = RlpNodeCodec; - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); - { - let mut t = TrieDBMut::new(&mut memdb, &mut root); - for x in &d { - t.insert(x, x).unwrap(); - } - } - - let t = TrieDB::new(&memdb, &root).unwrap(); - assert_eq!(d.iter().map(|i| i.clone().into_vec()).collect::>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::>()); - assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::>()); -} + #[test] + fn iterator() { + let d = vec![DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B")]; -#[test] -fn iterator_seek() { - use memorydb::*; - use super::TrieMut; - use super::triedbmut::*; + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + { + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + for x in &d { + t.insert(x, x).unwrap(); + } + } - let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + assert_eq!(d.iter().map(|i| i.clone().into_vec()).collect::>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::>()); + assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::>()); + } - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); - { - let mut t = TrieDBMut::new(&mut memdb, &mut root); - for x in &d { - t.insert(x, x).unwrap(); + #[test] + fn iterator_seek() { + let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + { + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + for x in &d { + t.insert(x, x).unwrap(); + } } + + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let mut iter = t.iter().unwrap(); + assert_eq!(iter.next(), Some(Ok((b"A".to_vec(), DBValue::from_slice(b"A"))))); + iter.seek(b"!").unwrap(); + assert_eq!(d, iter.map(|x| x.unwrap().1).collect::>()); + let mut iter = t.iter().unwrap(); + iter.seek(b"A").unwrap(); + assert_eq!(&d[1..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + let mut iter = t.iter().unwrap(); + iter.seek(b"AA").unwrap(); + assert_eq!(&d[2..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + let mut iter = t.iter().unwrap(); + iter.seek(b"A!").unwrap(); + assert_eq!(&d[1..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + let mut iter = t.iter().unwrap(); + iter.seek(b"AB").unwrap(); + assert_eq!(&d[3..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + let mut iter = t.iter().unwrap(); + iter.seek(b"AB!").unwrap(); + assert_eq!(&d[3..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + let mut iter = t.iter().unwrap(); + iter.seek(b"B").unwrap(); + assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::>()[..]); + let mut iter = t.iter().unwrap(); + iter.seek(b"C").unwrap(); + assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::>()[..]); } - let t = TrieDB::new(&memdb, &root).unwrap(); - let mut iter = t.iter().unwrap(); - assert_eq!(iter.next(), Some(Ok((b"A".to_vec(), DBValue::from_slice(b"A"))))); - iter.seek(b"!").unwrap(); - assert_eq!(d, iter.map(|x| x.unwrap().1).collect::>()); - let mut iter = t.iter().unwrap(); - iter.seek(b"A").unwrap(); - assert_eq!(&d[1..], &iter.map(|x| x.unwrap().1).collect::>()[..]); - let mut iter = t.iter().unwrap(); - iter.seek(b"AA").unwrap(); - assert_eq!(&d[2..], &iter.map(|x| x.unwrap().1).collect::>()[..]); - let mut iter = t.iter().unwrap(); - iter.seek(b"A!").unwrap(); - assert_eq!(&d[1..], &iter.map(|x| x.unwrap().1).collect::>()[..]); - let mut iter = t.iter().unwrap(); - iter.seek(b"AB").unwrap(); - assert_eq!(&d[3..], &iter.map(|x| x.unwrap().1).collect::>()[..]); - let mut iter = t.iter().unwrap(); - iter.seek(b"AB!").unwrap(); - assert_eq!(&d[3..], &iter.map(|x| x.unwrap().1).collect::>()[..]); - let mut iter = t.iter().unwrap(); - iter.seek(b"B").unwrap(); - assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::>()[..]); - let mut iter = t.iter().unwrap(); - iter.seek(b"C").unwrap(); - assert_eq!(&d[4..], &iter.map(|x| x.unwrap().1).collect::>()[..]); -} - -#[test] -fn get_len() { - use memorydb::*; - use super::TrieMut; - use super::triedbmut::*; + #[test] + fn get_len() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + { + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + t.insert(b"A", b"ABC").unwrap(); + t.insert(b"B", b"ABCBA").unwrap(); + } - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); - { - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(b"A", b"ABC").unwrap(); - t.insert(b"B", b"ABCBA").unwrap(); + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()), Ok(Some(3))); + assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()), Ok(Some(5))); + assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()), Ok(None)); } - let t = TrieDB::new(&memdb, &root).unwrap(); - assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()), Ok(Some(3))); - assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()), Ok(Some(5))); - assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()), Ok(None)); -} - -#[test] -fn debug_output_supports_pretty_print() { - use memorydb::*; - use super::TrieMut; - use super::triedbmut::*; + #[test] + fn debug_output_supports_pretty_print() { + let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; - let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; - - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); - let root = { - let mut t = TrieDBMut::new(&mut memdb, &mut root); - for x in &d { - t.insert(x, x).unwrap(); - } - t.root().clone() - }; - let t = TrieDB::new(&memdb, &root).unwrap(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let root = { + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + for x in &d { + t.insert(x, x).unwrap(); + } + t.root().clone() + }; + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); - assert_eq!(format!("{:?}", t), "TrieDB { hash_count: 0, root: Node::Extension { slice: 4, item: Node::Branch { nodes: [Node::Empty, Node::Branch { nodes: [Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Branch { nodes: [Node::Empty, Node::Leaf { slice: , value: [65, 65] }, Node::Leaf { slice: , value: [65, 66] }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: None }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: Some([65]) }, Node::Leaf { slice: , value: [66] }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: None } } }"); - assert_eq!(format!("{:#?}", t), + assert_eq!(format!("{:?}", t), "TrieDB { hash_count: 0, root: Node::Extension { slice: 4, item: Node::Branch { nodes: [Node::Empty, Node::Branch { nodes: [Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Branch { nodes: [Node::Empty, Node::Leaf { slice: , value: [65, 65] }, Node::Leaf { slice: , value: [65, 66] }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: None }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: Some([65]) }, Node::Leaf { slice: , value: [66] }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: None } } }"); + assert_eq!(format!("{:#?}", t), "TrieDB { hash_count: 0, root: Node::Extension { @@ -595,29 +599,28 @@ fn debug_output_supports_pretty_print() { } } }"); -} - -#[test] -fn test_lookup_with_corrupt_data_returns_decoder_error() { - use memorydb::*; - use super::TrieMut; - use super::triedbmut::*; - use rlp; - use ethereum_types::H512; - - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); - { - let mut t = TrieDBMut::new(&mut memdb, &mut root); - t.insert(b"A", b"ABC").unwrap(); - t.insert(b"B", b"ABCBA").unwrap(); } - let t = TrieDB::new(&memdb, &root).unwrap(); + #[test] + fn test_lookup_with_corrupt_data_returns_decoder_error() { + use rlp; + use ethereum_types::H512; + use std::marker::PhantomData; + + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + { + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + t.insert(b"A", b"ABC").unwrap(); + t.insert(b"B", b"ABCBA").unwrap(); + } - // query for an invalid data type to trigger an error - let q = rlp::decode::; - let lookup = Lookup{ db: t.db, query: q, hash: root }; - let query_result = lookup.look_up(NibbleSlice::new(b"A")); - assert_eq!(query_result.unwrap().unwrap().unwrap_err(), rlp::DecoderError::RlpIsTooShort); -} + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + + // query for an invalid data type to trigger an error + let q = rlp::decode::; + let lookup = Lookup::<_, RlpCodec, _>{ db: t.db, query: q, hash: root, marker: PhantomData }; + let query_result = lookup.look_up(NibbleSlice::new(b"A")); + assert_eq!(query_result.unwrap().unwrap().unwrap_err(), rlp::DecoderError::RlpIsTooShort); + } +} \ No newline at end of file diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 066ae3caf59..cd8d92ea001 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -22,16 +22,17 @@ use super::node::Node as RlpNode; use node_codec::NodeCodec; use super::node::NodeKey; -use hashdb::HashDB; use bytes::ToPretty; +use hashdb::HashDB; +use hashdb::{Hasher, DBValue}; use nibbleslice::NibbleSlice; use rlp::{Rlp, RlpStream, Encodable, Decodable}; -use hashdb::{Hasher, DBValue}; +use elastic_array::ElasticArray1024; use std::collections::{HashSet, VecDeque}; +use std::marker::PhantomData; use std::mem; use std::ops::Index; -use elastic_array::ElasticArray1024; // For lookups into the Node storage buffer. // This is deliberately non-copyable. @@ -86,29 +87,35 @@ enum Node { impl Node where H::Out: Decodable { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle { - RlpNode::try_decode_hash::(&node) + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle + where C: NodeCodec + { + C::try_decode_hash(&node) .map(NodeHandle::Hash) .unwrap_or_else(|| { - let child = Node::from_encoded(node, db, storage); + let child = Node::from_encoded::(node, db, storage); NodeHandle::InMemory(storage.alloc(Stored::New(child))) }) } // decode a node from rlp without getting its children. - fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self { - match RlpNode::decoded(data).expect("encoded bytes read from db; qed") { + fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self + where C: NodeCodec + { + match C::decode(data).expect("encoded bytes read from db; qed") { RlpNode::Empty => Node::Empty, RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)), RlpNode::Extension(key, cb) => { - Node::Extension(key.encoded(false), Self::inline_or_hash(cb, db, storage)) + Node::Extension( + key.encoded(false), + Self::inline_or_hash::(cb, db, storage)) } RlpNode::Branch(ref encoded_children, val) => { let mut child = |i:usize| { let raw = encoded_children[i]; - let encoded_child = RlpNode::new_encoded(raw); + let encoded_child = C::new_encoded(raw); if !encoded_child.is_empty() { - Some(Self::inline_or_hash(raw, db, storage)) + Some(Self::inline_or_hash::(raw, db, storage)) } else { None } @@ -128,30 +135,32 @@ impl Node where H::Out: Decodable { // encode a node to RLP // TODO: parallelize - fn into_rlp(self, mut child_cb: F) -> ElasticArray1024 - where F: FnMut(NodeHandle, &mut RlpStream) // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` + fn into_rlp(self, mut child_cb: F) -> ElasticArray1024 + where + F: FnMut(NodeHandle, &mut RlpStream), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` + C: NodeCodec, // where F: FnMut(NodeHandle, &mut ::StreamEncoding) { match self { Node::Empty => { - let mut stream = RlpNode::encoded_stream(); + let mut stream = C::encoded_stream(); stream.append_empty_data(); stream.drain() } Node::Leaf(partial, value) => { - let mut stream = RlpNode::encoded_list(2); + let mut stream = C::encoded_list(2); stream.append(&&*partial); stream.append(&&*value); stream.drain() } Node::Extension(partial, child) => { - let mut stream = RlpNode::encoded_list(2); + let mut stream = C::encoded_list(2); stream.append(&&*partial); child_cb(child, &mut stream); stream.drain() } Node::Branch(mut children, value) => { - let mut stream = RlpNode::encoded_list(17); + let mut stream = C::encoded_list(17); for child in children.iter_mut().map(Option::take) { if let Some(handle) = child { child_cb(handle, &mut stream); @@ -278,11 +287,14 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// use hashdb::*; /// use memorydb::*; /// use ethereum_types::H256; +/// use node_codec::RlpNodeCodec; +/// +/// type RlpCodec = RlpNodeCodec; /// /// fn main() { /// let mut memdb = MemoryDB::::new(); /// let mut root = H256::new(); -/// let mut t = TrieDBMut::new(&mut memdb, &mut root); +/// let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); /// assert!(t.is_empty()); /// assert_eq!(*t.root(), KECCAK_NULL_RLP); /// t.insert(b"foo", b"bar").unwrap(); @@ -292,7 +304,11 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// assert!(!t.contains(b"foo").unwrap()); /// } /// ``` -pub struct TrieDBMut<'a, H: Hasher + 'a> where H::Out: Decodable + Encodable { +pub struct TrieDBMut<'a, H, C> + where H: Hasher + 'a, + H::Out: Decodable + Encodable, + C: NodeCodec +{ storage: NodeStorage, db: &'a mut HashDB, root: &'a mut H::Out, @@ -301,9 +317,14 @@ pub struct TrieDBMut<'a, H: Hasher + 'a> where H::Out: Decodable + Encodable { /// The number of hash operations this trie has performed. /// Note that none are performed until changes are committed. hash_count: usize, + marker: PhantomData, // TODO: rpheimer: "we could have the NodeCodec trait take &self to its methods and then we don't need PhantomData. we can just store an instance of C: NodeCodec in the trie struct. If it's a ZST it won't have any additional overhead anyway" } -impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { +impl<'a, H, C> TrieDBMut<'a, H, C> + where H: Hasher, + H::Out: Decodable + Encodable, + C: NodeCodec +{ /// Create a new trie with backing database `db` and empty `root`. pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { *root = H::HASHED_NULL_RLP; @@ -316,6 +337,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { root_handle: root_handle, death_row: HashSet::new(), hash_count: 0, + marker: PhantomData, // TODO: prolly not needed } } @@ -334,6 +356,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { root_handle: root_handle, death_row: HashSet::new(), hash_count: 0, + marker: PhantomData, // TODO: prolly not needed }) } /// Get the backing database. @@ -349,7 +372,11 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { // cache a node by hash fn cache(&mut self, hash: H::Out) -> super::Result { let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; - let node = Node::from_encoded(&node_encoded, &*self.db, &mut self.storage); + let node = Node::from_encoded::( + &node_encoded, + &*self.db, + &mut self.storage + ); Ok(self.storage.alloc(Stored::Cached(node, hash))) } @@ -384,10 +411,11 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { let mut handle = handle; loop { let (mid, child) = match *handle { - NodeHandle::Hash(ref hash) => return Lookup { + NodeHandle::Hash(ref hash) => return Lookup{ db: &*self.db, query: DBValue::from_slice, hash: hash.clone(), + marker: PhantomData::, }.look_up(partial), NodeHandle::InMemory(ref handle) => match self.storage[handle] { Node::Empty => return Ok(None), @@ -822,7 +850,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { match self.storage.destroy(handle) { Stored::New(node) => { - let root_rlp = node.into_rlp(|child, stream| self.commit_node(child, stream)); + let root_rlp = node.into_rlp::<_, C>(|child, stream| self.commit_node(child, stream)); *self.root = self.db.insert(&root_rlp[..]); self.hash_count += 1; @@ -850,7 +878,7 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { NodeHandle::InMemory(h) => match self.storage.destroy(h) { Stored::Cached(_, h) => stream.append(&h), Stored::New(node) => { - let node_rlp = node.into_rlp(|child, stream| self.commit_node(child, stream)); + let node_rlp = node.into_rlp::<_, C>(|child, stream| self.commit_node(child, stream)); if node_rlp.len() >= 32 { let hash = self.db.insert(&node_rlp[..]); self.hash_count += 1; @@ -872,7 +900,11 @@ impl<'a, H: Hasher> TrieDBMut<'a, H> where H::Out: Decodable + Encodable { } } -impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> where H::Out: Decodable + Encodable { +impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> + where H: Hasher, + H::Out: Decodable + Encodable, + C: NodeCodec +{ type H = H; fn root(&mut self) -> &::Out { @@ -890,11 +922,12 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> where H::Out: Decodable + Encod } } - fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result, ::Out> where 'x: 'key { + fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result, ::Out> + where 'x: 'key + { self.lookup(NibbleSlice::new(key), &self.root_handle) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result, ::Out> { if value.is_empty() { return self.remove(key) } @@ -939,7 +972,11 @@ impl<'a, H: Hasher> TrieMut for TrieDBMut<'a, H> where H::Out: Decodable + Encod } } -impl<'a, H: Hasher> Drop for TrieDBMut<'a, H> where H::Out: Decodable + Encodable { +impl<'a, H, C> Drop for TrieDBMut<'a, H, C> + where H: Hasher, + H::Out: Decodable + Encodable, + C: NodeCodec +{ fn drop(&mut self) { self.commit(); } @@ -949,17 +986,20 @@ impl<'a, H: Hasher> Drop for TrieDBMut<'a, H> where H::Out: Decodable + Encodabl mod tests { extern crate triehash; - use self::triehash::trie_root; + use bytes::ToPretty; use hashdb::*; use memorydb::*; + use node_codec::RlpNodeCodec; + use rlp::{Decodable, Encodable}; + use self::triehash::trie_root; + use standardmap::*; use super::*; - use bytes::ToPretty; use super::super::TrieMut; - use standardmap::*; - use rlp::{Decodable, Encodable}; - fn populate_trie<'db, H: Hasher>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H> - where H::Out: Decodable + Encodable + type RlpCodec = RlpNodeCodec; + + fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec { let mut t = TrieDBMut::new(db, root); for i in 0..v.len() { @@ -970,7 +1010,7 @@ mod tests { t } - fn unpopulate_trie<'db, H: Hasher>(t: &mut TrieDBMut<'db, H>, v: &[(Vec, Vec)]) + fn unpopulate_trie<'db, H: Hasher, C: NodeCodec>(t: &mut TrieDBMut<'db, H, C>, v: &[(Vec, Vec)]) where H::Out: Decodable + Encodable { for i in v { @@ -999,7 +1039,7 @@ mod tests { let real = trie_root(x.clone()); let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); - let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + let mut memtrie = populate_trie::<_, RlpCodec>(&mut memdb, &mut root, &x); memtrie.commit(); if *memtrie.root() != real { @@ -1029,7 +1069,7 @@ mod tests { fn init() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); assert_eq!(*t.root(), KeccakHasher::HASHED_NULL_RLP); } @@ -1037,7 +1077,7 @@ mod tests { fn insert_on_empty() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } @@ -1048,12 +1088,13 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t1 = TrieDBMut::new(&mut memdb, &mut root); + let mut t1 = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t1.insert(&[0x01, 0x23], big_value).unwrap(); t1.insert(&[0x01, 0x34], big_value).unwrap(); let mut memdb2 = MemoryDB::::new(); let mut root2 = ::Out::default(); - let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); + let mut t2 = TrieDBMut::<_, RlpCodec>::new(&mut memdb2, &mut root2); + t2.insert(&[0x01], big_value).unwrap(); t2.insert(&[0x01, 0x23], big_value).unwrap(); t2.insert(&[0x01, 0x34], big_value).unwrap(); @@ -1064,7 +1105,7 @@ mod tests { fn insert_replace_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); @@ -1074,7 +1115,7 @@ mod tests { fn insert_make_branch_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1087,7 +1128,7 @@ mod tests { fn insert_into_branch_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -1102,7 +1143,7 @@ mod tests { fn insert_value_into_branch_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[], &[0x0]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1115,7 +1156,7 @@ mod tests { fn insert_split_leaf() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1128,7 +1169,7 @@ mod tests { fn insert_split_extenstion() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); @@ -1146,7 +1187,7 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1161,7 +1202,7 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value).unwrap(); t.insert(&[0x11u8, 0x23], big_value).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1174,7 +1215,7 @@ mod tests { fn test_at_empty() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let t = TrieDBMut::new(&mut memdb, &mut root); + let t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); assert_eq!(t.get(&[0x5]), Ok(None)); } @@ -1182,7 +1223,7 @@ mod tests { fn test_at_one() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); t.commit(); @@ -1193,7 +1234,7 @@ mod tests { fn test_at_three() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -1223,12 +1264,12 @@ mod tests { let real = trie_root(x.clone()); let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + let mut memtrie = populate_trie::<_, RlpCodec>(&mut memdb, &mut root, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); let mut memdb2 = MemoryDB::::new(); let mut root2 = ::Out::default(); - let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); + let mut memtrie_sorted = populate_trie::<_, RlpCodec>(&mut memdb2, &mut root2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -1251,12 +1292,12 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = TrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } { - let _ = TrieDBMut::from_existing(&mut db, &mut root); + let _ = TrieDBMut::<_, RlpCodec>::from_existing(&mut db, &mut root); } } @@ -1273,7 +1314,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // TODO: `t` was immutable – how could this pass? for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } @@ -1301,7 +1342,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // TODO: `t` was immutable – how could this pass? for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap().is_none()); assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); From 8bc455ca3e48e147defb14bf907257b5b603a66e Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 6 Jun 2018 08:24:44 +0200 Subject: [PATCH 039/160] Fix FatDB, FatDBMut, SecTrieDB, SecTrieDBMut Fix warnings --- util/patricia_trie/src/fatdb.rs | 44 +++++++++++----- util/patricia_trie/src/fatdbmut.rs | 20 +++++--- util/patricia_trie/src/lib.rs | 71 ++++++++++++-------------- util/patricia_trie/src/node.rs | 3 -- util/patricia_trie/src/node_codec.rs | 16 +----- util/patricia_trie/src/sectriedb.rs | 26 ++++++---- util/patricia_trie/src/sectriedbmut.rs | 20 +++++--- util/patricia_trie/src/triedbmut.rs | 3 +- 8 files changed, 110 insertions(+), 93 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index b818b2d3f85..328f25c684a 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -17,16 +17,21 @@ use hashdb::{HashDB, Hasher}; use super::{TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query}; use rlp::{Decodable, Encodable}; +use node_codec::NodeCodec; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// Additionaly it stores inserted hash-key mappings for later retrieval. /// /// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDB<'db, H: Hasher + 'db> { - raw: TrieDB<'db, H>, +pub struct FatDB<'db, H, C> + where H: Hasher + 'db, C: NodeCodec +{ + raw: TrieDB<'db, H, C>, } -impl<'db, H: Hasher> FatDB<'db, H> where H::Out: Decodable { +impl<'db, H, C> FatDB<'db, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. @@ -38,7 +43,9 @@ impl<'db, H: Hasher> FatDB<'db, H> where H::Out: Decodable { pub fn db(&self) -> &HashDB { self.raw.db() } } -impl<'db, H: Hasher> Trie for FatDB<'db, H> where H::Out: Decodable + Encodable { +impl<'db, H, C> Trie for FatDB<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +{ type H = H; fn root(&self) -> &::Out { self.raw.root() } @@ -54,19 +61,23 @@ impl<'db, H: Hasher> Trie for FatDB<'db, H> where H::Out: Decodable + Encodable } fn iter<'a>(&'a self) -> super::Result> + 'a>, ::Out> { - FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) + FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) } } /// Itarator over inserted pairs of key values. -pub struct FatDBIterator<'db, H: Hasher + 'db> { - trie_iterator: TrieDBIterator<'db, H>, - trie: &'db TrieDB<'db, H>, +pub struct FatDBIterator<'db, H, C> + where H: Hasher + 'db, C: NodeCodec + 'db +{ + trie_iterator: TrieDBIterator<'db, H, C>, + trie: &'db TrieDB<'db, H, C>, } -impl<'db, H: Hasher> FatDBIterator<'db, H> where H::Out: Decodable { +impl<'db, H, C> FatDBIterator<'db, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> super::Result { + pub fn new(trie: &'db TrieDB) -> super::Result { Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie: trie, @@ -74,14 +85,18 @@ impl<'db, H: Hasher> FatDBIterator<'db, H> where H::Out: Decodable { } } -impl<'db, H: Hasher> TrieIterator for FatDBIterator<'db, H> where H::Out: Decodable { +impl<'db, H, C> TrieIterator for FatDBIterator<'db, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { let hashed_key = H::hash(key); self.trie_iterator.seek(hashed_key.as_ref()) } } -impl<'db, H: Hasher> Iterator for FatDBIterator<'db, H> where H::Out: Decodable { +impl<'db, H, C> Iterator for FatDBIterator<'db, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ type Item = TrieItem<'db, H>; fn next(&mut self) -> Option { @@ -102,14 +117,15 @@ fn fatdb_to_trie() { use super::fatdbmut::FatDBMut; use super::TrieMut; use hashdb::KeccakHasher; + use node_codec::RlpNodeCodec; let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = FatDBMut::new(&mut memdb, &mut root); + let mut t = FatDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = FatDB::new(&memdb, &root).unwrap(); + let t = FatDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); assert_eq!( t.iter().unwrap().map(Result::unwrap).collect::>(), diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 41cd53c366f..d5f18f41805 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -19,16 +19,21 @@ use hashdb::{HashDB, DBValue, Hasher}; use super::{TrieDBMut, TrieMut}; use rlp::{Encodable, Decodable}; +use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// Additionaly it stores inserted hash-key mappings for later retrieval. /// /// Use it as a `Trie` or `TrieMut` trait object. -pub struct FatDBMut<'db, H: Hasher + 'db> where H::Out: Decodable + Encodable { - raw: TrieDBMut<'db, H>, +pub struct FatDBMut<'db, H, C> + where H: Hasher + 'db, H::Out: Decodable + Encodable, C: NodeCodec +{ + raw: TrieDBMut<'db, H, C>, } -impl<'db, H: Hasher> FatDBMut<'db, H> where H::Out: Decodable + Encodable { +impl<'db, H, C> FatDBMut<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +{ /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. @@ -54,7 +59,9 @@ impl<'db, H: Hasher> FatDBMut<'db, H> where H::Out: Decodable + Encodable { } } -impl<'db, H: Hasher> TrieMut for FatDBMut<'db, H> where H::Out: Decodable + Encodable { +impl<'db, H, C> TrieMut for FatDBMut<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +{ type H = H; fn root(&mut self) -> &::Out { self.raw.root() } @@ -103,13 +110,14 @@ fn fatdbmut_to_trie() { use super::Trie; use hashdb::KeccakHasher; use keccak; + use node_codec::RlpNodeCodec; let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = FatDBMut::new(&mut memdb, &mut root); + let mut t = FatDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = TrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 17c9938a20b..b62d634aa33 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -39,22 +39,22 @@ pub mod node; pub mod node_codec; pub mod triedb; pub mod triedbmut; -//pub mod sectriedb; -//pub mod sectriedbmut; +pub mod sectriedb; +pub mod sectriedbmut; pub mod recorder; -//mod fatdb; -//mod fatdbmut; +mod fatdb; +mod fatdbmut; mod lookup; mod nibbleslice; mod nibblevec; pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::triedbmut::TrieDBMut; -//pub use self::sectriedbmut::SecTrieDBMut; -//pub use self::sectriedb::SecTrieDB; -//pub use self::fatdb::{FatDB, FatDBIterator}; -//pub use self::fatdbmut::FatDBMut; +pub use self::sectriedbmut::SecTrieDBMut; +pub use self::sectriedb::SecTrieDB; +pub use self::fatdb::{FatDB, FatDBIterator}; +pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; use node_codec::NodeCodec; @@ -230,13 +230,9 @@ pub enum TrieKinds<'db, H: Hasher + 'db, C: NodeCodec> { /// A generic trie db. Generic(TrieDB<'db, H, C>), /// A secure trie db. - Secure(TrieDB<'db, H, C>), + Secure(SecTrieDB<'db, H, C>), /// A fat trie db. - Fat(TrieDB<'db, H, C>), -// /// A secure trie db. -// Secure(SecTrieDB<'db, H>), -// /// A fat trie db. -// Fat(FatDB<'db, H>), + Fat(FatDB<'db, H, C>), } // wrapper macro for making the match easier to deal with. @@ -275,44 +271,41 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> where H::Out } } -impl> TrieFactory where H::Out: rlp::Decodable + rlp::Encodable { +impl<'db, H, C> TrieFactory + where H: Hasher, H::Out: rlp::Decodable + rlp::Encodable, C: NodeCodec + 'db +{ /// Creates new factory. pub fn new(spec: TrieSpec) -> Self { TrieFactory { spec, mark_hash: PhantomData, mark_codec: PhantomData } } /// Create new immutable instance of Trie. - pub fn readonly<'db>(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { + pub fn readonly(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), - TrieSpec::Secure => Ok(TrieKinds::Secure(TrieDB::new(db, root)?)), - TrieSpec::Fat => Ok(TrieKinds::Fat(TrieDB::new(db, root)?)), -// TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), -// TrieSpec::Fat => Ok(TrieKinds::Fat(FatDB::new(db, root)?)), + TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), + TrieSpec::Fat => Ok(TrieKinds::Fat(FatDB::new(db, root)?)), } } -/* /// Create new mutable instance of Trie. - pub fn create<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { + pub fn create(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { match self.spec { - TrieSpec::Generic => Box::new(TrieDBMut::new(db, root)), - TrieSpec::Secure => Box::new(TrieDBMut::new(db, root)), - TrieSpec::Fat => Box::new(TrieDBMut::new(db, root)), -// TrieSpec::Secure => Box::new(SecTrieDBMut::new(db, root)), -// TrieSpec::Fat => Box::new(FatDBMut::new(db, root)), + TrieSpec::Generic => Box::new(TrieDBMut::<_, C>::new(db, root)), + TrieSpec::Secure => Box::new(SecTrieDBMut::<_, C>::new(db, root)), + TrieSpec::Fat => Box::new(FatDBMut::<_, C>::new(db, root)), } } -*/ -// /// Create new mutable instance of trie and check for errors. -// pub fn from_existing<'db>(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { -// match self.spec { -// TrieSpec::Generic => Ok(Box::new(TrieDBMut::from_existing(db, root)?)), -// TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::from_existing(db, root)?)), -// TrieSpec::Fat => Ok(Box::new(FatDBMut::from_existing(db, root)?)), -// } -// } - -// /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). -// pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } + + /// Create new mutable instance of trie and check for errors. + pub fn from_existing(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { + match self.spec { + TrieSpec::Generic => Ok(Box::new(TrieDBMut::<_, C>::from_existing(db, root)?)), + TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::<_, C>::from_existing(db, root)?)), + TrieSpec::Fat => Ok(Box::new(FatDBMut::<_, C>::from_existing(db, root)?)), + } + } + + /// Returns true iff the trie DB is a fat DB (allows enumeration of keys). + pub fn is_fat(&self) -> bool { self.spec == TrieSpec::Fat } } diff --git a/util/patricia_trie/src/node.rs b/util/patricia_trie/src/node.rs index d055e82f2ef..c4b4789b9c7 100644 --- a/util/patricia_trie/src/node.rs +++ b/util/patricia_trie/src/node.rs @@ -17,10 +17,7 @@ use elastic_array::ElasticArray36; use nibbleslice::NibbleSlice; use nibblevec::NibbleVec; -use bytes::*; -use rlp::{Rlp, RlpStream, Prototype, DecoderError, self}; use hashdb::DBValue; -use node_codec::NodeCodec; /// Partial node key type. pub type NodeKey = ElasticArray36; diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 8d678c3ec79..96445c7e8ec 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,23 +1,9 @@ use bytes::*; use nibbleslice::NibbleSlice; -//use nibblevec::NibbleVec; -use rlp::{self, Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; +use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; use hashdb::Hasher; use node::Node; use std::marker::PhantomData; -//pub trait OldNodeCodec<'a>: Sized { -// type Encoding; -// type StreamEncoding; -// -// fn encoded(&self) -> Bytes; -// fn decoded(data: &'a [u8]) -> Result; -// fn try_decode_hash(data: &[u8]) -> Option where O: Decodable; -// -// fn new_encoded(data: &'a[u8]) -> Self::Encoding; -// fn encoded_stream() -> Self::StreamEncoding; -// fn encoded_list(size: usize) -> Self::StreamEncoding; -//} - pub trait NodeCodec: Sized { fn encode(&Node) -> Bytes; diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index 37c3486e8a5..1f73d12a920 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -20,15 +20,20 @@ use hashdb::{HashDB, Hasher}; use super::triedb::TrieDB; use super::{Trie, TrieItem, TrieIterator, Query}; use rlp::{Decodable, Encodable}; +use node_codec::NodeCodec; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` trait object. You can use `raw()` to get the backing `TrieDB` object. -pub struct SecTrieDB<'db, H: Hasher + 'db> { - raw: TrieDB<'db, H> +pub struct SecTrieDB<'db, H, C> + where H: Hasher + 'db, C: NodeCodec +{ + raw: TrieDB<'db, H, C> } -impl<'db, H: Hasher> SecTrieDB<'db, H> where H::Out: Decodable { +impl<'db, H, C> SecTrieDB<'db, H, C> + where H: Hasher, H::Out: Decodable, C: NodeCodec +{ /// Create a new trie with the backing database `db` and empty `root` /// /// Initialise to the state entailed by the genesis block. @@ -39,17 +44,19 @@ impl<'db, H: Hasher> SecTrieDB<'db, H> where H::Out: Decodable { } /// Get a reference to the underlying raw `TrieDB` struct. - pub fn raw(&self) -> &TrieDB { + pub fn raw(&self) -> &TrieDB { &self.raw } /// Get a mutable reference to the underlying raw `TrieDB` struct. - pub fn raw_mut(&mut self) -> &mut TrieDB<'db, H> { + pub fn raw_mut(&mut self) -> &mut TrieDB<'db, H, C> { &mut self.raw } } -impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> where H::Out: Decodable + Encodable { +impl<'db, H, C> Trie for SecTrieDB<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +{ type H = H; fn root(&self) -> &::Out { self.raw.root() } @@ -58,7 +65,7 @@ impl<'db, H: Hasher> Trie for SecTrieDB<'db, H> where H::Out: Decodable + Encoda self.raw.contains(Self::H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> where 'a: 'key { self.raw.get_with(Self::H::hash(key).as_ref(), query) @@ -77,13 +84,14 @@ fn trie_to_sectrie() { use super::TrieMut; use hashdb::KeccakHasher; use keccak; + use node_codec::RlpNodeCodec; let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); t.insert(&keccak::keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); } - let t = SecTrieDB::new(&memdb, &root).unwrap(); + let t = SecTrieDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 425ae1e0c8d..f41afa3ee54 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -18,15 +18,20 @@ use hashdb::{HashDB, DBValue, Hasher}; use super::triedbmut::TrieDBMut; use super::TrieMut; use rlp::{Encodable, Decodable}; +use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. -pub struct SecTrieDBMut<'db, H: Hasher + 'db> where H::Out: Decodable + Encodable { - raw: TrieDBMut<'db, H> +pub struct SecTrieDBMut<'db, H, C> + where H: Hasher + 'db, H::Out: Decodable + Encodable, C: NodeCodec +{ + raw: TrieDBMut<'db, H, C> } -impl<'db, H: Hasher> SecTrieDBMut<'db, H> where H::Out: Decodable + Encodable { +impl<'db, H, C> SecTrieDBMut<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +{ /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. @@ -48,7 +53,9 @@ impl<'db, H: Hasher> SecTrieDBMut<'db, H> where H::Out: Decodable + Encodable { pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } -impl<'db, H: Hasher> TrieMut for SecTrieDBMut<'db, H> where H::Out: Decodable + Encodable { +impl<'db, H, C> TrieMut for SecTrieDBMut<'db, H, C> + where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +{ type H = H; fn root(&mut self) -> &::Out { self.raw.root() @@ -84,13 +91,14 @@ fn sectrie_to_trie() { use super::Trie; use hashdb::KeccakHasher; use keccak; + use node_codec::RlpNodeCodec; let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = SecTrieDBMut::new(&mut memdb, &mut root); + let mut t = SecTrieDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = TrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index cd8d92ea001..2b01b89d847 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -26,7 +26,7 @@ use bytes::ToPretty; use hashdb::HashDB; use hashdb::{Hasher, DBValue}; use nibbleslice::NibbleSlice; -use rlp::{Rlp, RlpStream, Encodable, Decodable}; +use rlp::{RlpStream, Encodable, Decodable}; use elastic_array::ElasticArray1024; use std::collections::{HashSet, VecDeque}; @@ -54,6 +54,7 @@ impl From for NodeHandle { } } +// TODO: is this used anywhere? //impl From for NodeHandle { // fn from(hash: H256) -> Self { // NodeHandle::Hash(hash) From 6d89eba7f88a0042b50351034cfa6fb5e6a205ca Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 6 Jun 2018 10:23:31 +0200 Subject: [PATCH 040/160] Make NodeCodec error type an associated type Add more comments on the bizzarre error message about adding type params to error --- util/patricia_trie/src/lookup.rs | 16 +++++++++++++++- util/patricia_trie/src/node_codec.rs | 6 ++++-- util/patricia_trie/src/triedb.rs | 9 ++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 458067f695a..76967253a26 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -57,7 +57,21 @@ impl<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> Lookup<'a, H, C, Q> where // without incrementing the depth. let mut node_data = &node_data[..]; loop { - match C::decode(node_data)? { +// match C::decode(node_data)? { // REVIEW: I can't figure out how to write a conversion for this. The error looks like this: + /* + error[E0277]: the trait bound `std::boxed::Box::Out>>: std::convert::From<>::E>` is not satisfied + --> util/patricia_trie/src/lookup.rs:61:11 + | + 61 | match C::decode(node_data)? { + | ^^^^^^^^^^^^^^^^^^^^^ the trait `std::convert::From<>::E>` is not implemented for `std::boxed::Box::Out>>` + | + = help: consider adding a `where std::boxed::Box::Out>>: std::convert::From<>::E>` bound + = note: required by `std::convert::From::from` + */ + /* + Need help writing the conversion! + */ + match C::decode(node_data).expect("FIXME: should use `?`") { Node::Leaf(slice, value) => { return Ok(match slice == key { true => Some(self.query.decode(value)), diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 96445c7e8ec..125a8ef1268 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -6,8 +6,9 @@ use node::Node; use std::marker::PhantomData; pub trait NodeCodec: Sized { + type E: ::std::error::Error; fn encode(&Node) -> Bytes; - fn decode(data: &[u8]) -> Result; // TODO: make the error generic here, perhaps an associated type on the trait + fn decode(data: &[u8]) -> Result; fn try_decode_hash(data: &[u8]) -> Option; // TODO: We don't want these here, but where do they go? Helper trait? @@ -19,6 +20,7 @@ pub trait NodeCodec: Sized { pub struct RlpNodeCodec {mark: PhantomData} impl NodeCodec for RlpNodeCodec where H::Out: Encodable + Decodable { + type E = DecoderError; fn encode(node: &Node) -> Bytes { match *node { Node::Leaf(ref slice, ref value) => { @@ -51,7 +53,7 @@ impl NodeCodec for RlpNodeCodec where H::Out: Encodable + Decod } } } - fn decode(data: &[u8]) -> Result { + fn decode(data: &[u8]) -> Result { let r = Rlp::new(data); match r.prototype()? { // either leaf or extension - decode first item with NibbleSlice::??? diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index ce69f652e17..96d9112d753 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -361,7 +361,9 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> where (Status::At, &OwnedNode::Leaf(_, ref v)) | (Status::At, &OwnedNode::Branch(_, Some(ref v))) => { return Some(Ok((self.key(), v.clone()))); }, - (Status::At, &OwnedNode::Extension(_, ref d)) => IterStep::Descend::(self.db.get_raw_or_lookup(&*d)), + (Status::At, &OwnedNode::Extension(_, ref d)) => { + IterStep::Descend::(self.db.get_raw_or_lookup(&*d)) + }, (Status::At, &OwnedNode::Branch(_, _)) => IterStep::Continue, (Status::AtChild(i), &OwnedNode::Branch(ref children, _)) if children[i].len() > 0 => { match i { @@ -401,6 +403,11 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> where = note: expected type `std::boxed::Box>` found type `std::boxed::Box::Out>>` */ + /* + The only way we end up here is when `self.db.get_raw_or_lookup()` fails. When it does it + returns `Box::new(TrieError::IncompleteDatabase(key))`, i.e. a proper `Box>`. + I'm missing something. + */ panic!("FIXME: this causes `expected type parameter, found associated type`") } IterStep::Continue => {}, From 3d69e55525be9e8461515a44e345afd2d817f5b1 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 6 Jun 2018 12:42:13 +0200 Subject: [PATCH 041/160] Minor doc changes --- util/patricia_trie/src/lib.rs | 6 +++--- util/patricia_trie/src/triedb.rs | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index b62d634aa33..98a2adadfe0 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -96,10 +96,10 @@ impl From for Box> { fn from(e: rlp::DecoderError) -> Self { Box::new(TrieError::DecoderError(e)) } } -/// Trie result type. Boxed to avoid copying around extra space for `H256`s on successful queries. +/// Trie result type. Boxed to avoid copying around extra space for the `Hasher`s `Out`s on successful queries. pub type Result = ::std::result::Result>>; -/// Trie-Item type. +/// Trie-Item type used for iterators over trie data. pub type TrieItem<'a, U> = Result<(Vec, DBValue), U>; /// Description of what kind of query will be made to the trie. @@ -193,7 +193,7 @@ pub trait TrieMut { fn remove(&mut self, key: &[u8]) -> Result, ::Out>; } -/// A trie iterator that also supports random access. +/// A trie iterator that also supports random access (`seek()`). pub trait TrieIterator: Iterator { /// Position the iterator on the first element with key > `key` fn seek(&mut self, key: &[u8]) -> Result<(), H::Out>; diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 96d9112d753..8a3c69ac99a 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -25,7 +25,9 @@ use bytes::Bytes; use rlp::{Decodable, Encodable}; use std::marker::PhantomData; -/// A `Trie` implementation using a generic `HashDB` backing database. +/// A `Trie` implementation using a generic `HashDB` backing database, a `Hasher` +/// implementation to generate keys and a `NodeCodec` implementation to encode/decode +/// the nodes. /// /// Use it as a `Trie` trait object. You can use `db()` to get the backing database object. /// Use `get` and `contains` to query values associated with keys in the trie. From e9054941dd9bb0b8200d6917312753837e1c1d9f Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 6 Jun 2018 12:53:09 +0200 Subject: [PATCH 042/160] Fix benches --- util/patricia_trie/benches/trie.rs | 55 ++++++++++++++++-------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/util/patricia_trie/benches/trie.rs b/util/patricia_trie/benches/trie.rs index f26febdb595..4f27796ea52 100644 --- a/util/patricia_trie/benches/trie.rs +++ b/util/patricia_trie/benches/trie.rs @@ -23,6 +23,7 @@ extern crate memorydb; extern crate patricia_trie as trie; extern crate keccak_hash; extern crate trie_standardmap; +extern crate hashdb; use ethcore_bytes::Bytes; use ethereum_types::H256; @@ -31,6 +32,10 @@ use memorydb::MemoryDB; use test::{Bencher, black_box}; use trie::{TrieDBMut, TrieDB, TrieMut, Trie}; use trie_standardmap::{Alphabet, ValueMode, StandardMap}; +use hashdb::{Hasher,KeccakHasher}; +use trie::node_codec::RlpNodeCodec; + +type RlpCodec = RlpNodeCodec; fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { assert!(min_count + diff_count <= 32); @@ -69,9 +74,9 @@ fn trie_insertions_32_mir_1k(b: &mut Bencher) { }; let d = st.make(); b.iter(&mut ||{ - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -87,17 +92,17 @@ fn trie_iter(b: &mut Bencher) { count: 1000, }; let d = st.make(); - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); { - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } } b.iter(&mut ||{ - let t = TrieDB::new(&memdb, &root).unwrap(); + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); for n in t.iter().unwrap() { black_box(n).unwrap(); } @@ -116,9 +121,9 @@ fn trie_insertions_32_ran_1k(b: &mut Bencher) { let d = st.make(); let mut r = H256::new(); b.iter(&mut ||{ - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -129,7 +134,7 @@ fn trie_insertions_32_ran_1k(b: &mut Bencher) { #[bench] fn trie_insertions_six_high(b: &mut Bencher) { let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = H256::new(); + let mut seed = ::Out::new(); for _ in 0..1000 { let k = random_bytes(6, 0, &mut seed); let v = random_value(&mut seed); @@ -137,9 +142,9 @@ fn trie_insertions_six_high(b: &mut Bencher) { } b.iter(||{ - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -157,9 +162,9 @@ fn trie_insertions_six_mid(b: &mut Bencher) { d.push((k, v)) } b.iter(||{ - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -170,7 +175,7 @@ fn trie_insertions_six_mid(b: &mut Bencher) { fn trie_insertions_random_mid(b: &mut Bencher) { let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = H256::new(); + let mut seed = ::Out::new(); for _ in 0..1000 { let k = random_word(alphabet, 1, 5, &mut seed); let v = random_value(&mut seed); @@ -178,9 +183,9 @@ fn trie_insertions_random_mid(b: &mut Bencher) { } b.iter(||{ - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -191,7 +196,7 @@ fn trie_insertions_random_mid(b: &mut Bencher) { fn trie_insertions_six_low(b: &mut Bencher) { let alphabet = b"abcdef"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = H256::new(); + let mut seed = ::Out::new(); for _ in 0..1000 { let k = random_word(alphabet, 6, 0, &mut seed); let v = random_value(&mut seed); @@ -199,9 +204,9 @@ fn trie_insertions_six_low(b: &mut Bencher) { } b.iter(||{ - let mut memdb = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut memdb, &mut root); + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::new(); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } From 2a579a3997bcbe07cb5a3ee072c939126f981e39 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 7 Jun 2018 12:46:49 +0200 Subject: [PATCH 043/160] Handle DecoderError better --- util/patricia_trie/src/lib.rs | 10 +++------- util/patricia_trie/src/lookup.rs | 11 +++++++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 98a2adadfe0..0e4f3049ebe 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -69,7 +69,7 @@ pub enum TrieError { /// Trie item not found in the database, IncompleteDatabase(T), /// Corrupt Trie item - DecoderError(rlp::DecoderError), + DecoderError(T), } impl fmt::Display for TrieError where T: std::fmt::Debug { @@ -77,7 +77,7 @@ impl fmt::Display for TrieError where T: std::fmt::Debug { match *self { TrieError::InvalidStateRoot(ref root) => write!(f, "Invalid state root: {:?}", root), TrieError::IncompleteDatabase(ref missing) => write!(f, "Database missing expected key: {:?}", missing), - TrieError::DecoderError(ref err) => write!(f, "Decoding failed with {}", err), + TrieError::DecoderError(ref err) => write!(f, "Decoding failed for {:?}", err), } } } @@ -87,15 +87,11 @@ impl error::Error for TrieError where T: std::fmt::Debug { match *self { TrieError::InvalidStateRoot(_) => "Invalid state root", TrieError::IncompleteDatabase(_) => "Incomplete database", - TrieError::DecoderError(ref e) => e.description(), + TrieError::DecoderError(_) => "Decoder error", } } } -impl From for Box> { - fn from(e: rlp::DecoderError) -> Self { Box::new(TrieError::DecoderError(e)) } -} - /// Trie result type. Boxed to avoid copying around extra space for the `Hasher`s `Out`s on successful queries. pub type Result = ::std::result::Result>>; diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 76967253a26..d228fde6d65 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -35,7 +35,13 @@ pub struct Lookup<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> { pub marker: PhantomData, // TODO: probably not needed when all is said and done? When Query is made generic? } -impl<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> Lookup<'a, H, C, Q> where H::Out: Decodable { +impl<'a, H, C, Q> Lookup<'a, H, C, Q> +where + H: Hasher + 'a, + H::Out: Decodable, + C: NodeCodec, + Q: Query +{ /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result, H::Out> { @@ -71,7 +77,8 @@ impl<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> Lookup<'a, H, C, Q> where /* Need help writing the conversion! */ - match C::decode(node_data).expect("FIXME: should use `?`") { +// match C::decode(node_data).expect("FIXME: should use `?`") { + match C::decode(node_data).map_err(|_| TrieError::DecoderError(hash))? { Node::Leaf(slice, value) => { return Ok(match slice == key { true => Some(self.query.decode(value)), From 12ea521c4907e387839b51b92faadb223d6adedb Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 7 Jun 2018 14:29:11 +0200 Subject: [PATCH 044/160] Include the DecoderError as a Box --- util/patricia_trie/src/lib.rs | 8 ++++---- util/patricia_trie/src/lookup.rs | 20 ++------------------ util/patricia_trie/src/node_codec.rs | 6 ++++-- util/patricia_trie/src/triedb.rs | 13 ++++++++----- util/patricia_trie/src/triedbmut.rs | 6 +++--- 5 files changed, 21 insertions(+), 32 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 0e4f3049ebe..9e56ea15e09 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -62,14 +62,14 @@ use node_codec::NodeCodec; /// /// These borrow the data within them to avoid excessive copying on every /// trie operation. -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug)] pub enum TrieError { /// Attempted to create a trie with a state root not in the DB. InvalidStateRoot(T), /// Trie item not found in the database, IncompleteDatabase(T), /// Corrupt Trie item - DecoderError(T), + DecoderError(T, Box), } impl fmt::Display for TrieError where T: std::fmt::Debug { @@ -77,7 +77,7 @@ impl fmt::Display for TrieError where T: std::fmt::Debug { match *self { TrieError::InvalidStateRoot(ref root) => write!(f, "Invalid state root: {:?}", root), TrieError::IncompleteDatabase(ref missing) => write!(f, "Database missing expected key: {:?}", missing), - TrieError::DecoderError(ref err) => write!(f, "Decoding failed for {:?}", err), + TrieError::DecoderError(ref hash, ref decoder_err) => write!(f, "Decoding failed for hash {:?}; err: {:?}", hash, decoder_err), } } } @@ -87,7 +87,7 @@ impl error::Error for TrieError where T: std::fmt::Debug { match *self { TrieError::InvalidStateRoot(_) => "Invalid state root", TrieError::IncompleteDatabase(_) => "Incomplete database", - TrieError::DecoderError(_) => "Decoder error", + TrieError::DecoderError(_, ref err) => err.description(), } } } diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index d228fde6d65..2af18255cb1 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -40,7 +40,7 @@ where H: Hasher + 'a, H::Out: Decodable, C: NodeCodec, - Q: Query + Q: Query, { /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. @@ -63,22 +63,7 @@ where // without incrementing the depth. let mut node_data = &node_data[..]; loop { -// match C::decode(node_data)? { // REVIEW: I can't figure out how to write a conversion for this. The error looks like this: - /* - error[E0277]: the trait bound `std::boxed::Box::Out>>: std::convert::From<>::E>` is not satisfied - --> util/patricia_trie/src/lookup.rs:61:11 - | - 61 | match C::decode(node_data)? { - | ^^^^^^^^^^^^^^^^^^^^^ the trait `std::convert::From<>::E>` is not implemented for `std::boxed::Box::Out>>` - | - = help: consider adding a `where std::boxed::Box::Out>>: std::convert::From<>::E>` bound - = note: required by `std::convert::From::from` - */ - /* - Need help writing the conversion! - */ -// match C::decode(node_data).expect("FIXME: should use `?`") { - match C::decode(node_data).map_err(|_| TrieError::DecoderError(hash))? { + match C::decode(node_data).map_err(|e| TrieError::DecoderError(hash, e.into()))? { Node::Leaf(slice, value) => { return Ok(match slice == key { true => Some(self.query.decode(value)), @@ -104,7 +89,6 @@ where } // check if new node data is inline or hash. -// if let Some(h) = Node::try_decode_hash::(&node_data) { if let Some(h) = C::try_decode_hash(&node_data) { hash = h; break diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 125a8ef1268..a574d08eaa1 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -6,7 +6,7 @@ use node::Node; use std::marker::PhantomData; pub trait NodeCodec: Sized { - type E: ::std::error::Error; + type E: ::std::error::Error + 'static; fn encode(&Node) -> Bytes; fn decode(data: &[u8]) -> Result; fn try_decode_hash(data: &[u8]) -> Option; @@ -19,7 +19,9 @@ pub trait NodeCodec: Sized { } pub struct RlpNodeCodec {mark: PhantomData} -impl NodeCodec for RlpNodeCodec where H::Out: Encodable + Decodable { +impl NodeCodec for RlpNodeCodec +where H::Out: Encodable + Decodable +{ type E = DecoderError; fn encode(node: &Node) -> Bytes { match *node { diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 8a3c69ac99a..95e3b39c0cc 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -103,7 +103,10 @@ impl<'db, H, C> TrieDB<'db, H, C> } impl<'db, H, C> Trie for TrieDB<'db, H, C> - where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher, + H::Out: Decodable + Encodable, + C: NodeCodec { type H = H; fn root(&self) -> &::Out { self.root } @@ -461,7 +464,7 @@ mod tests { let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); let mut iter = t.iter().unwrap(); - assert_eq!(iter.next(), Some(Ok((b"A".to_vec(), DBValue::from_slice(b"A"))))); + assert_eq!(iter.next().unwrap().unwrap(), (b"A".to_vec(), DBValue::from_slice(b"A"))); iter.seek(b"!").unwrap(); assert_eq!(d, iter.map(|x| x.unwrap().1).collect::>()); let mut iter = t.iter().unwrap(); @@ -498,9 +501,9 @@ mod tests { } let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); - assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()), Ok(Some(3))); - assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()), Ok(Some(5))); - assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()), Ok(None)); + assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); + assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()).unwrap(), Some(5)); + assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()).unwrap(), None); } #[test] diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 2b01b89d847..f50922d04e0 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -1217,7 +1217,7 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); let t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); - assert_eq!(t.get(&[0x5]), Ok(None)); + assert_eq!(t.get(&[0x5]).unwrap(), None); } #[test] @@ -1242,12 +1242,12 @@ mod tests { assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); - assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); + assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); t.commit(); assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); - assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); + assert_eq!(t.get(&[0x82, 0x23]).unwrap(), None); } #[test] From 829325e9f06d3642d20a4ed6b66f96225e8787b2 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 8 Jun 2018 11:15:03 +0200 Subject: [PATCH 045/160] Store a `String` in the TrieError::DecoderError variant instead of a boxed error --- util/patricia_trie/src/lib.rs | 17 +++++++++++++---- util/patricia_trie/src/lookup.rs | 3 ++- util/patricia_trie/src/node_codec.rs | 2 +- util/patricia_trie/src/triedbmut.rs | 3 +-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index b1ba4ee530d..ce8a7241460 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -32,7 +32,7 @@ extern crate trie_standardmap as standardmap; extern crate log; use std::{fmt, error}; -use hashdb::{HashDB, DBValue, Hasher}; +use hashdb::{HashDB, DBValue, Hasher, KeccakHasher}; use std::marker::PhantomData; pub mod node; @@ -58,18 +58,27 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; use node_codec::NodeCodec; +// REVIEW: is this overdoing it? +pub type KeccakRlpNodeCodec = node_codec::RlpNodeCodec; +pub type KeccakTrieResult = Result::Out>; + /// Trie Errors. /// /// These borrow the data within them to avoid excessive copying on every /// trie operation. -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq, Clone)] pub enum TrieError { /// Attempted to create a trie with a state root not in the DB. InvalidStateRoot(T), /// Trie item not found in the database, IncompleteDatabase(T), /// Corrupt Trie item - DecoderError(T, Box), + // TODO: what we'd really like to do here is include the `rlp::DecoderError` in the `TrieError` + // but if we add a `Box` here we run into issues in the `ethcore` crate: + // "the trait bound `std::error::Error + 'static: std::marker::Send` is not satisfied" + // Investigate if using `Box` would help here (it does compile). + // Another potential problem is that the PartialEq, Eq and Clone derives do not work. + DecoderError(T, String), } impl fmt::Display for TrieError where T: std::fmt::Debug { @@ -87,7 +96,7 @@ impl error::Error for TrieError where T: std::fmt::Debug { match *self { TrieError::InvalidStateRoot(_) => "Invalid state root", TrieError::IncompleteDatabase(_) => "Incomplete database", - TrieError::DecoderError(_, ref err) => err.description(), + TrieError::DecoderError(_, ref errString) => errString, } } } diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 9b972636cd2..39805058b73 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -63,7 +63,8 @@ where // without incrementing the depth. let mut node_data = &node_data[..]; loop { - match C::decode(node_data).map_err(|e| TrieError::DecoderError(hash, e.into()))? { + // TODO: see comment on `TrieError::DecoderError` for what to do here + match C::decode(node_data).map_err(|e| TrieError::DecoderError(hash, format!("{:?}",e)))? { Node::Leaf(slice, value) => { return Ok(match slice == key { true => Some(self.query.decode(value)), diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index a574d08eaa1..04a68a3f923 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,7 +1,7 @@ use bytes::*; use nibbleslice::NibbleSlice; use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; -use hashdb::Hasher; +use hashdb::{Hasher, KeccakHasher}; use node::Node; use std::marker::PhantomData; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 3d2bdf767b7..76726d3e11a 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -23,8 +23,7 @@ use node_codec::NodeCodec; use super::node::NodeKey; use bytes::ToPretty; -use hashdb::HashDB; -use hashdb::{Hasher, DBValue}; +use hashdb::{HashDB, Hasher, DBValue}; use nibbleslice::NibbleSlice; use rlp::{RlpStream, Encodable, Decodable}; From 494e58fb1571f010f45124212025cb0e53b132cb Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 06:10:21 +0200 Subject: [PATCH 046/160] WIP use generic hashdb in ethcore This is problematic because of AsHashDB taking a circular referecen to Hasher --- ethcore/src/account_db.rs | 38 +++++++----- ethcore/src/client/client.rs | 5 +- ethcore/src/client/error.rs | 7 ++- ethcore/src/error.rs | 4 +- ethcore/src/executed.rs | 6 +- ethcore/src/factory.rs | 5 +- ethcore/src/pod_account.rs | 6 +- ethcore/src/snapshot/error.rs | 7 ++- ethcore/src/snapshot/mod.rs | 13 +++-- ethcore/src/state/account.rs | 16 +++--- ethcore/src/state/backend.rs | 43 +++++++------- ethcore/src/state/mod.rs | 86 ++++++++++++++-------------- ethcore/src/state_db.rs | 43 +++++++------- util/hashdb/src/lib.rs | 15 +++-- util/journaldb/src/earlymergedb.rs | 1 + util/journaldb/src/lib.rs | 6 +- util/journaldb/src/traits.rs | 2 +- util/memorydb/src/lib.rs | 6 +- util/patricia_trie/src/lib.rs | 2 +- util/patricia_trie/src/node_codec.rs | 6 +- 20 files changed, 175 insertions(+), 142 deletions(-) diff --git a/ethcore/src/account_db.rs b/ethcore/src/account_db.rs index 8fc4b95c6ee..a6759836f6e 100644 --- a/ethcore/src/account_db.rs +++ b/ethcore/src/account_db.rs @@ -15,12 +15,12 @@ // along with Parity. If not, see . //! DB backend wrapper for Account trie -use std::collections::HashMap; -use hash::{KECCAK_NULL_RLP, keccak}; use ethereum_types::H256; +use hash::{KECCAK_NULL_RLP, keccak}; +use hashdb::{HashDB, KeccakHasher, AsHashDB}; use kvdb::DBValue; -use hashdb::HashDB; use rlp::NULL_RLP; +use std::collections::HashMap; #[cfg(test)] use ethereum_types::Address; @@ -44,7 +44,7 @@ fn combine_key<'a>(address_hash: &'a H256, key: &'a H256) -> H256 { /// A factory for different kinds of account dbs. #[derive(Debug, Clone)] pub enum Factory { - /// Mangle hashes based on address. + /// Mangle hashes based on address. This is the default. Mangled, /// Don't mangle hashes. Plain, @@ -57,7 +57,7 @@ impl Default for Factory { impl Factory { /// Create a read-only accountdb. /// This will panic when write operations are called. - pub fn readonly<'db>(&self, db: &'db HashDB, address_hash: H256) -> Box { + pub fn readonly<'db>(&self, db: &'db HashDB, address_hash: H256) -> Box + 'db> { match *self { Factory::Mangled => Box::new(AccountDB::from_hash(db, address_hash)), Factory::Plain => Box::new(Wrapping(db)), @@ -65,7 +65,7 @@ impl Factory { } /// Create a new mutable hashdb. - pub fn create<'db>(&self, db: &'db mut HashDB, address_hash: H256) -> Box { + pub fn create<'db>(&self, db: &'db mut HashDB, address_hash: H256) -> Box + 'db> { match *self { Factory::Mangled => Box::new(AccountDBMut::from_hash(db, address_hash)), Factory::Plain => Box::new(WrappingMut(db)), @@ -77,19 +77,25 @@ impl Factory { /// DB backend wrapper for Account trie /// Transforms trie node keys for the database pub struct AccountDB<'db> { - db: &'db HashDB, + db: &'db HashDB, address_hash: H256, } +// No: orphan rules +//impl<'db> AsHashDB for AccountDB<'db> { +// fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } +// fn as_hashdb_mut(&mut self) -> &mut HashDB { self.db.as_hashdb_mut() } +//} + impl<'db> AccountDB<'db> { /// Create a new AccountDB from an address. #[cfg(test)] - pub fn new(db: &'db HashDB, address: &Address) -> Self { + pub fn new(db: &'db HashDB, address: &Address) -> Self { Self::from_hash(db, keccak(address)) } /// Create a new AcountDB from an address' hash. - pub fn from_hash(db: &'db HashDB, address_hash: H256) -> Self { + pub fn from_hash(db: &'db HashDB, address_hash: H256) -> Self { AccountDB { db: db, address_hash: address_hash, @@ -98,6 +104,7 @@ impl<'db> AccountDB<'db> { } impl<'db> HashDB for AccountDB<'db>{ + type H = KeccakHasher; fn keys(&self) -> HashMap { unimplemented!() } @@ -131,19 +138,19 @@ impl<'db> HashDB for AccountDB<'db>{ /// DB backend wrapper for Account trie pub struct AccountDBMut<'db> { - db: &'db mut HashDB, + db: &'db mut HashDB, address_hash: H256, } impl<'db> AccountDBMut<'db> { /// Create a new AccountDB from an address. #[cfg(test)] - pub fn new(db: &'db mut HashDB, address: &Address) -> Self { + pub fn new(db: &'db mut HashDB, address: &Address) -> Self { Self::from_hash(db, keccak(address)) } /// Create a new AcountDB from an address' hash. - pub fn from_hash(db: &'db mut HashDB, address_hash: H256) -> Self { + pub fn from_hash(db: &'db mut HashDB, address_hash: H256) -> Self { AccountDBMut { db: db, address_hash: address_hash, @@ -157,6 +164,7 @@ impl<'db> AccountDBMut<'db> { } impl<'db> HashDB for AccountDBMut<'db>{ + type H = KeccakHasher; fn keys(&self) -> HashMap { unimplemented!() } @@ -202,9 +210,10 @@ impl<'db> HashDB for AccountDBMut<'db>{ } } -struct Wrapping<'db>(&'db HashDB); +struct Wrapping<'db>(&'db HashDB); impl<'db> HashDB for Wrapping<'db> { + type H = KeccakHasher; fn keys(&self) -> HashMap { unimplemented!() } @@ -236,9 +245,10 @@ impl<'db> HashDB for Wrapping<'db> { } } -struct WrappingMut<'db>(&'db mut HashDB); +struct WrappingMut<'db>(&'db mut HashDB); impl<'db> HashDB for WrappingMut<'db>{ + type H = KeccakHasher; fn keys(&self) -> HashMap { unimplemented!() } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index ea8fa88630e..c270eed035f 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -609,7 +609,9 @@ impl Importer { let call = move |addr, data| { let mut state_db = state_db.boxed_clone(); - let backend = ::state::backend::Proving::new(state_db.as_hashdb_mut()); + use ::state::backend::{Proving, Backend}; // TODO: really needed? Maybe, for inference to work? + let backend = Proving::new(state_db.as_hashdb_mut()); +// let backend = ::state::backend::Proving::new(state_db.as_hashdb_mut()); let transaction = client.contract_call_tx(BlockId::Hash(*header.parent_hash()), addr, data); @@ -1149,6 +1151,7 @@ impl Client { }, }; + use ::state::backend::Backend; snapshot::take_snapshot(&*self.engine, &self.chain.read(), start_hash, db.as_hashdb(), writer, p)?; Ok(()) diff --git a/ethcore/src/client/error.rs b/ethcore/src/client/error.rs index 0e6608c0ffa..67e74eddc7d 100644 --- a/ethcore/src/client/error.rs +++ b/ethcore/src/client/error.rs @@ -19,19 +19,20 @@ use util_error::UtilError; use kvdb; use trie::TrieError; +use hashdb::{Hasher, KeccakHasher}; /// Client configuration errors. #[derive(Debug)] pub enum Error { /// TrieDB-related error. - Trie(TrieError), + Trie(TrieError<::Out>), /// Database error Database(kvdb::Error), /// Util error Util(UtilError), } -impl From for Error { - fn from(err: TrieError) -> Self { +impl From> for Error { + fn from(err: TrieError) -> Self { Error::Trie(err) } } diff --git a/ethcore/src/error.rs b/ethcore/src/error.rs index ba53b9f93e7..2f2824015d1 100644 --- a/ethcore/src/error.rs +++ b/ethcore/src/error.rs @@ -231,6 +231,8 @@ impl From for TransactionImportError { } } +use hashdb::{Hasher, KeccakHasher}; +//Trie(TrieError<::Out>) #[doc = "Error concerning TrieDBs."]; error_chain! { types { Error, ErrorKind, ErrorResultExt, EthcoreResult; @@ -245,7 +247,7 @@ error_chain! { foreign_links { Io(IoError) #[doc = "Io create error"]; StdIo(::std::io::Error) #[doc = "Error concerning the Rust standard library's IO subsystem."]; - Trie(TrieError) #[doc = "Error concerning TrieDBs."]; + Trie(TrieError) #[doc = "Error concerning TrieDBs."]; Execution(ExecutionError) #[doc = "Error concerning EVM code execution."]; Block(BlockError) #[doc = "Error concerning block processing."]; Transaction(TransactionError) #[doc = "Error concerning transaction processing."]; diff --git a/ethcore/src/executed.rs b/ethcore/src/executed.rs index 3d0b9767c4a..d89b26c3534 100644 --- a/ethcore/src/executed.rs +++ b/ethcore/src/executed.rs @@ -117,9 +117,9 @@ pub enum ExecutionError { TransactionMalformed(String), } -impl From> for ExecutionError { - fn from(err: Box) -> Self { - ExecutionError::Internal(format!("{}", err)) +impl From>> for ExecutionError { + fn from(err: Box>) -> Self { + ExecutionError::Internal(format!("{:?}", err)) } } diff --git a/ethcore/src/factory.rs b/ethcore/src/factory.rs index b429073b30c..1b58effbcac 100644 --- a/ethcore/src/factory.rs +++ b/ethcore/src/factory.rs @@ -14,11 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use trie::TrieFactory; +use trie::{TrieFactory, KeccakRlpNodeCodec}; use account_db::Factory as AccountFactory; use evm::{Factory as EvmFactory, VMType}; use vm::{Vm, ActionParams, Schedule}; use wasm::WasmInterpreter; +use hashdb::KeccakHasher; const WASM_MAGIC_NUMBER: &'static [u8; 4] = b"\0asm"; @@ -54,7 +55,7 @@ pub struct Factories { /// factory for evm. pub vm: VmFactory, /// factory for tries. - pub trie: TrieFactory, + pub trie: TrieFactory, /// factory for account databases. pub accountdb: AccountFactory, } diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 281299b3b22..dacb2a56a2a 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -19,10 +19,10 @@ use std::collections::BTreeMap; use itertools::Itertools; use hash::{keccak}; use ethereum_types::{H256, U256}; -use hashdb::HashDB; +use hashdb::{HashDB, KeccakHasher}; use triehash::sec_trie_root; use bytes::Bytes; -use trie::TrieFactory; +use trie::{TrieFactory, KeccakRlpNodeCodec}; use state::Account; use ethjson; use types::account_diff::*; @@ -65,7 +65,7 @@ impl PodAccount { } /// Place additional data into given hash DB. - pub fn insert_additional(&self, db: &mut HashDB, factory: &TrieFactory) { + pub fn insert_additional(&self, db: &mut HashDB, factory: &TrieFactory) { match self.code { Some(ref c) if !c.is_empty() => { db.insert(c); } _ => {} diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index 36fb0927a6b..593659e5fd8 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -23,6 +23,7 @@ use ids::BlockId; use ethereum_types::H256; use trie::TrieError; use rlp::DecoderError; +use hashdb::{Hasher, KeccakHasher}; /// Snapshot-related errors. #[derive(Debug)] @@ -48,7 +49,7 @@ pub enum Error { /// Restoration aborted. RestorationAborted, /// Trie error. - Trie(TrieError), + Trie(TrieError<::Out>), /// Decoder error. Decoder(DecoderError), /// Io error. @@ -101,8 +102,8 @@ impl From<::std::io::Error> for Error { } } -impl From for Error { - fn from(err: TrieError) -> Self { +impl From> for Error { + fn from(err: TrieError) -> Self { Error::Trie(err) } } diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 30a61b779cc..17ccc260e93 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -31,7 +31,8 @@ use header::Header; use ids::BlockId; use ethereum_types::{H256, U256}; -use hashdb::HashDB; +//use hashdb::{HashDB, KeccakHasher}; +use hashdb::{AsHashDB, HashDB, KeccakHasher}; use kvdb::DBValue; use snappy; use bytes::Bytes; @@ -126,7 +127,7 @@ pub fn take_snapshot( engine: &EthEngine, chain: &BlockChain, block_at: H256, - state_db: &HashDB, + state_db: &HashDB, writer: W, p: &Progress ) -> Result<(), Error> { @@ -264,7 +265,7 @@ impl<'a> StateChunker<'a> { /// /// Returns a list of hashes of chunks created, or any error it may /// have encountered. -pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { +pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { let account_trie = TrieDB::new(db, &root)?; let mut chunker = StateChunker { @@ -304,7 +305,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, + db: Box>, state_root: H256, known_code: HashMap, // code hashes mapped to first account with this code. missing_code: HashMap>, // maps code hashes to lists of accounts missing that code. @@ -389,7 +390,7 @@ impl StateRebuilder { /// Finalize the restoration. Check for accounts missing code and make a dummy /// journal entry. /// Once all chunks have been fed, there should be nothing missing. - pub fn finalize(mut self, era: u64, id: H256) -> Result, ::error::Error> { + pub fn finalize(mut self, era: u64, id: H256) -> Result>, ::error::Error> { let missing = self.missing_code.keys().cloned().collect::>(); if !missing.is_empty() { return Err(Error::MissingCode(missing).into()) } @@ -414,7 +415,7 @@ struct RebuiltStatus { // rebuild a set of accounts and their storage. // returns a status detailing newly-loaded code and accounts missing code. fn rebuild_accounts( - db: &mut HashDB, + db: &mut HashDB, account_fat_rlps: Rlp, out_chunk: &mut [(H256, Bytes)], known_code: &HashMap, diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index a7a40e6a3bb..ba019372a70 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -22,11 +22,11 @@ use std::collections::{HashMap, BTreeMap}; use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use ethereum_types::{H256, U256, Address}; use error::Error; -use hashdb::HashDB; +use hashdb::{Hasher, HashDB, KeccakHasher}; use kvdb::DBValue; use bytes::{Bytes, ToPretty}; use trie; -use trie::{SecTrieDB, Trie, TrieFactory, TrieError}; +use trie::{SecTrieDB, Trie, TrieFactory, TrieError, KeccakRlpNodeCodec}; use pod_account::*; use rlp::{RlpStream, encode}; use lru_cache::LruCache; @@ -199,7 +199,7 @@ impl Account { /// Get (and cache) the contents of the trie's storage at `key`. /// Takes modified storage into account. - pub fn storage_at(&self, db: &HashDB, key: &H256) -> trie::Result { + pub fn storage_at(&self, db: &HashDB, key: &H256) -> trie::Result { if let Some(value) = self.cached_storage_at(key) { return Ok(value); } @@ -278,7 +278,7 @@ impl Account { } /// Provide a database to get `code_hash`. Should not be called if it is a contract without code. - pub fn cache_code(&mut self, db: &HashDB) -> Option> { + pub fn cache_code(&mut self, db: &HashDB) -> Option> { // TODO: fill out self.code_cache; trace!("Account::cache_code: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty()); @@ -307,7 +307,7 @@ impl Account { } /// Provide a database to get `code_size`. Should not be called if it is a contract without code. - pub fn cache_code_size(&mut self, db: &HashDB) -> bool { + pub fn cache_code_size(&mut self, db: &HashDB) -> bool { // TODO: fill out self.code_cache; trace!("Account::cache_code_size: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty()); self.code_size.is_some() || @@ -374,7 +374,7 @@ impl Account { } /// Commit the `storage_changes` to the backing DB and update `storage_root`. - pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> trie::Result<()> { + pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> trie::Result<(), ::Out> { let mut t = trie_factory.from_existing(db, &mut self.storage_root)?; for (k, v) in self.storage_changes.drain() { // cast key and value to trait type, @@ -390,7 +390,7 @@ impl Account { } /// Commit any unsaved code. `code_hash` will always return the hash of the `code_cache` after this. - pub fn commit_code(&mut self, db: &mut HashDB) { + pub fn commit_code(&mut self, db: &mut HashDB) { trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_filth == Filth::Dirty, self.code_cache.is_empty()); match (self.code_filth == Filth::Dirty, self.code_cache.is_empty()) { (true, true) => { @@ -472,7 +472,7 @@ impl Account { /// trie. /// `storage_key` is the hash of the desired storage key, meaning /// this will only work correctly under a secure trie. - pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box> { + pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box::Out>>> { use trie::{Trie, TrieDB}; use trie::recorder::Recorder; diff --git a/ethcore/src/state/backend.rs b/ethcore/src/state/backend.rs index 6b2e21cb465..5d0a018c584 100644 --- a/ethcore/src/state/backend.rs +++ b/ethcore/src/state/backend.rs @@ -27,16 +27,17 @@ use std::sync::Arc; use state::Account; use parking_lot::Mutex; use ethereum_types::{Address, H256}; -use memorydb::MemoryDB; -use hashdb::{AsHashDB, HashDB, DBValue}; +//use memorydb::MemoryDB; +use memorydb::KeccakMemoryDB; +use hashdb::{AsHashDB, HashDB, DBValue, Hasher, KeccakHasher}; /// State backend. See module docs for more details. pub trait Backend: Send { /// Treat the backend as a read-only hashdb. - fn as_hashdb(&self) -> &HashDB; + fn as_hashdb(&self) -> &HashDB; /// Treat the backend as a writeable hashdb. - fn as_hashdb_mut(&mut self) -> &mut HashDB; + fn as_hashdb_mut(&mut self) -> &mut HashDB; /// Add an account entry to the cache. fn add_to_account_cache(&mut self, addr: Address, data: Option, modified: bool); @@ -75,18 +76,19 @@ pub trait Backend: Send { // TODO: when account lookup moved into backends, this won't rely as tenuously on intended // usage. #[derive(Clone, PartialEq)] -pub struct ProofCheck(MemoryDB); +pub struct ProofCheck(KeccakMemoryDB); impl ProofCheck { /// Create a new `ProofCheck` backend from the given state items. pub fn new(proof: &[DBValue]) -> Self { - let mut db = MemoryDB::new(); + let mut db = KeccakMemoryDB::new(); for item in proof { db.insert(item); } ProofCheck(db) } } impl HashDB for ProofCheck { + type H = KeccakHasher; fn keys(&self) -> HashMap { self.0.keys() } fn get(&self, key: &H256) -> Option { self.0.get(key) @@ -108,8 +110,8 @@ impl HashDB for ProofCheck { } impl Backend for ProofCheck { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } fn add_to_account_cache(&mut self, _addr: Address, _data: Option, _modified: bool) {} fn cache_code(&self, _hash: H256, _code: Arc>) {} fn get_cached_account(&self, _addr: &Address) -> Option> { None } @@ -128,13 +130,14 @@ impl Backend for ProofCheck { /// The proof-of-execution can be extracted with `extract_proof`. /// /// This doesn't cache anything or rely on the canonical state caches. -pub struct Proving { +pub struct Proving> { base: H, // state we're proving values from. - changed: MemoryDB, // changed state via insertions. + changed: KeccakMemoryDB, // changed state via insertions. proof: Mutex>, } -impl HashDB for Proving { +impl + Send + Sync> HashDB for Proving { + type H = KeccakHasher; fn keys(&self) -> HashMap { let mut keys = self.base.as_hashdb().keys(); keys.extend(self.changed.keys()); @@ -171,12 +174,12 @@ impl HashDB for Proving { } } -impl Backend for Proving { - fn as_hashdb(&self) -> &HashDB { +impl + Send + Sync> Backend for Proving { + fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } @@ -197,13 +200,13 @@ impl Backend for Proving { fn is_known_null(&self, _: &Address) -> bool { false } } -impl Proving { +impl> Proving { /// Create a new `Proving` over a base database. /// This will store all values ever fetched from that base. pub fn new(base: H) -> Self { Proving { base: base, - changed: MemoryDB::new(), + changed: KeccakMemoryDB::new(), proof: Mutex::new(HashSet::new()), } } @@ -215,7 +218,7 @@ impl Proving { } } -impl Clone for Proving { +impl + Clone> Clone for Proving { fn clone(&self) -> Self { Proving { base: self.base.clone(), @@ -229,12 +232,12 @@ impl Clone for Proving { /// it. Doesn't cache anything. pub struct Basic(pub H); -impl Backend for Basic { - fn as_hashdb(&self) -> &HashDB { +impl + Send + Sync> Backend for Basic { + fn as_hashdb(&self) -> &HashDB { self.0.as_hashdb() } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { self.0.as_hashdb_mut() } diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index ccca20b71e1..31c2d3986aa 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -43,12 +43,12 @@ use state_db::StateDB; use factory::VmFactory; use ethereum_types::{H256, U256, Address}; -use hashdb::{HashDB, AsHashDB}; +use hashdb::{HashDB, AsHashDB, KeccakHasher}; use kvdb::DBValue; use bytes::Bytes; use trie; -use trie::{Trie, TrieError, TrieDB}; +use trie::{Trie, TrieError, TrieDB, KeccakTrieResult as TrieResult}; use trie::recorder::Recorder; mod account; @@ -225,7 +225,7 @@ pub fn check_proof( /// Prove a transaction on the given state. /// Returns `None` when the transacion could not be proved, /// and a proof otherwise. -pub fn prove_transaction( +pub fn prove_transaction + Send + Sync>( db: H, root: H256, transaction: &SignedTransaction, @@ -336,23 +336,23 @@ pub enum CleanupMode<'a> { /// Provides subset of `State` methods to query state information pub trait StateInfo { /// Get the nonce of account `a`. - fn nonce(&self, a: &Address) -> trie::Result; + fn nonce(&self, a: &Address) -> TrieResult; /// Get the balance of account `a`. - fn balance(&self, a: &Address) -> trie::Result; + fn balance(&self, a: &Address) -> TrieResult; /// Mutate storage of account `address` so that it is `value` for `key`. - fn storage_at(&self, address: &Address, key: &H256) -> trie::Result; + fn storage_at(&self, address: &Address, key: &H256) -> TrieResult; /// Get accounts' code. - fn code(&self, a: &Address) -> trie::Result>>; + fn code(&self, a: &Address) -> TrieResult>>; } impl StateInfo for State { - fn nonce(&self, a: &Address) -> trie::Result { State::nonce(self, a) } - fn balance(&self, a: &Address) -> trie::Result { State::balance(self, a) } - fn storage_at(&self, address: &Address, key: &H256) -> trie::Result { State::storage_at(self, address, key) } - fn code(&self, address: &Address) -> trie::Result>> { State::code(self, address) } + fn nonce(&self, a: &Address) -> TrieResult { State::nonce(self, a) } + fn balance(&self, a: &Address) -> TrieResult { State::balance(self, a) } + fn storage_at(&self, address: &Address, key: &H256) -> TrieResult { State::storage_at(self, address, key) } + fn code(&self, address: &Address) -> TrieResult>> { State::code(self, address) } } const SEC_TRIE_DB_UNWRAP_STR: &'static str = "A state can only be created with valid root. Creating a SecTrieDB with a valid root will not fail. \ @@ -379,7 +379,7 @@ impl State { } /// Creates new state with existing state root - pub fn from_existing(db: B, root: H256, account_start_nonce: U256, factories: Factories) -> Result, TrieError> { + pub fn from_existing(db: B, root: H256, account_start_nonce: U256, factories: Factories) -> Result, TrieError> { if !db.as_hashdb().contains(&root) { return Err(TrieError::InvalidStateRoot(root)); } @@ -481,7 +481,7 @@ impl State { } /// Destroy the current object and return single account data. - pub fn into_account(self, account: &Address) -> trie::Result<(Option>, HashMap)> { + pub fn into_account(self, account: &Address) -> TrieResult<(Option>, HashMap)> { // TODO: deconstruct without cloning. let account = self.require(account, true)?; Ok((account.code().clone(), account.storage_changes().clone())) @@ -504,43 +504,43 @@ impl State { } /// Determine whether an account exists. - pub fn exists(&self, a: &Address) -> trie::Result { + pub fn exists(&self, a: &Address) -> TrieResult { // Bloom filter does not contain empty accounts, so it is important here to // check if account exists in the database directly before EIP-161 is in effect. self.ensure_cached(a, RequireCache::None, false, |a| a.is_some()) } /// Determine whether an account exists and if not empty. - pub fn exists_and_not_null(&self, a: &Address) -> trie::Result { + pub fn exists_and_not_null(&self, a: &Address) -> TrieResult { self.ensure_cached(a, RequireCache::None, false, |a| a.map_or(false, |a| !a.is_null())) } /// Determine whether an account exists and has code or non-zero nonce. - pub fn exists_and_has_code_or_nonce(&self, a: &Address) -> trie::Result { + pub fn exists_and_has_code_or_nonce(&self, a: &Address) -> TrieResult { self.ensure_cached(a, RequireCache::CodeSize, false, |a| a.map_or(false, |a| a.code_hash() != KECCAK_EMPTY || *a.nonce() != self.account_start_nonce)) } /// Get the balance of account `a`. - pub fn balance(&self, a: &Address) -> trie::Result { + pub fn balance(&self, a: &Address) -> TrieResult { self.ensure_cached(a, RequireCache::None, true, |a| a.as_ref().map_or(U256::zero(), |account| *account.balance())) } /// Get the nonce of account `a`. - pub fn nonce(&self, a: &Address) -> trie::Result { + pub fn nonce(&self, a: &Address) -> TrieResult { self.ensure_cached(a, RequireCache::None, true, |a| a.as_ref().map_or(self.account_start_nonce, |account| *account.nonce())) } /// Get the storage root of account `a`. - pub fn storage_root(&self, a: &Address) -> trie::Result> { + pub fn storage_root(&self, a: &Address) -> TrieResult> { self.ensure_cached(a, RequireCache::None, true, |a| a.as_ref().and_then(|account| account.storage_root().cloned())) } /// Mutate storage of account `address` so that it is `value` for `key`. - pub fn storage_at(&self, address: &Address, key: &H256) -> trie::Result { + pub fn storage_at(&self, address: &Address, key: &H256) -> TrieResult { // Storage key search and update works like this: // 1. If there's an entry for the account in the local cache check for the key and return it if found. // 2. If there's an entry for the account in the global cache check for the key or load it into that account. @@ -602,25 +602,25 @@ impl State { } /// Get accounts' code. - pub fn code(&self, a: &Address) -> trie::Result>> { + pub fn code(&self, a: &Address) -> TrieResult>> { self.ensure_cached(a, RequireCache::Code, true, |a| a.as_ref().map_or(None, |a| a.code().clone())) } /// Get an account's code hash. - pub fn code_hash(&self, a: &Address) -> trie::Result { + pub fn code_hash(&self, a: &Address) -> TrieResult { self.ensure_cached(a, RequireCache::None, true, |a| a.as_ref().map_or(KECCAK_EMPTY, |a| a.code_hash())) } /// Get accounts' code size. - pub fn code_size(&self, a: &Address) -> trie::Result> { + pub fn code_size(&self, a: &Address) -> TrieResult> { self.ensure_cached(a, RequireCache::CodeSize, true, |a| a.as_ref().and_then(|a| a.code_size())) } /// Add `incr` to the balance of account `a`. - pub fn add_balance(&mut self, a: &Address, incr: &U256, cleanup_mode: CleanupMode) -> trie::Result<()> { + pub fn add_balance(&mut self, a: &Address, incr: &U256, cleanup_mode: CleanupMode) -> TrieResult<()> { trace!(target: "state", "add_balance({}, {}): {}", a, incr, self.balance(a)?); let is_value_transfer = !incr.is_zero(); if is_value_transfer || (cleanup_mode == CleanupMode::ForceCreate && !self.exists(a)?) { @@ -635,7 +635,7 @@ impl State { } /// Subtract `decr` from the balance of account `a`. - pub fn sub_balance(&mut self, a: &Address, decr: &U256, cleanup_mode: &mut CleanupMode) -> trie::Result<()> { + pub fn sub_balance(&mut self, a: &Address, decr: &U256, cleanup_mode: &mut CleanupMode) -> TrieResult<()> { trace!(target: "state", "sub_balance({}, {}): {}", a, decr, self.balance(a)?); if !decr.is_zero() || !self.exists(a)? { self.require(a, false)?.sub_balance(decr); @@ -647,19 +647,19 @@ impl State { } /// Subtracts `by` from the balance of `from` and adds it to that of `to`. - pub fn transfer_balance(&mut self, from: &Address, to: &Address, by: &U256, mut cleanup_mode: CleanupMode) -> trie::Result<()> { + pub fn transfer_balance(&mut self, from: &Address, to: &Address, by: &U256, mut cleanup_mode: CleanupMode) -> TrieResult<()> { self.sub_balance(from, by, &mut cleanup_mode)?; self.add_balance(to, by, cleanup_mode)?; Ok(()) } /// Increment the nonce of account `a` by 1. - pub fn inc_nonce(&mut self, a: &Address) -> trie::Result<()> { + pub fn inc_nonce(&mut self, a: &Address) -> TrieResult<()> { self.require(a, false).map(|mut x| x.inc_nonce()) } /// Mutate storage of account `a` so that it is `value` for `key`. - pub fn set_storage(&mut self, a: &Address, key: H256, value: H256) -> trie::Result<()> { + pub fn set_storage(&mut self, a: &Address, key: H256, value: H256) -> TrieResult<()> { trace!(target: "state", "set_storage({}:{:x} to {:x})", a, key, value); if self.storage_at(a, &key)? != value { self.require(a, false)?.set_storage(key, value) @@ -670,13 +670,13 @@ impl State { /// Initialise the code of account `a` so that it is `code`. /// NOTE: Account should have been created with `new_contract`. - pub fn init_code(&mut self, a: &Address, code: Bytes) -> trie::Result<()> { + pub fn init_code(&mut self, a: &Address, code: Bytes) -> TrieResult<()> { self.require_or_from(a, true, || Account::new_contract(0.into(), self.account_start_nonce), |_|{})?.init_code(code); Ok(()) } /// Reset the code of account `a` so that it is `code`. - pub fn reset_code(&mut self, a: &Address, code: Bytes) -> trie::Result<()> { + pub fn reset_code(&mut self, a: &Address, code: Bytes) -> TrieResult<()> { self.require_or_from(a, true, || Account::new_contract(0.into(), self.account_start_nonce), |_|{})?.reset_code(code); Ok(()) } @@ -753,7 +753,7 @@ impl State { } } - fn touch(&mut self, a: &Address) -> trie::Result<()> { + fn touch(&mut self, a: &Address) -> TrieResult<()> { self.require(a, false)?; Ok(()) } @@ -809,7 +809,7 @@ impl State { } /// Remove any touched empty or dust accounts. - pub fn kill_garbage(&mut self, touched: &HashSet
, remove_empty_touched: bool, min_balance: &Option, kill_contracts: bool) -> trie::Result<()> { + pub fn kill_garbage(&mut self, touched: &HashSet
, remove_empty_touched: bool, min_balance: &Option, kill_contracts: bool) -> TrieResult<()> { let to_kill: HashSet<_> = { self.cache.borrow().iter().filter_map(|(address, ref entry)| if touched.contains(address) && // Check all touched accounts @@ -850,7 +850,7 @@ impl State { } /// Populate a PodAccount map from this state, with another state as the account and storage query. - pub fn to_pod_diff(&mut self, query: &State) -> trie::Result { + pub fn to_pod_diff(&mut self, query: &State) -> TrieResult { assert!(self.checkpoints.borrow().is_empty()); // Merge PodAccount::to_pod for cache of self and `query`. @@ -858,7 +858,7 @@ impl State { .chain(query.cache.borrow().keys().cloned()) .collect::>(); - Ok(PodState::from(all_addresses.into_iter().fold(Ok(BTreeMap::new()), |m: trie::Result<_>, address| { + Ok(PodState::from(all_addresses.into_iter().fold(Ok(BTreeMap::new()), |m: TrieResult<_>, address| { let mut m = m?; let account = self.ensure_cached(&address, RequireCache::Code, true, |acc| { @@ -886,7 +886,7 @@ impl State { })?; if let Some((balance, nonce, storage_keys, code)) = account { - let storage = storage_keys.into_iter().fold(Ok(BTreeMap::new()), |s: trie::Result<_>, key| { + let storage = storage_keys.into_iter().fold(Ok(BTreeMap::new()), |s: TrieResult<_>, key| { let mut s = s?; s.insert(key, self.storage_at(&address, &key)?); @@ -904,14 +904,14 @@ impl State { /// Returns a `StateDiff` describing the difference from `orig` to `self`. /// Consumes self. - pub fn diff_from(&self, mut orig: State) -> trie::Result { + pub fn diff_from(&self, mut orig: State) -> TrieResult { let pod_state_post = self.to_pod(); let pod_state_pre = orig.to_pod_diff(self)?; Ok(pod_state::diff_pod(&pod_state_pre, &pod_state_post)) } // load required account data from the databases. - fn update_account_cache(require: RequireCache, account: &mut Account, state_db: &B, db: &HashDB) { + fn update_account_cache(require: RequireCache, account: &mut Account, state_db: &B, db: &HashDB) { if let RequireCache::None = require { return; } @@ -943,7 +943,7 @@ impl State { /// Check caches for required data /// First searches for account in the local, then the shared cache. /// Populates local cache if nothing found. - fn ensure_cached(&self, a: &Address, require: RequireCache, check_null: bool, f: F) -> trie::Result + fn ensure_cached(&self, a: &Address, require: RequireCache, check_null: bool, f: F) -> TrieResult where F: Fn(Option<&Account>) -> U { // check local cache first if let Some(ref mut maybe_acc) = self.cache.borrow_mut().get_mut(a) { @@ -984,13 +984,13 @@ impl State { } /// Pull account `a` in our cache from the trie DB. `require_code` requires that the code be cached, too. - fn require<'a>(&'a self, a: &Address, require_code: bool) -> trie::Result> { + fn require<'a>(&'a self, a: &Address, require_code: bool) -> TrieResult> { self.require_or_from(a, require_code, || Account::new_basic(0u8.into(), self.account_start_nonce), |_|{}) } /// Pull account `a` in our cache from the trie DB. `require_code` requires that the code be cached, too. /// If it doesn't exist, make account equal the evaluation of `default`. - fn require_or_from<'a, F, G>(&'a self, a: &Address, require_code: bool, default: F, not_default: G) -> trie::Result> + fn require_or_from<'a, F, G>(&'a self, a: &Address, require_code: bool, default: F, not_default: G) -> TrieResult> where F: FnOnce() -> Account, G: FnOnce(&mut Account), { let contains_key = self.cache.borrow().contains_key(a); @@ -1037,7 +1037,7 @@ impl State { } /// Replace account code and storage. Creates account if it does not exist. - pub fn patch_account(&self, a: &Address, code: Arc, storage: HashMap) -> trie::Result<()> { + pub fn patch_account(&self, a: &Address, code: Arc, storage: HashMap) -> TrieResult<()> { Ok(self.require(a, false)?.reset_code_and_storage(code, storage)) } } @@ -1049,7 +1049,7 @@ impl State { /// If the account doesn't exist in the trie, prove that and return defaults. /// Requires a secure trie to be used for accurate results. /// `account_key` == keccak(address) - pub fn prove_account(&self, account_key: H256) -> trie::Result<(Vec, BasicAccount)> { + pub fn prove_account(&self, account_key: H256) -> TrieResult<(Vec, BasicAccount)> { let mut recorder = Recorder::new(); let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; let maybe_account: Option = { @@ -1074,7 +1074,7 @@ impl State { /// Requires a secure trie to be used for correctness. /// `account_key` == keccak(address) /// `storage_key` == keccak(key) - pub fn prove_storage(&self, account_key: H256, storage_key: H256) -> trie::Result<(Vec, H256)> { + pub fn prove_storage(&self, account_key: H256, storage_key: H256) -> TrieResult<(Vec, H256)> { // TODO: probably could look into cache somehow but it's keyed by // address, not keccak(address). let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs index c3704828c25..d9832ce9154 100644 --- a/ethcore/src/state_db.rs +++ b/ethcore/src/state_db.rs @@ -16,22 +16,23 @@ //! State database abstraction. For more info, see the doc for `StateDB` -use std::collections::{VecDeque, HashSet}; -use std::sync::Arc; -use lru_cache::LruCache; -use memory_cache::MemoryLruCache; -use journaldb::JournalDB; -use kvdb::{KeyValueDB, DBTransaction}; +use bloom_journal::{Bloom, BloomJournal}; +use byteorder::{LittleEndian, ByteOrder}; +use db::COL_ACCOUNT_BLOOM; use ethereum_types::{H256, Address}; -use hashdb::HashDB; -use state::{self, Account}; -use header::BlockNumber; use hash::keccak; +use hashdb::{Hasher, KeccakHasher, HashDB, AsHashDB}; +use header::BlockNumber; +use journaldb::JournalDB; +use kvdb::{KeyValueDB, DBTransaction}; +use lru_cache::LruCache; +use memory_cache::MemoryLruCache; use parking_lot::Mutex; +use state::{self, Account}; +use std::collections::{VecDeque, HashSet}; +use std::sync::Arc; use util_error::UtilError; -use bloom_journal::{Bloom, BloomJournal}; -use db::COL_ACCOUNT_BLOOM; -use byteorder::{LittleEndian, ByteOrder}; + /// Value used to initialize bloom bitmap size. /// @@ -104,7 +105,7 @@ struct BlockChanges { /// `StateDB` is propagated into the global cache. pub struct StateDB { /// Backing database. - db: Box, + db: Box>, /// Shared canonical state cache. account_cache: Arc>, /// DB Code cache. Maps code hashes to shared bytes. @@ -129,7 +130,7 @@ impl StateDB { /// of the LRU cache in bytes. Actual used memory may (read: will) be higher due to bookkeeping. // TODO: make the cache size actually accurate by moving the account storage cache // into the `AccountCache` structure as its own `LruCache<(Address, H256), H256>`. - pub fn new(db: Box, cache_size: usize) -> StateDB { + pub fn new(db: Box>, cache_size: usize) -> StateDB { let bloom = Self::load_bloom(&**db.backing()); let acc_cache_size = cache_size * ACCOUNT_CACHE_RATIO / 100; let code_cache_size = cache_size - acc_cache_size; @@ -309,13 +310,15 @@ impl StateDB { } } + // TODO: needed? /// Conversion method to interpret self as `HashDB` reference - pub fn as_hashdb(&self) -> &HashDB { + pub fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } + // TODO: needed? /// Conversion method to interpret self as mutable `HashDB` reference - pub fn as_hashdb_mut(&mut self) -> &mut HashDB { + pub fn as_hashdb_mut(&mut self) -> &mut HashDB { self.db.as_hashdb_mut() } @@ -365,7 +368,7 @@ impl StateDB { } /// Returns underlying `JournalDB`. - pub fn journal_db(&self) -> &JournalDB { + pub fn journal_db(&self) -> &JournalDB { &*self.db } @@ -410,11 +413,9 @@ impl StateDB { } impl state::Backend for StateDB { - fn as_hashdb(&self) -> &HashDB { - self.db.as_hashdb() - } + fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { self.db.as_hashdb_mut() } diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index ba0ecdf3049..59b40cd3243 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -34,7 +34,7 @@ pub trait Hasher: Sync + Send { } // REVIEW: Where do the concrete Hasher implementations go? Own crate? -#[derive(Debug, Clone, PartialEq)] +#[derive(Default, Debug, Clone, PartialEq)] pub struct KeccakHasher; impl Hasher for KeccakHasher { type Out = H256; @@ -46,11 +46,14 @@ impl Hasher for KeccakHasher { out.into() } } +/// Convenience type for crates that need a `HashDB` with Keccak hashes +pub type KeccakHashDB = HashDB; /// `HashDB` value type. pub type DBValue = ElasticArray128; /// Trait modelling datastore keyed by a 32-byte Keccak hash. +// TODO: move `H: Hasher` to be a generic param instead of an associated type pub trait HashDB: Send + Sync { type H: Hasher; /// Get the keys in the database together with number of underlying references. @@ -85,20 +88,16 @@ pub trait AsHashDB { } impl> AsHashDB for T { - fn as_hashdb(&self) -> &HashDB { - self - } - fn as_hashdb_mut(&mut self) -> &mut HashDB { - self - } + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } } impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { fn as_hashdb(&self) -> &HashDB { &**self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { &mut **self } } +} \ No newline at end of file diff --git a/util/journaldb/src/earlymergedb.rs b/util/journaldb/src/earlymergedb.rs index cd40d246008..14cdeea85cf 100644 --- a/util/journaldb/src/earlymergedb.rs +++ b/util/journaldb/src/earlymergedb.rs @@ -104,6 +104,7 @@ enum RemoveFrom { /// ``` /// /// TODO: `store_reclaim_period` + pub struct EarlyMergeDB { overlay: MemoryDB, backing: Arc, diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index 1210e8f06f8..aa7813525b5 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -39,7 +39,7 @@ extern crate kvdb_memorydb; use std::{fmt, str}; use std::sync::Arc; -use hashdb::KeccakHasher; +use hashdb::{AsHashDB, HashDB, KeccakHasher}; /// Export the journaldb module. mod traits; @@ -54,6 +54,10 @@ pub mod overlaydb; /// Export the `JournalDB` trait. pub use self::traits::JournalDB; +/// Convenience type for crates that need a `JournalDB` with Keccak hashes +//pub type KeccakJournalDB = JournalDB; // TODO: use this in the `journaldb` crate +// TODO: do we need/want additional convenience exports like this? `KeccakArchiveDB`, `KeccakEarlymergeDB` etc? + /// Journal database operating strategy. #[derive(Debug, PartialEq, Clone, Copy)] pub enum Algorithm { diff --git a/util/journaldb/src/traits.rs b/util/journaldb/src/traits.rs index 985f00398b1..be97f92b370 100644 --- a/util/journaldb/src/traits.rs +++ b/util/journaldb/src/traits.rs @@ -25,7 +25,7 @@ use bytes::Bytes; /// A `HashDB` which can manage a short-term journal potentially containing many forks of mutually /// exclusive actions. -pub trait JournalDB: HashDB { +pub trait JournalDB: HashDB + AsHashDB { /// Return a copy of ourself, in a box. fn boxed_clone(&self) -> Box>; diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 0d07335dedb..17ce7370a38 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -23,13 +23,14 @@ extern crate rlp; #[cfg(test)] extern crate tiny_keccak; #[cfg(test)] extern crate ethereum_types; -use hashdb::{HashDB, Hasher, DBValue}; +use hashdb::{HashDB, Hasher, DBValue, KeccakHasher}; use heapsize::HeapSizeOf; use plain_hasher::H256FastMap; use rlp::NULL_RLP; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::mem; + /// Reference-counted memory-based `HashDB` implementation. /// /// Use `new()` to create a new database. Insert items with `insert()`, remove items @@ -79,6 +80,9 @@ pub struct MemoryDB { data: H256FastMap, } +/// Convenience type for crates that need a `MemoryDB` with Keccak hashes +pub type KeccakMemoryDB = MemoryDB; + impl MemoryDB { /// Create a new instance of the memory DB. pub fn new() -> MemoryDB { diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index ce8a7241460..658b754f7ff 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -96,7 +96,7 @@ impl error::Error for TrieError where T: std::fmt::Debug { match *self { TrieError::InvalidStateRoot(_) => "Invalid state root", TrieError::IncompleteDatabase(_) => "Incomplete database", - TrieError::DecoderError(_, ref errString) => errString, + TrieError::DecoderError(_, ref err_string) => err_string, } } } diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 04a68a3f923..2aca945763b 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,12 +1,12 @@ use bytes::*; use nibbleslice::NibbleSlice; use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; -use hashdb::{Hasher, KeccakHasher}; +use hashdb::Hasher; use node::Node; use std::marker::PhantomData; pub trait NodeCodec: Sized { - type E: ::std::error::Error + 'static; + type E: ::std::error::Error; fn encode(&Node) -> Bytes; fn decode(data: &[u8]) -> Result; fn try_decode_hash(data: &[u8]) -> Option; @@ -17,6 +17,8 @@ pub trait NodeCodec: Sized { fn encoded_list(size: usize) -> RlpStream; } + +#[derive(Default, Clone)] pub struct RlpNodeCodec {mark: PhantomData} impl NodeCodec for RlpNodeCodec From aaf5f4c8f337d2e977516b08fed5376ff8183765 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 07:19:12 +0200 Subject: [PATCH 047/160] Switch JournalDB to use type param'd HashDB --- util/journaldb/src/archivedb.rs | 5 ++--- util/journaldb/src/earlymergedb.rs | 5 ++--- util/journaldb/src/lib.rs | 4 ++-- util/journaldb/src/overlaydb.rs | 3 +-- util/journaldb/src/overlayrecentdb.rs | 5 ++--- util/journaldb/src/refcounteddb.rs | 5 ++--- util/journaldb/src/traits.rs | 4 ++-- 7 files changed, 13 insertions(+), 18 deletions(-) diff --git a/util/journaldb/src/archivedb.rs b/util/journaldb/src/archivedb.rs index de027685158..7aa4806fc07 100644 --- a/util/journaldb/src/archivedb.rs +++ b/util/journaldb/src/archivedb.rs @@ -62,8 +62,7 @@ impl ArchiveDB { } } -impl HashDB for ArchiveDB { - type H = KeccakHasher; +impl HashDB for ArchiveDB { fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| (H256::from_slice(&*key), 1)) @@ -109,7 +108,7 @@ impl HashDB for ArchiveDB { } impl JournalDB for ArchiveDB { - fn boxed_clone(&self) -> Box> { + fn boxed_clone(&self) -> Box { Box::new(ArchiveDB { overlay: self.overlay.clone(), backing: self.backing.clone(), diff --git a/util/journaldb/src/earlymergedb.rs b/util/journaldb/src/earlymergedb.rs index 14cdeea85cf..6b0681bce96 100644 --- a/util/journaldb/src/earlymergedb.rs +++ b/util/journaldb/src/earlymergedb.rs @@ -286,8 +286,7 @@ impl EarlyMergeDB { } } -impl HashDB for EarlyMergeDB { - type H = KeccakHasher; +impl HashDB for EarlyMergeDB { fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| (H256::from_slice(&*key), 1)) @@ -331,7 +330,7 @@ impl HashDB for EarlyMergeDB { } impl JournalDB for EarlyMergeDB { - fn boxed_clone(&self) -> Box> { + fn boxed_clone(&self) -> Box { Box::new(EarlyMergeDB { overlay: self.overlay.clone(), backing: self.backing.clone(), diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index aa7813525b5..1bf95e15e4f 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -55,7 +55,7 @@ pub mod overlaydb; pub use self::traits::JournalDB; /// Convenience type for crates that need a `JournalDB` with Keccak hashes -//pub type KeccakJournalDB = JournalDB; // TODO: use this in the `journaldb` crate +//pub type KeccakJournalDB = JournalDB; // TODO: use this in the `journaldb` crate // TODO: do we need/want additional convenience exports like this? `KeccakArchiveDB`, `KeccakEarlymergeDB` etc? /// Journal database operating strategy. @@ -145,7 +145,7 @@ impl fmt::Display for Algorithm { } /// Create a new `JournalDB` trait object over a generic key-value database. -pub fn new(backing: Arc<::kvdb::KeyValueDB>, algorithm: Algorithm, col: Option) -> Box> { +pub fn new(backing: Arc<::kvdb::KeyValueDB>, algorithm: Algorithm, col: Option) -> Box { match algorithm { Algorithm::Archive => Box::new(archivedb::ArchiveDB::new(backing, col)), Algorithm::EarlyMerge => Box::new(earlymergedb::EarlyMergeDB::new(backing, col)), diff --git a/util/journaldb/src/overlaydb.rs b/util/journaldb/src/overlaydb.rs index f2f0f638c4d..2878e463510 100644 --- a/util/journaldb/src/overlaydb.rs +++ b/util/journaldb/src/overlaydb.rs @@ -152,8 +152,7 @@ impl OverlayDB { } } -impl HashDB for OverlayDB { - type H = KeccakHasher; +impl HashDB for OverlayDB { fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| { diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index dcaae70a7b1..8a2f772f098 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -242,7 +242,7 @@ fn to_short_key(key: &H256) -> H256 { } impl JournalDB for OverlayRecentDB { - fn boxed_clone(&self) -> Box> { + fn boxed_clone(&self) -> Box { Box::new(self.clone()) } @@ -438,8 +438,7 @@ impl JournalDB for OverlayRecentDB { } } -impl HashDB for OverlayRecentDB { - type H = KeccakHasher; +impl HashDB for OverlayRecentDB { fn keys(&self) -> HashMap { let mut ret: HashMap = self.backing.iter(self.column) .map(|(key, _)| (H256::from_slice(&*key), 1)) diff --git a/util/journaldb/src/refcounteddb.rs b/util/journaldb/src/refcounteddb.rs index a37b011d555..54a54566f24 100644 --- a/util/journaldb/src/refcounteddb.rs +++ b/util/journaldb/src/refcounteddb.rs @@ -78,8 +78,7 @@ impl RefCountedDB { } } -impl HashDB for RefCountedDB { - type H = KeccakHasher; +impl HashDB for RefCountedDB { fn keys(&self) -> HashMap { self.forward.keys() } fn get(&self, key: &H256) -> Option { self.forward.get(key) } fn contains(&self, key: &H256) -> bool { self.forward.contains(key) } @@ -89,7 +88,7 @@ impl HashDB for RefCountedDB { } impl JournalDB for RefCountedDB { - fn boxed_clone(&self) -> Box> { + fn boxed_clone(&self) -> Box { Box::new(RefCountedDB { forward: self.forward.clone(), backing: self.backing.clone(), diff --git a/util/journaldb/src/traits.rs b/util/journaldb/src/traits.rs index be97f92b370..c7ae7eae0de 100644 --- a/util/journaldb/src/traits.rs +++ b/util/journaldb/src/traits.rs @@ -25,9 +25,9 @@ use bytes::Bytes; /// A `HashDB` which can manage a short-term journal potentially containing many forks of mutually /// exclusive actions. -pub trait JournalDB: HashDB + AsHashDB { +pub trait JournalDB: HashDB + AsHashDB { /// Return a copy of ourself, in a box. - fn boxed_clone(&self) -> Box>; + fn boxed_clone(&self) -> Box; /// Returns heap memory size used fn mem_used(&self) -> usize; From 80f5545309188f80e9a088c523c810cfd305561d Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 07:20:31 +0200 Subject: [PATCH 048/160] Switch to use type param instead of assoc type --- util/hashdb/src/lib.rs | 43 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 59b40cd3243..e80ad3e6458 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -47,57 +47,54 @@ impl Hasher for KeccakHasher { } } /// Convenience type for crates that need a `HashDB` with Keccak hashes -pub type KeccakHashDB = HashDB; +pub type KeccakHashDB = HashDB; /// `HashDB` value type. pub type DBValue = ElasticArray128; /// Trait modelling datastore keyed by a 32-byte Keccak hash. // TODO: move `H: Hasher` to be a generic param instead of an associated type -pub trait HashDB: Send + Sync { - type H: Hasher; +pub trait HashDB: Send + Sync + AsHashDB { /// Get the keys in the database together with number of underlying references. - fn keys(&self) -> HashMap<::Out, i32>; + fn keys(&self) -> HashMap; /// Look up a given hash into the bytes that hash to it, returning None if the /// hash is not known. - fn get(&self, key: &::Out) -> Option; + fn get(&self, key: &H::Out) -> Option; /// Check for the existance of a hash-key. - fn contains(&self, key: &::Out) -> bool; + fn contains(&self, key: &H::Out) -> bool; /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `remove()`s must be performed before the data /// is considered dead. - fn insert(&mut self, value: &[u8]) -> ::Out; + fn insert(&mut self, value: &[u8]) -> H::Out; /// Like `insert()`, except you provide the key and the data is all moved. - fn emplace(&mut self, key: ::Out, value: DBValue); + fn emplace(&mut self, key: H::Out, value: DBValue); /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may /// happen without the data being eventually being inserted into the DB. It can be "owed" more than once. - fn remove(&mut self, key: &::Out); + fn remove(&mut self, key: &H::Out); } /// Upcast trait. pub trait AsHashDB { /// Perform upcast to HashDB for anything that derives from HashDB. - fn as_hashdb(&self) -> &HashDB; + fn as_hashdb(&self) -> &HashDB; /// Perform mutable upcast to HashDB for anything that derives from HashDB. - fn as_hashdb_mut(&mut self) -> &mut HashDB; + fn as_hashdb_mut(&mut self) -> &mut HashDB; } -impl> AsHashDB for T { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +impl> AsHashDB for T { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } } -impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { - fn as_hashdb(&self) -> &HashDB { - &**self - } - fn as_hashdb_mut(&mut self) -> &mut HashDB { - &mut **self - } -} -} \ No newline at end of file +// TODO: This conflicts with the impl `for T`, see https://stackoverflow.com/questions/48432842/implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im +// I wonder why it didn't conflict before, when `H` was an associated type +//impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { +// fn as_hashdb(&self) -> &HashDB { &**self } +// fn as_hashdb_mut(&mut self) -> &mut HashDB { &mut **self } +//} + From 89e22d1588d01dee1f4e91ac5e05c728f7211b0c Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 07:37:16 +0200 Subject: [PATCH 049/160] Fix memorydb --- util/memorydb/src/lib.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 17ce7370a38..f69e0997e06 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -179,14 +179,13 @@ impl MemoryDB { } } -impl HashDB for MemoryDB { - type H = H; +impl HashDB for MemoryDB { // REVIEW: this method is what made it necessary to add a type param to H256FastMap, which I'd rather have avoided. // The problem is that the keys returned are `H256` and type inference fails on the `collect()` call. // I could not make it work without parameterizing H256FastMap too. It all sort of adds up as I could // avoid adding PhantomData to MemoryDB, but still quite annoying. What's a better way? - fn keys(&self) -> HashMap<::Out, i32> { + fn keys(&self) -> HashMap { self.data.iter() .filter_map(|(k, v)| if v.1 != 0 { Some((*k, v.1)) @@ -196,8 +195,8 @@ impl HashDB for MemoryDB { .collect() } - fn get(&self, key: &::Out) -> Option { - if key == &Self::H::HASHED_NULL_RLP { + fn get(&self, key: &H::Out) -> Option { + if key == &H::HASHED_NULL_RLP { return Some(DBValue::from_slice(&NULL_RLP)); } @@ -207,8 +206,8 @@ impl HashDB for MemoryDB { } } - fn contains(&self, key: &::Out) -> bool { - if key == &Self::H::HASHED_NULL_RLP { + fn contains(&self, key: &H::Out) -> bool { + if key == &H::HASHED_NULL_RLP { return true; } @@ -218,11 +217,11 @@ impl HashDB for MemoryDB { } } - fn insert(&mut self, value: &[u8]) -> ::Out { + fn insert(&mut self, value: &[u8]) -> H::Out { if value == &NULL_RLP { - return Self::H::HASHED_NULL_RLP.clone(); + return H::HASHED_NULL_RLP.clone(); } - let key = Self::H::hash(value); + let key = H::hash(value); match self.data.entry(key) { Entry::Occupied(mut entry) => { let &mut (ref mut old_value, ref mut rc) = entry.get_mut(); @@ -238,7 +237,7 @@ impl HashDB for MemoryDB { key } - fn emplace(&mut self, key:::Out, value: DBValue) { + fn emplace(&mut self, key:H::Out, value: DBValue) { if &*value == &NULL_RLP { return; } @@ -257,8 +256,8 @@ impl HashDB for MemoryDB { } } - fn remove(&mut self, key: &::Out) { - if key == &Self::H::HASHED_NULL_RLP { + fn remove(&mut self, key: &H::Out) { + if key == &H::HASHED_NULL_RLP { return; } From f374cc54e535b73dcfd9c5e3dac293bc35df40c4 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 07:37:50 +0200 Subject: [PATCH 050/160] Fix patricia_trie --- util/patricia_trie/src/fatdb.rs | 4 ++-- util/patricia_trie/src/fatdbmut.rs | 8 ++++---- util/patricia_trie/src/lib.rs | 6 +++--- util/patricia_trie/src/lookup.rs | 2 +- util/patricia_trie/src/sectriedb.rs | 2 +- util/patricia_trie/src/sectriedbmut.rs | 8 ++++---- util/patricia_trie/src/triedb.rs | 8 ++++---- util/patricia_trie/src/triedbmut.rs | 16 ++++++++-------- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index 849ca704343..d9515f7db63 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -35,12 +35,12 @@ impl<'db, H, C> FatDB<'db, H, C> /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { Ok(FatDB { raw: TrieDB::new(db, root)? }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { self.raw.db() } + pub fn db(&self) -> &HashDB { self.raw.db() } } impl<'db, H, C> Trie for FatDB<'db, H, C> diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 4b72e8fcc66..71151ba2eff 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -37,24 +37,24 @@ impl<'db, H, C> FatDBMut<'db, H, C> /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut HashDB, root: &'db mut H::Out) -> Self { + pub fn new(db: &'db mut HashDB, root: &'db mut H::Out) -> Self { FatDBMut { raw: TrieDBMut::new(db, root) } } /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { Ok(FatDBMut { raw: TrieDBMut::from_existing(db, root)? }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { + pub fn db(&self) -> &HashDB { self.raw.db() } /// Get the backing database. - pub fn db_mut(&mut self) -> &mut HashDB { + pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 658b754f7ff..591c4e79df9 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -285,7 +285,7 @@ impl<'db, H, C> TrieFactory } /// Create new immutable instance of Trie. - pub fn readonly(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { + pub fn readonly(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), @@ -294,7 +294,7 @@ impl<'db, H, C> TrieFactory } /// Create new mutable instance of Trie. - pub fn create(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { + pub fn create(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { match self.spec { TrieSpec::Generic => Box::new(TrieDBMut::<_, C>::new(db, root)), TrieSpec::Secure => Box::new(SecTrieDBMut::<_, C>::new(db, root)), @@ -303,7 +303,7 @@ impl<'db, H, C> TrieFactory } /// Create new mutable instance of trie and check for errors. - pub fn from_existing(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { + pub fn from_existing(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { match self.spec { TrieSpec::Generic => Ok(Box::new(TrieDBMut::<_, C>::from_existing(db, root)?)), TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::<_, C>::from_existing(db, root)?)), diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 39805058b73..fa050c8d28f 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -27,7 +27,7 @@ use std::marker::PhantomData; /// Trie lookup helper object. pub struct Lookup<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> { /// database to query from. - pub db: &'a HashDB, + pub db: &'a HashDB, /// Query object to record nodes and transform data. pub query: Q, /// Hash to start at diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index 0d672cdce25..ca0d9dd48c2 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -39,7 +39,7 @@ impl<'db, H, C> SecTrieDB<'db, H, C> /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. /// Returns an error if root does not exist. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { Ok(SecTrieDB { raw: TrieDB::new(db, root)? }) } diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 21d298c1711..43b6956b17a 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -35,22 +35,22 @@ impl<'db, H, C> SecTrieDBMut<'db, H, C> /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut HashDB, root: &'db mut H::Out) -> Self { + pub fn new(db: &'db mut HashDB, root: &'db mut H::Out) -> Self { SecTrieDBMut { raw: TrieDBMut::new(db, root) } } /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { Ok(SecTrieDBMut { raw: TrieDBMut::from_existing(db, root)? }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { self.raw.db() } + pub fn db(&self) -> &HashDB { self.raw.db() } /// Get the backing database. - pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } + pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } impl<'db, H, C> TrieMut for SecTrieDBMut<'db, H, C> diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index aafaed6dc98..d7eba098d72 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -59,7 +59,7 @@ use std::marker::PhantomData; pub struct TrieDB<'db, H, C> where H: Hasher + 'db, C: NodeCodec { - db: &'db HashDB, + db: &'db HashDB, root: &'db H::Out, /// The number of hashes performed so far in operations on this trie. hash_count: usize, @@ -71,7 +71,7 @@ impl<'db, H, C> TrieDB<'db, H, C> { /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist - pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { if !db.contains(root) { Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { @@ -80,7 +80,7 @@ impl<'db, H, C> TrieDB<'db, H, C> } /// Get the backing database. - pub fn db(&'db self) -> &'db HashDB { self.db } + pub fn db(&'db self) -> &'db HashDB { self.db } /// Get the data of the root node. fn root_data(&self) -> super::Result { @@ -396,7 +396,7 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> where let node = C::decode(&d).expect("rlp read from db; qed"); self.descend_into_node(node.into()) }, - IterStep::Descend::(Err(e)) => { + IterStep::Descend::(Err(_e)) => { // return Some(Err(e)) // <–– REVIEW: This causes a compiler error I can't figure out: /* error[E0308]: mismatched types diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 76726d3e11a..3ff1bd45ee7 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -87,7 +87,7 @@ enum Node { impl Node where H::Out: Decodable { // load an inline node into memory or get the hash to do the lookup later. - fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle + fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle where C: NodeCodec { C::try_decode_hash(&node) @@ -99,7 +99,7 @@ impl Node where H::Out: Decodable { } // decode a node from rlp without getting its children. - fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self + fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self where C: NodeCodec { match C::decode(data).expect("encoded bytes read from db; qed") { @@ -310,7 +310,7 @@ pub struct TrieDBMut<'a, H, C> C: NodeCodec { storage: NodeStorage, - db: &'a mut HashDB, + db: &'a mut HashDB, root: &'a mut H::Out, root_handle: NodeHandle, death_row: HashSet, @@ -326,7 +326,7 @@ impl<'a, H, C> TrieDBMut<'a, H, C> C: NodeCodec { /// Create a new trie with backing database `db` and empty `root`. - pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { + pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { *root = H::HASHED_NULL_RLP; let root_handle = NodeHandle::Hash(H::HASHED_NULL_RLP); @@ -343,7 +343,7 @@ impl<'a, H, C> TrieDBMut<'a, H, C> /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. - pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> super::Result { + pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> super::Result { if !db.contains(root) { return Err(Box::new(TrieError::InvalidStateRoot(*root))); } @@ -360,12 +360,12 @@ impl<'a, H, C> TrieDBMut<'a, H, C> }) } /// Get the backing database. - pub fn db(&self) -> &HashDB { + pub fn db(&self) -> &HashDB { self.db } /// Get the backing database mutably. - pub fn db_mut(&mut self) -> &mut HashDB { + pub fn db_mut(&mut self) -> &mut HashDB { self.db } @@ -998,7 +998,7 @@ mod tests { type RlpCodec = RlpNodeCodec; - fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> + fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec { let mut t = TrieDBMut::new(db, root); From e6e35b7913aa05cc233168ae052678895e88435f Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 10:21:33 +0200 Subject: [PATCH 051/160] Fix ethcore/src with generic HashDB Add concrete impls for AsHashDB --- ethcore/src/account_db.rs | 57 ++++++++++++++++----------- ethcore/src/client/client.rs | 4 +- ethcore/src/client/error.rs | 6 ++- ethcore/src/client/evm_test_client.rs | 4 +- ethcore/src/client/test_client.rs | 8 ++-- ethcore/src/executed.rs | 6 ++- ethcore/src/pod_account.rs | 2 +- ethcore/src/snapshot/account.rs | 6 +-- ethcore/src/snapshot/error.rs | 6 ++- ethcore/src/snapshot/mod.rs | 16 ++++---- ethcore/src/snapshot/tests/helpers.rs | 16 ++++---- ethcore/src/state/account.rs | 16 ++++---- ethcore/src/state/backend.rs | 32 +++++++++------ ethcore/src/state/mod.rs | 9 ++--- ethcore/src/state_db.rs | 14 +++---- util/hashdb/src/lib.rs | 16 ++++---- util/journaldb/src/lib.rs | 28 +++++++++++-- util/journaldb/src/traits.rs | 2 +- util/memorydb/src/lib.rs | 7 +++- 19 files changed, 152 insertions(+), 103 deletions(-) diff --git a/ethcore/src/account_db.rs b/ethcore/src/account_db.rs index a6759836f6e..2713559f5dc 100644 --- a/ethcore/src/account_db.rs +++ b/ethcore/src/account_db.rs @@ -57,7 +57,7 @@ impl Default for Factory { impl Factory { /// Create a read-only accountdb. /// This will panic when write operations are called. - pub fn readonly<'db>(&self, db: &'db HashDB, address_hash: H256) -> Box + 'db> { + pub fn readonly<'db>(&self, db: &'db HashDB, address_hash: H256) -> Box + 'db> { match *self { Factory::Mangled => Box::new(AccountDB::from_hash(db, address_hash)), Factory::Plain => Box::new(Wrapping(db)), @@ -65,7 +65,7 @@ impl Factory { } /// Create a new mutable hashdb. - pub fn create<'db>(&self, db: &'db mut HashDB, address_hash: H256) -> Box + 'db> { + pub fn create<'db>(&self, db: &'db mut HashDB, address_hash: H256) -> Box + 'db> { match *self { Factory::Mangled => Box::new(AccountDBMut::from_hash(db, address_hash)), Factory::Plain => Box::new(WrappingMut(db)), @@ -77,25 +77,19 @@ impl Factory { /// DB backend wrapper for Account trie /// Transforms trie node keys for the database pub struct AccountDB<'db> { - db: &'db HashDB, + db: &'db HashDB, address_hash: H256, } -// No: orphan rules -//impl<'db> AsHashDB for AccountDB<'db> { -// fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } -// fn as_hashdb_mut(&mut self) -> &mut HashDB { self.db.as_hashdb_mut() } -//} - impl<'db> AccountDB<'db> { /// Create a new AccountDB from an address. #[cfg(test)] - pub fn new(db: &'db HashDB, address: &Address) -> Self { + pub fn new(db: &'db HashDB, address: &Address) -> Self { Self::from_hash(db, keccak(address)) } /// Create a new AcountDB from an address' hash. - pub fn from_hash(db: &'db HashDB, address_hash: H256) -> Self { + pub fn from_hash(db: &'db HashDB, address_hash: H256) -> Self { AccountDB { db: db, address_hash: address_hash, @@ -103,8 +97,12 @@ impl<'db> AccountDB<'db> { } } -impl<'db> HashDB for AccountDB<'db>{ - type H = KeccakHasher; +impl<'db> AsHashDB for AccountDB<'db> { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +impl<'db> HashDB for AccountDB<'db> { fn keys(&self) -> HashMap { unimplemented!() } @@ -138,19 +136,19 @@ impl<'db> HashDB for AccountDB<'db>{ /// DB backend wrapper for Account trie pub struct AccountDBMut<'db> { - db: &'db mut HashDB, + db: &'db mut HashDB, address_hash: H256, } impl<'db> AccountDBMut<'db> { /// Create a new AccountDB from an address. #[cfg(test)] - pub fn new(db: &'db mut HashDB, address: &Address) -> Self { + pub fn new(db: &'db mut HashDB, address: &Address) -> Self { Self::from_hash(db, keccak(address)) } /// Create a new AcountDB from an address' hash. - pub fn from_hash(db: &'db mut HashDB, address_hash: H256) -> Self { + pub fn from_hash(db: &'db mut HashDB, address_hash: H256) -> Self { AccountDBMut { db: db, address_hash: address_hash, @@ -163,8 +161,7 @@ impl<'db> AccountDBMut<'db> { } } -impl<'db> HashDB for AccountDBMut<'db>{ - type H = KeccakHasher; +impl<'db> HashDB for AccountDBMut<'db>{ fn keys(&self) -> HashMap { unimplemented!() } @@ -210,10 +207,19 @@ impl<'db> HashDB for AccountDBMut<'db>{ } } -struct Wrapping<'db>(&'db HashDB); +impl<'db> AsHashDB for AccountDBMut<'db> { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +struct Wrapping<'db>(&'db HashDB); + +impl<'db> AsHashDB for Wrapping<'db> { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} -impl<'db> HashDB for Wrapping<'db> { - type H = KeccakHasher; +impl<'db> HashDB for Wrapping<'db> { fn keys(&self) -> HashMap { unimplemented!() } @@ -245,10 +251,13 @@ impl<'db> HashDB for Wrapping<'db> { } } -struct WrappingMut<'db>(&'db mut HashDB); +struct WrappingMut<'db>(&'db mut HashDB); +impl<'db> AsHashDB for WrappingMut<'db> { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} -impl<'db> HashDB for WrappingMut<'db>{ - type H = KeccakHasher; +impl<'db> HashDB for WrappingMut<'db>{ fn keys(&self) -> HashMap { unimplemented!() } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index c270eed035f..b21e1aec969 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -609,9 +609,7 @@ impl Importer { let call = move |addr, data| { let mut state_db = state_db.boxed_clone(); - use ::state::backend::{Proving, Backend}; // TODO: really needed? Maybe, for inference to work? - let backend = Proving::new(state_db.as_hashdb_mut()); -// let backend = ::state::backend::Proving::new(state_db.as_hashdb_mut()); + let backend = ::state::backend::Proving::new(state_db.as_hashdb_mut()); let transaction = client.contract_call_tx(BlockId::Hash(*header.parent_hash()), addr, data); diff --git a/ethcore/src/client/error.rs b/ethcore/src/client/error.rs index 67e74eddc7d..3c3d866c80b 100644 --- a/ethcore/src/client/error.rs +++ b/ethcore/src/client/error.rs @@ -24,7 +24,7 @@ use hashdb::{Hasher, KeccakHasher}; #[derive(Debug)] pub enum Error { /// TrieDB-related error. - Trie(TrieError<::Out>), + Trie(TrieError<::Out>), // TODO: this doesn't look right – need to make `Error` generic too? :/ /// Database error Database(kvdb::Error), /// Util error @@ -33,7 +33,9 @@ pub enum Error { impl From> for Error { fn from(err: TrieError) -> Self { - Error::Trie(err) + // Error::Trie(err) + // Error::Trie(::Out::new()) // TODO: fix this + Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // TODO: fix this } } diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index fbf57cbdc72..36c9945df1e 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -25,12 +25,14 @@ use {state, state_db, client, executive, trace, transaction, db, spec, pod_state use factory::Factories; use evm::{VMType, FinalizationResult}; use vm::{self, ActionParams}; +use hashdb::{Hasher, KeccakHasher}; /// EVM test Error. #[derive(Debug)] pub enum EvmTestError { /// Trie integrity error. - Trie(trie::TrieError), + // Trie(trie::TrieError), + Trie(trie::TrieError<::Out>), // TODO: this doesn't look right. /// EVM error. Evm(vm::Error), /// Initialization error. diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 4bd76b4495c..ae372054a04 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -581,10 +581,10 @@ impl Call for TestBlockChainClient { } impl StateInfo for () { - fn nonce(&self, _address: &Address) -> trie::Result { unimplemented!() } - fn balance(&self, _address: &Address) -> trie::Result { unimplemented!() } - fn storage_at(&self, _address: &Address, _key: &H256) -> trie::Result { unimplemented!() } - fn code(&self, _address: &Address) -> trie::Result>> { unimplemented!() } + fn nonce(&self, _address: &Address) -> trie::KeccakTrieResult { unimplemented!() } + fn balance(&self, _address: &Address) -> trie::KeccakTrieResult { unimplemented!() } + fn storage_at(&self, _address: &Address, _key: &H256) -> trie::KeccakTrieResult { unimplemented!() } + fn code(&self, _address: &Address) -> trie::KeccakTrieResult>> { unimplemented!() } } impl StateClient for TestBlockChainClient { diff --git a/ethcore/src/executed.rs b/ethcore/src/executed.rs index d89b26c3534..33a024c66d5 100644 --- a/ethcore/src/executed.rs +++ b/ethcore/src/executed.rs @@ -118,8 +118,10 @@ pub enum ExecutionError { } impl From>> for ExecutionError { - fn from(err: Box>) -> Self { - ExecutionError::Internal(format!("{:?}", err)) + // TODO: sort out the error types + fn from(_err: Box>) -> Self { + // ExecutionError::Internal(format!("{:?}", err)) + ExecutionError::Internal(format!("TODO: sort out the trie error types")) } } diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index dacb2a56a2a..08f22bd65d2 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -65,7 +65,7 @@ impl PodAccount { } /// Place additional data into given hash DB. - pub fn insert_additional(&self, db: &mut HashDB, factory: &TrieFactory) { + pub fn insert_additional(&self, db: &mut HashDB, factory: &TrieFactory) { match self.code { Some(ref c) if !c.is_empty() => { db.insert(c); } _ => {} diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 71c10068fe7..47469063b86 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -24,7 +24,7 @@ use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; use ethereum_types::{H256, U256}; use hashdb::HashDB; use bytes::Bytes; -use trie::{TrieDB, Trie}; +use trie::{TrieDB, Trie, KeccakRlpNodeCodec}; use rlp::{RlpStream, Rlp}; use std::collections::HashSet; @@ -67,7 +67,7 @@ impl CodeState { // account address hash, account properties and the storage. Each item contains at most `max_storage_items` // storage records split according to snapshot format definition. pub fn to_fat_rlps(account_hash: &H256, acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet, first_chunk_size: usize, max_chunk_size: usize) -> Result, Error> { - let db = TrieDB::new(acct_db, &acc.storage_root)?; + let db = TrieDB::<_, KeccakRlpNodeCodec>::new(acct_db, &acc.storage_root)?; let mut chunks = Vec::new(); let mut db_iter = db.iter()?; let mut target_chunk_size = first_chunk_size; @@ -183,7 +183,7 @@ pub fn from_fat_rlp( { let mut storage_trie = if storage_root.is_zero() { - TrieDBMut::new(acct_db, &mut storage_root) + TrieDBMut::<_, KeccakRlpNodeCodec>::new(acct_db, &mut storage_root) } else { TrieDBMut::from_existing(acct_db, &mut storage_root)? }; diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index 593659e5fd8..d21f276cc56 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -49,7 +49,7 @@ pub enum Error { /// Restoration aborted. RestorationAborted, /// Trie error. - Trie(TrieError<::Out>), + Trie(TrieError<::Out>), // TODO: this doesn't look right. /// Decoder error. Decoder(DecoderError), /// Io error. @@ -104,7 +104,9 @@ impl From<::std::io::Error> for Error { impl From> for Error { fn from(err: TrieError) -> Self { - Error::Trie(err) + // Error::Trie(err) + // Error::Trie(::Out::new()) // TODO: fix this + Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // TODO: fix this } } diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 17ccc260e93..03953dc0854 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -39,7 +39,7 @@ use bytes::Bytes; use parking_lot::Mutex; use journaldb::{self, Algorithm, JournalDB}; use kvdb::KeyValueDB; -use trie::{TrieDB, TrieDBMut, Trie, TrieMut}; +use trie::{TrieDB, TrieDBMut, Trie, TrieMut, KeccakRlpNodeCodec}; use rlp::{RlpStream, Rlp}; use bloom_journal::Bloom; @@ -127,7 +127,7 @@ pub fn take_snapshot( engine: &EthEngine, chain: &BlockChain, block_at: H256, - state_db: &HashDB, + state_db: &HashDB, writer: W, p: &Progress ) -> Result<(), Error> { @@ -265,8 +265,8 @@ impl<'a> StateChunker<'a> { /// /// Returns a list of hashes of chunks created, or any error it may /// have encountered. -pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { - let account_trie = TrieDB::new(db, &root)?; +pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { + let account_trie = TrieDB::::new(db, &root)?; let mut chunker = StateChunker { hashes: Vec::new(), @@ -305,7 +305,7 @@ pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex< /// Used to rebuild the state trie piece by piece. pub struct StateRebuilder { - db: Box>, + db: Box, state_root: H256, known_code: HashMap, // code hashes mapped to first account with this code. missing_code: HashMap>, // maps code hashes to lists of accounts missing that code. @@ -363,7 +363,7 @@ impl StateRebuilder { // batch trie writes { let mut account_trie = if self.state_root != KECCAK_NULL_RLP { - TrieDBMut::from_existing(self.db.as_hashdb_mut(), &mut self.state_root)? + TrieDBMut::<_, KeccakRlpNodeCodec>::from_existing(self.db.as_hashdb_mut(), &mut self.state_root)? } else { TrieDBMut::new(self.db.as_hashdb_mut(), &mut self.state_root) }; @@ -390,7 +390,7 @@ impl StateRebuilder { /// Finalize the restoration. Check for accounts missing code and make a dummy /// journal entry. /// Once all chunks have been fed, there should be nothing missing. - pub fn finalize(mut self, era: u64, id: H256) -> Result>, ::error::Error> { + pub fn finalize(mut self, era: u64, id: H256) -> Result, ::error::Error> { let missing = self.missing_code.keys().cloned().collect::>(); if !missing.is_empty() { return Err(Error::MissingCode(missing).into()) } @@ -415,7 +415,7 @@ struct RebuiltStatus { // rebuild a set of accounts and their storage. // returns a status detailing newly-loaded code and accounts missing code. fn rebuild_accounts( - db: &mut HashDB, + db: &mut HashDB, account_fat_rlps: Rlp, out_chunk: &mut [(H256, Bytes)], known_code: &HashMap, diff --git a/ethcore/src/snapshot/tests/helpers.rs b/ethcore/src/snapshot/tests/helpers.rs index 516e438abdb..eccdc4ecc85 100644 --- a/ethcore/src/snapshot/tests/helpers.rs +++ b/ethcore/src/snapshot/tests/helpers.rs @@ -35,9 +35,9 @@ use rand::Rng; use kvdb::{KeyValueDB, DBValue}; use ethereum_types::H256; -use hashdb::HashDB; +use hashdb::{HashDB, KeccakHasher}; use journaldb; -use trie::{SecTrieDBMut, TrieMut, TrieDB, TrieDBMut, Trie}; +use trie::{SecTrieDBMut, TrieMut, TrieDB, TrieDBMut, Trie, KeccakRlpNodeCodec}; use self::trie_standardmap::{Alphabet, StandardMap, ValueMode}; // the proportion of accounts we will alter each tick. @@ -60,10 +60,10 @@ impl StateProducer { /// Tick the state producer. This alters the state, writing new data into /// the database. - pub fn tick(&mut self, rng: &mut R, db: &mut HashDB) { + pub fn tick(&mut self, rng: &mut R, db: &mut HashDB) { // modify existing accounts. let mut accounts_to_modify: Vec<_> = { - let trie = TrieDB::new(&*db, &self.state_root).unwrap(); + let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(&*db, &self.state_root).unwrap(); let temp = trie.iter().unwrap() // binding required due to complicated lifetime stuff .filter(|_| rng.gen::() < ACCOUNT_CHURN) .map(Result::unwrap) @@ -82,7 +82,7 @@ impl StateProducer { } // sweep again to alter account trie. - let mut trie = TrieDBMut::from_existing(db, &mut self.state_root).unwrap(); + let mut trie = TrieDBMut::<_, KeccakRlpNodeCodec>::from_existing(db, &mut self.state_root).unwrap(); for (address_hash, account_data) in accounts_to_modify { trie.insert(&address_hash[..], &account_data).unwrap(); @@ -117,9 +117,9 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { }; { let mut trie = if *root == KECCAK_NULL_RLP { - SecTrieDBMut::new(&mut db, root) + SecTrieDBMut::<_, KeccakRlpNodeCodec>::new(&mut db, root) } else { - SecTrieDBMut::from_existing(&mut db, root).unwrap() + SecTrieDBMut::<_, KeccakRlpNodeCodec>::from_existing(&mut db, root).unwrap() }; for (k, v) in map.make_with(seed) { @@ -129,7 +129,7 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { } /// Compare two state dbs. -pub fn compare_dbs(one: &HashDB, two: &HashDB) { +pub fn compare_dbs(one: &HashDB, two: &HashDB) { let keys = one.keys(); for key in keys.keys() { diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index ba019372a70..e99aa9dd14d 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -199,11 +199,11 @@ impl Account { /// Get (and cache) the contents of the trie's storage at `key`. /// Takes modified storage into account. - pub fn storage_at(&self, db: &HashDB, key: &H256) -> trie::Result { + pub fn storage_at(&self, db: &HashDB, key: &H256) -> trie::Result { if let Some(value) = self.cached_storage_at(key) { return Ok(value); } - let db = SecTrieDB::new(db, &self.storage_root)?; + let db = SecTrieDB::<_, KeccakRlpNodeCodec>::new(db, &self.storage_root)?; let panicky_decoder = |bytes:&[u8]| ::rlp::decode(&bytes).expect("decoding db value failed"); let item: U256 = db.get_with(key, panicky_decoder)?.unwrap_or_else(U256::zero); let value: H256 = item.into(); @@ -278,7 +278,7 @@ impl Account { } /// Provide a database to get `code_hash`. Should not be called if it is a contract without code. - pub fn cache_code(&mut self, db: &HashDB) -> Option> { + pub fn cache_code(&mut self, db: &HashDB) -> Option> { // TODO: fill out self.code_cache; trace!("Account::cache_code: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty()); @@ -307,7 +307,7 @@ impl Account { } /// Provide a database to get `code_size`. Should not be called if it is a contract without code. - pub fn cache_code_size(&mut self, db: &HashDB) -> bool { + pub fn cache_code_size(&mut self, db: &HashDB) -> bool { // TODO: fill out self.code_cache; trace!("Account::cache_code_size: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty()); self.code_size.is_some() || @@ -374,7 +374,7 @@ impl Account { } /// Commit the `storage_changes` to the backing DB and update `storage_root`. - pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> trie::Result<(), ::Out> { + pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> trie::Result<(), ::Out> { let mut t = trie_factory.from_existing(db, &mut self.storage_root)?; for (k, v) in self.storage_changes.drain() { // cast key and value to trait type, @@ -390,7 +390,7 @@ impl Account { } /// Commit any unsaved code. `code_hash` will always return the hash of the `code_cache` after this. - pub fn commit_code(&mut self, db: &mut HashDB) { + pub fn commit_code(&mut self, db: &mut HashDB) { trace!("Commiting code of {:?} - {:?}, {:?}", self, self.code_filth == Filth::Dirty, self.code_cache.is_empty()); match (self.code_filth == Filth::Dirty, self.code_cache.is_empty()) { (true, true) => { @@ -472,13 +472,13 @@ impl Account { /// trie. /// `storage_key` is the hash of the desired storage key, meaning /// this will only work correctly under a secure trie. - pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box::Out>>> { + pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box::Out>>> { use trie::{Trie, TrieDB}; use trie::recorder::Recorder; let mut recorder = Recorder::new(); - let trie = TrieDB::new(db, &self.storage_root)?; + let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(db, &self.storage_root)?; let item: U256 = { let panicky_decoder = |bytes:&[u8]| ::rlp::decode(bytes).expect("decoding db value failed"); let query = (&mut recorder, panicky_decoder); diff --git a/ethcore/src/state/backend.rs b/ethcore/src/state/backend.rs index 5d0a018c584..011546cd8b9 100644 --- a/ethcore/src/state/backend.rs +++ b/ethcore/src/state/backend.rs @@ -34,10 +34,10 @@ use hashdb::{AsHashDB, HashDB, DBValue, Hasher, KeccakHasher}; /// State backend. See module docs for more details. pub trait Backend: Send { /// Treat the backend as a read-only hashdb. - fn as_hashdb(&self) -> &HashDB; + fn as_hashdb(&self) -> &HashDB; /// Treat the backend as a writeable hashdb. - fn as_hashdb_mut(&mut self) -> &mut HashDB; + fn as_hashdb_mut(&mut self) -> &mut HashDB; /// Add an account entry to the cache. fn add_to_account_cache(&mut self, addr: Address, data: Option, modified: bool); @@ -87,8 +87,7 @@ impl ProofCheck { } } -impl HashDB for ProofCheck { - type H = KeccakHasher; +impl HashDB for ProofCheck { fn keys(&self) -> HashMap { self.0.keys() } fn get(&self, key: &H256) -> Option { self.0.get(key) @@ -109,9 +108,14 @@ impl HashDB for ProofCheck { fn remove(&mut self, _key: &H256) { } } +impl AsHashDB for ProofCheck { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + impl Backend for ProofCheck { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } fn add_to_account_cache(&mut self, _addr: Address, _data: Option, _modified: bool) {} fn cache_code(&self, _hash: H256, _code: Arc>) {} fn get_cached_account(&self, _addr: &Address) -> Option> { None } @@ -136,8 +140,12 @@ pub struct Proving> { proof: Mutex>, } -impl + Send + Sync> HashDB for Proving { - type H = KeccakHasher; +impl + Send + Sync> AsHashDB for Proving { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +impl + Send + Sync> HashDB for Proving { fn keys(&self) -> HashMap { let mut keys = self.base.as_hashdb().keys(); keys.extend(self.changed.keys()); @@ -175,11 +183,11 @@ impl + Send + Sync> HashDB for Proving { } impl + Send + Sync> Backend for Proving { - fn as_hashdb(&self) -> &HashDB { + fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } @@ -233,11 +241,11 @@ impl + Clone> Clone for Proving { pub struct Basic(pub H); impl + Send + Sync> Backend for Basic { - fn as_hashdb(&self) -> &HashDB { + fn as_hashdb(&self) -> &HashDB { self.0.as_hashdb() } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { self.0.as_hashdb_mut() } diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 31c2d3986aa..9a59fe26b93 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -47,8 +47,7 @@ use hashdb::{HashDB, AsHashDB, KeccakHasher}; use kvdb::DBValue; use bytes::Bytes; -use trie; -use trie::{Trie, TrieError, TrieDB, KeccakTrieResult as TrieResult}; +use trie::{Trie, TrieError, TrieDB, KeccakTrieResult as TrieResult, KeccakRlpNodeCodec}; use trie::recorder::Recorder; mod account; @@ -911,7 +910,7 @@ impl State { } // load required account data from the databases. - fn update_account_cache(require: RequireCache, account: &mut Account, state_db: &B, db: &HashDB) { + fn update_account_cache(require: RequireCache, account: &mut Account, state_db: &B, db: &HashDB) { if let RequireCache::None = require { return; } @@ -1051,7 +1050,7 @@ impl State { /// `account_key` == keccak(address) pub fn prove_account(&self, account_key: H256) -> TrieResult<(Vec, BasicAccount)> { let mut recorder = Recorder::new(); - let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; + let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(self.db.as_hashdb(), &self.root)?; let maybe_account: Option = { let panicky_decoder = |bytes: &[u8]| { ::rlp::decode(bytes).expect(&format!("prove_account, could not query trie for account key={}", &account_key)) @@ -1077,7 +1076,7 @@ impl State { pub fn prove_storage(&self, account_key: H256, storage_key: H256) -> TrieResult<(Vec, H256)> { // TODO: probably could look into cache somehow but it's keyed by // address, not keccak(address). - let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; + let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(self.db.as_hashdb(), &self.root)?; let from_rlp = |b: &[u8]| Account::from_rlp(b).expect("decoding db value failed"); let acc = match trie.get_with(&account_key, from_rlp)? { Some(acc) => acc, diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs index d9832ce9154..072f660ad7c 100644 --- a/ethcore/src/state_db.rs +++ b/ethcore/src/state_db.rs @@ -105,7 +105,7 @@ struct BlockChanges { /// `StateDB` is propagated into the global cache. pub struct StateDB { /// Backing database. - db: Box>, + db: Box, /// Shared canonical state cache. account_cache: Arc>, /// DB Code cache. Maps code hashes to shared bytes. @@ -130,7 +130,7 @@ impl StateDB { /// of the LRU cache in bytes. Actual used memory may (read: will) be higher due to bookkeeping. // TODO: make the cache size actually accurate by moving the account storage cache // into the `AccountCache` structure as its own `LruCache<(Address, H256), H256>`. - pub fn new(db: Box>, cache_size: usize) -> StateDB { + pub fn new(db: Box, cache_size: usize) -> StateDB { let bloom = Self::load_bloom(&**db.backing()); let acc_cache_size = cache_size * ACCOUNT_CACHE_RATIO / 100; let code_cache_size = cache_size - acc_cache_size; @@ -312,13 +312,13 @@ impl StateDB { // TODO: needed? /// Conversion method to interpret self as `HashDB` reference - pub fn as_hashdb(&self) -> &HashDB { + pub fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } // TODO: needed? /// Conversion method to interpret self as mutable `HashDB` reference - pub fn as_hashdb_mut(&mut self) -> &mut HashDB { + pub fn as_hashdb_mut(&mut self) -> &mut HashDB { self.db.as_hashdb_mut() } @@ -368,7 +368,7 @@ impl StateDB { } /// Returns underlying `JournalDB`. - pub fn journal_db(&self) -> &JournalDB { + pub fn journal_db(&self) -> &JournalDB { &*self.db } @@ -413,9 +413,9 @@ impl StateDB { } impl state::Backend for StateDB { - fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } + fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } - fn as_hashdb_mut(&mut self) -> &mut HashDB { + fn as_hashdb_mut(&mut self) -> &mut HashDB { self.db.as_hashdb_mut() } diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index e80ad3e6458..0d92150844b 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -86,15 +86,15 @@ pub trait AsHashDB { fn as_hashdb_mut(&mut self) -> &mut HashDB; } -impl> AsHashDB for T { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } -} +//impl> AsHashDB for T { +// fn as_hashdb(&self) -> &HashDB { self } +// fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +//} // TODO: This conflicts with the impl `for T`, see https://stackoverflow.com/questions/48432842/implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im // I wonder why it didn't conflict before, when `H` was an associated type -//impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { -// fn as_hashdb(&self) -> &HashDB { &**self } -// fn as_hashdb_mut(&mut self) -> &mut HashDB { &mut **self } -//} +impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { + fn as_hashdb(&self) -> &HashDB { &**self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { &mut **self } +} diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index 1bf95e15e4f..b346a8adbe5 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -54,9 +54,31 @@ pub mod overlaydb; /// Export the `JournalDB` trait. pub use self::traits::JournalDB; -/// Convenience type for crates that need a `JournalDB` with Keccak hashes -//pub type KeccakJournalDB = JournalDB; // TODO: use this in the `journaldb` crate -// TODO: do we need/want additional convenience exports like this? `KeccakArchiveDB`, `KeccakEarlymergeDB` etc? +use archivedb::ArchiveDB; +impl AsHashDB for ArchiveDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} +use earlymergedb::EarlyMergeDB; +impl AsHashDB for EarlyMergeDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} +use overlayrecentdb::OverlayRecentDB; +impl AsHashDB for OverlayRecentDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} +use refcounteddb::RefCountedDB; +impl AsHashDB for RefCountedDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} +use overlaydb::OverlayDB; +impl AsHashDB for OverlayDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} /// Journal database operating strategy. #[derive(Debug, PartialEq, Clone, Copy)] diff --git a/util/journaldb/src/traits.rs b/util/journaldb/src/traits.rs index c7ae7eae0de..e5b9d816033 100644 --- a/util/journaldb/src/traits.rs +++ b/util/journaldb/src/traits.rs @@ -25,7 +25,7 @@ use bytes::Bytes; /// A `HashDB` which can manage a short-term journal potentially containing many forks of mutually /// exclusive actions. -pub trait JournalDB: HashDB + AsHashDB { +pub trait JournalDB: HashDB { /// Return a copy of ourself, in a box. fn boxed_clone(&self) -> Box; diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index f69e0997e06..c85ee4866f5 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -23,7 +23,7 @@ extern crate rlp; #[cfg(test)] extern crate tiny_keccak; #[cfg(test)] extern crate ethereum_types; -use hashdb::{HashDB, Hasher, DBValue, KeccakHasher}; +use hashdb::{HashDB, Hasher, DBValue, KeccakHasher, AsHashDB}; use heapsize::HeapSizeOf; use plain_hasher::H256FastMap; use rlp::NULL_RLP; @@ -273,6 +273,11 @@ impl HashDB for MemoryDB { } } +impl AsHashDB for MemoryDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + #[cfg(test)] mod tests { use super::*; From 7d2a414d309a7da44c6e55695cb96d192a618236 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 11:43:11 +0200 Subject: [PATCH 052/160] Fix running tests depending on the parent `test-helpers` feature --- ethcore/light/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index 3853ad910ff..46e7ed800e5 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -39,6 +39,7 @@ error-chain = { version = "0.11", default-features = false } [dev-dependencies] kvdb-memorydb = { path = "../../util/kvdb-memorydb" } +ethcore = { path = "..", features = ["test-helpers"] } tempdir = "0.3" [features] From ae55ca4f169391b869a30c87a635f6011c342885 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 11:43:30 +0200 Subject: [PATCH 053/160] Fix ethcore/light --- ethcore/light/src/cht.rs | 22 +++++++++++----------- ethcore/light/src/client/header_chain.rs | 9 +++++---- ethcore/light/src/on_demand/request.rs | 21 +++++++++++---------- ethcore/light/src/provider.rs | 2 +- ethcore/light/src/transaction_queue.rs | 3 ++- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index 805cca3cbc9..bc2c4f601cb 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -25,10 +25,10 @@ use ethcore::ids::BlockId; use ethereum_types::{H256, U256}; -use hashdb::HashDB; +use hashdb::{HashDB, KeccakHasher}; use memorydb::MemoryDB; use bytes::Bytes; -use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder}; +use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder, KeccakRlpNodeCodec}; use rlp::{RlpStream, Rlp}; // encode a key. @@ -50,13 +50,13 @@ pub const SIZE: u64 = 2048; /// A canonical hash trie. This is generic over any database it can query. /// See module docs for more details. #[derive(Debug, Clone)] -pub struct CHT { +pub struct CHT> { db: DB, root: H256, // the root of this CHT. number: u64, } -impl CHT { +impl> CHT { /// Query the root of the CHT. pub fn root(&self) -> H256 { self.root } @@ -66,11 +66,11 @@ impl CHT { /// Generate an inclusion proof for the entry at a specific block. /// Nodes before level `from_level` will be omitted. /// Returns an error on an incomplete trie, and `Ok(None)` on an unprovable request. - pub fn prove(&self, num: u64, from_level: u32) -> trie::Result>> { + pub fn prove(&self, num: u64, from_level: u32) -> trie::KeccakTrieResult>> { if block_to_cht_number(num) != Some(self.number) { return Ok(None) } let mut recorder = Recorder::with_depth(from_level); - let t = TrieDB::new(&self.db, &self.root)?; + let t = TrieDB::<_, KeccakRlpNodeCodec>::new(&self.db, &self.root)?; t.get_with(&key!(num), &mut recorder)?; Ok(Some(recorder.drain().into_iter().map(|x| x.data).collect())) @@ -90,10 +90,10 @@ pub struct BlockInfo { /// Build an in-memory CHT from a closure which provides necessary information /// about blocks. If the fetcher ever fails to provide the info, the CHT /// will not be generated. -pub fn build(cht_num: u64, mut fetcher: F) -> Option> +pub fn build(cht_num: u64, mut fetcher: F) -> Option>> where F: FnMut(BlockId) -> Option { - let mut db = MemoryDB::new(); + let mut db = MemoryDB::::new(); // start from the last block by number and work backwards. let last_num = start_number(cht_num + 1) - 1; @@ -102,7 +102,7 @@ pub fn build(cht_num: u64, mut fetcher: F) -> Option> let mut root = H256::default(); { - let mut t = TrieDBMut::new(&mut db, &mut root); + let mut t = TrieDBMut::<_, KeccakRlpNodeCodec>::new(&mut db, &mut root); for blk_num in (0..SIZE).map(|n| last_num - n) { let info = match fetcher(id) { Some(info) => info, @@ -147,10 +147,10 @@ pub fn compute_root(cht_num: u64, iterable: I) -> Option /// verify the given trie branch and extract the canonical hash and total difficulty. // TODO: better support for partially-checked queries. pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256)> { - let mut db = MemoryDB::new(); + let mut db = MemoryDB::::new(); for node in proof { db.insert(&node[..]); } - let res = match TrieDB::new(&db, &root) { + let res = match TrieDB::<_, KeccakRlpNodeCodec>::new(&db, &root) { Err(_) => return None, Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| { let rlp = Rlp::new(val); diff --git a/ethcore/light/src/client/header_chain.rs b/ethcore/light/src/client/header_chain.rs index cb370da2a75..37c2812d146 100644 --- a/ethcore/light/src/client/header_chain.rs +++ b/ethcore/light/src/client/header_chain.rs @@ -41,11 +41,12 @@ use ethcore::engines::epoch::{ PendingTransition as PendingEpochTransition }; -use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; -use heapsize::HeapSizeOf; use ethereum_types::{H256, H264, U256}; -use plain_hasher::H256FastMap; +use hashdb::KeccakHasher; +use heapsize::HeapSizeOf; use kvdb::{DBTransaction, KeyValueDB}; +use plain_hasher::H256FastMap; +use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; use cache::Cache; use parking_lot::{Mutex, RwLock}; @@ -207,7 +208,7 @@ pub struct HeaderChain { genesis_header: encoded::Header, // special-case the genesis. candidates: RwLock>, best_block: RwLock, - live_epoch_proofs: RwLock>, + live_epoch_proofs: RwLock>, db: Arc, col: Option, cache: Arc>, diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 4cac6b629db..f0b5b01c8dd 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -33,11 +33,11 @@ use request::{self as net_request, IncompleteRequest, CompleteRequest, Output, O use rlp::{RlpStream, Rlp}; use ethereum_types::{H256, U256, Address}; use parking_lot::Mutex; -use hashdb::HashDB; +use hashdb::{HashDB, Hasher, KeccakHasher}; use kvdb::DBValue; use bytes::Bytes; use memorydb::MemoryDB; -use trie::{Trie, TrieDB, TrieError}; +use trie::{Trie, TrieDB, TrieError, KeccakRlpNodeCodec}; const SUPPLIED_MATCHES: &'static str = "supplied responses always match produced requests; enforced by `check_response`; qed"; @@ -627,7 +627,7 @@ pub enum Error { /// Empty response. Empty, /// Trie lookup error (result of bad proof) - Trie(TrieError), + Trie(TrieError<::Out>), // TODO: fix error handling /// Bad inclusion proof BadProof, /// Header by number instead of hash. @@ -650,9 +650,10 @@ impl From<::rlp::DecoderError> for Error { } } -impl From> for Error { - fn from(err: Box) -> Self { - Error::Trie(*err) +impl From>> for Error { + fn from(err: Box>) -> Self { + // Error::Trie(*err) + Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // TODO: error handling zomg } } @@ -828,7 +829,7 @@ impl Account { let mut db = MemoryDB::new(); for node in proof { db.insert(&node[..]); } - match TrieDB::new(&db, &state_root).and_then(|t| t.get(&keccak(&self.address)))? { + match TrieDB::<_, KeccakRlpNodeCodec>::new(&db, &state_root).and_then(|t| t.get(&keccak(&self.address)))? { Some(val) => { let rlp = Rlp::new(&val); Ok(Some(BasicAccount { @@ -939,7 +940,7 @@ mod tests { use trie::recorder::Recorder; use hash::keccak; - use ethcore::client::{BlockChainClient, BlockInfo, TestBlockChainClient, EachBlockWith}; + use ::ethcore::client::{BlockChainClient, BlockInfo, TestBlockChainClient, EachBlockWith}; use ethcore::header::Header; use ethcore::encoded; use ethcore::receipt::{Receipt, TransactionOutcome}; @@ -1051,7 +1052,7 @@ mod tests { stream.out() }; { - let mut trie = SecTrieDBMut::new(&mut db, &mut root); + let mut trie = SecTrieDBMut::<_, KeccakRlpNodeCodec>::new(&mut db, &mut root); for _ in 0..100 { let address = Address::random(); trie.insert(&*address, &rand_acc()).unwrap(); @@ -1061,7 +1062,7 @@ mod tests { } let proof = { - let trie = SecTrieDB::new(&db, &root).unwrap(); + let trie = SecTrieDB::<_, KeccakRlpNodeCodec>::new(&db, &root).unwrap(); let mut recorder = Recorder::new(); trie.get_with(&*addr, &mut recorder).unwrap().unwrap(); diff --git a/ethcore/light/src/provider.rs b/ethcore/light/src/provider.rs index 0e518ea7723..5d9b0d0b564 100644 --- a/ethcore/light/src/provider.rs +++ b/ethcore/light/src/provider.rs @@ -383,7 +383,7 @@ impl AsLightClient for LightProvider { #[cfg(test)] mod tests { - use ethcore::client::{EachBlockWith, TestBlockChainClient}; + use ::ethcore::client::{EachBlockWith, TestBlockChainClient}; use super::Provider; #[test] diff --git a/ethcore/light/src/transaction_queue.rs b/ethcore/light/src/transaction_queue.rs index e8880037a17..6803cc6af64 100644 --- a/ethcore/light/src/transaction_queue.rs +++ b/ethcore/light/src/transaction_queue.rs @@ -30,6 +30,7 @@ use std::collections::hash_map::Entry; use transaction::{self, Condition, PendingTransaction, SignedTransaction}; use ethereum_types::{H256, U256, Address}; use plain_hasher::H256FastMap; +use hashdb::KeccakHasher; // Knowledge of an account's current nonce. #[derive(Debug, Clone, PartialEq, Eq)] @@ -138,7 +139,7 @@ type Listener = Box; #[derive(Default)] pub struct TransactionQueue { by_account: HashMap, - by_hash: H256FastMap, + by_hash: H256FastMap, listeners: Vec, } From 6b4095e5976fa59ccaaabd7a9d092c762f7403ef Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 11:58:16 +0200 Subject: [PATCH 054/160] "Handle" Trie errors --- ethcore/private-tx/src/error.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethcore/private-tx/src/error.rs b/ethcore/private-tx/src/error.rs index 0456b330530..abcf1142d49 100644 --- a/ethcore/private-tx/src/error.rs +++ b/ethcore/private-tx/src/error.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethereum_types::Address; +use ethereum_types::{Address, H256}; use rlp::DecoderError; use trie::TrieError; use ethcore::account_provider::SignError; @@ -26,7 +26,7 @@ error_chain! { foreign_links { Io(::std::io::Error) #[doc = "Error concerning the Rust standard library's IO subsystem."]; Decoder(DecoderError) #[doc = "RLP decoding error."]; - Trie(TrieError) #[doc = "Error concerning TrieDBs."]; + Trie(TrieError) #[doc = "Error concerning TrieDBs."]; // REVIEW: error_chain requires concrete types, but this probably is not ok. What's a better way? } errors { From 4288d494621f850617038e79e4c64f2f4aaa3b57 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 12:11:33 +0200 Subject: [PATCH 055/160] Add `version` key for new generic crates --- ethcore/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 2b6ab9ebc3a..cedb2b99284 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -17,9 +17,9 @@ ethash = { path = "../ethash" } ethcore-bloom-journal = { path = "../util/bloom" } ethcore-bytes = { path = "../util/bytes" } fetch = { path = "../util/fetch" } -hashdb = { path = "../util/hashdb" } -memorydb = { path = "../util/memorydb" } -patricia-trie = { path = "../util/patricia_trie" } +hashdb = { version = "0.2", path = "../util/hashdb" } +memorydb = { version = "0.2", path = "../util/memorydb" } +patricia-trie = { version = "0.2", path = "../util/patricia_trie" } ethcore-crypto = { path = "crypto" } error-chain = { version = "0.11", default-features = false } ethcore-io = { path = "../util/io" } From 1d5f5e2c0b937a78d14d19b11cc60e21eda203ad Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 12:12:37 +0200 Subject: [PATCH 056/160] New cargo lockfile --- Cargo.lock | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5e11a352ad..d82a24433a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -555,7 +555,7 @@ dependencies = [ "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-machine 0.1.0", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "patricia-trie 0.1.0", + "patricia-trie 0.2.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", @@ -640,7 +640,7 @@ dependencies = [ "memory-cache 0.1.0", "memorydb 0.2.0", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "patricia-trie 0.1.0", + "patricia-trie 0.2.0", "plain_hasher 0.2.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", @@ -774,7 +774,7 @@ dependencies = [ "keccak-hash 0.1.2", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "patricia-trie 0.1.0", + "patricia-trie 0.2.0", "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", "rlp_derive 0.1.0", @@ -875,6 +875,7 @@ dependencies = [ "ethcore-transaction 0.1.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethkey 0.3.0", + "hashdb 0.2.0", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", @@ -2231,7 +2232,7 @@ dependencies = [ "parity-updater 1.12.0", "parity-version 1.12.0", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "patricia-trie 0.1.0", + "patricia-trie 0.2.0", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", @@ -2398,7 +2399,7 @@ version = "0.1.0" [[package]] name = "patricia-trie" -version = "0.1.0" +version = "0.2.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", @@ -3617,7 +3618,7 @@ dependencies = [ "ethjson 0.1.0", "keccak-hash 0.1.2", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "patricia-trie 0.1.0", + "patricia-trie 0.2.0", "rlp 0.2.1", ] From 33f4e796c693f9e49a209e893a6168c54dff47f9 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 12:13:00 +0200 Subject: [PATCH 057/160] Fix tests --- ethcore/service/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ethcore/service/Cargo.toml b/ethcore/service/Cargo.toml index 3a10849b61f..1c0d1af3c6e 100644 --- a/ethcore/service/Cargo.toml +++ b/ethcore/service/Cargo.toml @@ -18,3 +18,4 @@ trace-time = { path = "../../util/trace-time" } [dev-dependencies] tempdir = "0.3" kvdb-rocksdb = { path = "../../util/kvdb-rocksdb" } +ethcore = { path = "..", features = ["test-helpers"] } \ No newline at end of file From a567462daa25058a55926b9a9504cb84a04fad7f Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 12:13:17 +0200 Subject: [PATCH 058/160] Add review comment about error_chain and generic errors --- ethcore/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/error.rs b/ethcore/src/error.rs index 2f2824015d1..c5f293d1475 100644 --- a/ethcore/src/error.rs +++ b/ethcore/src/error.rs @@ -247,7 +247,7 @@ error_chain! { foreign_links { Io(IoError) #[doc = "Io create error"]; StdIo(::std::io::Error) #[doc = "Error concerning the Rust standard library's IO subsystem."]; - Trie(TrieError) #[doc = "Error concerning TrieDBs."]; + Trie(TrieError) #[doc = "Error concerning TrieDBs."]; // REVIEW: error_chain requires concrete types, but this probably is not ok. What's a better way? Execution(ExecutionError) #[doc = "Error concerning EVM code execution."]; Block(BlockError) #[doc = "Error concerning block processing."]; Transaction(TransactionError) #[doc = "Error concerning transaction processing."]; From b146e3c92703743d9b7dce96c2d43e17c9abe53d Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 12:13:50 +0200 Subject: [PATCH 059/160] Bump version --- util/patricia_trie/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index fdb09f90348..6e4b422bbf3 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "patricia-trie" -version = "0.1.0" +version = "0.2.0" authors = ["Parity Technologies "] description = "Merkle-Patricia Trie (Ethereum Style)" license = "GPL-3.0" From f3dd59b57f09035411a5b1a9b732acb94f9bfb7c Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 12:19:37 +0200 Subject: [PATCH 060/160] Generic hasdb --- ethcore/sync/Cargo.toml | 3 ++- ethcore/sync/src/chain/mod.rs | 3 ++- ethcore/sync/src/lib.rs | 1 + ethcore/sync/src/transactions_stats.rs | 5 +++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 66ee5662158..aae6cbf177a 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -16,7 +16,8 @@ ethcore-light = { path = "../light" } ethcore-transaction = { path = "../transaction" } ethcore = { path = ".." } ethereum-types = "0.3" -plain_hasher = { path = "../../util/plain_hasher" } +hashdb = { version = "0.2", path = "../../util/hashdb" } +plain_hasher = { version = "0.2", path = "../../util/plain_hasher" } rlp = { path = "../../util/rlp" } rustc-hex = "1.0" keccak-hash = { path = "../../util/hash" } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 8f0aff7514b..01b312eaf51 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -97,6 +97,7 @@ use std::collections::{HashSet, HashMap}; use std::cmp; use std::time::{Duration, Instant}; use hash::keccak; +use hashdb::KeccakHasher; use heapsize::HeapSizeOf; use ethereum_types::{H256, U256}; use plain_hasher::H256FastMap; @@ -485,7 +486,7 @@ impl ChainSync { } /// Returns transactions propagation statistics - pub fn transactions_stats(&self) -> &H256FastMap { + pub fn transactions_stats(&self) -> &H256FastMap { self.transactions_stats.stats() } diff --git a/ethcore/sync/src/lib.rs b/ethcore/sync/src/lib.rs index 35483f4ec3a..636ad6cce14 100644 --- a/ethcore/sync/src/lib.rs +++ b/ethcore/sync/src/lib.rs @@ -30,6 +30,7 @@ extern crate ethcore_transaction as transaction; extern crate ethcore; extern crate ethereum_types; extern crate env_logger; +extern crate hashdb; extern crate plain_hasher; extern crate rand; extern crate semver; diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index c45b1ad8b3c..546287407f7 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -18,6 +18,7 @@ use api::TransactionStats; use std::collections::{HashSet, HashMap}; use ethereum_types::{H256, H512}; use plain_hasher::H256FastMap; +use hashdb::KeccakHasher; type NodeId = H512; type BlockNumber = u64; @@ -51,7 +52,7 @@ impl<'a> From<&'a Stats> for TransactionStats { #[derive(Debug, Default)] pub struct TransactionsStats { - pending_transactions: H256FastMap, + pending_transactions: H256FastMap, } impl TransactionsStats { @@ -69,7 +70,7 @@ impl TransactionsStats { self.pending_transactions.get(hash) } - pub fn stats(&self) -> &H256FastMap { + pub fn stats(&self) -> &H256FastMap { &self.pending_transactions } From 0374aa80bcddc72d583c85277db02a752cf9ec3e Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 13:01:22 +0200 Subject: [PATCH 061/160] Fix merge issues --- ethcore/light/Cargo.toml | 1 - ethcore/service/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index d0de0e3fe09..ab347c62e83 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -40,7 +40,6 @@ error-chain = { version = "0.11", default-features = false } [dev-dependencies] ethcore = { path = "..", features = ["test-helpers"] } kvdb-memorydb = { path = "../../util/kvdb-memorydb" } -ethcore = { path = "..", features = ["test-helpers"] } tempdir = "0.3" [features] diff --git a/ethcore/service/Cargo.toml b/ethcore/service/Cargo.toml index ab3cb274be0..634ee55dba4 100644 --- a/ethcore/service/Cargo.toml +++ b/ethcore/service/Cargo.toml @@ -19,4 +19,3 @@ trace-time = { path = "../../util/trace-time" } ethcore = { path = "..", features = ["test-helpers"] } tempdir = "0.3" kvdb-rocksdb = { path = "../../util/kvdb-rocksdb" } -ethcore = { path = "..", features = ["test-helpers"] } \ No newline at end of file From 1a1f98e03ae7a6e8f3d00468094f9f89858d7e58 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 14:24:25 +0200 Subject: [PATCH 062/160] cleanup --- ethcore/src/client/client.rs | 1 - ethcore/src/error.rs | 2 -- ethcore/src/snapshot/mod.rs | 3 +-- ethcore/src/state/backend.rs | 3 +-- ethcore/src/state_db.rs | 2 +- 5 files changed, 3 insertions(+), 8 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 0dc545f81b5..2d0201067f1 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -1149,7 +1149,6 @@ impl Client { }, }; - use ::state::backend::Backend; snapshot::take_snapshot(&*self.engine, &self.chain.read(), start_hash, db.as_hashdb(), writer, p)?; Ok(()) diff --git a/ethcore/src/error.rs b/ethcore/src/error.rs index c5f293d1475..94c768dedd2 100644 --- a/ethcore/src/error.rs +++ b/ethcore/src/error.rs @@ -231,8 +231,6 @@ impl From for TransactionImportError { } } -use hashdb::{Hasher, KeccakHasher}; -//Trie(TrieError<::Out>) #[doc = "Error concerning TrieDBs."]; error_chain! { types { Error, ErrorKind, ErrorResultExt, EthcoreResult; diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 03953dc0854..7807f85f71c 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -31,8 +31,7 @@ use header::Header; use ids::BlockId; use ethereum_types::{H256, U256}; -//use hashdb::{HashDB, KeccakHasher}; -use hashdb::{AsHashDB, HashDB, KeccakHasher}; +use hashdb::{HashDB, KeccakHasher}; use kvdb::DBValue; use snappy; use bytes::Bytes; diff --git a/ethcore/src/state/backend.rs b/ethcore/src/state/backend.rs index 011546cd8b9..2d9b0550365 100644 --- a/ethcore/src/state/backend.rs +++ b/ethcore/src/state/backend.rs @@ -27,9 +27,8 @@ use std::sync::Arc; use state::Account; use parking_lot::Mutex; use ethereum_types::{Address, H256}; -//use memorydb::MemoryDB; use memorydb::KeccakMemoryDB; -use hashdb::{AsHashDB, HashDB, DBValue, Hasher, KeccakHasher}; +use hashdb::{AsHashDB, HashDB, DBValue, KeccakHasher}; /// State backend. See module docs for more details. pub trait Backend: Send { diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs index 072f660ad7c..b65f2dfc47d 100644 --- a/ethcore/src/state_db.rs +++ b/ethcore/src/state_db.rs @@ -21,7 +21,7 @@ use byteorder::{LittleEndian, ByteOrder}; use db::COL_ACCOUNT_BLOOM; use ethereum_types::{H256, Address}; use hash::keccak; -use hashdb::{Hasher, KeccakHasher, HashDB, AsHashDB}; +use hashdb::{KeccakHasher, HashDB}; use header::BlockNumber; use journaldb::JournalDB; use kvdb::{KeyValueDB, DBTransaction}; From 5d1261256694dffbe3ed734af1803a81c0a147ff Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 14:29:55 +0200 Subject: [PATCH 063/160] Fix build error --- ethcore/src/json_tests/executive.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 5bda6c55a39..db233e6639f 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -80,7 +80,7 @@ impl<'a, T: 'a, V: 'a, B: 'a> TestExt<'a, T, V, B> address: Address, tracer: &'a mut T, vm_tracer: &'a mut V, - ) -> trie::Result { + ) -> trie::KeccakTrieResult { let static_call = false; Ok(TestExt { nonce: state.nonce(&address)?, From b9f17605419d6d125e338440acbf26d73ede8cdf Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 11 Jun 2018 16:06:30 +0200 Subject: [PATCH 064/160] Fix json_tests --- ethcore/src/json_tests/trie.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index fae7cc7380a..51693aa561c 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -15,17 +15,18 @@ // along with Parity. If not, see . use ethjson; -use trie::{TrieFactory, TrieSpec}; +use trie::{TrieFactory, TrieSpec, KeccakRlpNodeCodec}; use ethereum_types::H256; -use memorydb::MemoryDB; +use memorydb::KeccakMemoryDB; + fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { let tests = ethjson::trie::Test::load(json).unwrap(); - let factory = TrieFactory::new(trie); + let factory = TrieFactory::<_, KeccakRlpNodeCodec>::new(trie); let mut result = vec![]; for (name, test) in tests.into_iter() { - let mut memdb = MemoryDB::new(); + let mut memdb = KeccakMemoryDB::new(); let mut root = H256::default(); let mut t = factory.create(&mut memdb, &mut root); From 468525e0d17ce61b02c54c7b520257651a0bbd6d Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 13 Jun 2018 11:10:50 +0200 Subject: [PATCH 065/160] Minor cleanup --- util/patricia_trie/src/lib.rs | 2 +- util/patricia_trie/src/triedbmut.rs | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 591c4e79df9..139f624d5a7 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -73,7 +73,7 @@ pub enum TrieError { /// Trie item not found in the database, IncompleteDatabase(T), /// Corrupt Trie item - // TODO: what we'd really like to do here is include the `rlp::DecoderError` in the `TrieError` + // REVIEW: what we'd really like to do here is include the `rlp::DecoderError` in the `TrieError` // but if we add a `Box` here we run into issues in the `ethcore` crate: // "the trait bound `std::error::Error + 'static: std::marker::Send` is not satisfied" // Investigate if using `Box` would help here (it does compile). diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 3ff1bd45ee7..b8499b8b1a9 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -53,13 +53,6 @@ impl From for NodeHandle { } } -// TODO: is this used anywhere? -//impl From for NodeHandle { -// fn from(hash: H256) -> Self { -// NodeHandle::Hash(hash) -// } -//} - fn empty_children() -> Box<[Option>; 16]> { Box::new([ None, None, None, None, None, None, None, None, @@ -139,7 +132,6 @@ impl Node where H::Out: Decodable { where F: FnMut(NodeHandle, &mut RlpStream), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` C: NodeCodec, -// where F: FnMut(NodeHandle, &mut ::StreamEncoding) { match self { Node::Empty => { @@ -1314,7 +1306,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // TODO: `t` was immutable – how could this pass? + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // REVIEW: `t` was immutable – how could this pass? for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } @@ -1342,7 +1334,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // TODO: `t` was immutable – how could this pass? + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // REVIEW: `t` was immutable – how could this pass? for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap().is_none()); assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); From 071fa8ff42fa4113d490488ca83bb9971c4931dd Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 13 Jun 2018 12:58:47 +0200 Subject: [PATCH 066/160] cleanup --- util/patricia_trie/src/triedb.rs | 1 - util/patricia_trie/src/triedbmut.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index d7eba098d72..c57adb1c1d6 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -232,7 +232,6 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> where H::Out: Deco fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> super::Result<(), H::Out> { loop { let (data, mid) = { -// let node = Node::decoded(&node_data).expect("rlp read from db; qed"); let node = C::decode(&node_data).expect("rlp read from db; qed"); match node { Node::Leaf(slice, _) => { diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index b8499b8b1a9..10efd1b4969 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -329,7 +329,7 @@ impl<'a, H, C> TrieDBMut<'a, H, C> root_handle: root_handle, death_row: HashSet::new(), hash_count: 0, - marker: PhantomData, // TODO: prolly not needed + marker: PhantomData, } } @@ -348,7 +348,7 @@ impl<'a, H, C> TrieDBMut<'a, H, C> root_handle: root_handle, death_row: HashSet::new(), hash_count: 0, - marker: PhantomData, // TODO: prolly not needed + marker: PhantomData, }) } /// Get the backing database. From 03d266d8267712fa57bfa050e80805ada84b2dc8 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 13 Jun 2018 14:41:18 +0200 Subject: [PATCH 067/160] Add more questions to reviewers wrt error handling in ethcore --- ethcore/light/src/on_demand/request.rs | 5 +++-- ethcore/src/client/error.rs | 5 ++--- ethcore/src/client/evm_test_client.rs | 4 ++-- ethcore/src/executed.rs | 8 +++----- ethcore/src/snapshot/error.rs | 6 +++--- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index f0b5b01c8dd..90ab21f01d2 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -627,7 +627,8 @@ pub enum Error { /// Empty response. Empty, /// Trie lookup error (result of bad proof) - Trie(TrieError<::Out>), // TODO: fix error handling + // Trie(TrieError), + Trie(TrieError<::Out>), // REVIEW: how do I fix this without making `Error` generic also? /// Bad inclusion proof BadProof, /// Header by number instead of hash. @@ -653,7 +654,7 @@ impl From<::rlp::DecoderError> for Error { impl From>> for Error { fn from(err: Box>) -> Self { // Error::Trie(*err) - Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // TODO: error handling zomg + Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // REVIEW: how do I fix this without making `Error` generic also? } } diff --git a/ethcore/src/client/error.rs b/ethcore/src/client/error.rs index 3c3d866c80b..787a9fa5368 100644 --- a/ethcore/src/client/error.rs +++ b/ethcore/src/client/error.rs @@ -24,7 +24,7 @@ use hashdb::{Hasher, KeccakHasher}; #[derive(Debug)] pub enum Error { /// TrieDB-related error. - Trie(TrieError<::Out>), // TODO: this doesn't look right – need to make `Error` generic too? :/ + Trie(TrieError<::Out>), // REVIEW: this doesn't look right – need to make `Error` generic too? /// Database error Database(kvdb::Error), /// Util error @@ -34,8 +34,7 @@ pub enum Error { impl From> for Error { fn from(err: TrieError) -> Self { // Error::Trie(err) - // Error::Trie(::Out::new()) // TODO: fix this - Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // TODO: fix this + Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // REVIEW: how do I fix this without making `Error` generic also? } } diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index 36c9945df1e..fda2534c532 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -31,8 +31,8 @@ use hashdb::{Hasher, KeccakHasher}; #[derive(Debug)] pub enum EvmTestError { /// Trie integrity error. - // Trie(trie::TrieError), - Trie(trie::TrieError<::Out>), // TODO: this doesn't look right. + // Trie(trie::TrieError), + Trie(trie::TrieError<::Out>), // REVIEW: how do I fix this without making `EvmTestError` generic also? /// EVM error. Evm(vm::Error), /// Initialization error. diff --git a/ethcore/src/executed.rs b/ethcore/src/executed.rs index 33a024c66d5..8ed2bca28bf 100644 --- a/ethcore/src/executed.rs +++ b/ethcore/src/executed.rs @@ -117,11 +117,9 @@ pub enum ExecutionError { TransactionMalformed(String), } -impl From>> for ExecutionError { - // TODO: sort out the error types - fn from(_err: Box>) -> Self { - // ExecutionError::Internal(format!("{:?}", err)) - ExecutionError::Internal(format!("TODO: sort out the trie error types")) +impl From>> for ExecutionError where T: fmt::Debug { + fn from(err: Box>) -> Self { + ExecutionError::Internal(format!("{:?}", err)) } } diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index d21f276cc56..c6856301123 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -49,7 +49,8 @@ pub enum Error { /// Restoration aborted. RestorationAborted, /// Trie error. - Trie(TrieError<::Out>), // TODO: this doesn't look right. + // Trie(TrieError), + Trie(TrieError<::Out>), // REVIEW: how do I fix this without making snapshot `Error` generic also? /// Decoder error. Decoder(DecoderError), /// Io error. @@ -105,8 +106,7 @@ impl From<::std::io::Error> for Error { impl From> for Error { fn from(err: TrieError) -> Self { // Error::Trie(err) - // Error::Trie(::Out::new()) // TODO: fix this - Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // TODO: fix this + Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // REVIEW: how do I fix this without making snapshot `Error` generic also? } } From 5db5de901017b4938fd2d13a82677c51fc13ad58 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 14 Jun 2018 09:35:22 +0200 Subject: [PATCH 068/160] Remove `Decodable` bound and reduce the nr of `Encodable` bounds required --- util/hashdb/src/lib.rs | 9 +------ util/patricia_trie/src/fatdb.rs | 29 ++++++++++++++------ util/patricia_trie/src/fatdbmut.rs | 17 +++++++++--- util/patricia_trie/src/lib.rs | 7 +++-- util/patricia_trie/src/lookup.rs | 2 -- util/patricia_trie/src/sectriedb.rs | 13 ++++++--- util/patricia_trie/src/sectriedbmut.rs | 17 +++++++++--- util/patricia_trie/src/triedb.rs | 32 +++++++++++++--------- util/patricia_trie/src/triedbmut.rs | 37 +++++++++++++------------- 9 files changed, 100 insertions(+), 63 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 0d92150844b..2584c13fc1d 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -53,7 +53,6 @@ pub type KeccakHashDB = HashDB; pub type DBValue = ElasticArray128; /// Trait modelling datastore keyed by a 32-byte Keccak hash. -// TODO: move `H: Hasher` to be a generic param instead of an associated type pub trait HashDB: Send + Sync + AsHashDB { /// Get the keys in the database together with number of underlying references. fn keys(&self) -> HashMap; @@ -86,13 +85,7 @@ pub trait AsHashDB { fn as_hashdb_mut(&mut self) -> &mut HashDB; } -//impl> AsHashDB for T { -// fn as_hashdb(&self) -> &HashDB { self } -// fn as_hashdb_mut(&mut self) -> &mut HashDB { self } -//} - -// TODO: This conflicts with the impl `for T`, see https://stackoverflow.com/questions/48432842/implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im -// I wonder why it didn't conflict before, when `H` was an associated type +// TODO: This conflicts with an impl `for T`, see https://stackoverflow.com/questions/48432842/implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { fn as_hashdb(&self) -> &HashDB { &**self } fn as_hashdb_mut(&mut self) -> &mut HashDB { &mut **self } diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index d9515f7db63..f75ebf99787 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -16,7 +16,6 @@ use hashdb::{HashDB, Hasher}; use super::{TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query}; -use rlp::{Decodable, Encodable}; use node_codec::NodeCodec; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -24,13 +23,17 @@ use node_codec::NodeCodec; /// /// Use it as a `Trie` or `TrieMut` trait object. pub struct FatDB<'db, H, C> - where H: Hasher + 'db, C: NodeCodec +where + H: Hasher + 'db, + C: NodeCodec { raw: TrieDB<'db, H, C>, } impl<'db, H, C> FatDB<'db, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. @@ -44,7 +47,9 @@ impl<'db, H, C> FatDB<'db, H, C> } impl<'db, H, C> Trie for FatDB<'db, H, C> - where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { type H = H; @@ -67,14 +72,18 @@ impl<'db, H, C> Trie for FatDB<'db, H, C> /// Itarator over inserted pairs of key values. pub struct FatDBIterator<'db, H, C> - where H: Hasher + 'db, C: NodeCodec + 'db +where + H: Hasher + 'db, + C: NodeCodec + 'db { trie_iterator: TrieDBIterator<'db, H, C>, trie: &'db TrieDB<'db, H, C>, } impl<'db, H, C> FatDBIterator<'db, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { /// Creates new iterator. pub fn new(trie: &'db TrieDB) -> super::Result { @@ -86,7 +95,9 @@ impl<'db, H, C> FatDBIterator<'db, H, C> } impl<'db, H, C> TrieIterator for FatDBIterator<'db, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { let hashed_key = H::hash(key); @@ -95,7 +106,9 @@ impl<'db, H, C> TrieIterator for FatDBIterator<'db, H, C> } impl<'db, H, C> Iterator for FatDBIterator<'db, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { type Item = TrieItem<'db, H>; diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 71151ba2eff..7e663d07d81 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -18,7 +18,7 @@ //use keccak::keccak; use hashdb::{HashDB, DBValue, Hasher}; use super::{TrieDBMut, TrieMut}; -use rlp::{Encodable, Decodable}; +use rlp::Encodable; use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -26,13 +26,19 @@ use node_codec::NodeCodec; /// /// Use it as a `Trie` or `TrieMut` trait object. pub struct FatDBMut<'db, H, C> - where H: Hasher + 'db, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher + 'db, + H::Out: Encodable, + C: NodeCodec { raw: TrieDBMut<'db, H, C>, } impl<'db, H, C> FatDBMut<'db, H, C> - where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher, + H::Out: Encodable, + C: NodeCodec { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. @@ -60,7 +66,10 @@ impl<'db, H, C> FatDBMut<'db, H, C> } impl<'db, H, C> TrieMut for FatDBMut<'db, H, C> - where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher, + H::Out: Encodable, + C: NodeCodec { type H = H; fn root(&mut self) -> &::Out { self.raw.root() } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 139f624d5a7..8a91e76aec4 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -251,7 +251,7 @@ macro_rules! wrapper { } } -impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> where H::Out: rlp::Decodable + rlp::Encodable { +impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { type H = H; fn root(&self) -> &::Out { wrapper!(self, root,) @@ -277,7 +277,10 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> where H::Out } impl<'db, H, C> TrieFactory - where H: Hasher, H::Out: rlp::Decodable + rlp::Encodable, C: NodeCodec + 'db +where + H: Hasher, + H::Out: rlp::Encodable, + C: NodeCodec + 'db { /// Creates new factory. pub fn new(spec: TrieSpec) -> Self { diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index fa050c8d28f..3c61947fc02 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -18,7 +18,6 @@ use hashdb::{HashDB, Hasher}; use nibbleslice::NibbleSlice; -use rlp::Decodable; use super::node::Node; use node_codec::NodeCodec; use super::{TrieError, Query}; @@ -38,7 +37,6 @@ pub struct Lookup<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> { impl<'a, H, C, Q> Lookup<'a, H, C, Q> where H: Hasher + 'a, - H::Out: Decodable, C: NodeCodec, Q: Query, { diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index ca0d9dd48c2..ff982eadf9a 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -19,20 +19,23 @@ use hashdb::{HashDB, Hasher}; use super::triedb::TrieDB; use super::{Trie, TrieItem, TrieIterator, Query}; -use rlp::{Decodable, Encodable}; use node_codec::NodeCodec; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` trait object. You can use `raw()` to get the backing `TrieDB` object. pub struct SecTrieDB<'db, H, C> - where H: Hasher + 'db, C: NodeCodec +where + H: Hasher + 'db, + C: NodeCodec { raw: TrieDB<'db, H, C> } impl<'db, H, C> SecTrieDB<'db, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { /// Create a new trie with the backing database `db` and empty `root` /// @@ -55,7 +58,9 @@ impl<'db, H, C> SecTrieDB<'db, H, C> } impl<'db, H, C> Trie for SecTrieDB<'db, H, C> - where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { type H = H; diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 43b6956b17a..f9560149fc3 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -17,20 +17,26 @@ use hashdb::{HashDB, DBValue, Hasher}; use super::triedbmut::TrieDBMut; use super::TrieMut; -use rlp::{Encodable, Decodable}; +use rlp::Encodable; use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. /// /// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing `TrieDBMut` object. pub struct SecTrieDBMut<'db, H, C> - where H: Hasher + 'db, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher + 'db, + H::Out: Encodable, + C: NodeCodec { raw: TrieDBMut<'db, H, C> } impl<'db, H, C> SecTrieDBMut<'db, H, C> - where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher, + H::Out: Encodable, + C: NodeCodec { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. @@ -54,7 +60,10 @@ impl<'db, H, C> SecTrieDBMut<'db, H, C> } impl<'db, H, C> TrieMut for SecTrieDBMut<'db, H, C> - where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec +where + H: Hasher, + H::Out: Encodable, + C: NodeCodec { type H = H; fn root(&mut self) -> &::Out { diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index c57adb1c1d6..8d0fae39367 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -22,7 +22,6 @@ use node_codec::NodeCodec; use super::lookup::Lookup; use super::{Trie, TrieItem, TrieError, TrieIterator, Query}; use bytes::Bytes; -use rlp::{Decodable, Encodable}; use std::marker::PhantomData; /// A `Trie` implementation using a generic `HashDB` backing database, a `Hasher` @@ -57,7 +56,9 @@ use std::marker::PhantomData; /// } /// ``` pub struct TrieDB<'db, H, C> - where H: Hasher + 'db, C: NodeCodec +where + H: Hasher + 'db, + C: NodeCodec { db: &'db HashDB, root: &'db H::Out, @@ -67,7 +68,9 @@ pub struct TrieDB<'db, H, C> } impl<'db, H, C> TrieDB<'db, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist @@ -105,7 +108,6 @@ impl<'db, H, C> TrieDB<'db, H, C> impl<'db, H, C> Trie for TrieDB<'db, H, C> where H: Hasher, - H::Out: Decodable + Encodable, C: NodeCodec { type H = H; @@ -129,14 +131,18 @@ where // This is for pretty debug output only struct TrieAwareDebugNode<'db, 'a, H, C> -where H: Hasher + 'db, C: NodeCodec + 'db +where + H: Hasher + 'db, + C: NodeCodec + 'db { trie: &'db TrieDB<'db, H, C>, key: &'a[u8] } impl<'db, 'a, H, C> fmt::Debug for TrieAwareDebugNode<'db, 'a, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Ok(node) = self.trie.get_raw_or_lookup(self.key) { @@ -173,7 +179,9 @@ impl<'db, 'a, H, C> fmt::Debug for TrieAwareDebugNode<'db, 'a, H, C> } impl<'db, H, C> fmt::Debug for TrieDB<'db, H, C> - where H: Hasher, H::Out: Decodable, C: NodeCodec +where + H: Hasher, + C: NodeCodec { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let root_rlp = self.db.get(self.root).expect("Trie root not found!"); @@ -221,7 +229,7 @@ pub struct TrieDBIterator<'a, H: Hasher + 'a, C: NodeCodec + 'a> { key_nibbles: Bytes, } -impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> where H::Out: Decodable { +impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { /// Create a new iterator. pub fn new(db: &'a TrieDB) -> super::Result, H::Out> { let mut r = TrieDBIterator { db, trail: vec![], key_nibbles: Vec::new() }; @@ -295,7 +303,6 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> where H::Out: Deco /// Descend into a payload. fn descend(&mut self, d: &[u8]) -> super::Result<(), H::Out> { let node_data = &self.db.get_raw_or_lookup(d)?; -// let node = Node::decoded(&node_data).expect("rlp read from db; qed"); let node = C::decode(&node_data).expect("rlp read from db; qed"); Ok(self.descend_into_node(node.into())) } @@ -326,7 +333,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> where H::Out: Deco } } -impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C> where H::Out: Decodable { +impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C> { /// Position the iterator on the first element with key >= `key` fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { self.trail.clear(); @@ -336,11 +343,12 @@ impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C } } -impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> where H::Out: Decodable { +impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { type Item = TrieItem<'a, H>; fn next(&mut self) -> Option { - enum IterStep { + // enum IterStep { + enum IterStep { Continue, PopTrail, Descend(super::Result), diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 10efd1b4969..f793a03255b 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -25,7 +25,7 @@ use super::node::NodeKey; use bytes::ToPretty; use hashdb::{HashDB, Hasher, DBValue}; use nibbleslice::NibbleSlice; -use rlp::{RlpStream, Encodable, Decodable}; +use rlp::{RlpStream, Encodable}; use elastic_array::ElasticArray1024; use std::collections::{HashSet, VecDeque}; @@ -78,7 +78,8 @@ enum Node { Branch(Box<[Option>; 16]>, Option) } -impl Node where H::Out: Decodable { +// impl Node where H::Out: Decodable { +impl Node { // load an inline node into memory or get the hash to do the lookup later. fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle where C: NodeCodec @@ -297,9 +298,10 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// } /// ``` pub struct TrieDBMut<'a, H, C> - where H: Hasher + 'a, - H::Out: Decodable + Encodable, - C: NodeCodec +where + H: Hasher + 'a, + H::Out: Encodable, + C: NodeCodec { storage: NodeStorage, db: &'a mut HashDB, @@ -313,9 +315,9 @@ pub struct TrieDBMut<'a, H, C> } impl<'a, H, C> TrieDBMut<'a, H, C> - where H: Hasher, - H::Out: Decodable + Encodable, - C: NodeCodec +where H: Hasher, + H::Out: Encodable, + C: NodeCodec { /// Create a new trie with backing database `db` and empty `root`. pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { @@ -859,12 +861,8 @@ impl<'a, H, C> TrieDBMut<'a, H, C> /// commit a node, hashing it, committing it to the db, /// and writing it to the rlp stream as necessary. + // REVIEW: this is causing the `Encodable` bound all the way upstream, precisely it's the `stream.append()` call requires it fn commit_node(&mut self, handle: NodeHandle, stream: &mut RlpStream) { -// fn commit_node( -// &mut self, handle: NodeHandle, -// stream: &mut ::StreamEncoding -// ) -// { match handle { NodeHandle::Hash(h) => stream.append(&h), NodeHandle::InMemory(h) => match self.storage.destroy(h) { @@ -893,9 +891,9 @@ impl<'a, H, C> TrieDBMut<'a, H, C> } impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> - where H: Hasher, - H::Out: Decodable + Encodable, - C: NodeCodec +where H: Hasher, + H::Out:Encodable, + C: NodeCodec { type H = H; @@ -965,9 +963,10 @@ impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> } impl<'a, H, C> Drop for TrieDBMut<'a, H, C> - where H: Hasher, - H::Out: Decodable + Encodable, - C: NodeCodec +where + H: Hasher, + H::Out: Encodable, + C: NodeCodec { fn drop(&mut self) { self.commit(); From b2e8b93a0a6dbe8c696faa995eedc6650351dce1 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 14 Jun 2018 11:51:37 +0200 Subject: [PATCH 069/160] Add a rlp::Stream trait with the stuff we need in patricia_trie Implement Stream for RlpStream Add assoc type to NodeCodec with Stream bound Remove dependency on RlpStream from patricia_trie --- util/patricia_trie/src/node_codec.rs | 12 +-- util/patricia_trie/src/triedbmut.rs | 14 ++-- util/rlp/src/impls.rs | 2 +- util/rlp/src/lib.rs | 2 +- util/rlp/src/stream.rs | 107 +++++++++++++++++++-------- util/rlp/src/traits.rs | 2 +- util/rlp/tests/tests.rs | 2 +- util/triehash/src/lib.rs | 1 + 8 files changed, 96 insertions(+), 46 deletions(-) diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 2aca945763b..0f1dee657fe 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,21 +1,21 @@ -use bytes::*; +use bytes::Bytes; use nibbleslice::NibbleSlice; use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; use hashdb::Hasher; use node::Node; use std::marker::PhantomData; +use rlp::Stream; pub trait NodeCodec: Sized { type E: ::std::error::Error; + type S: Stream; fn encode(&Node) -> Bytes; fn decode(data: &[u8]) -> Result; fn try_decode_hash(data: &[u8]) -> Option; - // TODO: We don't want these here, but where do they go? Helper trait? fn new_encoded<'a>(data: &'a [u8]) -> Rlp<'a>; - fn encoded_stream() -> RlpStream; - fn encoded_list(size: usize) -> RlpStream; - + fn encoded_stream() -> Self::S; + fn encoded_list(size: usize) -> Self::S; } #[derive(Default, Clone)] @@ -25,6 +25,7 @@ impl NodeCodec for RlpNodeCodec where H::Out: Encodable + Decodable { type E = DecoderError; + type S = RlpStream; fn encode(node: &Node) -> Bytes { match *node { Node::Leaf(ref slice, ref value) => { @@ -103,5 +104,4 @@ where H::Out: Encodable + Decodable fn encoded_list(size: usize) -> RlpStream{ RlpStream::new_list(size) } - } \ No newline at end of file diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index f793a03255b..cb3e00d904c 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -25,7 +25,7 @@ use super::node::NodeKey; use bytes::ToPretty; use hashdb::{HashDB, Hasher, DBValue}; use nibbleslice::NibbleSlice; -use rlp::{RlpStream, Encodable}; +use rlp::Encodable; use elastic_array::ElasticArray1024; use std::collections::{HashSet, VecDeque}; @@ -33,6 +33,8 @@ use std::marker::PhantomData; use std::mem; use std::ops::Index; +use rlp::Stream; // TODO: move to own crate? + // For lookups into the Node storage buffer. // This is deliberately non-copyable. #[derive(Debug)] @@ -78,7 +80,6 @@ enum Node { Branch(Box<[Option>; 16]>, Option) } -// impl Node where H::Out: Decodable { impl Node { // load an inline node into memory or get the hash to do the lookup later. fn inline_or_hash(node: &[u8], db: &HashDB, storage: &mut NodeStorage) -> NodeHandle @@ -130,9 +131,10 @@ impl Node { // encode a node to RLP // TODO: parallelize fn into_rlp(self, mut child_cb: F) -> ElasticArray1024 - where - F: FnMut(NodeHandle, &mut RlpStream), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` - C: NodeCodec, + where + // F: FnMut(NodeHandle, &mut RlpStream), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` + F: FnMut(NodeHandle, &mut >::S), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` + C: NodeCodec, { match self { Node::Empty => { @@ -862,7 +864,7 @@ where H: Hasher, /// commit a node, hashing it, committing it to the db, /// and writing it to the rlp stream as necessary. // REVIEW: this is causing the `Encodable` bound all the way upstream, precisely it's the `stream.append()` call requires it - fn commit_node(&mut self, handle: NodeHandle, stream: &mut RlpStream) { + fn commit_node(&mut self, handle: NodeHandle, stream: &mut >::S) { match handle { NodeHandle::Hash(h) => stream.append(&h), NodeHandle::InMemory(h) => match self.storage.destroy(h) { diff --git a/util/rlp/src/impls.rs b/util/rlp/src/impls.rs index 573f2c0781e..ff5507ccc1e 100644 --- a/util/rlp/src/impls.rs +++ b/util/rlp/src/impls.rs @@ -10,7 +10,7 @@ use std::{cmp, mem, str}; use byteorder::{ByteOrder, BigEndian}; use bigint::{U128, U256, H64, H128, H160, H256, H512, H520, Bloom}; use traits::{Encodable, Decodable}; -use stream::RlpStream; +use stream::{Stream, RlpStream}; use {Rlp, DecoderError}; pub fn decode_usize(bytes: &[u8]) -> Result { diff --git a/util/rlp/src/lib.rs b/util/rlp/src/lib.rs index b416b1c25b0..8fc953337c1 100644 --- a/util/rlp/src/lib.rs +++ b/util/rlp/src/lib.rs @@ -49,7 +49,7 @@ use elastic_array::ElasticArray1024; pub use error::DecoderError; pub use traits::{Decodable, Encodable}; pub use rlpin::{Rlp, RlpIterator, PayloadInfo, Prototype}; -pub use stream::RlpStream; +pub use stream::{RlpStream, Stream}; /// The RLP encoded empty data (used to mean "null value"). pub const NULL_RLP: [u8; 1] = [0x80; 1]; diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index 550ede03991..5e25d9c5cc1 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -41,6 +41,53 @@ impl Default for RlpStream { } } +pub trait Stream { + fn append_empty_data(&mut self) -> &mut Self; + fn drain(self) -> ElasticArray1024; + fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable; + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; +} + +impl Stream for RlpStream { + fn append_empty_data(&mut self) -> &mut Self { + // self push raw item + self.buffer.push(0x80); + + // try to finish and prepend the length + self.note_appended(1); + + // return chainable self + self + } + + fn drain(self) -> ElasticArray1024 { + match self.is_finished() { + true => self.buffer, + false => panic!() + } + } + + fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { + self.finished_list = false; + value.rlp_append(self); + if !self.finished_list { + self.note_appended(1); + } + self + } + + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self { + // push raw items + self.buffer.append_slice(bytes); + + // try to finish and prepend the length + self.note_appended(item_count); + + // return chainable self + self + } +} + impl RlpStream { /// Initializes instance of empty `Stream`. pub fn new() -> Self { @@ -71,14 +118,14 @@ impl RlpStream { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { - self.finished_list = false; - value.rlp_append(self); - if !self.finished_list { - self.note_appended(1); - } - self - } + // pub fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { + // self.finished_list = false; + // value.rlp_append(self); + // if !self.finished_list { + // self.note_appended(1); + // } + // self + // } /// Appends list of values to the end of stream, chainable. pub fn append_list<'a, E, K>(&'a mut self, values: &[K]) -> &'a mut Self where E: Encodable, K: Borrow { @@ -158,28 +205,28 @@ impl RlpStream { /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); /// } /// ``` - pub fn append_empty_data(&mut self) -> &mut RlpStream { - // self push raw item - self.buffer.push(0x80); + // pub fn append_empty_data(&mut self) -> &mut RlpStream { + // // self push raw item + // self.buffer.push(0x80); - // try to finish and prepend the length - self.note_appended(1); + // // try to finish and prepend the length + // self.note_appended(1); - // return chainable self - self - } + // // return chainable self + // self + // } /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. - pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { - // push raw items - self.buffer.append_slice(bytes); + // pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { + // // push raw items + // self.buffer.append_slice(bytes); - // try to finish and prepend the length - self.note_appended(item_count); + // // try to finish and prepend the length + // self.note_appended(item_count); - // return chainable self - self - } + // // return chainable self + // self + // } /// Appends raw (pre-serialised) RLP data. Checks for size oveflow. pub fn append_raw_checked<'a>(&'a mut self, bytes: &[u8], item_count: usize, max_size: usize) -> bool { @@ -301,12 +348,12 @@ impl RlpStream { } /// Drain the object and return the underlying ElasticArray. - pub fn drain(self) -> ElasticArray1024 { - match self.is_finished() { - true => self.buffer, - false => panic!() - } - } + // pub fn drain(self) -> ElasticArray1024 { + // match self.is_finished() { + // true => self.buffer, + // false => panic!() + // } + // } /// Finalize current ubnbound list. Panics if no unbounded list has been opened. pub fn complete_unbounded_list(&mut self) { diff --git a/util/rlp/src/traits.rs b/util/rlp/src/traits.rs index 1596009e75f..c8dd80ac287 100644 --- a/util/rlp/src/traits.rs +++ b/util/rlp/src/traits.rs @@ -8,7 +8,7 @@ //! Common RLP traits use elastic_array::ElasticArray1024; -use {DecoderError, Rlp, RlpStream}; +use {DecoderError, Rlp, RlpStream, Stream}; /// RLP decodable trait pub trait Decodable: Sized { diff --git a/util/rlp/tests/tests.rs b/util/rlp/tests/tests.rs index 041c267667d..ad6a698840d 100644 --- a/util/rlp/tests/tests.rs +++ b/util/rlp/tests/tests.rs @@ -11,7 +11,7 @@ extern crate rlp; use std::{fmt, cmp}; use bigint::{U256, H160}; -use rlp::{Encodable, Decodable, Rlp, RlpStream, DecoderError}; +use rlp::{Encodable, Decodable, Rlp, Stream, RlpStream, DecoderError}; #[test] fn rlp_at() { diff --git a/util/triehash/src/lib.rs b/util/triehash/src/lib.rs index c78ed0ca1ba..2754d91d442 100644 --- a/util/triehash/src/lib.rs +++ b/util/triehash/src/lib.rs @@ -29,6 +29,7 @@ use elastic_array::{ElasticArray4, ElasticArray8}; use ethereum_types::H256; use hash::keccak; use rlp::RlpStream; +use rlp::Stream; // TODO: move to own crate? fn shared_prefix_len(first: &[T], second: &[T]) -> usize { let len = cmp::min(first.len(), second.len()); From 4f172f7163c9c329530aca6d2b92fe83cb2ad69b Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 14 Jun 2018 12:34:36 +0200 Subject: [PATCH 070/160] Get rid of more RLP dependent code --- util/patricia_trie/src/node_codec.rs | 17 +++---------- util/patricia_trie/src/triedbmut.rs | 35 +++++++++++++------------ util/rlp/src/stream.rs | 38 +++++++++++++++++++--------- 3 files changed, 47 insertions(+), 43 deletions(-) diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 0f1dee657fe..2efc18c9f2c 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -13,9 +13,7 @@ pub trait NodeCodec: Sized { fn decode(data: &[u8]) -> Result; fn try_decode_hash(data: &[u8]) -> Option; - fn new_encoded<'a>(data: &'a [u8]) -> Rlp<'a>; - fn encoded_stream() -> Self::S; - fn encoded_list(size: usize) -> Self::S; + fn is_empty_node(data: &[u8]) -> bool; } #[derive(Default, Clone)] @@ -93,15 +91,8 @@ where H::Out: Encodable + Decodable } } - fn new_encoded<'a>(data: &'a [u8]) -> Rlp<'a> { - Rlp::new(data) - } - - fn encoded_stream() -> RlpStream { - RlpStream::new() - } - - fn encoded_list(size: usize) -> RlpStream{ - RlpStream::new_list(size) + fn is_empty_node(data: &[u8]) -> bool { + // REVIEW: Could also be `Rlp::new(data).is_empty()` – better? + data.len() != 0 && (data[0] == 0xC0 || data[0] == 0x80) } } \ No newline at end of file diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index cb3e00d904c..63eb7d13e10 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -33,7 +33,7 @@ use std::marker::PhantomData; use std::mem; use std::ops::Index; -use rlp::Stream; // TODO: move to own crate? +use rlp::Stream; // TODO: this should move to own crate, not really rlp-related // For lookups into the Node storage buffer. // This is deliberately non-copyable. @@ -93,7 +93,7 @@ impl Node { }) } - // decode a node from rlp without getting its children. + // decode a node from encoded bytes without getting its children. fn from_encoded(data: &[u8], db: &HashDB, storage: &mut NodeStorage) -> Self where C: NodeCodec { @@ -108,8 +108,7 @@ impl Node { RlpNode::Branch(ref encoded_children, val) => { let mut child = |i:usize| { let raw = encoded_children[i]; - let encoded_child = C::new_encoded(raw); - if !encoded_child.is_empty() { + if !C::is_empty_node(raw) { Some(Self::inline_or_hash::(raw, db, storage)) } else { None @@ -128,9 +127,9 @@ impl Node { } } - // encode a node to RLP + // // TODO: parallelize - fn into_rlp(self, mut child_cb: F) -> ElasticArray1024 + fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where // F: FnMut(NodeHandle, &mut RlpStream), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` F: FnMut(NodeHandle, &mut >::S), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` @@ -138,24 +137,24 @@ impl Node { { match self { Node::Empty => { - let mut stream = C::encoded_stream(); + let mut stream = >::S::new(); stream.append_empty_data(); stream.drain() } Node::Leaf(partial, value) => { - let mut stream = C::encoded_list(2); + let mut stream = >::S::new_list(2); stream.append(&&*partial); stream.append(&&*value); stream.drain() } Node::Extension(partial, child) => { - let mut stream = C::encoded_list(2); + let mut stream = >::S::new_list(2); stream.append(&&*partial); child_cb(child, &mut stream); stream.drain() } Node::Branch(mut children, value) => { - let mut stream = C::encoded_list(17); + let mut stream = >::S::new_list(17); for child in children.iter_mut().map(Option::take) { if let Some(handle) = child { child_cb(handle, &mut stream); @@ -846,11 +845,11 @@ where H: Hasher, match self.storage.destroy(handle) { Stored::New(node) => { - let root_rlp = node.into_rlp::<_, C>(|child, stream| self.commit_node(child, stream)); - *self.root = self.db.insert(&root_rlp[..]); + let encoded_root = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); + *self.root = self.db.insert(&encoded_root[..]); self.hash_count += 1; - trace!(target: "trie", "root node rlp: {:?}", (&root_rlp[..]).pretty()); + trace!(target: "trie", "encoded root node: {:?}", (&encoded_root[..]).pretty()); self.root_handle = NodeHandle::Hash(*self.root); } Stored::Cached(node, hash) => { @@ -862,7 +861,7 @@ where H: Hasher, } /// commit a node, hashing it, committing it to the db, - /// and writing it to the rlp stream as necessary. + /// and writing it to the encoded stream as necessary. // REVIEW: this is causing the `Encodable` bound all the way upstream, precisely it's the `stream.append()` call requires it fn commit_node(&mut self, handle: NodeHandle, stream: &mut >::S) { match handle { @@ -870,13 +869,13 @@ where H: Hasher, NodeHandle::InMemory(h) => match self.storage.destroy(h) { Stored::Cached(_, h) => stream.append(&h), Stored::New(node) => { - let node_rlp = node.into_rlp::<_, C>(|child, stream| self.commit_node(child, stream)); - if node_rlp.len() >= 32 { - let hash = self.db.insert(&node_rlp[..]); + let encoded_node = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); + if encoded_node.len() >= 32 { + let hash = self.db.insert(&encoded_node[..]); self.hash_count += 1; stream.append(&hash) } else { - stream.append_raw(&node_rlp, 1) + stream.append_raw(&encoded_node, 1) } } } diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index 5e25d9c5cc1..1cf1208e0b8 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -42,6 +42,8 @@ impl Default for RlpStream { } pub trait Stream { + fn new() -> Self; + fn new_list(len: usize) -> Self; fn append_empty_data(&mut self) -> &mut Self; fn drain(self) -> ElasticArray1024; fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable; @@ -49,6 +51,18 @@ pub trait Stream { } impl Stream for RlpStream { + fn new() -> Self { + RlpStream { + unfinished_lists: ElasticArray16::new(), + buffer: ElasticArray1024::new(), + finished_list: false, + } + } + fn new_list(len: usize) -> Self { + let mut stream = RlpStream::new(); + stream.begin_list(len); + stream + } fn append_empty_data(&mut self) -> &mut Self { // self push raw item self.buffer.push(0x80); @@ -90,20 +104,20 @@ impl Stream for RlpStream { impl RlpStream { /// Initializes instance of empty `Stream`. - pub fn new() -> Self { - RlpStream { - unfinished_lists: ElasticArray16::new(), - buffer: ElasticArray1024::new(), - finished_list: false, - } - } + // pub fn new() -> Self { + // RlpStream { + // unfinished_lists: ElasticArray16::new(), + // buffer: ElasticArray1024::new(), + // finished_list: false, + // } + // } /// Initializes the `Stream` as a list. - pub fn new_list(len: usize) -> Self { - let mut stream = RlpStream::new(); - stream.begin_list(len); - stream - } + // pub fn new_list(len: usize) -> Self { + // let mut stream = RlpStream::new(); + // stream.begin_list(len); + // stream + // } /// Appends value to the end of stream, chainable. /// From a6fd313a02d6acca4d602a7703b718de03da26ad Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 14 Jun 2018 13:08:06 +0200 Subject: [PATCH 071/160] Add special-cased `append_bytes` to `Stream` to work around the `Encodable` bound --- util/patricia_trie/src/fatdbmut.rs | 4 ---- util/patricia_trie/src/sectriedbmut.rs | 4 ---- util/patricia_trie/src/triedb.rs | 2 +- util/patricia_trie/src/triedbmut.rs | 23 +++++++++++------------ util/rlp/src/stream.rs | 26 ++++++++++++-------------- 5 files changed, 24 insertions(+), 35 deletions(-) diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 7e663d07d81..50c9ded2703 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -18,7 +18,6 @@ //use keccak::keccak; use hashdb::{HashDB, DBValue, Hasher}; use super::{TrieDBMut, TrieMut}; -use rlp::Encodable; use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -28,7 +27,6 @@ use node_codec::NodeCodec; pub struct FatDBMut<'db, H, C> where H: Hasher + 'db, - H::Out: Encodable, C: NodeCodec { raw: TrieDBMut<'db, H, C>, @@ -37,7 +35,6 @@ where impl<'db, H, C> FatDBMut<'db, H, C> where H: Hasher, - H::Out: Encodable, C: NodeCodec { /// Create a new trie with the backing database `db` and empty `root` @@ -68,7 +65,6 @@ where impl<'db, H, C> TrieMut for FatDBMut<'db, H, C> where H: Hasher, - H::Out: Encodable, C: NodeCodec { type H = H; diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index f9560149fc3..8144ad8f72a 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -17,7 +17,6 @@ use hashdb::{HashDB, DBValue, Hasher}; use super::triedbmut::TrieDBMut; use super::TrieMut; -use rlp::Encodable; use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -26,7 +25,6 @@ use node_codec::NodeCodec; pub struct SecTrieDBMut<'db, H, C> where H: Hasher + 'db, - H::Out: Encodable, C: NodeCodec { raw: TrieDBMut<'db, H, C> @@ -35,7 +33,6 @@ where impl<'db, H, C> SecTrieDBMut<'db, H, C> where H: Hasher, - H::Out: Encodable, C: NodeCodec { /// Create a new trie with the backing database `db` and empty `root` @@ -62,7 +59,6 @@ where impl<'db, H, C> TrieMut for SecTrieDBMut<'db, H, C> where H: Hasher, - H::Out: Encodable, C: NodeCodec { type H = H; diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 8d0fae39367..df7ebfcfbd1 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -303,7 +303,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { /// Descend into a payload. fn descend(&mut self, d: &[u8]) -> super::Result<(), H::Out> { let node_data = &self.db.get_raw_or_lookup(d)?; - let node = C::decode(&node_data).expect("rlp read from db; qed"); + let node = C::decode(&node_data).expect("encoded node read from db; qed"); Ok(self.descend_into_node(node.into())) } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 63eb7d13e10..3defe3c018b 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -25,7 +25,6 @@ use super::node::NodeKey; use bytes::ToPretty; use hashdb::{HashDB, Hasher, DBValue}; use nibbleslice::NibbleSlice; -use rlp::Encodable; use elastic_array::ElasticArray1024; use std::collections::{HashSet, VecDeque}; @@ -143,13 +142,13 @@ impl Node { } Node::Leaf(partial, value) => { let mut stream = >::S::new_list(2); - stream.append(&&*partial); - stream.append(&&*value); + stream.append_bytes(&&*partial); + stream.append_bytes(&&*value); stream.drain() } Node::Extension(partial, child) => { let mut stream = >::S::new_list(2); - stream.append(&&*partial); + stream.append_bytes(&&*partial); child_cb(child, &mut stream); stream.drain() } @@ -163,7 +162,7 @@ impl Node { } } if let Some(value) = value { - stream.append(&&*value); + stream.append_bytes(&&*value); } else { stream.append_empty_data(); } @@ -301,7 +300,6 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { pub struct TrieDBMut<'a, H, C> where H: Hasher + 'a, - H::Out: Encodable, C: NodeCodec { storage: NodeStorage, @@ -317,7 +315,6 @@ where impl<'a, H, C> TrieDBMut<'a, H, C> where H: Hasher, - H::Out: Encodable, C: NodeCodec { /// Create a new trie with backing database `db` and empty `root`. @@ -865,15 +862,19 @@ where H: Hasher, // REVIEW: this is causing the `Encodable` bound all the way upstream, precisely it's the `stream.append()` call requires it fn commit_node(&mut self, handle: NodeHandle, stream: &mut >::S) { match handle { - NodeHandle::Hash(h) => stream.append(&h), + NodeHandle::Hash(h) => { + stream.append_bytes(&h.as_ref()) + }, NodeHandle::InMemory(h) => match self.storage.destroy(h) { - Stored::Cached(_, h) => stream.append(&h), + Stored::Cached(_, h) => { + stream.append_bytes(&h.as_ref()) + }, Stored::New(node) => { let encoded_node = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); if encoded_node.len() >= 32 { let hash = self.db.insert(&encoded_node[..]); self.hash_count += 1; - stream.append(&hash) + stream.append_bytes(&hash.as_ref()) } else { stream.append_raw(&encoded_node, 1) } @@ -893,7 +894,6 @@ where H: Hasher, impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> where H: Hasher, - H::Out:Encodable, C: NodeCodec { type H = H; @@ -966,7 +966,6 @@ where H: Hasher, impl<'a, H, C> Drop for TrieDBMut<'a, H, C> where H: Hasher, - H::Out: Encodable, C: NodeCodec { fn drop(&mut self) { diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index 1cf1208e0b8..65918092f88 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -46,7 +46,7 @@ pub trait Stream { fn new_list(len: usize) -> Self; fn append_empty_data(&mut self) -> &mut Self; fn drain(self) -> ElasticArray1024; - fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable; + fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; } @@ -81,12 +81,10 @@ impl Stream for RlpStream { } } - fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { + fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self { self.finished_list = false; - value.rlp_append(self); - if !self.finished_list { - self.note_appended(1); - } + self.encoder().encode_value(bytes); + if !self.finished_list { self.note_appended(1) } self } @@ -132,14 +130,14 @@ impl RlpStream { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } /// ``` - // pub fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { - // self.finished_list = false; - // value.rlp_append(self); - // if !self.finished_list { - // self.note_appended(1); - // } - // self - // } + pub fn append<'a, E>(&'a mut self, value: &E) -> &'a mut Self where E: Encodable { + self.finished_list = false; + value.rlp_append(self); + if !self.finished_list { + self.note_appended(1); + } + self + } /// Appends list of values to the end of stream, chainable. pub fn append_list<'a, E, K>(&'a mut self, values: &[K]) -> &'a mut Self where E: Encodable, K: Borrow { From 073c2132ba51e95af1d25b90405a5373d23950ad Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 14 Jun 2018 20:41:47 +0200 Subject: [PATCH 072/160] Add trie-node-codec crate and move Stream trait there --- Cargo.lock | 12 ++++++++++++ Cargo.toml | 1 + util/patricia_trie/Cargo.toml | 1 + util/patricia_trie/src/lib.rs | 13 +++++++------ util/patricia_trie/src/node_codec.rs | 2 +- util/patricia_trie/src/triedbmut.rs | 6 ++---- util/rlp/Cargo.toml | 1 + util/rlp/src/impls.rs | 2 +- util/rlp/src/lib.rs | 6 ++++-- util/rlp/src/stream.rs | 29 +++++++++++++++++++--------- util/rlp/src/traits.rs | 3 ++- util/rlp/tests/tests.rs | 4 +++- util/trie-node-codec/Cargo.toml | 9 +++++++++ util/trie-node-codec/src/lib.rs | 27 ++++++++++++++++++++++++++ util/triehash/Cargo.toml | 1 + util/triehash/src/lib.rs | 3 ++- 16 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 util/trie-node-codec/Cargo.toml create mode 100644 util/trie-node-codec/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6a75e318163..8403776c44a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2419,6 +2419,7 @@ dependencies = [ "memorydb 0.2.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", + "trie-node-codec 0.1.0", "trie-standardmap 0.1.0", "triehash 0.1.0", ] @@ -2752,6 +2753,7 @@ dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-node-codec 0.1.0", ] [[package]] @@ -3471,6 +3473,15 @@ dependencies = [ "protobuf 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "trie-node-codec" +version = "0.1.0" +dependencies = [ + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore-bytes 0.1.0", + "hashdb 0.2.0", +] + [[package]] name = "trie-standardmap" version = "0.1.0" @@ -3489,6 +3500,7 @@ dependencies = [ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", "rlp 0.2.1", + "trie-node-codec 0.1.0", "trie-standardmap 0.1.0", ] diff --git a/Cargo.toml b/Cargo.toml index 8df37d5942b..24b7e9bbbd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,6 +135,7 @@ members = [ "transaction-pool", "whisper", "whisper/cli", + "util/trie-node-codec", ] [patch.crates-io] diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 6e4b422bbf3..4b1ccbed9df 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -17,6 +17,7 @@ rlp = { version = "0.2.1", path = "../rlp" } triehash = { version = "0.1.0", path = "../triehash" } memorydb = { version = "0.2", path = "../memorydb" } ethcore-logger = { version = "1.9.0", path = "../../logger" } +trie-node-codec = { version = "0.1", path = "../trie-node-codec" } [dev-dependencies] trie-standardmap = { path = "../trie-standardmap" } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 8a91e76aec4..7cb26a0427d 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -15,15 +15,16 @@ // along with Parity. If not, see . //! Trie interface and implementation. -extern crate rand; +extern crate elastic_array; +extern crate ethcore_bytes as bytes; +extern crate ethcore_logger; extern crate ethereum_types; -extern crate keccak_hash as keccak; -extern crate rlp; extern crate hashdb; -extern crate ethcore_bytes as bytes; -extern crate elastic_array; +extern crate keccak_hash as keccak; extern crate memorydb; -extern crate ethcore_logger; +extern crate rand; +extern crate rlp; +extern crate trie_node_codec; #[cfg(test)] extern crate trie_standardmap as standardmap; diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 2efc18c9f2c..ab6bcb196a5 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -4,7 +4,7 @@ use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; use hashdb::Hasher; use node::Node; use std::marker::PhantomData; -use rlp::Stream; +use trie_node_codec::Stream; pub trait NodeCodec: Sized { type E: ::std::error::Error; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 3defe3c018b..fa593a1686b 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -31,8 +31,7 @@ use std::collections::{HashSet, VecDeque}; use std::marker::PhantomData; use std::mem; use std::ops::Index; - -use rlp::Stream; // TODO: this should move to own crate, not really rlp-related +use trie_node_codec::Stream; // For lookups into the Node storage buffer. // This is deliberately non-copyable. @@ -130,8 +129,7 @@ impl Node { // TODO: parallelize fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where - // F: FnMut(NodeHandle, &mut RlpStream), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` - F: FnMut(NodeHandle, &mut >::S), // REVIEW: how can I use the NodeCodec associated type instead? Causes lifetime issues in `commit_node()` + F: FnMut(NodeHandle, &mut >::S), C: NodeCodec, { match self { diff --git a/util/rlp/Cargo.toml b/util/rlp/Cargo.toml index 3bde7206ff8..19f4e18a50f 100644 --- a/util/rlp/Cargo.toml +++ b/util/rlp/Cargo.toml @@ -7,6 +7,7 @@ version = "0.2.1" authors = ["Parity Technologies "] [dependencies] +trie-node-codec = { path = "../trie-node-codec"} elastic-array = "0.10" ethereum-types = "0.3" rustc-hex = "1.0" diff --git a/util/rlp/src/impls.rs b/util/rlp/src/impls.rs index ff5507ccc1e..573f2c0781e 100644 --- a/util/rlp/src/impls.rs +++ b/util/rlp/src/impls.rs @@ -10,7 +10,7 @@ use std::{cmp, mem, str}; use byteorder::{ByteOrder, BigEndian}; use bigint::{U128, U256, H64, H128, H160, H256, H512, H520, Bloom}; use traits::{Encodable, Decodable}; -use stream::{Stream, RlpStream}; +use stream::RlpStream; use {Rlp, DecoderError}; pub fn decode_usize(bytes: &[u8]) -> Result { diff --git a/util/rlp/src/lib.rs b/util/rlp/src/lib.rs index 8fc953337c1..f7c35068b90 100644 --- a/util/rlp/src/lib.rs +++ b/util/rlp/src/lib.rs @@ -36,6 +36,7 @@ extern crate byteorder; extern crate ethereum_types as bigint; extern crate elastic_array; extern crate rustc_hex; +extern crate trie_node_codec; mod traits; mod error; @@ -43,13 +44,14 @@ mod rlpin; mod stream; mod impls; -use std::borrow::Borrow; use elastic_array::ElasticArray1024; +use std::borrow::Borrow; +use trie_node_codec::Stream; pub use error::DecoderError; pub use traits::{Decodable, Encodable}; pub use rlpin::{Rlp, RlpIterator, PayloadInfo, Prototype}; -pub use stream::{RlpStream, Stream}; +pub use stream::RlpStream; /// The RLP encoded empty data (used to mean "null value"). pub const NULL_RLP: [u8; 1] = [0x80; 1]; diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index 65918092f88..95776eb4cc7 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -10,6 +10,7 @@ use std::borrow::Borrow; use byteorder::{ByteOrder, BigEndian}; use elastic_array::{ElasticArray16, ElasticArray1024}; use traits::Encodable; +use trie_node_codec::Stream; #[derive(Debug, Copy, Clone)] struct ListInfo { @@ -41,14 +42,14 @@ impl Default for RlpStream { } } -pub trait Stream { - fn new() -> Self; - fn new_list(len: usize) -> Self; - fn append_empty_data(&mut self) -> &mut Self; - fn drain(self) -> ElasticArray1024; - fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; - fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; -} +// pub trait Stream { +// fn new() -> Self; +// fn new_list(len: usize) -> Self; +// fn append_empty_data(&mut self) -> &mut Self; +// fn drain(self) -> ElasticArray1024; +// fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; +// fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; +// } impl Stream for RlpStream { fn new() -> Self { @@ -121,7 +122,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; + /// extern crate trie_node_codec; /// use rlp::*; + /// use trie_node_codec::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -159,7 +162,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; + /// extern crate trie_node_codec; /// use rlp::*; + /// use trie_node_codec::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -208,7 +213,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; + /// extern crate trie_node_codec; /// use rlp::*; + /// use trie_node_codec::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -273,8 +280,10 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; + /// extern crate trie_node_codec; /// use rlp::*; - /// + /// use trie_node_codec::Stream; + /// /// fn main () { /// let mut stream = RlpStream::new_list(3); /// stream.append(&"cat"); @@ -295,7 +304,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; + /// extern crate trie_node_codec; /// use rlp::*; + /// use trie_node_codec::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); diff --git a/util/rlp/src/traits.rs b/util/rlp/src/traits.rs index c8dd80ac287..8230e8308e4 100644 --- a/util/rlp/src/traits.rs +++ b/util/rlp/src/traits.rs @@ -8,7 +8,8 @@ //! Common RLP traits use elastic_array::ElasticArray1024; -use {DecoderError, Rlp, RlpStream, Stream}; +use {DecoderError, Rlp, RlpStream}; +use trie_node_codec::Stream; /// RLP decodable trait pub trait Decodable: Sized { diff --git a/util/rlp/tests/tests.rs b/util/rlp/tests/tests.rs index ad6a698840d..a08c23d7f55 100644 --- a/util/rlp/tests/tests.rs +++ b/util/rlp/tests/tests.rs @@ -8,10 +8,12 @@ extern crate ethereum_types as bigint; extern crate rlp; +extern crate trie_node_codec; use std::{fmt, cmp}; use bigint::{U256, H160}; -use rlp::{Encodable, Decodable, Rlp, Stream, RlpStream, DecoderError}; +use rlp::{Encodable, Decodable, Rlp, RlpStream, DecoderError}; +use trie_node_codec::Stream; #[test] fn rlp_at() { diff --git a/util/trie-node-codec/Cargo.toml b/util/trie-node-codec/Cargo.toml new file mode 100644 index 00000000000..d8c44d21cce --- /dev/null +++ b/util/trie-node-codec/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "trie-node-codec" +version = "0.1.0" +authors = ["David Palm "] + +[dependencies] +hashdb = { version = "0.2", path = "../hashdb" } +ethcore-bytes = { version = "0.1.0", path = "../bytes" } +elastic-array = "0.10" diff --git a/util/trie-node-codec/src/lib.rs b/util/trie-node-codec/src/lib.rs new file mode 100644 index 00000000000..c40ef16c480 --- /dev/null +++ b/util/trie-node-codec/src/lib.rs @@ -0,0 +1,27 @@ +extern crate hashdb; +extern crate ethcore_bytes; +extern crate elastic_array; + +use hashdb::Hasher; +use ethcore_bytes::Bytes; +use elastic_array::ElasticArray1024; + +pub trait Stream { + fn new() -> Self; + fn new_list(len: usize) -> Self; + fn append_empty_data(&mut self) -> &mut Self; + fn drain(self) -> ElasticArray1024; // TODO: add as assoc type? Makes the types kind of hairy and requires some extra trait bounds, but not sure if it's worth it. Needs AsRef I think. + fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; +} + +pub trait NodeCodec: Sized { + type E: ::std::error::Error; + type S: Stream; + type Node; + fn encode(Self::Node) -> Bytes; + fn decode(data: &[u8]) -> Result; + fn try_decode_hash(data: &[u8]) -> Option; + + fn is_empty_node(data: &[u8]) -> bool; +} \ No newline at end of file diff --git a/util/triehash/Cargo.toml b/util/triehash/Cargo.toml index ee42b9d8253..3210897e4b1 100644 --- a/util/triehash/Cargo.toml +++ b/util/triehash/Cargo.toml @@ -10,6 +10,7 @@ elastic-array = "0.10" rlp = { version = "0.2.1", path = "../rlp" } ethereum-types = "0.3" keccak-hash = { version = "0.1", path = "../hash" } +trie-node-codec = { version = "0.1", path = "../trie-node-codec" } [dev-dependencies] trie-standardmap = { path = "../trie-standardmap" } diff --git a/util/triehash/src/lib.rs b/util/triehash/src/lib.rs index 2754d91d442..cfffdee7783 100644 --- a/util/triehash/src/lib.rs +++ b/util/triehash/src/lib.rs @@ -22,6 +22,7 @@ extern crate elastic_array; extern crate ethereum_types; extern crate keccak_hash as hash; extern crate rlp; +extern crate trie_node_codec; use std::collections::BTreeMap; use std::cmp; @@ -29,7 +30,7 @@ use elastic_array::{ElasticArray4, ElasticArray8}; use ethereum_types::H256; use hash::keccak; use rlp::RlpStream; -use rlp::Stream; // TODO: move to own crate? +use trie_node_codec::Stream; fn shared_prefix_len(first: &[T], second: &[T]) -> usize { let len = cmp::min(first.len(), second.len()); From cf51e79c6d5d8a0011f7710642eb624490280724 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 12:26:12 +0200 Subject: [PATCH 073/160] Rename trie-node-codec to stream-encoder TODOs and cleanup --- Cargo.lock | 22 +++++++++----------- Cargo.toml | 2 +- util/hashdb/src/lib.rs | 5 +++-- util/patricia_trie/Cargo.toml | 2 +- util/patricia_trie/src/lib.rs | 2 +- util/patricia_trie/src/node_codec.rs | 6 +++--- util/patricia_trie/src/triedbmut.rs | 2 +- util/rlp/Cargo.toml | 2 +- util/rlp/src/lib.rs | 4 ++-- util/rlp/src/stream.rs | 14 ++++++------- util/rlp/src/traits.rs | 2 +- util/rlp/tests/tests.rs | 4 ++-- util/stream-encoder/Cargo.toml | 9 ++++++++ util/stream-encoder/src/lib.rs | 31 ++++++++++++++++++++++++++++ util/trie-node-codec/Cargo.toml | 9 -------- util/trie-node-codec/src/lib.rs | 27 ------------------------ util/triehash/Cargo.toml | 2 +- util/triehash/src/lib.rs | 4 ++-- 18 files changed, 76 insertions(+), 73 deletions(-) create mode 100644 util/stream-encoder/Cargo.toml create mode 100644 util/stream-encoder/src/lib.rs delete mode 100644 util/trie-node-codec/Cargo.toml delete mode 100644 util/trie-node-codec/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8403776c44a..c10f3fce9cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2419,7 +2419,7 @@ dependencies = [ "memorydb 0.2.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", - "trie-node-codec 0.1.0", + "stream-encoder 0.1.0", "trie-standardmap 0.1.0", "triehash 0.1.0", ] @@ -2753,7 +2753,7 @@ dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-node-codec 0.1.0", + "stream-encoder 0.1.0", ] [[package]] @@ -3056,6 +3056,13 @@ dependencies = [ name = "stop-guard" version = "0.1.0" +[[package]] +name = "stream-encoder" +version = "0.1.0" +dependencies = [ + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "strsim" version = "0.6.0" @@ -3473,15 +3480,6 @@ dependencies = [ "protobuf 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "trie-node-codec" -version = "0.1.0" -dependencies = [ - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-bytes 0.1.0", - "hashdb 0.2.0", -] - [[package]] name = "trie-standardmap" version = "0.1.0" @@ -3500,7 +3498,7 @@ dependencies = [ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", "rlp 0.2.1", - "trie-node-codec 0.1.0", + "stream-encoder 0.1.0", "trie-standardmap 0.1.0", ] diff --git a/Cargo.toml b/Cargo.toml index 24b7e9bbbd3..87ec010e9ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,7 +135,7 @@ members = [ "transaction-pool", "whisper", "whisper/cli", - "util/trie-node-codec", + "util/stream-encoder", ] [patch.crates-io] diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 2584c13fc1d..8117e12d530 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//! Database of byte-slices keyed to their Keccak hash. +//! Database of byte-slices keyed to their hash. extern crate elastic_array; extern crate ethereum_types; extern crate heapsize; @@ -33,7 +33,7 @@ pub trait Hasher: Sync + Send { fn hash(x: &[u8]) -> Self::Out; } -// REVIEW: Where do the concrete Hasher implementations go? Own crate? +// TODO: Move concrete impl to own "façade" crate #[derive(Default, Debug, Clone, PartialEq)] pub struct KeccakHasher; impl Hasher for KeccakHasher { @@ -77,6 +77,7 @@ pub trait HashDB: Send + Sync + AsHashDB { fn remove(&mut self, key: &H::Out); } +//TODO: investigate if we can/should get rid of this trait altogether. There used to be a `impl AsHashDB for T` but that does not work with generics so we currently have concrete impls in `ethcore` for this – perhaps we can just impl the needed methods where needed and get rid of this? /// Upcast trait. pub trait AsHashDB { /// Perform upcast to HashDB for anything that derives from HashDB. diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 4b1ccbed9df..11c8948ffa3 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -17,7 +17,7 @@ rlp = { version = "0.2.1", path = "../rlp" } triehash = { version = "0.1.0", path = "../triehash" } memorydb = { version = "0.2", path = "../memorydb" } ethcore-logger = { version = "1.9.0", path = "../../logger" } -trie-node-codec = { version = "0.1", path = "../trie-node-codec" } +stream-encoder = { path = "../stream-encoder" } [dev-dependencies] trie-standardmap = { path = "../trie-standardmap" } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 7cb26a0427d..19e1f378ff4 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -24,7 +24,7 @@ extern crate keccak_hash as keccak; extern crate memorydb; extern crate rand; extern crate rlp; -extern crate trie_node_codec; +extern crate stream_encoder; #[cfg(test)] extern crate trie_standardmap as standardmap; diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index ab6bcb196a5..7f7bea80985 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,10 +1,10 @@ use bytes::Bytes; use nibbleslice::NibbleSlice; -use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable, Encodable}; +use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable}; use hashdb::Hasher; use node::Node; use std::marker::PhantomData; -use trie_node_codec::Stream; +use stream_encoder::Stream; pub trait NodeCodec: Sized { type E: ::std::error::Error; @@ -20,7 +20,7 @@ pub trait NodeCodec: Sized { pub struct RlpNodeCodec {mark: PhantomData} impl NodeCodec for RlpNodeCodec -where H::Out: Encodable + Decodable +where H::Out: Decodable { type E = DecoderError; type S = RlpStream; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index fa593a1686b..57578f37c6c 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -31,7 +31,7 @@ use std::collections::{HashSet, VecDeque}; use std::marker::PhantomData; use std::mem; use std::ops::Index; -use trie_node_codec::Stream; +use stream_encoder::Stream; // For lookups into the Node storage buffer. // This is deliberately non-copyable. diff --git a/util/rlp/Cargo.toml b/util/rlp/Cargo.toml index 19f4e18a50f..38afd24d9b2 100644 --- a/util/rlp/Cargo.toml +++ b/util/rlp/Cargo.toml @@ -7,7 +7,7 @@ version = "0.2.1" authors = ["Parity Technologies "] [dependencies] -trie-node-codec = { path = "../trie-node-codec"} +stream-encoder = { path = "../stream-encoder"} elastic-array = "0.10" ethereum-types = "0.3" rustc-hex = "1.0" diff --git a/util/rlp/src/lib.rs b/util/rlp/src/lib.rs index f7c35068b90..b084acd0773 100644 --- a/util/rlp/src/lib.rs +++ b/util/rlp/src/lib.rs @@ -36,7 +36,7 @@ extern crate byteorder; extern crate ethereum_types as bigint; extern crate elastic_array; extern crate rustc_hex; -extern crate trie_node_codec; +extern crate stream_encoder; mod traits; mod error; @@ -46,7 +46,7 @@ mod impls; use elastic_array::ElasticArray1024; use std::borrow::Borrow; -use trie_node_codec::Stream; +use stream_encoder::Stream; pub use error::DecoderError; pub use traits::{Decodable, Encodable}; diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index 95776eb4cc7..8bab965b5a7 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -10,7 +10,7 @@ use std::borrow::Borrow; use byteorder::{ByteOrder, BigEndian}; use elastic_array::{ElasticArray16, ElasticArray1024}; use traits::Encodable; -use trie_node_codec::Stream; +use stream_encoder::Stream; #[derive(Debug, Copy, Clone)] struct ListInfo { @@ -162,9 +162,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate trie_node_codec; + /// extern crate stream_encoder; /// use rlp::*; - /// use trie_node_codec::Stream; + /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -280,9 +280,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate trie_node_codec; + /// extern crate stream_encoder; /// use rlp::*; - /// use trie_node_codec::Stream; + /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(3); @@ -304,9 +304,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate trie_node_codec; + /// extern crate stream_encoder; /// use rlp::*; - /// use trie_node_codec::Stream; + /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); diff --git a/util/rlp/src/traits.rs b/util/rlp/src/traits.rs index 8230e8308e4..7f0dee76cf0 100644 --- a/util/rlp/src/traits.rs +++ b/util/rlp/src/traits.rs @@ -9,7 +9,7 @@ //! Common RLP traits use elastic_array::ElasticArray1024; use {DecoderError, Rlp, RlpStream}; -use trie_node_codec::Stream; +use stream_encoder::Stream; /// RLP decodable trait pub trait Decodable: Sized { diff --git a/util/rlp/tests/tests.rs b/util/rlp/tests/tests.rs index a08c23d7f55..e2ce9dfeb67 100644 --- a/util/rlp/tests/tests.rs +++ b/util/rlp/tests/tests.rs @@ -8,12 +8,12 @@ extern crate ethereum_types as bigint; extern crate rlp; -extern crate trie_node_codec; +extern crate stream_encoder; use std::{fmt, cmp}; use bigint::{U256, H160}; use rlp::{Encodable, Decodable, Rlp, RlpStream, DecoderError}; -use trie_node_codec::Stream; +use stream_encoder::Stream; #[test] fn rlp_at() { diff --git a/util/stream-encoder/Cargo.toml b/util/stream-encoder/Cargo.toml new file mode 100644 index 00000000000..aab4a4a33fb --- /dev/null +++ b/util/stream-encoder/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "stream-encoder" +version = "0.1.0" +authors = ["Parity Technologies "] +description = "trait for encoded streams" +license = "GPL-3.0" + +[dependencies] +elastic-array = "0.10" diff --git a/util/stream-encoder/src/lib.rs b/util/stream-encoder/src/lib.rs new file mode 100644 index 00000000000..709be0b8d7a --- /dev/null +++ b/util/stream-encoder/src/lib.rs @@ -0,0 +1,31 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Trait for encoded streams of bytes, used by RLP and other encoding schemes +extern crate elastic_array; + +use elastic_array::ElasticArray1024; + +/// Trait with common encoding operations +// TODO: document this +pub trait Stream { + fn new() -> Self; + fn new_list(len: usize) -> Self; + fn append_empty_data(&mut self) -> &mut Self; + fn drain(self) -> ElasticArray1024; // TODO: add as assoc type? Makes the types kind of hairy and requires some extra trait bounds, but not sure if it's worth it. Needs AsRef I think. + fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; +} \ No newline at end of file diff --git a/util/trie-node-codec/Cargo.toml b/util/trie-node-codec/Cargo.toml deleted file mode 100644 index d8c44d21cce..00000000000 --- a/util/trie-node-codec/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "trie-node-codec" -version = "0.1.0" -authors = ["David Palm "] - -[dependencies] -hashdb = { version = "0.2", path = "../hashdb" } -ethcore-bytes = { version = "0.1.0", path = "../bytes" } -elastic-array = "0.10" diff --git a/util/trie-node-codec/src/lib.rs b/util/trie-node-codec/src/lib.rs deleted file mode 100644 index c40ef16c480..00000000000 --- a/util/trie-node-codec/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -extern crate hashdb; -extern crate ethcore_bytes; -extern crate elastic_array; - -use hashdb::Hasher; -use ethcore_bytes::Bytes; -use elastic_array::ElasticArray1024; - -pub trait Stream { - fn new() -> Self; - fn new_list(len: usize) -> Self; - fn append_empty_data(&mut self) -> &mut Self; - fn drain(self) -> ElasticArray1024; // TODO: add as assoc type? Makes the types kind of hairy and requires some extra trait bounds, but not sure if it's worth it. Needs AsRef I think. - fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; - fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; -} - -pub trait NodeCodec: Sized { - type E: ::std::error::Error; - type S: Stream; - type Node; - fn encode(Self::Node) -> Bytes; - fn decode(data: &[u8]) -> Result; - fn try_decode_hash(data: &[u8]) -> Option; - - fn is_empty_node(data: &[u8]) -> bool; -} \ No newline at end of file diff --git a/util/triehash/Cargo.toml b/util/triehash/Cargo.toml index 3210897e4b1..7468ef6acb5 100644 --- a/util/triehash/Cargo.toml +++ b/util/triehash/Cargo.toml @@ -10,7 +10,7 @@ elastic-array = "0.10" rlp = { version = "0.2.1", path = "../rlp" } ethereum-types = "0.3" keccak-hash = { version = "0.1", path = "../hash" } -trie-node-codec = { version = "0.1", path = "../trie-node-codec" } +stream-encoder = { path = "../stream-encoder" } [dev-dependencies] trie-standardmap = { path = "../trie-standardmap" } diff --git a/util/triehash/src/lib.rs b/util/triehash/src/lib.rs index cfffdee7783..e6fefb09378 100644 --- a/util/triehash/src/lib.rs +++ b/util/triehash/src/lib.rs @@ -22,7 +22,7 @@ extern crate elastic_array; extern crate ethereum_types; extern crate keccak_hash as hash; extern crate rlp; -extern crate trie_node_codec; +extern crate stream_encoder; use std::collections::BTreeMap; use std::cmp; @@ -30,7 +30,7 @@ use elastic_array::{ElasticArray4, ElasticArray8}; use ethereum_types::H256; use hash::keccak; use rlp::RlpStream; -use trie_node_codec::Stream; +use stream_encoder::Stream; fn shared_prefix_len(first: &[T], second: &[T]) -> usize { let len = cmp::min(first.len(), second.len()); From 574a22a4282a3d86cfa277acec62b3e9050195d5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 12:56:50 +0200 Subject: [PATCH 074/160] Address some grumbles More TODOs and REVIEWs --- util/patricia_trie/src/lib.rs | 12 ++++++++--- util/patricia_trie/src/node_codec.rs | 31 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 19e1f378ff4..e734d479737 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -59,7 +59,7 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; use node_codec::NodeCodec; -// REVIEW: is this overdoing it? +// TODO: Move to façade crate and rename to just `RlpNodeCodec` pub type KeccakRlpNodeCodec = node_codec::RlpNodeCodec; pub type KeccakTrieResult = Result::Out>; @@ -121,8 +121,7 @@ pub trait Query { fn decode(self, data: &[u8]) -> Self::Item; /// Record that a node has been passed through. - #[allow(unused_variables)] - fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) {} + fn record(&mut self, _hash: &H::Out, _data: &[u8], _depth: u32) {} } impl<'a, H: Hasher> Query for &'a mut Recorder { @@ -153,6 +152,13 @@ pub trait Trie { fn root(&self) -> &::Out; /// Is the trie empty? + // TODO: The `Hasher` should not carry RLP-specifics; the null node should live in `NodeCodec`. Not sure what the best way to deal with this is. + // rpheimer: specifically, this should draw from the node codec, because non-rlp-encoded tries will have + // non-rlp-encoded null nodes. so the hash will be different. this is an architectural issue: + // `trait NodeCodec` should have the `const HASH_NULL_NODE: H::Out`. This will + // make `impl NodeCodec for MyGenericCodec` impossible without better const + // function evaluation, so we should limit ourselves only to + // `impl NodeCodec for MyGenericCodec` or similar fn is_empty(&self) -> bool { *self.root() == Self::H::HASHED_NULL_RLP } /// Does the trie contain a given key? diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 7f7bea80985..93b8c821c19 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -1,3 +1,22 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Generic trait for trie node encoding/decoding. Takes a `hashdb::Hasher` +//! to parametrize the hashes used in the codec; takes a `stream_encoder::Stream` +//! implementation to do streaming encoding. use bytes::Bytes; use nibbleslice::NibbleSlice; use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable}; @@ -6,19 +25,29 @@ use node::Node; use std::marker::PhantomData; use stream_encoder::Stream; +/// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { type E: ::std::error::Error; type S: Stream; + + /// Encode a Node to bytes (`Vec`). fn encode(&Node) -> Bytes; + + /// Decode bytes to a `Node`. Returns `Result` on failure. fn decode(data: &[u8]) -> Result; + + /// Decode bytes to the `Hasher`s output type. Assumes 32 bytes long hashes! Returns `None` on failure. fn try_decode_hash(data: &[u8]) -> Option; + // Check if the provided bytes correspond to the codecs "empty" node. fn is_empty_node(data: &[u8]) -> bool; } #[derive(Default, Clone)] +// TODO: move this to façade crate pub struct RlpNodeCodec {mark: PhantomData} +// TODO: move this to façade crate impl NodeCodec for RlpNodeCodec where H::Out: Decodable { @@ -84,7 +113,7 @@ where H::Out: Decodable } fn try_decode_hash(data: &[u8]) -> Option { let r = Rlp::new(data); - if r.is_data() && r.size() == 32 { + if r.is_data() && r.size() == 32 { // REVIEW: this makes an assumption on the length of the `Hasher` output type – should it be `std::mem::size_of(H::Out)`? Some(r.as_val().expect("Hash is the correct size of 32 bytes; qed")) } else { None From 42df94e50f9203edfa70a8e9e9615c5e661c38a5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 13:08:36 +0200 Subject: [PATCH 075/160] Add `LENGTH` const to `Hasher` and use it to remove the assumption of 32 bytes long hashes --- util/hashdb/src/lib.rs | 3 ++- util/patricia_trie/src/node_codec.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 8117e12d530..43146d39e8c 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -30,6 +30,7 @@ use tiny_keccak::Keccak; pub trait Hasher: Sync + Send { type Out: AsRef<[u8]> + Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? --> */ + HeapSizeOf; const HASHED_NULL_RLP: Self::Out; + const LENGTH: usize; fn hash(x: &[u8]) -> Self::Out; } @@ -39,7 +40,7 @@ pub struct KeccakHasher; impl Hasher for KeccakHasher { type Out = H256; const HASHED_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); - + const LENGTH: usize = 32; fn hash(x: &[u8]) -> Self::Out { let mut out = [0;32]; Keccak::keccak256(x, &mut out); diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 93b8c821c19..3c0fd76139b 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -113,8 +113,8 @@ where H::Out: Decodable } fn try_decode_hash(data: &[u8]) -> Option { let r = Rlp::new(data); - if r.is_data() && r.size() == 32 { // REVIEW: this makes an assumption on the length of the `Hasher` output type – should it be `std::mem::size_of(H::Out)`? - Some(r.as_val().expect("Hash is the correct size of 32 bytes; qed")) + if r.is_data() && r.size() == H::LENGTH { + Some(r.as_val().expect("Hash is the correct size; qed")) } else { None } From f081b357f043dadf46a884347b7bf360bc8347df Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 13:49:16 +0200 Subject: [PATCH 076/160] remove solved review comment --- util/patricia_trie/src/triedbmut.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 57578f37c6c..68de9e0fc96 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -857,7 +857,6 @@ where H: Hasher, /// commit a node, hashing it, committing it to the db, /// and writing it to the encoded stream as necessary. - // REVIEW: this is causing the `Encodable` bound all the way upstream, precisely it's the `stream.append()` call requires it fn commit_node(&mut self, handle: NodeHandle, stream: &mut >::S) { match handle { NodeHandle::Hash(h) => { From c6333af9a38c5728942327f3729af175daa6fdd3 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 14:26:44 +0200 Subject: [PATCH 077/160] Add stream_encoder::Stream where needed --- Cargo.lock | 7 +++++++ ethcore/Cargo.toml | 1 + ethcore/light/Cargo.toml | 1 + ethcore/light/src/cht.rs | 1 + ethcore/light/src/client/header_chain.rs | 1 + ethcore/light/src/lib.rs | 1 + ethcore/light/src/net/mod.rs | 1 + ethcore/light/src/net/status.rs | 1 + ethcore/light/src/net/tests/mod.rs | 2 ++ ethcore/light/src/on_demand/request.rs | 1 + ethcore/light/src/provider.rs | 2 +- ethcore/light/src/types/request/mod.rs | 3 +++ ethcore/src/block.rs | 1 + ethcore/src/blockchain/blockchain.rs | 1 + ethcore/src/client/test_client.rs | 1 + ethcore/src/encoded.rs | 1 + ethcore/src/engines/authority_round/mod.rs | 1 + ethcore/src/engines/tendermint/message.rs | 1 + ethcore/src/engines/tendermint/mod.rs | 1 + ethcore/src/engines/validator_set/safe_contract.rs | 1 + ethcore/src/executive.rs | 1 + ethcore/src/header.rs | 1 + ethcore/src/lib.rs | 1 + ethcore/src/pod_account.rs | 1 + ethcore/src/snapshot/account.rs | 1 + ethcore/src/snapshot/block.rs | 1 + ethcore/src/snapshot/consensus/authority.rs | 1 + ethcore/src/snapshot/consensus/work.rs | 1 + ethcore/src/snapshot/io.rs | 1 + ethcore/src/snapshot/mod.rs | 1 + ethcore/src/spec/seal.rs | 1 + ethcore/src/spec/spec.rs | 1 + ethcore/src/state/account.rs | 1 + ethcore/src/test_helpers.rs | 1 + ethcore/transaction/Cargo.toml | 1 + ethcore/transaction/src/lib.rs | 1 + ethcore/transaction/src/transaction.rs | 1 + ethcore/types/Cargo.toml | 1 + ethcore/types/src/lib.rs | 2 ++ ethcore/types/src/snapshot_manifest.rs | 1 + util/network-devp2p/Cargo.toml | 1 + util/network-devp2p/src/connection.rs | 1 + util/network-devp2p/src/discovery.rs | 1 + util/network-devp2p/src/handshake.rs | 1 + util/network-devp2p/src/lib.rs | 1 + util/network-devp2p/src/session.rs | 1 + util/rlp_compress/Cargo.toml | 1 + util/rlp_compress/src/lib.rs | 2 ++ util/version/Cargo.toml | 1 + util/version/src/lib.rs | 2 ++ 50 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c10f3fce9cc..85782e60080 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,6 +255,7 @@ dependencies = [ "rlp 0.2.1", "rlp_derive 0.1.0", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-encoder 0.1.0", ] [[package]] @@ -565,6 +566,7 @@ dependencies = [ "snappy 0.1.0 (git+https://github.com/paritytech/rust-snappy)", "stats 0.1.0", "stop-guard 0.1.0", + "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "trace-time 0.1.0", "trie-standardmap 0.1.0", @@ -649,6 +651,7 @@ dependencies = [ "serde_derive 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", + "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "triehash 0.1.0", "vm 0.1.0", @@ -746,6 +749,7 @@ dependencies = [ "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "snappy 0.1.0 (git+https://github.com/paritytech/rust-snappy)", + "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -905,6 +909,7 @@ dependencies = [ "keccak-hash 0.1.2", "rlp 0.2.1", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-encoder 0.1.0", "unexpected 0.1.0", ] @@ -2324,6 +2329,7 @@ dependencies = [ "ethcore-bytes 0.1.0", "rlp 0.2.1", "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-encoder 0.1.0", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "vergen 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2763,6 +2769,7 @@ dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", + "stream-encoder 0.1.0", ] [[package]] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index cedb2b99284..472f7746155 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -67,6 +67,7 @@ keccak-hash = { path = "../util/hash" } triehash = { path = "../util/triehash" } unexpected = { path = "../util/unexpected" } journaldb = { path = "../util/journaldb" } +stream-encoder = { path = "../util/stream-encoder" } [dev-dependencies] tempdir = "0.3" diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index ab347c62e83..609d4e1c1d2 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -36,6 +36,7 @@ triehash = { path = "../../util/triehash" } kvdb = { path = "../../util/kvdb" } memory-cache = { path = "../../util/memory_cache" } error-chain = { version = "0.11", default-features = false } +stream-encoder = { path = "../../util/stream-encoder" } [dev-dependencies] ethcore = { path = "..", features = ["test-helpers"] } diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index bc2c4f601cb..1ab88b34c05 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -30,6 +30,7 @@ use memorydb::MemoryDB; use bytes::Bytes; use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder, KeccakRlpNodeCodec}; use rlp::{RlpStream, Rlp}; +use stream_encoder::Stream; // encode a key. macro_rules! key { diff --git a/ethcore/light/src/client/header_chain.rs b/ethcore/light/src/client/header_chain.rs index 37c2812d146..3df994fb7b3 100644 --- a/ethcore/light/src/client/header_chain.rs +++ b/ethcore/light/src/client/header_chain.rs @@ -47,6 +47,7 @@ use heapsize::HeapSizeOf; use kvdb::{DBTransaction, KeyValueDB}; use plain_hasher::H256FastMap; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; +use stream_encoder::Stream; use cache::Cache; use parking_lot::{Mutex, RwLock}; diff --git a/ethcore/light/src/lib.rs b/ethcore/light/src/lib.rs index d7469fdcee0..7a630c22976 100644 --- a/ethcore/light/src/lib.rs +++ b/ethcore/light/src/lib.rs @@ -69,6 +69,7 @@ extern crate patricia_trie as trie; extern crate plain_hasher; extern crate rand; extern crate rlp; +extern crate stream_encoder; extern crate parking_lot; #[macro_use] extern crate rlp_derive; diff --git a/ethcore/light/src/net/mod.rs b/ethcore/light/src/net/mod.rs index 179f7090036..5a3a12ec4a6 100644 --- a/ethcore/light/src/net/mod.rs +++ b/ethcore/light/src/net/mod.rs @@ -23,6 +23,7 @@ use transaction::UnverifiedTransaction; use io::TimerToken; use network::{NetworkProtocolHandler, NetworkContext, PeerId}; use rlp::{RlpStream, Rlp}; +use stream_encoder::Stream; use ethereum_types::{H256, U256}; use kvdb::DBValue; use parking_lot::{Mutex, RwLock}; diff --git a/ethcore/light/src/net/status.rs b/ethcore/light/src/net/status.rs index d89db173cb8..dcc14f87c0e 100644 --- a/ethcore/light/src/net/status.rs +++ b/ethcore/light/src/net/status.rs @@ -17,6 +17,7 @@ //! Peer status and capabilities. use rlp::{DecoderError, Encodable, Decodable, RlpStream, Rlp}; +use stream_encoder::Stream; use ethereum_types::{H256, U256}; use super::request_credits::FlowParams; diff --git a/ethcore/light/src/net/tests/mod.rs b/ethcore/light/src/net/tests/mod.rs index e182f38b8a6..b2f159932a4 100644 --- a/ethcore/light/src/net/tests/mod.rs +++ b/ethcore/light/src/net/tests/mod.rs @@ -32,6 +32,7 @@ use request; use request::*; use rlp::{Rlp, RlpStream}; +use stream_encoder::Stream; use ethereum_types::{H256, U256, Address}; use std::sync::Arc; @@ -642,6 +643,7 @@ fn proof_of_execution() { fn id_guard() { use super::request_set::RequestSet; use super::ReqId; + use stream_encoder::Stream; let capabilities = capabilities(); diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 90ab21f01d2..4238b2af6c8 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -31,6 +31,7 @@ use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY, KECCAK_EMPTY_LIST_RLP, keccak}; use request::{self as net_request, IncompleteRequest, CompleteRequest, Output, OutputKind, Field}; use rlp::{RlpStream, Rlp}; +use stream_encoder::Stream; use ethereum_types::{H256, U256, Address}; use parking_lot::Mutex; use hashdb::{HashDB, Hasher, KeccakHasher}; diff --git a/ethcore/light/src/provider.rs b/ethcore/light/src/provider.rs index 4344c363aa6..e75915e8cf8 100644 --- a/ethcore/light/src/provider.rs +++ b/ethcore/light/src/provider.rs @@ -386,7 +386,7 @@ impl AsLightClient for LightProvider { #[cfg(test)] mod tests { - use ::ethcore::client::{EachBlockWith, TestBlockChainClient}; + use ethcore::client::{EachBlockWith, TestBlockChainClient}; use super::Provider; #[test] diff --git a/ethcore/light/src/types/request/mod.rs b/ethcore/light/src/types/request/mod.rs index 538aa0c6bf4..96a7227cb22 100644 --- a/ethcore/light/src/types/request/mod.rs +++ b/ethcore/light/src/types/request/mod.rs @@ -672,6 +672,7 @@ pub mod header { use super::{Field, HashOrNumber, NoSuchOutput, OutputKind, Output}; use ethcore::encoded; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; + use stream_encoder::Stream; /// Potentially incomplete headers request. #[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] @@ -1026,6 +1027,7 @@ pub mod block_body { use super::{Field, NoSuchOutput, OutputKind, Output}; use ethcore::encoded; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; + use stream_encoder::Stream; use ethereum_types::H256; /// Potentially incomplete block body request. @@ -1634,6 +1636,7 @@ pub mod epoch_signal { mod tests { use super::*; use ethcore::header::Header; + use stream_encoder::Stream; fn check_roundtrip(val: T) where T: ::rlp::Encodable + ::rlp::Decodable + PartialEq + ::std::fmt::Debug diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 682171170e1..c705005423b 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -23,6 +23,7 @@ use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP}; use triehash::ordered_trie_root; use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError, encode_list}; +use stream_encoder::Stream; use ethereum_types::{H256, U256, Address, Bloom}; use bytes::Bytes; use unexpected::{Mismatch, OutOfBounds}; diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index ee781ebe534..89e6df6c144 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -48,6 +48,7 @@ use engines::epoch::{Transition as EpochTransition, PendingTransition as Pending use rayon::prelude::*; use ansi_term::Colour; use kvdb::{DBTransaction, KeyValueDB}; +use stream_encoder::Stream; const LOG_BLOOMS_LEVELS: usize = 3; const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 69aff39409b..814dc7d20ca 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -30,6 +30,7 @@ use kvdb::DBValue; use kvdb_memorydb; use bytes::Bytes; use rlp::{Rlp, RlpStream}; +use stream_encoder::Stream; use ethkey::{Generator, Random}; use ethcore_miner::pool::VerifiedTransaction; use transaction::{self, Transaction, LocalizedTransaction, SignedTransaction, Action}; diff --git a/ethcore/src/encoded.rs b/ethcore/src/encoded.rs index 5bd723f0e24..19436dfd8be 100644 --- a/ethcore/src/encoded.rs +++ b/ethcore/src/encoded.rs @@ -29,6 +29,7 @@ use hash::keccak; use header::{BlockNumber, Header as FullHeader}; use heapsize::HeapSizeOf; use rlp::{self, Rlp, RlpStream}; +use stream_encoder::Stream; use transaction::UnverifiedTransaction; use views::{self, BlockView, HeaderView, BodyView}; diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index e2e88c8178d..492dc0d51f3 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -44,6 +44,7 @@ use ethkey::{self, Signature}; use io::{IoContext, IoHandler, TimerToken, IoService}; use itertools::{self, Itertools}; use rlp::{encode, Decodable, DecoderError, Encodable, RlpStream, Rlp}; +use stream_encoder::Stream; use ethereum_types::{H256, H520, Address, U128, U256}; use parking_lot::{Mutex, RwLock}; use unexpected::{Mismatch, OutOfBounds}; diff --git a/ethcore/src/engines/tendermint/message.rs b/ethcore/src/engines/tendermint/message.rs index ba8e4390ec1..3b5c78a08a7 100644 --- a/ethcore/src/engines/tendermint/message.rs +++ b/ethcore/src/engines/tendermint/message.rs @@ -26,6 +26,7 @@ use header::Header; use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError}; use ethkey::{recover, public_to_address}; use super::super::vote_collector::Message; +use stream_encoder::Stream; /// Message transmitted between consensus participants. #[derive(Debug, PartialEq, Eq, Clone, Hash, Default)] diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index 967ef482a69..1fe8b4640f9 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -50,6 +50,7 @@ use super::vote_collector::VoteCollector; use self::message::*; use self::params::TendermintParams; use machine::{AuxiliaryData, EthereumMachine}; +use stream_encoder::Stream; #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] pub enum Step { diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index e55a0e3e36a..d341b0514a5 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -34,6 +34,7 @@ use header::Header; use ids::BlockId; use log_entry::LogEntry; use receipt::Receipt; +use stream_encoder::Stream; use super::{SystemCall, ValidatorSet}; use super::simple_list::SimpleList; diff --git a/ethcore/src/executive.rs b/ethcore/src/executive.rs index f375f3c2e88..491d126afec 100644 --- a/ethcore/src/executive.rs +++ b/ethcore/src/executive.rs @@ -33,6 +33,7 @@ use trace::{self, Tracer, VMTracer}; use transaction::{Action, SignedTransaction}; use crossbeam; pub use executed::{Executed, ExecutionResult}; +use stream_encoder::Stream; #[cfg(debug_assertions)] /// Roughly estimate what stack size each level of evm depth will use. (Debug build) diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index 5aa4be32375..ad383f1ab85 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -22,6 +22,7 @@ use heapsize::HeapSizeOf; use ethereum_types::{H256, U256, Address, Bloom}; use bytes::Bytes; use rlp::{Rlp, RlpStream, Encodable, DecoderError, Decodable}; +use stream_encoder::Stream; pub use types::BlockNumber; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 51e75d4b40a..961958e214d 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -98,6 +98,7 @@ extern crate kvdb; extern crate kvdb_memorydb; extern crate util_error; extern crate snappy; +extern crate stream_encoder; extern crate ethabi; extern crate rustc_hex; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 08f22bd65d2..f2c5199c578 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -27,6 +27,7 @@ use state::Account; use ethjson; use types::account_diff::*; use rlp::{self, RlpStream}; +use stream_encoder::Stream; #[derive(Debug, Clone, PartialEq, Eq)] /// An account, expressed as Plain-Old-Data (hence the name). diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 47469063b86..eb3044bf7aa 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -26,6 +26,7 @@ use hashdb::HashDB; use bytes::Bytes; use trie::{TrieDB, Trie, KeccakRlpNodeCodec}; use rlp::{RlpStream, Rlp}; +use stream_encoder::Stream; use std::collections::HashSet; diff --git a/ethcore/src/snapshot/block.rs b/ethcore/src/snapshot/block.rs index cec2e491a70..77944eed99d 100644 --- a/ethcore/src/snapshot/block.rs +++ b/ethcore/src/snapshot/block.rs @@ -25,6 +25,7 @@ use rlp::{DecoderError, RlpStream, Rlp}; use ethereum_types::H256; use bytes::Bytes; use triehash::ordered_trie_root; +use stream_encoder::Stream; const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; diff --git a/ethcore/src/snapshot/consensus/authority.rs b/ethcore/src/snapshot/consensus/authority.rs index 0bb6d232955..922c45bc7ba 100644 --- a/ethcore/src/snapshot/consensus/authority.rs +++ b/ethcore/src/snapshot/consensus/authority.rs @@ -37,6 +37,7 @@ use rlp::{RlpStream, Rlp}; use ethereum_types::{H256, U256}; use kvdb::KeyValueDB; use bytes::Bytes; +use stream_encoder::Stream; /// Snapshot creation and restoration for PoA chains. /// Chunk format: diff --git a/ethcore/src/snapshot/consensus/work.rs b/ethcore/src/snapshot/consensus/work.rs index 31c7b51ec55..bdfe60ead4a 100644 --- a/ethcore/src/snapshot/consensus/work.rs +++ b/ethcore/src/snapshot/consensus/work.rs @@ -35,6 +35,7 @@ use kvdb::KeyValueDB; use bytes::Bytes; use rlp::{RlpStream, Rlp}; use rand::OsRng; +use stream_encoder::Stream; /// Snapshot creation and restoration for PoW chains. /// This includes blocks from the head of the chain as a diff --git a/ethcore/src/snapshot/io.rs b/ethcore/src/snapshot/io.rs index 7d2cbcf92cf..cb9c9105abb 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/src/snapshot/io.rs @@ -28,6 +28,7 @@ use std::path::{Path, PathBuf}; use bytes::Bytes; use ethereum_types::H256; use rlp::{RlpStream, Rlp}; +use stream_encoder::Stream; use super::ManifestData; diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 7807f85f71c..f329f4dd227 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -41,6 +41,7 @@ use kvdb::KeyValueDB; use trie::{TrieDB, TrieDBMut, Trie, TrieMut, KeccakRlpNodeCodec}; use rlp::{RlpStream, Rlp}; use bloom_journal::Bloom; +use stream_encoder::Stream; use self::io::SnapshotWriter; diff --git a/ethcore/src/spec/seal.rs b/ethcore/src/spec/seal.rs index 0ed41acc846..c9f875b5292 100644 --- a/ethcore/src/spec/seal.rs +++ b/ethcore/src/spec/seal.rs @@ -19,6 +19,7 @@ use rlp::RlpStream; use ethereum_types::{H64, H256, H520}; use ethjson; +use stream_encoder::Stream; /// Classic ethereum seal. pub struct Ethereum { diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index 6f785fe7fba..8a1035d20d2 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -45,6 +45,7 @@ use spec::seal::Generic as GenericSeal; use state::backend::Basic as BasicBackend; use state::{Backend, State, Substate}; use trace::{NoopTracer, NoopVMTracer}; +use stream_encoder::Stream; pub use ethash::OptimizeFor; diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index e99aa9dd14d..9a796de214b 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -31,6 +31,7 @@ use pod_account::*; use rlp::{RlpStream, encode}; use lru_cache::LruCache; use basic_account::BasicAccount; +use stream_encoder::Stream; use std::cell::{RefCell, Cell}; diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index 4a83752c046..ec9f29d1213 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -37,6 +37,7 @@ use state::*; use std::sync::Arc; use transaction::{Action, Transaction, SignedTransaction}; use views::BlockView; +use stream_encoder::Stream; /// Creates test block with corresponding header pub fn create_test_block(header: &Header) -> Bytes { diff --git a/ethcore/transaction/Cargo.toml b/ethcore/transaction/Cargo.toml index 79e7282c388..317030dbfbd 100644 --- a/ethcore/transaction/Cargo.toml +++ b/ethcore/transaction/Cargo.toml @@ -13,6 +13,7 @@ keccak-hash = { path = "../../util/hash" } rlp = { path = "../../util/rlp" } unexpected = { path = "../../util/unexpected" } ethereum-types = "0.3" +stream-encoder = { path = "../../util/stream-encoder" } [dev-dependencies] rustc-hex= "1.0" diff --git a/ethcore/transaction/src/lib.rs b/ethcore/transaction/src/lib.rs index 829613cf9c7..48d1c23d462 100644 --- a/ethcore/transaction/src/lib.rs +++ b/ethcore/transaction/src/lib.rs @@ -24,6 +24,7 @@ extern crate heapsize; extern crate keccak_hash as hash; extern crate rlp; extern crate unexpected; +extern crate stream_encoder; #[cfg(test)] extern crate rustc_hex; diff --git a/ethcore/transaction/src/transaction.rs b/ethcore/transaction/src/transaction.rs index 27b8b346cf2..add4525cb90 100644 --- a/ethcore/transaction/src/transaction.rs +++ b/ethcore/transaction/src/transaction.rs @@ -25,6 +25,7 @@ use evm::Schedule; use hash::keccak; use heapsize::HeapSizeOf; use rlp::{self, RlpStream, Rlp, DecoderError, Encodable}; +use stream_encoder::Stream; type Bytes = Vec; type BlockNumber = u64; diff --git a/ethcore/types/Cargo.toml b/ethcore/types/Cargo.toml index 92cc74551d3..29ca4317a09 100644 --- a/ethcore/types/Cargo.toml +++ b/ethcore/types/Cargo.toml @@ -12,6 +12,7 @@ ethereum-types = "0.3" ethjson = { path = "../../json" } keccak-hash = { path = "../../util/hash" } heapsize = "0.4" +stream-encoder = { path = "../../util/stream-encoder" } [dev-dependencies] rustc-hex= "1.0" diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index 5ac8ff12add..de23cdfa2a1 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -20,6 +20,8 @@ extern crate ethereum_types; extern crate ethcore_bytes as bytes; extern crate ethjson; extern crate rlp; +extern crate stream_encoder; + #[macro_use] extern crate rlp_derive; extern crate keccak_hash as hash; diff --git a/ethcore/types/src/snapshot_manifest.rs b/ethcore/types/src/snapshot_manifest.rs index 40ff4c532f7..0709c74230b 100644 --- a/ethcore/types/src/snapshot_manifest.rs +++ b/ethcore/types/src/snapshot_manifest.rs @@ -19,6 +19,7 @@ use ethereum_types::H256; use rlp::{Rlp, RlpStream, DecoderError}; use bytes::Bytes; +use stream_encoder::Stream; /// Manifest data. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/util/network-devp2p/Cargo.toml b/util/network-devp2p/Cargo.toml index 4a5d2d942ec..e185ca83029 100644 --- a/util/network-devp2p/Cargo.toml +++ b/util/network-devp2p/Cargo.toml @@ -35,6 +35,7 @@ serde = "1.0" serde_json = "1.0" serde_derive = "1.0" error-chain = { version = "0.11", default-features = false } +stream-encoder = { path = "../stream-encoder" } [dev-dependencies] tempdir = "0.3" diff --git a/util/network-devp2p/src/connection.rs b/util/network-devp2p/src/connection.rs index 37824ae5d75..b08dccc99e2 100644 --- a/util/network-devp2p/src/connection.rs +++ b/util/network-devp2p/src/connection.rs @@ -36,6 +36,7 @@ use tiny_keccak::Keccak; use bytes::{Buf, BufMut}; use ethkey::crypto; use network::{Error, ErrorKind}; +use stream_encoder::Stream; const ENCRYPTED_HEADER_LEN: usize = 32; const RECEIVE_PAYLOAD: Duration = Duration::from_secs(30); diff --git a/util/network-devp2p/src/discovery.rs b/util/network-devp2p/src/discovery.rs index 5f8f0cdbc2a..ea170fe50a8 100644 --- a/util/network-devp2p/src/discovery.rs +++ b/util/network-devp2p/src/discovery.rs @@ -26,6 +26,7 @@ use node_table::*; use network::{Error, ErrorKind}; use ethkey::{Secret, KeyPair, sign, recover}; use network::IpFilter; +use stream_encoder::Stream; use PROTOCOL_VERSION; diff --git a/util/network-devp2p/src/handshake.rs b/util/network-devp2p/src/handshake.rs index 18869de55fc..64948a11028 100644 --- a/util/network-devp2p/src/handshake.rs +++ b/util/network-devp2p/src/handshake.rs @@ -28,6 +28,7 @@ use ethkey::{KeyPair, Public, Secret, recover, sign, Generator, Random}; use ethkey::crypto::{ecdh, ecies}; use network::{Error, ErrorKind}; use host::HostInfo; +use stream_encoder::Stream; #[derive(PartialEq, Eq, Debug)] enum HandshakeState { diff --git a/util/network-devp2p/src/lib.rs b/util/network-devp2p/src/lib.rs index 01fc1fe25f4..a09b12d6b43 100644 --- a/util/network-devp2p/src/lib.rs +++ b/util/network-devp2p/src/lib.rs @@ -85,6 +85,7 @@ extern crate keccak_hash as hash; extern crate serde; extern crate serde_json; extern crate snappy; +extern crate stream_encoder; #[macro_use] extern crate error_chain; diff --git a/util/network-devp2p/src/session.rs b/util/network-devp2p/src/session.rs index a405ad469dd..3d9fc91700d 100644 --- a/util/network-devp2p/src/session.rs +++ b/util/network-devp2p/src/session.rs @@ -32,6 +32,7 @@ use network::SessionCapabilityInfo; use host::*; use node_table::NodeId; use snappy; +use stream_encoder::Stream; // Timeout must be less than (interval - 1). const PING_TIMEOUT: Duration = Duration::from_secs(60); diff --git a/util/rlp_compress/Cargo.toml b/util/rlp_compress/Cargo.toml index d5f85425f23..847354ccbec 100644 --- a/util/rlp_compress/Cargo.toml +++ b/util/rlp_compress/Cargo.toml @@ -7,3 +7,4 @@ authors = ["Parity Technologies "] rlp = { path = "../rlp" } elastic-array = "0.10" lazy_static = "1.0" +stream-encoder = { path = "../stream-encoder" } diff --git a/util/rlp_compress/src/lib.rs b/util/rlp_compress/src/lib.rs index af5b09aac79..e600c7b9521 100644 --- a/util/rlp_compress/src/lib.rs +++ b/util/rlp_compress/src/lib.rs @@ -10,6 +10,7 @@ extern crate elastic_array; #[macro_use] extern crate lazy_static; extern crate rlp; +extern crate stream_encoder; mod common; @@ -18,6 +19,7 @@ use std::collections::HashMap; use elastic_array::ElasticArray1024; use rlp::{Rlp, RlpStream}; use common::{SNAPSHOT_SWAPPER, BLOCKS_SWAPPER}; +use stream_encoder::Stream; pub fn snapshot_swapper() -> &'static Swapper<'static> { &SNAPSHOT_SWAPPER as &Swapper diff --git a/util/version/Cargo.toml b/util/version/Cargo.toml index c7bd4f581e3..9273d84db26 100644 --- a/util/version/Cargo.toml +++ b/util/version/Cargo.toml @@ -23,6 +23,7 @@ kovan = { forkBlock = 6600000, critical = false } [dependencies] ethcore-bytes = { path = "../bytes" } rlp = { path = "../rlp" } +stream-encoder = { path = "../stream-encoder" } target_info = "0.1" [build-dependencies] diff --git a/util/version/src/lib.rs b/util/version/src/lib.rs index 77fc71c70f3..01ffd8b360f 100644 --- a/util/version/src/lib.rs +++ b/util/version/src/lib.rs @@ -19,10 +19,12 @@ extern crate target_info; extern crate ethcore_bytes as bytes; extern crate rlp; +extern crate stream_encoder; use target_info::Target; use bytes::Bytes; use rlp::RlpStream; +use stream_encoder::Stream; mod vergen { #![allow(unused)] From c8a1c939df6b87b3c1cd4ff9fd66fa0be91d5438 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 14:57:35 +0200 Subject: [PATCH 078/160] Fix tests in ethcore --- ethcore/src/blockchain/extras.rs | 1 + ethcore/src/snapshot/tests/proof_of_work.rs | 1 + ethcore/src/snapshot/tests/state.rs | 1 + ethcore/src/tests/client.rs | 1 + ethcore/src/trace/types/error.rs | 1 + ethcore/src/trace/types/flat.rs | 1 + ethcore/src/verification/verification.rs | 2 ++ 7 files changed, 8 insertions(+) diff --git a/ethcore/src/blockchain/extras.rs b/ethcore/src/blockchain/extras.rs index 30dbec707e5..ab4edb858e1 100644 --- a/ethcore/src/blockchain/extras.rs +++ b/ethcore/src/blockchain/extras.rs @@ -280,6 +280,7 @@ pub struct EpochTransitions { #[cfg(test)] mod tests { use rlp::*; + use stream_encoder::Stream; use super::BlockReceipts; #[test] diff --git a/ethcore/src/snapshot/tests/proof_of_work.rs b/ethcore/src/snapshot/tests/proof_of_work.rs index e689edf80fa..64e0fd4a62e 100644 --- a/ethcore/src/snapshot/tests/proof_of_work.rs +++ b/ethcore/src/snapshot/tests/proof_of_work.rs @@ -116,6 +116,7 @@ fn chunk_and_restore_4k() { #[test] fn checks_flag() { use rlp::RlpStream; + use stream_encoder::Stream; use ethereum_types::H256; let mut stream = RlpStream::new_list(5); diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index 12f19e8c27c..fa7d5047615 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -95,6 +95,7 @@ fn snap_and_restore() { fn get_code_from_prev_chunk() { use std::collections::HashSet; use rlp::RlpStream; + use stream_encoder::Stream; use ethereum_types::{H256, U256}; use hashdb::HashDB; diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index e18b4db9832..cafefcbe562 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -37,6 +37,7 @@ use ethkey::KeyPair; use transaction::{PendingTransaction, Transaction, Action, Condition}; use miner::MinerService; use rlp::{RlpStream, EMPTY_LIST_RLP}; +use stream_encoder::Stream; use tempdir::TempDir; #[test] diff --git a/ethcore/src/trace/types/error.rs b/ethcore/src/trace/types/error.rs index a934443c5db..f449fc62c29 100644 --- a/ethcore/src/trace/types/error.rs +++ b/ethcore/src/trace/types/error.rs @@ -138,6 +138,7 @@ impl Decodable for Error { #[cfg(test)] mod tests { use rlp::*; + use stream_encoder::Stream; use super::Error; #[test] diff --git a/ethcore/src/trace/types/flat.rs b/ethcore/src/trace/types/flat.rs index 8610692200c..ab1a73886c5 100644 --- a/ethcore/src/trace/types/flat.rs +++ b/ethcore/src/trace/types/flat.rs @@ -137,6 +137,7 @@ impl Into> for FlatBlockTraces { #[cfg(test)] mod tests { use rlp::*; + use stream_encoder::Stream; use super::{FlatBlockTraces, FlatTransactionTraces, FlatTrace}; use trace::trace::{Action, Res, CallResult, Call, Suicide, Reward}; use evm::CallType; diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 1275b5c5c3f..3980a074ede 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -365,6 +365,7 @@ mod tests { use transaction::{SignedTransaction, Transaction, UnverifiedTransaction, Action}; use types::log_entry::{LogEntry, LocalizedLogEntry}; use rlp; + use stream_encoder::Stream; use triehash::ordered_trie_root; fn check_ok(result: Result<(), Error>) { @@ -545,6 +546,7 @@ mod tests { #[test] fn test_verify_block() { use rlp::RlpStream; + use stream_encoder::Stream; // Test against morden let mut good = Header::new(); From a66596d2e94be6af312ce9aa3e7051b5f6fd0f8d Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 16:00:03 +0200 Subject: [PATCH 079/160] A few grumbles and made more tests passing --- Cargo.lock | 1 + util/memorydb/src/lib.rs | 16 ---------------- util/patricia_trie/src/fatdbmut.rs | 2 -- whisper/Cargo.toml | 1 + whisper/src/lib.rs | 1 + whisper/src/message.rs | 1 + whisper/src/net/mod.rs | 1 + 7 files changed, 5 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85782e60080..ae9b8159ffe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2370,6 +2370,7 @@ dependencies = [ "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-encoder 0.1.0", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index c85ee4866f5..c797c9d4389 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -336,20 +336,4 @@ mod tests { assert_eq!(overlay.get(&insert_key).unwrap(), &(DBValue::from_slice(b"arf"), 2)); assert_eq!(overlay.get(&negative_remove_key).unwrap(), &(DBValue::from_slice(b"negative"), -2)); } - -// #[test] -// fn other_hashers() { -// struct DummyHasher; -// impl Hasher for DummyHasher { -// type Out = ethereum_types::H160; -// const HASHED_NULL_RLP: ethereum_types::H160= ethereum_types::H160([0; 20]); -// fn hash(_x: &[u8]) -> Self::Out { ethereum_types::H160(*b"01010202010102020101") } -// } -// impl HeapSizeOf for DummyHasher { fn heap_size_of_children(&self) -> usize { 0 } } -// -// let mut db = MemoryDB::::new(); -// // TODO: Fails. Trying to use a type that isn't H256 fails because of the tight coupling between memorydb and plain_hasher (specifically the assert on key length == 32) -// let key = db.insert(b"32103210321032103210321032103210"); -// assert_eq!(key, ethereum_types::H264(*b"010102020101020201010202010102025")); -// } } diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 50c9ded2703..7ce1450035f 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//use ethereum_types::H256; -//use keccak::keccak; use hashdb::{HashDB, DBValue, Hasher}; use super::{TrieDBMut, TrieMut}; use node_codec::NodeCodec; diff --git a/whisper/Cargo.toml b/whisper/Cargo.toml index e503a74fd6d..0cfab7ae7cd 100644 --- a/whisper/Cargo.toml +++ b/whisper/Cargo.toml @@ -24,6 +24,7 @@ serde_json = "1.0" slab = "0.3" smallvec = "0.4" tiny-keccak = "1.4" +stream-encoder = { path = "../util/stream-encoder" } jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.11" } jsonrpc-macros = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.11" } diff --git a/whisper/src/lib.rs b/whisper/src/lib.rs index 190169b2c21..bb72d54a2f8 100644 --- a/whisper/src/lib.rs +++ b/whisper/src/lib.rs @@ -32,6 +32,7 @@ extern crate serde; extern crate slab; extern crate smallvec; extern crate tiny_keccak; +extern crate stream_encoder; extern crate jsonrpc_core; extern crate jsonrpc_pubsub; diff --git a/whisper/src/message.rs b/whisper/src/message.rs index 95c21125513..9560a430397 100644 --- a/whisper/src/message.rs +++ b/whisper/src/message.rs @@ -21,6 +21,7 @@ use std::time::{self, SystemTime, Duration, Instant}; use ethereum_types::{H256, H512}; use rlp::{self, DecoderError, RlpStream, Rlp}; +use stream_encoder::Stream; use smallvec::SmallVec; use tiny_keccak::{keccak256, Keccak}; diff --git a/whisper/src/net/mod.rs b/whisper/src/net/mod.rs index 6ec3b08a548..766d622e39f 100644 --- a/whisper/src/net/mod.rs +++ b/whisper/src/net/mod.rs @@ -27,6 +27,7 @@ use network::{self, NetworkContext, NodeId, PeerId, ProtocolId, TimerToken}; use ordered_float::OrderedFloat; use parking_lot::{Mutex, RwLock}; use rlp::{DecoderError, RlpStream, Rlp}; +use stream_encoder::Stream; use message::{Message, Error as MessageError}; From c5c602127f49eaab6fc66fa66e1048058b0329a1 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 15 Jun 2018 16:07:21 +0200 Subject: [PATCH 080/160] fix more tests --- Cargo.lock | 2 ++ ethcore/sync/Cargo.toml | 1 + ethcore/sync/src/blocks.rs | 1 + ethcore/sync/src/chain/mod.rs | 1 + ethcore/sync/src/chain/propagator.rs | 1 + ethcore/sync/src/chain/requester.rs | 1 + ethcore/sync/src/chain/supplier.rs | 1 + ethcore/sync/src/lib.rs | 1 + rpc/Cargo.toml | 1 + rpc/src/lib.rs | 1 + rpc/src/v1/helpers/dispatch.rs | 1 + 11 files changed, 12 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ae9b8159ffe..a339cef1d46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -893,6 +893,7 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-encoder 0.1.0", "trace-time 0.1.0", "triehash 0.1.0", ] @@ -2255,6 +2256,7 @@ dependencies = [ "serde_derive 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", + "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index aae6cbf177a..8d8b529ebfc 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -31,6 +31,7 @@ heapsize = "0.4" semver = "0.9" smallvec = { version = "0.4", features = ["heapsizeof"] } parking_lot = "0.5" +stream-encoder = { path = "../../util/stream-encoder" } trace-time = { path = "../../util/trace-time" } ipnetwork = "0.12.6" diff --git a/ethcore/sync/src/blocks.rs b/ethcore/sync/src/blocks.rs index 8485b1d75e6..cd0fe72a0e0 100644 --- a/ethcore/sync/src/blocks.rs +++ b/ethcore/sync/src/blocks.rs @@ -23,6 +23,7 @@ use ethereum_types::H256; use triehash::ordered_trie_root; use bytes::Bytes; use rlp::{Rlp, RlpStream, DecoderError}; +use stream_encoder::Stream; use network; use ethcore::encoded::Block; use ethcore::views::{HeaderView, BodyView}; diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index afedbde3788..1c891a5bfac 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -104,6 +104,7 @@ use plain_hasher::H256FastMap; use parking_lot::RwLock; use bytes::Bytes; use rlp::{Rlp, RlpStream, DecoderError}; +use stream_encoder::Stream; use network::{self, PeerId, PacketId}; use ethcore::header::{BlockNumber}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockQueueInfo}; diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index 75cf550f28b..222a2dceaf8 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -24,6 +24,7 @@ use rlp::{Encodable, RlpStream}; use sync_io::SyncIo; use std::cmp; use std::collections::HashSet; +use stream_encoder::Stream; use transaction::SignedTransaction; use super::{ diff --git a/ethcore/sync/src/chain/requester.rs b/ethcore/sync/src/chain/requester.rs index a85874d2927..105cf4a76d8 100644 --- a/ethcore/sync/src/chain/requester.rs +++ b/ethcore/sync/src/chain/requester.rs @@ -22,6 +22,7 @@ use ethereum_types::H256; use network::{PeerId, PacketId}; use rlp::RlpStream; use std::time::Instant; +use stream_encoder::Stream; use sync_io::SyncIo; use super::{ diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index e0245fdbcd9..9afa4936d11 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -22,6 +22,7 @@ use network::{self, PeerId}; use parking_lot::RwLock; use rlp::{Rlp, RlpStream}; use std::cmp; +use stream_encoder::Stream; use sync_io::SyncIo; use super::{ diff --git a/ethcore/sync/src/lib.rs b/ethcore/sync/src/lib.rs index 636ad6cce14..11e088aa771 100644 --- a/ethcore/sync/src/lib.rs +++ b/ethcore/sync/src/lib.rs @@ -41,6 +41,7 @@ extern crate ipnetwork; extern crate keccak_hash as hash; extern crate triehash; extern crate kvdb; +extern crate stream_encoder; extern crate ethcore_light as light; diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index d227f45f535..455df7600a2 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -62,6 +62,7 @@ parity-version = { path = "../util/version" } patricia-trie = { path = "../util/patricia_trie" } rlp = { path = "../util/rlp" } stats = { path = "../util/stats" } +stream-encoder = { path = "../util/stream-encoder" } vm = { path = "../ethcore/vm" } [dev-dependencies] diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 2d49a8c7717..276f5d81c56 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -65,6 +65,7 @@ extern crate parity_reactor; extern crate parity_updater as updater; extern crate parity_version as version; extern crate rlp; +extern crate stream_encoder; extern crate stats; extern crate keccak_hash as hash; extern crate hardware_wallet; diff --git a/rpc/src/v1/helpers/dispatch.rs b/rpc/src/v1/helpers/dispatch.rs index 81fc182e4bc..3b97cea380f 100644 --- a/rpc/src/v1/helpers/dispatch.rs +++ b/rpc/src/v1/helpers/dispatch.rs @@ -25,6 +25,7 @@ use light::client::LightChainClient; use light::on_demand::{request, OnDemand}; use light::TransactionQueue as LightTransactionQueue; use rlp; +use stream_encoder::Stream; use hash::keccak; use ethereum_types::{H256, H520, Address, U256}; use bytes::Bytes; From f6cd87bfa2c554aaadc6985218a03447ccdfe300 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 17:39:08 +0200 Subject: [PATCH 081/160] grumble: Idiomatic use of Result --- util/patricia_trie/src/fatdb.rs | 14 ++++++------- util/patricia_trie/src/fatdbmut.rs | 12 +++++------ util/patricia_trie/src/lookup.rs | 6 +++--- util/patricia_trie/src/sectriedb.rs | 13 ++++++------ util/patricia_trie/src/sectriedbmut.rs | 13 ++++++------ util/patricia_trie/src/triedb.rs | 22 ++++++++++---------- util/patricia_trie/src/triedbmut.rs | 28 +++++++++++++------------- 7 files changed, 53 insertions(+), 55 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index f75ebf99787..aa2adc7e0fa 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use hashdb::{HashDB, Hasher}; -use super::{TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query}; +use super::{Result, TrieDB, Trie, TrieDBIterator, TrieItem, TrieIterator, Query}; use node_codec::NodeCodec; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -38,7 +38,7 @@ where /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { Ok(FatDB { raw: TrieDB::new(db, root)? }) } @@ -55,17 +55,17 @@ where fn root(&self) -> &::Out { self.raw.root() } - fn contains(&self, key: &[u8]) -> super::Result::Out> { + fn contains(&self, key: &[u8]) -> Result::Out> { self.raw.contains(Self::H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> where 'a: 'key { self.raw.get_with(Self::H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> super::Result> + 'a>, ::Out> { + fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) } } @@ -86,7 +86,7 @@ where C: NodeCodec { /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> super::Result { + pub fn new(trie: &'db TrieDB) -> Result { Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie: trie, @@ -99,7 +99,7 @@ where H: Hasher, C: NodeCodec { - fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out> { let hashed_key = H::hash(key); self.trie_iterator.seek(hashed_key.as_ref()) } diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 7ce1450035f..762125b7b2c 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use hashdb::{HashDB, DBValue, Hasher}; -use super::{TrieDBMut, TrieMut}; +use super::{Result, TrieDBMut, TrieMut}; use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -45,7 +45,7 @@ where /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { Ok(FatDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -70,17 +70,17 @@ where fn is_empty(&self) -> bool { self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> super::Result::Out> { + fn contains(&self, key: &[u8]) -> Result::Out> { self.raw.contains(Self::H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result, ::Out> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> where 'a: 'key { self.raw.get(Self::H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result, ::Out> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out> { let hash = Self::H::hash(key); let out = self.raw.insert(hash.as_ref(), value)?; let db = self.raw.db_mut(); @@ -93,7 +93,7 @@ where Ok(out) } - fn remove(&mut self, key: &[u8]) -> super::Result, ::Out> { + fn remove(&mut self, key: &[u8]) -> Result, ::Out> { let hash = Self::H::hash(key); let out = self.raw.remove(hash.as_ref())?; diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 3c61947fc02..b5430dc8eaf 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -18,9 +18,9 @@ use hashdb::{HashDB, Hasher}; use nibbleslice::NibbleSlice; -use super::node::Node; +use node::Node; use node_codec::NodeCodec; -use super::{TrieError, Query}; +use super::{Result, TrieError, Query}; use std::marker::PhantomData; /// Trie lookup helper object. @@ -42,7 +42,7 @@ where { /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. - pub fn look_up(mut self, mut key: NibbleSlice) -> super::Result, H::Out> { + pub fn look_up(mut self, mut key: NibbleSlice) -> Result, H::Out> { let mut hash = self.hash; // this loop iterates through non-inline nodes. diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index ff982eadf9a..b856c689879 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -18,7 +18,7 @@ //use keccak::keccak; use hashdb::{HashDB, Hasher}; use super::triedb::TrieDB; -use super::{Trie, TrieItem, TrieIterator, Query}; +use super::{Result, Trie, TrieItem, TrieIterator, Query}; use node_codec::NodeCodec; /// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -42,7 +42,7 @@ where /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. /// Returns an error if root does not exist. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { Ok(SecTrieDB { raw: TrieDB::new(db, root)? }) } @@ -66,17 +66,17 @@ where fn root(&self) -> &::Out { self.raw.root() } - fn contains(&self, key: &[u8]) -> super::Result::Out> { + fn contains(&self, key: &[u8]) -> Result::Out> { self.raw.contains(Self::H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, ::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> where 'a: 'key { self.raw.get_with(Self::H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> super::Result> + 'a>, ::Out> { + fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { TrieDB::iter(&self.raw) } } @@ -85,8 +85,7 @@ where fn trie_to_sectrie() { use memorydb::MemoryDB; use hashdb::DBValue; - use super::triedbmut::TrieDBMut; - use super::TrieMut; + use super::{TrieMut, TrieDBMut}; use hashdb::KeccakHasher; use keccak; use node_codec::RlpNodeCodec; diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 8144ad8f72a..07d7d02c93c 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -15,8 +15,7 @@ // along with Parity. If not, see . use hashdb::{HashDB, DBValue, Hasher}; -use super::triedbmut::TrieDBMut; -use super::TrieMut; +use super::{Result, TrieMut, TrieDBMut}; use node_codec::NodeCodec; /// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. @@ -45,7 +44,7 @@ where /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> super::Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { Ok(SecTrieDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -70,21 +69,21 @@ where self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> super::Result::Out> { + fn contains(&self, key: &[u8]) -> Result::Out> { self.raw.contains(&Self::H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> super::Result, ::Out> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> where 'a: 'key { self.raw.get(&Self::H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result, ::Out> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out> { self.raw.insert(&Self::H::hash(key).as_ref(), value) } - fn remove(&mut self, key: &[u8]) -> super::Result, ::Out> { + fn remove(&mut self, key: &[u8]) -> Result, ::Out> { self.raw.remove(&Self::H::hash(key).as_ref()) } } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index df7ebfcfbd1..7301dfc0d9a 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -20,7 +20,7 @@ use nibbleslice::NibbleSlice; use super::node::{Node, OwnedNode}; use node_codec::NodeCodec; use super::lookup::Lookup; -use super::{Trie, TrieItem, TrieError, TrieIterator, Query}; +use super::{Result, Trie, TrieItem, TrieError, TrieIterator, Query}; use bytes::Bytes; use std::marker::PhantomData; @@ -74,7 +74,7 @@ where { /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist - pub fn new(db: &'db HashDB, root: &'db H::Out) -> super::Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { if !db.contains(root) { Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { @@ -86,7 +86,7 @@ where pub fn db(&'db self) -> &'db HashDB { self.db } /// Get the data of the root node. - fn root_data(&self) -> super::Result { + fn root_data(&self) -> Result { self.db .get(self.root) .ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) @@ -95,7 +95,7 @@ where /// Given some node-describing data `node`, return the actual node RLP. /// This could be a simple identity operation in the case that the node is sufficiently small, but /// may require a database lookup. - fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> super::Result { + fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> Result { match C::try_decode_hash(node) { Some(key) => { self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) @@ -113,7 +113,7 @@ where type H = H; fn root(&self) -> &::Out { self.root } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> super::Result, H::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out> where 'a: 'key { Lookup { @@ -124,7 +124,7 @@ where }.look_up(NibbleSlice::new(key)) } - fn iter<'a>(&'a self) -> super::Result> + 'a>, H::Out> { + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out> { TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>) } } @@ -231,13 +231,13 @@ pub struct TrieDBIterator<'a, H: Hasher + 'a, C: NodeCodec + 'a> { impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> super::Result, H::Out> { + pub fn new(db: &'a TrieDB) -> Result, H::Out> { let mut r = TrieDBIterator { db, trail: vec![], key_nibbles: Vec::new() }; db.root_data().and_then(|root| r.descend(&root))?; Ok(r) } - fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> super::Result<(), H::Out> { + fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> Result<(), H::Out> { loop { let (data, mid) = { let node = C::decode(&node_data).expect("rlp read from db; qed"); @@ -301,7 +301,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { } /// Descend into a payload. - fn descend(&mut self, d: &[u8]) -> super::Result<(), H::Out> { + fn descend(&mut self, d: &[u8]) -> Result<(), H::Out> { let node_data = &self.db.get_raw_or_lookup(d)?; let node = C::decode(&node_data).expect("encoded node read from db; qed"); Ok(self.descend_into_node(node.into())) @@ -335,7 +335,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C> { /// Position the iterator on the first element with key >= `key` - fn seek(&mut self, key: &[u8]) -> super::Result<(), H::Out> { + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out> { self.trail.clear(); self.key_nibbles.clear(); let root_rlp = self.db.root_data()?; @@ -351,7 +351,7 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { enum IterStep { Continue, PopTrail, - Descend(super::Result), + Descend(Result), } loop { let iter_step = { diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 68de9e0fc96..df0ce529aa3 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -16,7 +16,7 @@ //! In-memory trie representation. -use super::{TrieError, TrieMut}; +use super::{Result, TrieError, TrieMut}; use super::lookup::Lookup; use super::node::Node as RlpNode; use node_codec::NodeCodec; @@ -333,7 +333,7 @@ where H: Hasher, /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. - pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> super::Result { + pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> Result { if !db.contains(root) { return Err(Box::new(TrieError::InvalidStateRoot(*root))); } @@ -360,7 +360,7 @@ where H: Hasher, } // cache a node by hash - fn cache(&mut self, hash: H::Out) -> super::Result { + fn cache(&mut self, hash: H::Out) -> Result { let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; let node = Node::from_encoded::( &node_encoded, @@ -372,8 +372,8 @@ where H: Hasher, // inspect a node, choosing either to replace, restore, or delete it. // if restored or replaced, returns the new node along with a flag of whether it was changed. - fn inspect(&mut self, stored: Stored, inspector: F) -> super::Result, bool)>, H::Out> - where F: FnOnce(&mut Self, Node) -> super::Result, H::Out> { + fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out> + where F: FnOnce(&mut Self, Node) -> Result, H::Out> { Ok(match stored { Stored::New(node) => match inspector(self, node)? { Action::Restore(node) => Some((Stored::New(node), false)), @@ -395,7 +395,7 @@ where H: Hasher, } // walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> super::Result, H::Out> + fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out> where 'x: 'key { let mut handle = handle; @@ -444,7 +444,7 @@ where H: Hasher, } /// insert a key, value pair into the trie, creating new nodes if necessary. - fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> super::Result<(StorageHandle, bool), H::Out> { + fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out> { let h = match handle { NodeHandle::InMemory(h) => h, NodeHandle::Hash(h) => self.cache(h)?, @@ -458,7 +458,7 @@ where H: Hasher, } /// the insertion inspector. - fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> super::Result, H::Out> { + fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out> { trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty()); Ok(match node { @@ -619,7 +619,7 @@ where H: Hasher, } /// Remove a node from the trie based on key. - fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> super::Result, H::Out> { + fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), NodeHandle::Hash(h) => { @@ -634,7 +634,7 @@ where H: Hasher, } /// the removal inspector - fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> super::Result, H::Out> { + fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out> { Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete, (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), @@ -720,7 +720,7 @@ where H: Hasher, /// _invalid state_ means: /// - Branch node where there is only a single entry; /// - Extension node followed by anything other than a Branch node. - fn fix(&mut self, node: Node) -> super::Result, H::Out> { + fn fix(&mut self, node: Node) -> Result, H::Out> { match node { Node::Branch(mut children, value) => { // if only a single value, transmute to leaf/extension and feed through fixed. @@ -910,13 +910,13 @@ where H: Hasher, } } - fn get<'x, 'key>(&'x self, key: &'key [u8]) -> super::Result, ::Out> + fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, ::Out> where 'x: 'key { self.lookup(NibbleSlice::new(key), &self.root_handle) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result, ::Out> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out> { if value.is_empty() { return self.remove(key) } let mut old_val = None; @@ -937,7 +937,7 @@ where H: Hasher, Ok(old_val) } - fn remove(&mut self, key: &[u8]) -> super::Result, ::Out> { + fn remove(&mut self, key: &[u8]) -> Result, ::Out> { trace!(target: "trie", "remove: key={:?}", key.pretty()); let root_handle = self.root_handle(); From f57565c9a503581192f209184b6f8ebce990fac4 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 20:47:45 +0200 Subject: [PATCH 082/160] Add keccak-hasher crate that implements Hasher with the Keccak-256 hash --- util/keccak-hasher/Cargo.toml | 11 +++++++++ util/keccak-hasher/src/lib.rs | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 util/keccak-hasher/Cargo.toml create mode 100644 util/keccak-hasher/src/lib.rs diff --git a/util/keccak-hasher/Cargo.toml b/util/keccak-hasher/Cargo.toml new file mode 100644 index 00000000000..7b5d0175889 --- /dev/null +++ b/util/keccak-hasher/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "keccak-hasher" +version = "0.1.0" +authors = ["Parity Technologies "] +description = "Keccak-256 implementation of the Hasher trait" +license = "GPL-3.0" + +[dependencies] +hashdb = { path = "../hashdb" } +ethereum-types = "0.3" +tiny-keccak = "1.4.2" \ No newline at end of file diff --git a/util/keccak-hasher/src/lib.rs b/util/keccak-hasher/src/lib.rs new file mode 100644 index 00000000000..5b255671814 --- /dev/null +++ b/util/keccak-hasher/src/lib.rs @@ -0,0 +1,42 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Hasher implementation for the Keccak-256 hash +extern crate hashdb; +extern crate ethereum_types; +extern crate tiny_keccak; + +use hashdb::Hasher; +use ethereum_types::H256; +use tiny_keccak::Keccak; + +#[derive(Default, Debug, Clone, PartialEq)] +pub struct KeccakHasher; +impl Hasher for KeccakHasher { + type Out = H256; + // TODO: this is a problem: we do not want the Hasher to make assumptions about the encoding scheme. + const HASHED_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); + const LENGTH: usize = 32; + fn hash(x: &[u8]) -> Self::Out { + let mut out = [0;32]; + Keccak::keccak256(x, &mut out); + out.into() + } +} + +// Move concrete impl to own "façade" crate? +// /// Convenience type for crates that need a `HashDB` with Keccak hashes +// pub type KeccakHashDB = HashDB; From c2c7789feeba8c2ec0eff2122ab1e4db2c7979d5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 20:49:22 +0200 Subject: [PATCH 083/160] Remove KeccakHasher impl from hashdb --- util/hashdb/Cargo.toml | 5 +---- util/hashdb/src/lib.rs | 20 -------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/util/hashdb/Cargo.toml b/util/hashdb/Cargo.toml index 962f0e7f694..f5e63fb1ba5 100644 --- a/util/hashdb/Cargo.toml +++ b/util/hashdb/Cargo.toml @@ -7,7 +7,4 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" -heapsize = "0.4" -# TODO: Used by the `KeccakHasher` impl which I think should move to own crate? -ethereum-types = "0.3" -tiny-keccak = "1.4.2" \ No newline at end of file +heapsize = "0.4" \ No newline at end of file diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 43146d39e8c..47b97bb1cc0 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -16,16 +16,12 @@ //! Database of byte-slices keyed to their hash. extern crate elastic_array; -extern crate ethereum_types; extern crate heapsize; -extern crate tiny_keccak; use elastic_array::ElasticArray128; -use ethereum_types::H256; use heapsize::HeapSizeOf; use std::collections::HashMap; use std::{fmt::Debug, hash::Hash}; -use tiny_keccak::Keccak; pub trait Hasher: Sync + Send { type Out: AsRef<[u8]> + Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? --> */ + HeapSizeOf; @@ -34,22 +30,6 @@ pub trait Hasher: Sync + Send { fn hash(x: &[u8]) -> Self::Out; } -// TODO: Move concrete impl to own "façade" crate -#[derive(Default, Debug, Clone, PartialEq)] -pub struct KeccakHasher; -impl Hasher for KeccakHasher { - type Out = H256; - const HASHED_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); - const LENGTH: usize = 32; - fn hash(x: &[u8]) -> Self::Out { - let mut out = [0;32]; - Keccak::keccak256(x, &mut out); - out.into() - } -} -/// Convenience type for crates that need a `HashDB` with Keccak hashes -pub type KeccakHashDB = HashDB; - /// `HashDB` value type. pub type DBValue = ElasticArray128; From 046b5d6a876fc16a752bbe6e300ee2c0f0ccf88d Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 20:49:56 +0200 Subject: [PATCH 084/160] memorydb: Use keccak_hasher::KeccakHasher --- util/memorydb/Cargo.toml | 1 + util/memorydb/src/lib.rs | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/util/memorydb/Cargo.toml b/util/memorydb/Cargo.toml index 6bf29e5722f..aa27a0a7218 100644 --- a/util/memorydb/Cargo.toml +++ b/util/memorydb/Cargo.toml @@ -11,6 +11,7 @@ heapsize = "0.4" hashdb = { version = "0.2.0", path = "../hashdb" } plain_hasher = { path = "../plain_hasher" } rlp = { version = "0.2.1", path = "../rlp" } +keccak-hasher = { path = "../keccak-hasher" } [dev-dependencies] tiny-keccak = "1.4.2" diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index c797c9d4389..d5ee1030cc3 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -18,12 +18,14 @@ extern crate elastic_array; extern crate hashdb; extern crate heapsize; +extern crate keccak_hasher; extern crate plain_hasher; extern crate rlp; #[cfg(test)] extern crate tiny_keccak; #[cfg(test)] extern crate ethereum_types; -use hashdb::{HashDB, Hasher, DBValue, KeccakHasher, AsHashDB}; +use hashdb::{HashDB, Hasher, DBValue, AsHashDB}; +use keccak_hasher::KeccakHasher; use heapsize::HeapSizeOf; use plain_hasher::H256FastMap; use rlp::NULL_RLP; @@ -41,8 +43,11 @@ use std::mem; /// # Example /// ```rust /// extern crate hashdb; +/// extern crate keccak_hasher; /// extern crate memorydb; +/// /// use hashdb::*; +/// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// fn main() { /// let mut m = MemoryDB::::new(); @@ -96,9 +101,13 @@ impl MemoryDB { /// # Examples /// ```rust /// extern crate hashdb; + /// extern crate keccak_hasher; /// extern crate memorydb; + /// /// use hashdb::*; + /// use keccak_hasher::KeccakHasher; /// use memorydb::*; + /// /// fn main() { /// let mut m = MemoryDB::::new(); /// let hello_bytes = "Hello world!".as_bytes(); @@ -283,7 +292,7 @@ mod tests { use super::*; use tiny_keccak::Keccak; use ethereum_types::H256; - use hashdb::KeccakHasher; + use keccak_hasher::KeccakHasher; #[test] fn memorydb_remove_and_purge() { From 91fd334702a0cab55e9b4a3dc434216198c90bb9 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 20:50:26 +0200 Subject: [PATCH 085/160] =?UTF-8?q?patricia=5Ftrie:=20Use=20keccak=5Fhashe?= =?UTF-8?q?r::KeccakHasher=20Add=20recorder=20tests=20back=20=E2=80=93=20o?= =?UTF-8?q?ops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/patricia_trie/Cargo.toml | 1 + util/patricia_trie/benches/trie.rs | 4 +++- util/patricia_trie/src/fatdb.rs | 2 +- util/patricia_trie/src/fatdbmut.rs | 2 +- util/patricia_trie/src/lib.rs | 8 ++++++-- util/patricia_trie/src/recorder.rs | 13 +++++++------ util/patricia_trie/src/sectriedb.rs | 2 +- util/patricia_trie/src/sectriedbmut.rs | 2 +- util/patricia_trie/src/triedb.rs | 9 +++++---- util/patricia_trie/src/triedbmut.rs | 9 +++++---- 10 files changed, 31 insertions(+), 21 deletions(-) diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 11c8948ffa3..e54baf1e99a 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -18,6 +18,7 @@ triehash = { version = "0.1.0", path = "../triehash" } memorydb = { version = "0.2", path = "../memorydb" } ethcore-logger = { version = "1.9.0", path = "../../logger" } stream-encoder = { path = "../stream-encoder" } +keccak-hasher = { path = "../keccak-hasher" } [dev-dependencies] trie-standardmap = { path = "../trie-standardmap" } diff --git a/util/patricia_trie/benches/trie.rs b/util/patricia_trie/benches/trie.rs index 4f27796ea52..3744930d13c 100644 --- a/util/patricia_trie/benches/trie.rs +++ b/util/patricia_trie/benches/trie.rs @@ -21,6 +21,7 @@ extern crate ethcore_bytes; extern crate ethereum_types; extern crate memorydb; extern crate patricia_trie as trie; +extern crate keccak_hasher; extern crate keccak_hash; extern crate trie_standardmap; extern crate hashdb; @@ -32,7 +33,8 @@ use memorydb::MemoryDB; use test::{Bencher, black_box}; use trie::{TrieDBMut, TrieDB, TrieMut, Trie}; use trie_standardmap::{Alphabet, ValueMode, StandardMap}; -use hashdb::{Hasher,KeccakHasher}; +use hashdb::Hasher; +use keccak_hasher::KeccakHasher; use trie::node_codec::RlpNodeCodec; type RlpCodec = RlpNodeCodec; diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index aa2adc7e0fa..75f3bb5d122 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -129,7 +129,7 @@ fn fatdb_to_trie() { use hashdb::DBValue; use super::fatdbmut::FatDBMut; use super::TrieMut; - use hashdb::KeccakHasher; + use keccak_hasher::KeccakHasher; use node_codec::RlpNodeCodec; let mut memdb = MemoryDB::::new(); diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 762125b7b2c..a110fa3f82e 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -111,7 +111,7 @@ fn fatdbmut_to_trie() { use memorydb::MemoryDB; use super::TrieDB; use super::Trie; - use hashdb::KeccakHasher; + use keccak_hasher::KeccakHasher; use keccak; use node_codec::RlpNodeCodec; diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index e734d479737..85d4920570f 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -21,6 +21,7 @@ extern crate ethcore_logger; extern crate ethereum_types; extern crate hashdb; extern crate keccak_hash as keccak; +extern crate keccak_hasher; extern crate memorydb; extern crate rand; extern crate rlp; @@ -33,7 +34,8 @@ extern crate trie_standardmap as standardmap; extern crate log; use std::{fmt, error}; -use hashdb::{HashDB, DBValue, Hasher, KeccakHasher}; +use hashdb::{HashDB, DBValue, Hasher}; +use keccak_hasher::KeccakHasher; use std::marker::PhantomData; pub mod node; @@ -60,8 +62,10 @@ pub use self::recorder::Recorder; use node_codec::NodeCodec; // TODO: Move to façade crate and rename to just `RlpNodeCodec` -pub type KeccakRlpNodeCodec = node_codec::RlpNodeCodec; +pub type KeccakRlpNodeCodec = node_codec::RlpNodeCodec; pub type KeccakTrieResult = Result::Out>; +#[cfg(test)] +type RlpCodec = node_codec::RlpNodeCodec; /// Trie Errors. /// diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 14b304d83d9..4a5d1f46d0c 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -84,13 +84,12 @@ impl Recorder { } } -/* - +// TODO: WUT! #[cfg(test)] mod tests { use super::*; use keccak::keccak; - use hashdb::KeccakHasher; + use keccak_hasher::KeccakHasher; #[test] fn basic_recorder() { @@ -149,13 +148,14 @@ mod tests { fn trie_record() { use super::super::{TrieDB, TrieDBMut, Trie, TrieMut}; use memorydb::MemoryDB; + use RlpCodec; let mut db = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut x = TrieDBMut::new(&mut db, &mut root); + let mut x = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); x.insert(b"dog", b"cat").unwrap(); x.insert(b"lunch", b"time").unwrap(); @@ -167,7 +167,7 @@ mod tests { x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); } - let trie = TrieDB::new(&db, &root).unwrap(); + let trie = TrieDB::<_, RlpCodec>::new(&db, &root).unwrap(); let mut recorder = Recorder::::new(); trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); @@ -215,5 +215,6 @@ mod tests { ] ]); } -} +/* */ +} diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index b856c689879..1949096281d 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -86,7 +86,7 @@ fn trie_to_sectrie() { use memorydb::MemoryDB; use hashdb::DBValue; use super::{TrieMut, TrieDBMut}; - use hashdb::KeccakHasher; + use keccak_hasher::KeccakHasher; use keccak; use node_codec::RlpNodeCodec; diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 07d7d02c93c..f71c2524991 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -93,7 +93,7 @@ fn sectrie_to_trie() { use memorydb::*; use super::triedb::*; use super::Trie; - use hashdb::KeccakHasher; + use keccak_hasher::KeccakHasher; use keccak; use node_codec::RlpNodeCodec; diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 7301dfc0d9a..ad8af7b31b4 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -35,11 +35,13 @@ use std::marker::PhantomData; /// ``` /// extern crate patricia_trie as trie; /// extern crate hashdb; +/// extern crate keccak_hasher; /// extern crate memorydb; /// extern crate ethereum_types; /// /// use trie::*; /// use hashdb::*; +/// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; /// use node_codec::RlpNodeCodec; @@ -430,13 +432,12 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { #[cfg(test)] mod tests { - use node_codec::RlpNodeCodec; - use hashdb::{Hasher, KeccakHasher, DBValue}; + use hashdb::{Hasher, DBValue}; + use keccak_hasher::KeccakHasher; use memorydb::*; use super::{TrieDB, Trie, Lookup, NibbleSlice}; use super::super::{TrieMut, triedbmut::*}; - - type RlpCodec = RlpNodeCodec; + use RlpCodec; #[test] fn iterator() { diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index df0ce529aa3..8989562260a 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -268,14 +268,16 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// # Example /// ``` /// extern crate patricia_trie as trie; -/// extern crate keccak_hash; /// extern crate hashdb; +/// extern crate keccak_hash; +/// extern crate keccak_hasher; /// extern crate memorydb; /// extern crate ethereum_types; /// /// use keccak_hash::KECCAK_NULL_RLP; /// use trie::*; /// use hashdb::*; +/// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; /// use node_codec::RlpNodeCodec; @@ -976,15 +978,14 @@ mod tests { use bytes::ToPretty; use hashdb::*; + use keccak_hasher::KeccakHasher; use memorydb::*; - use node_codec::RlpNodeCodec; use rlp::{Decodable, Encodable}; use self::triehash::trie_root; use standardmap::*; use super::*; use super::super::TrieMut; - - type RlpCodec = RlpNodeCodec; + use RlpCodec; fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec From 57b79b89d852291c6b48f1e29b392f7ec998a25a Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 20:51:30 +0200 Subject: [PATCH 086/160] Add keccak-hasher crate to main Cargo.toml --- Cargo.lock | 13 +++++++++++-- Cargo.toml | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a339cef1d46..5d6a1ead8a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1208,9 +1208,7 @@ name = "hashdb" version = "0.2.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1477,6 +1475,15 @@ dependencies = [ "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "keccak-hasher" +version = "0.1.0" +dependencies = [ + "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.2.0", + "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -1663,6 +1670,7 @@ dependencies = [ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.1.0", "plain_hasher 0.2.0", "rlp 0.2.1", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2424,6 +2432,7 @@ dependencies = [ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", "keccak-hash 0.1.2", + "keccak-hasher 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memorydb 0.2.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 87ec010e9ea..61ee060f00c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,6 +136,7 @@ members = [ "whisper", "whisper/cli", "util/stream-encoder", + "util/keccak-hasher", ] [patch.crates-io] From f3da9b129c435c2ed66afd3f411b19a58df90915 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 21:04:42 +0200 Subject: [PATCH 087/160] journaldb: Use keccak-hasher --- util/journaldb/Cargo.toml | 1 + util/journaldb/src/archivedb.rs | 14 ++++++++------ util/journaldb/src/earlymergedb.rs | 16 +++++++++------- util/journaldb/src/lib.rs | 4 +++- util/journaldb/src/overlaydb.rs | 1 + util/journaldb/src/overlayrecentdb.rs | 18 ++++++++++-------- util/journaldb/src/refcounteddb.rs | 16 +++++++++------- util/journaldb/src/traits.rs | 9 +++++---- 8 files changed, 46 insertions(+), 33 deletions(-) diff --git a/util/journaldb/Cargo.toml b/util/journaldb/Cargo.toml index 9cd94d59b72..56412533c7b 100644 --- a/util/journaldb/Cargo.toml +++ b/util/journaldb/Cargo.toml @@ -10,6 +10,7 @@ ethcore-bytes = { path = "../bytes" } ethereum-types = "0.3" hashdb = { version = "0.2.0", path = "../hashdb" } heapsize = "0.4" +keccak-hasher = { path = "../keccak-hasher" } kvdb = { path = "../kvdb" } log = "0.3" memorydb = { version="0.2.0", path = "../memorydb" } diff --git a/util/journaldb/src/archivedb.rs b/util/journaldb/src/archivedb.rs index 7aa4806fc07..2978e86c21a 100644 --- a/util/journaldb/src/archivedb.rs +++ b/util/journaldb/src/archivedb.rs @@ -19,15 +19,17 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; -use rlp::{encode, decode}; + +use bytes::Bytes; +use error::{BaseDataError, UtilError}; +use ethereum_types::H256; use hashdb::*; -use super::memorydb::*; +use keccak_hasher::KeccakHasher; +use kvdb::{KeyValueDB, DBTransaction}; +use rlp::{encode, decode}; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; +use super::memorydb::*; use traits::JournalDB; -use kvdb::{KeyValueDB, DBTransaction}; -use ethereum_types::H256; -use error::{BaseDataError, UtilError}; -use bytes::Bytes; /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay /// and latent-removal semantics. diff --git a/util/journaldb/src/earlymergedb.rs b/util/journaldb/src/earlymergedb.rs index 6b0681bce96..77adeac7156 100644 --- a/util/journaldb/src/earlymergedb.rs +++ b/util/journaldb/src/earlymergedb.rs @@ -19,17 +19,19 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; -use parking_lot::RwLock; -use heapsize::HeapSizeOf; -use rlp::{encode, decode}; + +use bytes::Bytes; +use error::{BaseDataError, UtilError}; +use ethereum_types::H256; use hashdb::*; +use heapsize::HeapSizeOf; +use keccak_hasher::KeccakHasher; +use kvdb::{KeyValueDB, DBTransaction}; use memorydb::*; +use parking_lot::RwLock; +use rlp::{encode, decode}; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; use super::traits::JournalDB; -use kvdb::{KeyValueDB, DBTransaction}; -use ethereum_types::H256; -use error::{BaseDataError, UtilError}; -use bytes::Bytes; use util::{DatabaseKey, DatabaseValueView, DatabaseValueRef}; #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index b346a8adbe5..c5c1f90fd68 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -23,6 +23,7 @@ extern crate log; extern crate ethereum_types; extern crate ethcore_bytes as bytes; extern crate hashdb; +extern crate keccak_hasher; extern crate kvdb; extern crate memorydb; extern crate parking_lot; @@ -39,7 +40,8 @@ extern crate kvdb_memorydb; use std::{fmt, str}; use std::sync::Arc; -use hashdb::{AsHashDB, HashDB, KeccakHasher}; +use hashdb::{AsHashDB, HashDB}; +use keccak_hasher::KeccakHasher; /// Export the journaldb module. mod traits; diff --git a/util/journaldb/src/overlaydb.rs b/util/journaldb/src/overlaydb.rs index 2878e463510..1e01edfead7 100644 --- a/util/journaldb/src/overlaydb.rs +++ b/util/journaldb/src/overlaydb.rs @@ -23,6 +23,7 @@ use error::{Result, BaseDataError}; use ethereum_types::H256; use rlp::{Rlp, RlpStream, Encodable, DecoderError, Decodable, encode, decode}; use hashdb::*; +use keccak_hasher::KeccakHasher; use memorydb::*; use kvdb::{KeyValueDB, DBTransaction}; diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index 8a2f772f098..c4027e29c3c 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -19,18 +19,20 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::sync::Arc; -use parking_lot::RwLock; -use heapsize::HeapSizeOf; -use rlp::{Rlp, RlpStream, encode, decode, DecoderError, Decodable, Encodable}; + +use bytes::Bytes; +use error::{BaseDataError, UtilError}; +use ethereum_types::H256; use hashdb::*; +use heapsize::HeapSizeOf; +use keccak_hasher::KeccakHasher; +use kvdb::{KeyValueDB, DBTransaction}; use memorydb::*; +use parking_lot::RwLock; +use plain_hasher::H256FastMap; +use rlp::{Rlp, RlpStream, encode, decode, DecoderError, Decodable, Encodable}; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; -use kvdb::{KeyValueDB, DBTransaction}; use super::JournalDB; -use ethereum_types::H256; -use plain_hasher::H256FastMap; -use error::{BaseDataError, UtilError}; -use bytes::Bytes; use util::DatabaseKey; /// Implementation of the `JournalDB` trait for a disk-backed database with a memory overlay diff --git a/util/journaldb/src/refcounteddb.rs b/util/journaldb/src/refcounteddb.rs index 54a54566f24..bca8d93055e 100644 --- a/util/journaldb/src/refcounteddb.rs +++ b/util/journaldb/src/refcounteddb.rs @@ -18,17 +18,19 @@ use std::collections::HashMap; use std::sync::Arc; -use heapsize::HeapSizeOf; -use rlp::{encode, decode}; + +use bytes::Bytes; +use error::UtilError; +use ethereum_types::H256; use hashdb::*; -use overlaydb::OverlayDB; +use heapsize::HeapSizeOf; +use keccak_hasher::KeccakHasher; +use kvdb::{KeyValueDB, DBTransaction}; use memorydb::MemoryDB; +use overlaydb::OverlayDB; +use rlp::{encode, decode}; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; use super::traits::JournalDB; -use kvdb::{KeyValueDB, DBTransaction}; -use ethereum_types::H256; -use error::UtilError; -use bytes::Bytes; use util::{DatabaseKey, DatabaseValueView, DatabaseValueRef}; /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay diff --git a/util/journaldb/src/traits.rs b/util/journaldb/src/traits.rs index e5b9d816033..470761614de 100644 --- a/util/journaldb/src/traits.rs +++ b/util/journaldb/src/traits.rs @@ -16,12 +16,13 @@ //! Disk-backed `HashDB` implementation. -use std::sync::Arc; +use bytes::Bytes; +use error::UtilError; +use ethereum_types::H256; use hashdb::*; +use keccak_hasher::KeccakHasher; use kvdb::{self, DBTransaction}; -use ethereum_types::H256; -use error::UtilError; -use bytes::Bytes; +use std::sync::Arc; /// A `HashDB` which can manage a short-term journal potentially containing many forks of mutually /// exclusive actions. From 5f474420856d3936c246c841fd20e700fa01c211 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 21:05:11 +0200 Subject: [PATCH 088/160] cargo lock changes --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 5d6a1ead8a2..e7b12f1d7c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1360,6 +1360,7 @@ dependencies = [ "hashdb 0.2.0", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", + "keccak-hasher 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", From b0c715b05fcadb1d8ac742012671e7db08c9c2de Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 21:08:11 +0200 Subject: [PATCH 089/160] rlp: Fix doc tests --- util/rlp/src/stream.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index 8bab965b5a7..eeccff42df4 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -122,9 +122,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate trie_node_codec; + /// extern crate stream_encoder; /// use rlp::*; - /// use trie_node_codec::Stream; + /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -213,9 +213,9 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate trie_node_codec; + /// extern crate stream_encoder; /// use rlp::*; - /// use trie_node_codec::Stream; + /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); From 730099a7d82ca8e597b04a0fa7192bf9f74633ae Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 21:49:31 +0200 Subject: [PATCH 090/160] ethcore: Use keccak-hasher --- ethcore/Cargo.toml | 1 + ethcore/light/Cargo.toml | 1 + ethcore/light/src/cht.rs | 3 ++- ethcore/light/src/client/header_chain.rs | 2 +- ethcore/light/src/lib.rs | 1 + ethcore/light/src/on_demand/request.rs | 3 ++- ethcore/light/src/transaction_queue.rs | 2 +- ethcore/src/account_db.rs | 3 ++- ethcore/src/client/client.rs | 2 +- ethcore/src/client/error.rs | 4 +++- ethcore/src/client/evm_test_client.rs | 3 ++- ethcore/src/factory.rs | 2 +- ethcore/src/lib.rs | 1 + ethcore/src/pod_account.rs | 3 ++- ethcore/src/snapshot/error.rs | 3 ++- ethcore/src/snapshot/mod.rs | 3 ++- ethcore/src/snapshot/tests/helpers.rs | 3 ++- ethcore/src/state/account.rs | 3 ++- ethcore/src/state/backend.rs | 3 ++- ethcore/src/state/mod.rs | 3 ++- ethcore/src/state_db.rs | 3 ++- ethcore/sync/src/chain/mod.rs | 2 +- ethcore/sync/src/transactions_stats.rs | 2 +- 23 files changed, 37 insertions(+), 19 deletions(-) diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 472f7746155..68b7f7a7cea 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -68,6 +68,7 @@ triehash = { path = "../util/triehash" } unexpected = { path = "../util/unexpected" } journaldb = { path = "../util/journaldb" } stream-encoder = { path = "../util/stream-encoder" } +keccak-hasher = { path = "../util/keccak-hasher" } [dev-dependencies] tempdir = "0.3" diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index 609d4e1c1d2..98d304011f5 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -32,6 +32,7 @@ serde_derive = "1.0" parking_lot = "0.5" stats = { path = "../../util/stats" } keccak-hash = { path = "../../util/hash" } +keccak-hasher = { path = "../../util/keccak-hasher" } triehash = { path = "../../util/triehash" } kvdb = { path = "../../util/kvdb" } memory-cache = { path = "../../util/memory_cache" } diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index 1ab88b34c05..022ea2ea9e9 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -25,7 +25,8 @@ use ethcore::ids::BlockId; use ethereum_types::{H256, U256}; -use hashdb::{HashDB, KeccakHasher}; +use hashdb::HashDB; +use keccak_hasher::KeccakHasher; use memorydb::MemoryDB; use bytes::Bytes; use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder, KeccakRlpNodeCodec}; diff --git a/ethcore/light/src/client/header_chain.rs b/ethcore/light/src/client/header_chain.rs index 3df994fb7b3..945a84f95a5 100644 --- a/ethcore/light/src/client/header_chain.rs +++ b/ethcore/light/src/client/header_chain.rs @@ -42,7 +42,7 @@ use ethcore::engines::epoch::{ }; use ethereum_types::{H256, H264, U256}; -use hashdb::KeccakHasher; +use keccak_hasher::KeccakHasher; use heapsize::HeapSizeOf; use kvdb::{DBTransaction, KeyValueDB}; use plain_hasher::H256FastMap; diff --git a/ethcore/light/src/lib.rs b/ethcore/light/src/lib.rs index 7a630c22976..6e8f8e2c887 100644 --- a/ethcore/light/src/lib.rs +++ b/ethcore/light/src/lib.rs @@ -64,6 +64,7 @@ extern crate hashdb; extern crate heapsize; extern crate futures; extern crate itertools; +extern crate keccak_hasher; extern crate memorydb; extern crate patricia_trie as trie; extern crate plain_hasher; diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 4238b2af6c8..5ef8a651384 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -34,7 +34,8 @@ use rlp::{RlpStream, Rlp}; use stream_encoder::Stream; use ethereum_types::{H256, U256, Address}; use parking_lot::Mutex; -use hashdb::{HashDB, Hasher, KeccakHasher}; +use hashdb::{HashDB, Hasher}; +use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::Bytes; use memorydb::MemoryDB; diff --git a/ethcore/light/src/transaction_queue.rs b/ethcore/light/src/transaction_queue.rs index 6803cc6af64..290dacc5386 100644 --- a/ethcore/light/src/transaction_queue.rs +++ b/ethcore/light/src/transaction_queue.rs @@ -30,7 +30,7 @@ use std::collections::hash_map::Entry; use transaction::{self, Condition, PendingTransaction, SignedTransaction}; use ethereum_types::{H256, U256, Address}; use plain_hasher::H256FastMap; -use hashdb::KeccakHasher; +use keccak_hasher::KeccakHasher; // Knowledge of an account's current nonce. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/ethcore/src/account_db.rs b/ethcore/src/account_db.rs index 2713559f5dc..dd05d22cbc6 100644 --- a/ethcore/src/account_db.rs +++ b/ethcore/src/account_db.rs @@ -17,7 +17,8 @@ //! DB backend wrapper for Account trie use ethereum_types::H256; use hash::{KECCAK_NULL_RLP, keccak}; -use hashdb::{HashDB, KeccakHasher, AsHashDB}; +use hashdb::{HashDB, AsHashDB}; +use keccak_hasher::KeccakHasher; use kvdb::DBValue; use rlp::NULL_RLP; use std::collections::HashMap; diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index f1e5d882ead..6e331f341d4 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -1710,7 +1710,7 @@ impl BlockChainClient for Client { fn list_storage(&self, id: BlockId, account: &Address, after: Option<&H256>, count: u64) -> Option> { if !self.factories.trie.is_fat() { - trace!(target: "fatdb", "list_stroage: Not a fat DB"); + trace!(target: "fatdb", "list_storage: Not a fat DB"); return None; } diff --git a/ethcore/src/client/error.rs b/ethcore/src/client/error.rs index 787a9fa5368..a9f753db653 100644 --- a/ethcore/src/client/error.rs +++ b/ethcore/src/client/error.rs @@ -19,7 +19,9 @@ use util_error::UtilError; use kvdb; use trie::TrieError; -use hashdb::{Hasher, KeccakHasher}; +use hashdb::Hasher; +use keccak_hasher::KeccakHasher; + /// Client configuration errors. #[derive(Debug)] pub enum Error { diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index fda2534c532..d4ed9a52063 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -25,7 +25,8 @@ use {state, state_db, client, executive, trace, transaction, db, spec, pod_state use factory::Factories; use evm::{VMType, FinalizationResult}; use vm::{self, ActionParams}; -use hashdb::{Hasher, KeccakHasher}; +use hashdb::Hasher; +use keccak_hasher::KeccakHasher; /// EVM test Error. #[derive(Debug)] diff --git a/ethcore/src/factory.rs b/ethcore/src/factory.rs index 1b58effbcac..ec183bcd89e 100644 --- a/ethcore/src/factory.rs +++ b/ethcore/src/factory.rs @@ -19,7 +19,7 @@ use account_db::Factory as AccountFactory; use evm::{Factory as EvmFactory, VMType}; use vm::{Vm, ActionParams, Schedule}; use wasm::WasmInterpreter; -use hashdb::KeccakHasher; +use keccak_hasher::KeccakHasher; const WASM_MAGIC_NUMBER: &'static [u8; 4] = b"\0asm"; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 961958e214d..b57a89b71a3 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -88,6 +88,7 @@ extern crate rayon; extern crate rlp; extern crate rlp_compress; extern crate keccak_hash as hash; +extern crate keccak_hasher; extern crate heapsize; extern crate memorydb; extern crate patricia_trie as trie; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index f2c5199c578..43cb6a4b272 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -19,7 +19,8 @@ use std::collections::BTreeMap; use itertools::Itertools; use hash::{keccak}; use ethereum_types::{H256, U256}; -use hashdb::{HashDB, KeccakHasher}; +use hashdb::HashDB; +use keccak_hasher::KeccakHasher; use triehash::sec_trie_root; use bytes::Bytes; use trie::{TrieFactory, KeccakRlpNodeCodec}; diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index c6856301123..201f6c30a2c 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -23,7 +23,8 @@ use ids::BlockId; use ethereum_types::H256; use trie::TrieError; use rlp::DecoderError; -use hashdb::{Hasher, KeccakHasher}; +use hashdb::Hasher; +use keccak_hasher::KeccakHasher; /// Snapshot-related errors. #[derive(Debug)] diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index f329f4dd227..e47229c51cb 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -31,7 +31,8 @@ use header::Header; use ids::BlockId; use ethereum_types::{H256, U256}; -use hashdb::{HashDB, KeccakHasher}; +use hashdb::HashDB; +use keccak_hasher::KeccakHasher; use kvdb::DBValue; use snappy; use bytes::Bytes; diff --git a/ethcore/src/snapshot/tests/helpers.rs b/ethcore/src/snapshot/tests/helpers.rs index eccdc4ecc85..e5893263b96 100644 --- a/ethcore/src/snapshot/tests/helpers.rs +++ b/ethcore/src/snapshot/tests/helpers.rs @@ -35,7 +35,8 @@ use rand::Rng; use kvdb::{KeyValueDB, DBValue}; use ethereum_types::H256; -use hashdb::{HashDB, KeccakHasher}; +use hashdb::HashDB; +use keccak_hasher::KeccakHasher; use journaldb; use trie::{SecTrieDBMut, TrieMut, TrieDB, TrieDBMut, Trie, KeccakRlpNodeCodec}; use self::trie_standardmap::{Alphabet, StandardMap, ValueMode}; diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index 9a796de214b..71fb5211af3 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -22,7 +22,8 @@ use std::collections::{HashMap, BTreeMap}; use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use ethereum_types::{H256, U256, Address}; use error::Error; -use hashdb::{Hasher, HashDB, KeccakHasher}; +use hashdb::{Hasher, HashDB}; +use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::{Bytes, ToPretty}; use trie; diff --git a/ethcore/src/state/backend.rs b/ethcore/src/state/backend.rs index 2d9b0550365..6ec4f77ac0e 100644 --- a/ethcore/src/state/backend.rs +++ b/ethcore/src/state/backend.rs @@ -28,7 +28,8 @@ use state::Account; use parking_lot::Mutex; use ethereum_types::{Address, H256}; use memorydb::KeccakMemoryDB; -use hashdb::{AsHashDB, HashDB, DBValue, KeccakHasher}; +use hashdb::{AsHashDB, HashDB, DBValue}; +use keccak_hasher::KeccakHasher; /// State backend. See module docs for more details. pub trait Backend: Send { diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 9a59fe26b93..12a85c698c2 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -43,7 +43,8 @@ use state_db::StateDB; use factory::VmFactory; use ethereum_types::{H256, U256, Address}; -use hashdb::{HashDB, AsHashDB, KeccakHasher}; +use hashdb::{HashDB, AsHashDB}; +use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::Bytes; diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs index b65f2dfc47d..80174d71723 100644 --- a/ethcore/src/state_db.rs +++ b/ethcore/src/state_db.rs @@ -21,7 +21,8 @@ use byteorder::{LittleEndian, ByteOrder}; use db::COL_ACCOUNT_BLOOM; use ethereum_types::{H256, Address}; use hash::keccak; -use hashdb::{KeccakHasher, HashDB}; +use hashdb::HashDB; +use keccak_hasher::KeccakHasher; use header::BlockNumber; use journaldb::JournalDB; use kvdb::{KeyValueDB, DBTransaction}; diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 1c891a5bfac..97d3ff5df00 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -97,7 +97,7 @@ use std::collections::{HashSet, HashMap}; use std::cmp; use std::time::{Duration, Instant}; use hash::keccak; -use hashdb::KeccakHasher; +use keccak_hasher::KeccakHasher; use heapsize::HeapSizeOf; use ethereum_types::{H256, U256}; use plain_hasher::H256FastMap; diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index 546287407f7..ad2954a5df7 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -18,7 +18,7 @@ use api::TransactionStats; use std::collections::{HashSet, HashMap}; use ethereum_types::{H256, H512}; use plain_hasher::H256FastMap; -use hashdb::KeccakHasher; +use keccak_hasher::KeccakHasher; type NodeId = H512; type BlockNumber = u64; From 0e5fdfacd4d816eda4cae3db48178a4bd23c0cda Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 21:51:38 +0200 Subject: [PATCH 091/160] cargo lock --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index e7b12f1d7c9..8c3318c86ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -543,6 +543,7 @@ dependencies = [ "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.1.2", + "keccak-hasher 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", "kvdb-rocksdb 0.1.0", @@ -636,6 +637,7 @@ dependencies = [ "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", + "keccak-hasher 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", From 5ac48b3c0b4aebb6877efe49b4e9ad37d9564b71 Mon Sep 17 00:00:00 2001 From: David Palm Date: Sat, 16 Jun 2018 22:07:24 +0200 Subject: [PATCH 092/160] sync: Use keccak-hasher --- ethcore/sync/Cargo.toml | 1 + ethcore/sync/src/lib.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 8d8b529ebfc..f9c75fa3d5c 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -21,6 +21,7 @@ plain_hasher = { version = "0.2", path = "../../util/plain_hasher" } rlp = { path = "../../util/rlp" } rustc-hex = "1.0" keccak-hash = { path = "../../util/hash" } +keccak-hasher = { path = "../../util/keccak-hasher" } triehash = { path = "../../util/triehash" } kvdb = { path = "../../util/kvdb" } macros = { path = "../../util/macros" } diff --git a/ethcore/sync/src/lib.rs b/ethcore/sync/src/lib.rs index 11e088aa771..708c3b89a92 100644 --- a/ethcore/sync/src/lib.rs +++ b/ethcore/sync/src/lib.rs @@ -39,6 +39,7 @@ extern crate smallvec; extern crate rlp; extern crate ipnetwork; extern crate keccak_hash as hash; +extern crate keccak_hasher; extern crate triehash; extern crate kvdb; extern crate stream_encoder; From 67bf64483673339fec0281fc19cc075fda2a0c64 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 18 Jun 2018 10:18:06 +0200 Subject: [PATCH 093/160] =?UTF-8?q?Add=20patricia-trie-ethereum=20fa=C3=A7?= =?UTF-8?q?ade=20crate=20for=20keccak/rlp=20concrete=20impls=20of=20patric?= =?UTF-8?q?ia=5Ftrie?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 14 +++ Cargo.toml | 1 + util/patricia-trie-ethereum/Cargo.toml | 14 +++ util/patricia-trie-ethereum/src/lib.rs | 117 +++++++++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 util/patricia-trie-ethereum/Cargo.toml create mode 100644 util/patricia-trie-ethereum/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8c3318c86ac..b37582e958d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -884,6 +884,7 @@ dependencies = [ "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", + "keccak-hasher 0.1.0", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2438,6 +2439,7 @@ dependencies = [ "keccak-hasher 0.1.0", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "memorydb 0.2.0", + "patricia-trie-ethereum 0.1.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", "stream-encoder 0.1.0", @@ -2445,6 +2447,18 @@ dependencies = [ "triehash 0.1.0", ] +[[package]] +name = "patricia-trie-ethereum" +version = "0.1.0" +dependencies = [ + "ethcore-bytes 0.1.0", + "hashdb 0.2.0", + "keccak-hasher 0.1.0", + "patricia-trie 0.2.0", + "rlp 0.2.1", + "stream-encoder 0.1.0", +] + [[package]] name = "percent-encoding" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 61ee060f00c..8a01c33b915 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,6 +137,7 @@ members = [ "whisper/cli", "util/stream-encoder", "util/keccak-hasher", + "util/patricia-trie-ethereum", ] [patch.crates-io] diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml new file mode 100644 index 00000000000..8be0cd69663 --- /dev/null +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "patricia-trie-ethereum" +version = "0.1.0" +authors = ["Parity Technologies "] +description = "Merkle-Patricia Trie (Ethereum Style)" +license = "GPL-3.0" + +[dependencies] + patricia-trie = { path = "../patricia_trie" } + keccak-hasher = { path = "../keccak-hasher" } + hashdb = { path = "../hashdb" } + rlp = { path = "../rlp" } + ethcore-bytes = { path = "../bytes" } + stream-encoder = { path = "../stream-encoder" } diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs new file mode 100644 index 00000000000..d331c63b8da --- /dev/null +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -0,0 +1,117 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Façade crate for `patricia_trie` for Ethereum specific impls + +pub extern crate patricia_trie as trie; +extern crate ethcore_bytes; +extern crate hashdb; +extern crate keccak_hasher; +extern crate rlp; +extern crate stream_encoder; + +use ethcore_bytes::Bytes; +use hashdb::Hasher; +use keccak_hasher::KeccakHasher; +use rlp::{Decodable, RlpStream, DecoderError, Rlp, Prototype}; +use std::marker::PhantomData; +use stream_encoder::Stream; +use trie::node::Node; // TODO: re-export from root? +pub use trie::{Lookup, NibbleSlice, node_codec::NodeCodec}; + +pub type KeccakTrieResult = trie::Result::Out>; +pub type RlpCodec = RlpNodeCodec; + +#[derive(Default, Clone)] +pub struct RlpNodeCodec {mark: PhantomData} + +impl NodeCodec for RlpNodeCodec +where H::Out: Decodable +{ + type E = DecoderError; + type S = RlpStream; + fn encode(node: &Node) -> Bytes { + match *node { + Node::Leaf(ref slice, ref value) => { + let mut stream = RlpStream::new_list(2); + stream.append(&&*slice.encoded(true)); + stream.append(value); + stream.out() + }, + Node::Extension(ref slice, ref raw_rlp) => { + let mut stream = RlpStream::new_list(2); + stream.append(&&*slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + stream.out() + }, + Node::Branch(ref nodes, ref value) => { + let mut stream = RlpStream::new_list(17); + for i in 0..16 { + stream.append_raw(nodes[i], 1); + } + match *value { + Some(ref n) => { stream.append(n); }, + None => { stream.append_empty_data(); }, + } + stream.out() + }, + Node::Empty => { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.out() + } + } + } + fn decode(data: &[u8]) -> Result { + let r = Rlp::new(data); + match r.prototype()? { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { + (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), + (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes = [&[] as &[u8]; 16]; + for i in 0..16 { + nodes[i] = r.at(i)?.as_raw(); + } + Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) + }, + // an empty branch index. + Prototype::Data(0) => Ok(Node::Empty), + // something went wrong. + _ => Err(DecoderError::Custom("Rlp is not valid.")) + } + } + fn try_decode_hash(data: &[u8]) -> Option { + let r = Rlp::new(data); + if r.is_data() && r.size() == H::LENGTH { + Some(r.as_val().expect("Hash is the correct size; qed")) + } else { + None + } + } + + fn is_empty_node(data: &[u8]) -> bool { + // REVIEW: Could also be `Rlp::new(data).is_empty()` – better? + data.len() != 0 && (data[0] == 0xC0 || data[0] == 0x80) + } +} From b8e97641e1a1f6ce35af29cd69e65af56d825285 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 18 Jun 2018 11:00:54 +0200 Subject: [PATCH 094/160] Remove all Rlp dependencies from patricia_trie Make tests more explicit about dependencies Make tests use patricia-trie-ethereum --- util/patricia_trie/Cargo.toml | 15 ++--- util/patricia_trie/src/fatdb.rs | 36 ++++++----- util/patricia_trie/src/fatdbmut.rs | 29 +++++---- util/patricia_trie/src/lib.rs | 38 ++++++----- util/patricia_trie/src/node_codec.rs | 87 +------------------------- util/patricia_trie/src/recorder.rs | 7 +-- util/patricia_trie/src/sectriedb.rs | 33 +++++----- util/patricia_trie/src/sectriedbmut.rs | 33 +++++----- util/patricia_trie/src/triedb.rs | 12 ++-- util/patricia_trie/src/triedbmut.rs | 16 +++-- 10 files changed, 116 insertions(+), 190 deletions(-) diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index e54baf1e99a..f25a2586311 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -2,7 +2,7 @@ name = "patricia-trie" version = "0.2.0" authors = ["Parity Technologies "] -description = "Merkle-Patricia Trie (Ethereum Style)" +description = "Merkle-Patricia Trie generic over key hasher and node encoding" license = "GPL-3.0" [dependencies] @@ -10,15 +10,16 @@ elastic-array = "0.10" log = "0.3" rand = "0.4" ethcore-bytes = { version = "0.1.0", path = "../bytes" } -ethereum-types = "0.3" -keccak-hash = { version = "0.1.0", path = "../hash" } hashdb = { version = "0.2", path = "../hashdb" } -rlp = { version = "0.2.1", path = "../rlp" } -triehash = { version = "0.1.0", path = "../triehash" } -memorydb = { version = "0.2", path = "../memorydb" } ethcore-logger = { version = "1.9.0", path = "../../logger" } stream-encoder = { path = "../stream-encoder" } -keccak-hasher = { path = "../keccak-hasher" } [dev-dependencies] +triehash = { version = "0.1.0", path = "../triehash" } +memorydb = { version = "0.2", path = "../memorydb" } +ethereum-types = "0.3" +keccak-hash = { version = "0.1.0", path = "../hash" } +keccak-hasher = { path = "../keccak-hasher" } +rlp = { version = "0.2.1", path = "../rlp" } trie-standardmap = { path = "../trie-standardmap" } +patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } \ No newline at end of file diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index 75f3bb5d122..5e0852c8c73 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -123,24 +123,26 @@ where } } -#[test] -fn fatdb_to_trie() { +#[cfg(test)] +mod test { use memorydb::MemoryDB; - use hashdb::DBValue; - use super::fatdbmut::FatDBMut; - use super::TrieMut; + use hashdb::{Hasher, DBValue}; use keccak_hasher::KeccakHasher; - use node_codec::RlpNodeCodec; - - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); - { - let mut t = FatDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + use ethtrie::RlpCodec; + use ethtrie::trie::{Trie, TrieMut, FatDB, FatDBMut}; + + #[test] + fn fatdb_to_trie() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = FatDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + } + let t = FatDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); + assert_eq!( + t.iter().unwrap().map(Result::unwrap).collect::>(), + vec![(vec![0x01u8, 0x23], DBValue::from_slice(&[0x01u8, 0x23] as &[u8]))]); } - let t = FatDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); - assert_eq!( - t.iter().unwrap().map(Result::unwrap).collect::>(), - vec![(vec![0x01u8, 0x23], DBValue::from_slice(&[0x01u8, 0x23] as &[u8]))]); } diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index a110fa3f82e..f9d2ba141ec 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -106,21 +106,24 @@ where } } -#[test] -fn fatdbmut_to_trie() { +#[cfg(test)] +mod test { + use hashdb::{Hasher, DBValue}; use memorydb::MemoryDB; - use super::TrieDB; - use super::Trie; + use ethtrie::trie::{Trie, TrieMut, TrieDB, FatDBMut}; + use ethtrie::RlpCodec; use keccak_hasher::KeccakHasher; use keccak; - use node_codec::RlpNodeCodec; - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); - { - let mut t = FatDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + #[test] + fn fatdbmut_to_trie() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = FatDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + } + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } - let t = TrieDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -} +} \ No newline at end of file diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 85d4920570f..1007783a650 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -18,24 +18,32 @@ extern crate elastic_array; extern crate ethcore_bytes as bytes; extern crate ethcore_logger; -extern crate ethereum_types; extern crate hashdb; -extern crate keccak_hash as keccak; -extern crate keccak_hasher; -extern crate memorydb; extern crate rand; -extern crate rlp; extern crate stream_encoder; -#[cfg(test)] -extern crate trie_standardmap as standardmap; - #[macro_use] extern crate log; +#[cfg(test)] +extern crate ethereum_types; +#[cfg(test)] +extern crate trie_standardmap as standardmap; +#[cfg(test)] +extern crate patricia_trie_ethereum as ethtrie; +#[cfg(test)] +extern crate memorydb; +#[cfg(test)] +extern crate rlp; +#[cfg(test)] +extern crate keccak_hash as keccak; +#[cfg(test)] +extern crate keccak_hasher; +#[cfg(test)] +extern crate triehash; + use std::{fmt, error}; use hashdb::{HashDB, DBValue, Hasher}; -use keccak_hasher::KeccakHasher; use std::marker::PhantomData; pub mod node; @@ -49,8 +57,8 @@ pub mod recorder; mod fatdb; mod fatdbmut; mod lookup; -mod nibbleslice; mod nibblevec; +mod nibbleslice; pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::triedbmut::TrieDBMut; @@ -59,14 +67,10 @@ pub use self::sectriedb::SecTrieDB; pub use self::fatdb::{FatDB, FatDBIterator}; pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; +pub use self::lookup::Lookup; +pub use self::nibbleslice::NibbleSlice; use node_codec::NodeCodec; -// TODO: Move to façade crate and rename to just `RlpNodeCodec` -pub type KeccakRlpNodeCodec = node_codec::RlpNodeCodec; -pub type KeccakTrieResult = Result::Out>; -#[cfg(test)] -type RlpCodec = node_codec::RlpNodeCodec; - /// Trie Errors. /// /// These borrow the data within them to avoid excessive copying on every @@ -290,7 +294,7 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { impl<'db, H, C> TrieFactory where H: Hasher, - H::Out: rlp::Encodable, + H::Out:, C: NodeCodec + 'db { /// Creates new factory. diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 3c0fd76139b..694028b0896 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -17,12 +17,10 @@ //! Generic trait for trie node encoding/decoding. Takes a `hashdb::Hasher` //! to parametrize the hashes used in the codec; takes a `stream_encoder::Stream` //! implementation to do streaming encoding. + use bytes::Bytes; -use nibbleslice::NibbleSlice; -use rlp::{Prototype, Rlp, RlpStream, DecoderError, Decodable}; use hashdb::Hasher; use node::Node; -use std::marker::PhantomData; use stream_encoder::Stream; /// Trait for trie node encoding/decoding @@ -42,86 +40,3 @@ pub trait NodeCodec: Sized { // Check if the provided bytes correspond to the codecs "empty" node. fn is_empty_node(data: &[u8]) -> bool; } - -#[derive(Default, Clone)] -// TODO: move this to façade crate -pub struct RlpNodeCodec {mark: PhantomData} - -// TODO: move this to façade crate -impl NodeCodec for RlpNodeCodec -where H::Out: Decodable -{ - type E = DecoderError; - type S = RlpStream; - fn encode(node: &Node) -> Bytes { - match *node { - Node::Leaf(ref slice, ref value) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(true)); - stream.append(value); - stream.out() - }, - Node::Extension(ref slice, ref raw_rlp) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(false)); - stream.append_raw(raw_rlp, 1); - stream.out() - }, - Node::Branch(ref nodes, ref value) => { - let mut stream = RlpStream::new_list(17); - for i in 0..16 { - stream.append_raw(nodes[i], 1); - } - match *value { - Some(ref n) => { stream.append(n); }, - None => { stream.append_empty_data(); }, - } - stream.out() - }, - Node::Empty => { - let mut stream = RlpStream::new(); - stream.append_empty_data(); - stream.out() - } - } - } - fn decode(data: &[u8]) -> Result { - let r = Rlp::new(data); - match r.prototype()? { - // either leaf or extension - decode first item with NibbleSlice::??? - // and use is_leaf return to figure out which. - // if leaf, second item is a value (is_data()) - // if extension, second item is a node (either SHA3 to be looked up and - // fed back into this function or inline RLP which can be fed back into this function). - Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { - (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), - (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), - }, - // branch - first 16 are nodes, 17th is a value (or empty). - Prototype::List(17) => { - let mut nodes = [&[] as &[u8]; 16]; - for i in 0..16 { - nodes[i] = r.at(i)?.as_raw(); - } - Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) - }, - // an empty branch index. - Prototype::Data(0) => Ok(Node::Empty), - // something went wrong. - _ => Err(DecoderError::Custom("Rlp is not valid.")) - } - } - fn try_decode_hash(data: &[u8]) -> Option { - let r = Rlp::new(data); - if r.is_data() && r.size() == H::LENGTH { - Some(r.as_val().expect("Hash is the correct size; qed")) - } else { - None - } - } - - fn is_empty_node(data: &[u8]) -> bool { - // REVIEW: Could also be `Rlp::new(data).is_empty()` – better? - data.len() != 0 && (data[0] == 0xC0 || data[0] == 0x80) - } -} \ No newline at end of file diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 4a5d1f46d0c..2c6f203976d 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -84,7 +84,6 @@ impl Recorder { } } -// TODO: WUT! #[cfg(test)] mod tests { use super::*; @@ -146,9 +145,9 @@ mod tests { #[test] fn trie_record() { - use super::super::{TrieDB, TrieDBMut, Trie, TrieMut}; + use ethtrie::trie::{TrieDB, TrieDBMut, Trie, TrieMut, Recorder}; use memorydb::MemoryDB; - use RlpCodec; + use ethtrie::RlpCodec; let mut db = MemoryDB::::new(); @@ -215,6 +214,4 @@ mod tests { ] ]); } -/* -*/ } diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index 1949096281d..4c9308a6602 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -//use ethereum_types::H256; -//use keccak::keccak; use hashdb::{HashDB, Hasher}; use super::triedb::TrieDB; use super::{Result, Trie, TrieItem, TrieIterator, Query}; @@ -81,21 +79,24 @@ where } } -#[test] -fn trie_to_sectrie() { +#[cfg(test)] +mod test { use memorydb::MemoryDB; - use hashdb::DBValue; - use super::{TrieMut, TrieDBMut}; - use keccak_hasher::KeccakHasher; + use hashdb::{Hasher, DBValue}; use keccak; - use node_codec::RlpNodeCodec; + use keccak_hasher::KeccakHasher; + use ethtrie::RlpCodec; + use ethtrie::trie::{Trie, TrieMut, TrieDBMut, SecTrieDB}; - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); - { - let mut t = TrieDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); - t.insert(&keccak::keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); + #[test] + fn trie_to_sectrie() { + let mut db = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); + t.insert(&keccak::keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); + } + let t = SecTrieDB::<_, RlpCodec>::new(&db, &root).unwrap(); + assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } - let t = SecTrieDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); -} +} \ No newline at end of file diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index f71c2524991..9ba1005ce61 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -88,21 +88,24 @@ where } } -#[test] -fn sectrie_to_trie() { - use memorydb::*; - use super::triedb::*; - use super::Trie; - use keccak_hasher::KeccakHasher; +#[cfg(test)] +mod test { + use memorydb::MemoryDB; + use hashdb::{Hasher, DBValue}; use keccak; - use node_codec::RlpNodeCodec; - - let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); - { - let mut t = SecTrieDBMut::<_, RlpNodeCodec<_>>::new(&mut memdb, &mut root); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + use keccak_hasher::KeccakHasher; + use ethtrie::RlpCodec; + use ethtrie::trie::{Trie, TrieMut, TrieDB, SecTrieDBMut}; + + #[test] + fn sectrie_to_trie() { + let mut memdb = MemoryDB::::new(); + let mut root = ::Out::default(); + { + let mut t = SecTrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + } + let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } - let t = TrieDB::<_, RlpNodeCodec<_>>::new(&memdb, &root).unwrap(); - assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index ad8af7b31b4..15318757ea9 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -34,6 +34,7 @@ use std::marker::PhantomData; /// # Example /// ``` /// extern crate patricia_trie as trie; +/// extern crate patricia_trie_ethereum as ethtrie; /// extern crate hashdb; /// extern crate keccak_hasher; /// extern crate memorydb; @@ -44,7 +45,7 @@ use std::marker::PhantomData; /// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; -/// use node_codec::RlpNodeCodec; +/// use ethtrie::RlpNodeCodec; /// /// type RlpCodec = RlpNodeCodec; /// @@ -435,9 +436,9 @@ mod tests { use hashdb::{Hasher, DBValue}; use keccak_hasher::KeccakHasher; use memorydb::*; - use super::{TrieDB, Trie, Lookup, NibbleSlice}; - use super::super::{TrieMut, triedbmut::*}; - use RlpCodec; + use ethtrie::trie::{Trie, TrieMut, TrieDB, Lookup}; + use ethtrie::trie::triedbmut::*; + use ethtrie::RlpCodec; #[test] fn iterator() { @@ -626,6 +627,7 @@ mod tests { use rlp; use ethereum_types::H512; use std::marker::PhantomData; + use ethtrie::trie::NibbleSlice; let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); @@ -639,7 +641,7 @@ mod tests { // query for an invalid data type to trigger an error let q = rlp::decode::; - let lookup = Lookup::<_, RlpCodec, _>{ db: t.db, query: q, hash: root, marker: PhantomData }; + let lookup = Lookup::<_, RlpCodec, _>{ db: t.db(), query: q, hash: root, marker: PhantomData }; let query_result = lookup.look_up(NibbleSlice::new(b"A")); assert_eq!(query_result.unwrap().unwrap().unwrap_err(), rlp::DecoderError::RlpIsTooShort); } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 8989562260a..48ebe0e5d52 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -268,6 +268,7 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// # Example /// ``` /// extern crate patricia_trie as trie; +/// extern crate patricia_trie_ethereum as ethtrie; /// extern crate hashdb; /// extern crate keccak_hash; /// extern crate keccak_hasher; @@ -280,7 +281,7 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; -/// use node_codec::RlpNodeCodec; +/// use ethtrie::RlpNodeCodec; /// /// type RlpCodec = RlpNodeCodec; /// @@ -974,18 +975,15 @@ where #[cfg(test)] mod tests { - extern crate triehash; - use bytes::ToPretty; - use hashdb::*; + use hashdb::{DBValue, Hasher, HashDB}; use keccak_hasher::KeccakHasher; - use memorydb::*; + use memorydb::MemoryDB; use rlp::{Decodable, Encodable}; - use self::triehash::trie_root; + use triehash::trie_root; use standardmap::*; - use super::*; - use super::super::TrieMut; - use RlpCodec; + use ethtrie::trie::{TrieMut, TrieDBMut}; + use ethtrie::{NodeCodec, RlpCodec}; fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec From c1eee0cc64a5298696a1a4fa452877d3ed405e4a Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 18 Jun 2018 11:24:54 +0200 Subject: [PATCH 095/160] cleanup nibbleslice --- util/patricia_trie/src/nibbleslice.rs | 43 +++++++++++++++------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/util/patricia_trie/src/nibbleslice.rs b/util/patricia_trie/src/nibbleslice.rs index 41530498108..8cb03e53dd6 100644 --- a/util/patricia_trie/src/nibbleslice.rs +++ b/util/patricia_trie/src/nibbleslice.rs @@ -67,31 +67,29 @@ impl<'a> Iterator for NibbleSliceIterator<'a> { } } -impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { +impl<'a> NibbleSlice<'a> { /// Create a new nibble slice with the given byte-slice. pub fn new(data: &'a [u8]) -> Self { NibbleSlice::new_offset(data, 0) } /// Create a new nibble slice with the given byte-slice with a nibble offset. - pub fn new_offset(data: &'a [u8], offset: usize) -> Self { NibbleSlice{data: data, offset: offset, data_encode_suffix: &b""[..], offset_encode_suffix: 0} } + pub fn new_offset(data: &'a [u8], offset: usize) -> Self { + NibbleSlice { + data, + offset, + data_encode_suffix: &b""[..], + offset_encode_suffix: 0 + } + } /// Create a composed nibble slice; one followed by the other. - pub fn new_composed(a: &'a NibbleSlice, b: &'a NibbleSlice) -> Self { NibbleSlice{data: a.data, offset: a.offset, data_encode_suffix: b.data, offset_encode_suffix: b.offset} } - - /*pub fn new_composed_bytes_offset(a: &NibbleSlice, b: &NibbleSlice) -> (Bytes, usize) { - let r: Vec::with_capacity((a.len() + b.len() + 1) / 2); - let mut i = (a.len() + b.len()) % 2; - while i < a.len() { - match i % 2 { - 0 => , - 1 => , - } - i += 1; + pub fn new_composed(a: &NibbleSlice<'a>, b: &NibbleSlice<'a>) -> Self { + NibbleSlice { + data: a.data, + offset: a.offset, + data_encode_suffix: b.data, + offset_encode_suffix: b.offset } - while i < a.len() + b.len() { - i += 1; - } - (r, a.len() + b.len()) - }*/ + } /// Get an iterator for the series of nibbles. pub fn iter(&'a self) -> NibbleSliceIterator<'a> { @@ -132,7 +130,14 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { } /// Return object which represents a view on to this slice (further) offset by `i` nibbles. - pub fn mid(&'view self, i: usize) -> NibbleSlice<'a> { NibbleSlice{ data: self.data, offset: self.offset + i, data_encode_suffix: &b""[..], offset_encode_suffix: 0 } } + pub fn mid(&self, i: usize) -> NibbleSlice<'a> { + NibbleSlice { + data: self.data, + offset: self.offset + i, + data_encode_suffix: &b""[..], + offset_encode_suffix: 0 + } + } /// Do we start with the same nibbles as the whole of `them`? pub fn starts_with(&self, them: &Self) -> bool { self.common_prefix(them) == them.len() } From 60e7c90c7bcbd24651cfd8072147093e0c037ceb Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 18 Jun 2018 11:55:00 +0200 Subject: [PATCH 096/160] Fix ethcore tests --- Cargo.lock | 1 + ethcore/Cargo.toml | 7 ++++--- ethcore/light/src/cht.rs | 11 ++++++----- ethcore/light/src/on_demand/request.rs | 9 +++++---- ethcore/src/client/test_client.rs | 10 +++++----- ethcore/src/factory.rs | 5 +++-- ethcore/src/json_tests/executive.rs | 2 +- ethcore/src/json_tests/trie.rs | 5 +++-- ethcore/src/lib.rs | 1 + ethcore/src/pod_account.rs | 5 +++-- ethcore/src/snapshot/account.rs | 7 ++++--- ethcore/src/snapshot/mod.rs | 7 ++++--- ethcore/src/snapshot/tests/helpers.rs | 11 ++++++----- ethcore/src/state/account.rs | 9 +++++---- ethcore/src/state/mod.rs | 7 ++++--- 15 files changed, 55 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b37582e958d..abba5df83e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -558,6 +558,7 @@ dependencies = [ "parity-machine 0.1.0", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", + "patricia-trie-ethereum 0.1.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 68b7f7a7cea..1c3b8acea42 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -17,9 +17,10 @@ ethash = { path = "../ethash" } ethcore-bloom-journal = { path = "../util/bloom" } ethcore-bytes = { path = "../util/bytes" } fetch = { path = "../util/fetch" } -hashdb = { version = "0.2", path = "../util/hashdb" } -memorydb = { version = "0.2", path = "../util/memorydb" } -patricia-trie = { version = "0.2", path = "../util/patricia_trie" } +hashdb = { path = "../util/hashdb" } +memorydb = { path = "../util/memorydb" } +patricia-trie = { path = "../util/patricia_trie" } +patricia-trie-ethereum = { path = "../util/patricia-trie-ethereum" } ethcore-crypto = { path = "crypto" } error-chain = { version = "0.11", default-features = false } ethcore-io = { path = "../util/io" } diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index 022ea2ea9e9..b510089195b 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -29,7 +29,8 @@ use hashdb::HashDB; use keccak_hasher::KeccakHasher; use memorydb::MemoryDB; use bytes::Bytes; -use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder, KeccakRlpNodeCodec}; +use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder}; +use ethtrie::RlpCodec; use rlp::{RlpStream, Rlp}; use stream_encoder::Stream; @@ -68,11 +69,11 @@ impl> CHT { /// Generate an inclusion proof for the entry at a specific block. /// Nodes before level `from_level` will be omitted. /// Returns an error on an incomplete trie, and `Ok(None)` on an unprovable request. - pub fn prove(&self, num: u64, from_level: u32) -> trie::KeccakTrieResult>> { + pub fn prove(&self, num: u64, from_level: u32) -> ethtrie::KeccakTrieResult>> { if block_to_cht_number(num) != Some(self.number) { return Ok(None) } let mut recorder = Recorder::with_depth(from_level); - let t = TrieDB::<_, KeccakRlpNodeCodec>::new(&self.db, &self.root)?; + let t = TrieDB::<_, RlpCodec>::new(&self.db, &self.root)?; t.get_with(&key!(num), &mut recorder)?; Ok(Some(recorder.drain().into_iter().map(|x| x.data).collect())) @@ -104,7 +105,7 @@ pub fn build(cht_num: u64, mut fetcher: F) -> Option::new(&mut db, &mut root); + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); for blk_num in (0..SIZE).map(|n| last_num - n) { let info = match fetcher(id) { Some(info) => info, @@ -152,7 +153,7 @@ pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256) let mut db = MemoryDB::::new(); for node in proof { db.insert(&node[..]); } - let res = match TrieDB::<_, KeccakRlpNodeCodec>::new(&db, &root) { + let res = match TrieDB::<_, RlpCodec>::new(&db, &root) { Err(_) => return None, Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| { let rlp = Rlp::new(val); diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 5ef8a651384..7ebffe93b89 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -39,7 +39,8 @@ use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::Bytes; use memorydb::MemoryDB; -use trie::{Trie, TrieDB, TrieError, KeccakRlpNodeCodec}; +use trie::{Trie, TrieDB, TrieError}; +use ethtrie::RlpCodec; const SUPPLIED_MATCHES: &'static str = "supplied responses always match produced requests; enforced by `check_response`; qed"; @@ -832,7 +833,7 @@ impl Account { let mut db = MemoryDB::new(); for node in proof { db.insert(&node[..]); } - match TrieDB::<_, KeccakRlpNodeCodec>::new(&db, &state_root).and_then(|t| t.get(&keccak(&self.address)))? { + match TrieDB::<_, RlpCodec>::new(&db, &state_root).and_then(|t| t.get(&keccak(&self.address)))? { Some(val) => { let rlp = Rlp::new(&val); Ok(Some(BasicAccount { @@ -1055,7 +1056,7 @@ mod tests { stream.out() }; { - let mut trie = SecTrieDBMut::<_, KeccakRlpNodeCodec>::new(&mut db, &mut root); + let mut trie = SecTrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); for _ in 0..100 { let address = Address::random(); trie.insert(&*address, &rand_acc()).unwrap(); @@ -1065,7 +1066,7 @@ mod tests { } let proof = { - let trie = SecTrieDB::<_, KeccakRlpNodeCodec>::new(&db, &root).unwrap(); + let trie = SecTrieDB::<_, RlpCodec>::new(&db, &root).unwrap(); let mut recorder = Recorder::new(); trie.get_with(&*addr, &mut recorder).unwrap().unwrap(); diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 814dc7d20ca..5aacbe80bbb 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -63,7 +63,7 @@ use state_db::StateDB; use header::Header; use encoded; use engines::EthEngine; -use trie; +use ethtrie; use state::StateInfo; use views::BlockView; @@ -582,10 +582,10 @@ impl Call for TestBlockChainClient { } impl StateInfo for () { - fn nonce(&self, _address: &Address) -> trie::KeccakTrieResult { unimplemented!() } - fn balance(&self, _address: &Address) -> trie::KeccakTrieResult { unimplemented!() } - fn storage_at(&self, _address: &Address, _key: &H256) -> trie::KeccakTrieResult { unimplemented!() } - fn code(&self, _address: &Address) -> trie::KeccakTrieResult>> { unimplemented!() } + fn nonce(&self, _address: &Address) -> ethtrie::KeccakTrieResult { unimplemented!() } + fn balance(&self, _address: &Address) -> ethtrie::KeccakTrieResult { unimplemented!() } + fn storage_at(&self, _address: &Address, _key: &H256) -> ethtrie::KeccakTrieResult { unimplemented!() } + fn code(&self, _address: &Address) -> ethtrie::KeccakTrieResult>> { unimplemented!() } } impl StateClient for TestBlockChainClient { diff --git a/ethcore/src/factory.rs b/ethcore/src/factory.rs index ec183bcd89e..2eff7d7605a 100644 --- a/ethcore/src/factory.rs +++ b/ethcore/src/factory.rs @@ -14,7 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use trie::{TrieFactory, KeccakRlpNodeCodec}; +use trie::TrieFactory; +use ethtrie::RlpCodec; use account_db::Factory as AccountFactory; use evm::{Factory as EvmFactory, VMType}; use vm::{Vm, ActionParams, Schedule}; @@ -55,7 +56,7 @@ pub struct Factories { /// factory for evm. pub vm: VmFactory, /// factory for tries. - pub trie: TrieFactory, + pub trie: TrieFactory, /// factory for account databases. pub accountdb: AccountFactory, } diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index db233e6639f..7ae2bb61c4f 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -80,7 +80,7 @@ impl<'a, T: 'a, V: 'a, B: 'a> TestExt<'a, T, V, B> address: Address, tracer: &'a mut T, vm_tracer: &'a mut V, - ) -> trie::KeccakTrieResult { + ) -> ethtrie::KeccakTrieResult { let static_call = false; Ok(TestExt { nonce: state.nonce(&address)?, diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index 51693aa561c..cd340faf5ce 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -15,14 +15,15 @@ // along with Parity. If not, see . use ethjson; -use trie::{TrieFactory, TrieSpec, KeccakRlpNodeCodec}; +use trie::{TrieFactory, TrieSpec}; +use ethtrie::RlpCodec; use ethereum_types::H256; use memorydb::KeccakMemoryDB; fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { let tests = ethjson::trie::Test::load(json).unwrap(); - let factory = TrieFactory::<_, KeccakRlpNodeCodec>::new(trie); + let factory = TrieFactory::<_, RlpCodec>::new(trie); let mut result = vec![]; for (name, test) in tests.into_iter() { diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index b57a89b71a3..130cf368c96 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -92,6 +92,7 @@ extern crate keccak_hasher; extern crate heapsize; extern crate memorydb; extern crate patricia_trie as trie; +extern crate patricia_trie_ethereum as ethtrie; extern crate triehash; extern crate ansi_term; extern crate unexpected; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 43cb6a4b272..0915628abc0 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -23,7 +23,8 @@ use hashdb::HashDB; use keccak_hasher::KeccakHasher; use triehash::sec_trie_root; use bytes::Bytes; -use trie::{TrieFactory, KeccakRlpNodeCodec}; +use trie::TrieFactory; +use ethtrie::RlpCodec; use state::Account; use ethjson; use types::account_diff::*; @@ -67,7 +68,7 @@ impl PodAccount { } /// Place additional data into given hash DB. - pub fn insert_additional(&self, db: &mut HashDB, factory: &TrieFactory) { + pub fn insert_additional(&self, db: &mut HashDB, factory: &TrieFactory) { match self.code { Some(ref c) if !c.is_empty() => { db.insert(c); } _ => {} diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index eb3044bf7aa..3985e1cec02 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -24,7 +24,8 @@ use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; use ethereum_types::{H256, U256}; use hashdb::HashDB; use bytes::Bytes; -use trie::{TrieDB, Trie, KeccakRlpNodeCodec}; +use trie::{TrieDB, Trie}; +use ethtrie::RlpCodec; use rlp::{RlpStream, Rlp}; use stream_encoder::Stream; @@ -68,7 +69,7 @@ impl CodeState { // account address hash, account properties and the storage. Each item contains at most `max_storage_items` // storage records split according to snapshot format definition. pub fn to_fat_rlps(account_hash: &H256, acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet, first_chunk_size: usize, max_chunk_size: usize) -> Result, Error> { - let db = TrieDB::<_, KeccakRlpNodeCodec>::new(acct_db, &acc.storage_root)?; + let db = TrieDB::<_, RlpCodec>::new(acct_db, &acc.storage_root)?; let mut chunks = Vec::new(); let mut db_iter = db.iter()?; let mut target_chunk_size = first_chunk_size; @@ -184,7 +185,7 @@ pub fn from_fat_rlp( { let mut storage_trie = if storage_root.is_zero() { - TrieDBMut::<_, KeccakRlpNodeCodec>::new(acct_db, &mut storage_root) + TrieDBMut::<_, RlpCodec>::new(acct_db, &mut storage_root) } else { TrieDBMut::from_existing(acct_db, &mut storage_root)? }; diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index e47229c51cb..4b4d6e0e2fe 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -39,7 +39,8 @@ use bytes::Bytes; use parking_lot::Mutex; use journaldb::{self, Algorithm, JournalDB}; use kvdb::KeyValueDB; -use trie::{TrieDB, TrieDBMut, Trie, TrieMut, KeccakRlpNodeCodec}; +use trie::{TrieDB, TrieDBMut, Trie, TrieMut}; +use ethtrie::RlpCodec; use rlp::{RlpStream, Rlp}; use bloom_journal::Bloom; use stream_encoder::Stream; @@ -267,7 +268,7 @@ impl<'a> StateChunker<'a> { /// Returns a list of hashes of chunks created, or any error it may /// have encountered. pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { - let account_trie = TrieDB::::new(db, &root)?; + let account_trie = TrieDB::::new(db, &root)?; let mut chunker = StateChunker { hashes: Vec::new(), @@ -364,7 +365,7 @@ impl StateRebuilder { // batch trie writes { let mut account_trie = if self.state_root != KECCAK_NULL_RLP { - TrieDBMut::<_, KeccakRlpNodeCodec>::from_existing(self.db.as_hashdb_mut(), &mut self.state_root)? + TrieDBMut::<_, RlpCodec>::from_existing(self.db.as_hashdb_mut(), &mut self.state_root)? } else { TrieDBMut::new(self.db.as_hashdb_mut(), &mut self.state_root) }; diff --git a/ethcore/src/snapshot/tests/helpers.rs b/ethcore/src/snapshot/tests/helpers.rs index e5893263b96..7d6c809ca70 100644 --- a/ethcore/src/snapshot/tests/helpers.rs +++ b/ethcore/src/snapshot/tests/helpers.rs @@ -38,7 +38,8 @@ use ethereum_types::H256; use hashdb::HashDB; use keccak_hasher::KeccakHasher; use journaldb; -use trie::{SecTrieDBMut, TrieMut, TrieDB, TrieDBMut, Trie, KeccakRlpNodeCodec}; +use trie::{SecTrieDBMut, TrieMut, TrieDB, TrieDBMut, Trie}; +use ethtrie::RlpCodec; use self::trie_standardmap::{Alphabet, StandardMap, ValueMode}; // the proportion of accounts we will alter each tick. @@ -64,7 +65,7 @@ impl StateProducer { pub fn tick(&mut self, rng: &mut R, db: &mut HashDB) { // modify existing accounts. let mut accounts_to_modify: Vec<_> = { - let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(&*db, &self.state_root).unwrap(); + let trie = TrieDB::<_, RlpCodec>::new(&*db, &self.state_root).unwrap(); let temp = trie.iter().unwrap() // binding required due to complicated lifetime stuff .filter(|_| rng.gen::() < ACCOUNT_CHURN) .map(Result::unwrap) @@ -83,7 +84,7 @@ impl StateProducer { } // sweep again to alter account trie. - let mut trie = TrieDBMut::<_, KeccakRlpNodeCodec>::from_existing(db, &mut self.state_root).unwrap(); + let mut trie = TrieDBMut::<_, RlpCodec>::from_existing(db, &mut self.state_root).unwrap(); for (address_hash, account_data) in accounts_to_modify { trie.insert(&address_hash[..], &account_data).unwrap(); @@ -118,9 +119,9 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { }; { let mut trie = if *root == KECCAK_NULL_RLP { - SecTrieDBMut::<_, KeccakRlpNodeCodec>::new(&mut db, root) + SecTrieDBMut::<_, RlpCodec>::new(&mut db, root) } else { - SecTrieDBMut::<_, KeccakRlpNodeCodec>::from_existing(&mut db, root).unwrap() + SecTrieDBMut::<_, RlpCodec>::from_existing(&mut db, root).unwrap() }; for (k, v) in map.make_with(seed) { diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index 71fb5211af3..511b94dcfc0 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -27,7 +27,8 @@ use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::{Bytes, ToPretty}; use trie; -use trie::{SecTrieDB, Trie, TrieFactory, TrieError, KeccakRlpNodeCodec}; +use trie::{SecTrieDB, Trie, TrieFactory, TrieError}; +use ethtrie::RlpCodec; use pod_account::*; use rlp::{RlpStream, encode}; use lru_cache::LruCache; @@ -205,7 +206,7 @@ impl Account { if let Some(value) = self.cached_storage_at(key) { return Ok(value); } - let db = SecTrieDB::<_, KeccakRlpNodeCodec>::new(db, &self.storage_root)?; + let db = SecTrieDB::<_, RlpCodec>::new(db, &self.storage_root)?; let panicky_decoder = |bytes:&[u8]| ::rlp::decode(&bytes).expect("decoding db value failed"); let item: U256 = db.get_with(key, panicky_decoder)?.unwrap_or_else(U256::zero); let value: H256 = item.into(); @@ -376,7 +377,7 @@ impl Account { } /// Commit the `storage_changes` to the backing DB and update `storage_root`. - pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> trie::Result<(), ::Out> { + pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> trie::Result<(), ::Out> { let mut t = trie_factory.from_existing(db, &mut self.storage_root)?; for (k, v) in self.storage_changes.drain() { // cast key and value to trait type, @@ -480,7 +481,7 @@ impl Account { let mut recorder = Recorder::new(); - let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(db, &self.storage_root)?; + let trie = TrieDB::<_, RlpCodec>::new(db, &self.storage_root)?; let item: U256 = { let panicky_decoder = |bytes:&[u8]| ::rlp::decode(bytes).expect("decoding db value failed"); let query = (&mut recorder, panicky_decoder); diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 12a85c698c2..cd47846f065 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -48,7 +48,8 @@ use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::Bytes; -use trie::{Trie, TrieError, TrieDB, KeccakTrieResult as TrieResult, KeccakRlpNodeCodec}; +use trie::{Trie, TrieError, TrieDB}; +use ethtrie::{RlpCodec, KeccakTrieResult as TrieResult}; use trie::recorder::Recorder; mod account; @@ -1051,7 +1052,7 @@ impl State { /// `account_key` == keccak(address) pub fn prove_account(&self, account_key: H256) -> TrieResult<(Vec, BasicAccount)> { let mut recorder = Recorder::new(); - let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(self.db.as_hashdb(), &self.root)?; + let trie = TrieDB::<_, RlpCodec>::new(self.db.as_hashdb(), &self.root)?; let maybe_account: Option = { let panicky_decoder = |bytes: &[u8]| { ::rlp::decode(bytes).expect(&format!("prove_account, could not query trie for account key={}", &account_key)) @@ -1077,7 +1078,7 @@ impl State { pub fn prove_storage(&self, account_key: H256, storage_key: H256) -> TrieResult<(Vec, H256)> { // TODO: probably could look into cache somehow but it's keyed by // address, not keccak(address). - let trie = TrieDB::<_, KeccakRlpNodeCodec>::new(self.db.as_hashdb(), &self.root)?; + let trie = TrieDB::<_, RlpCodec>::new(self.db.as_hashdb(), &self.root)?; let from_rlp = |b: &[u8]| Account::from_rlp(b).expect("decoding db value failed"); let acc = match trie.get_with(&account_key, from_rlp)? { Some(acc) => acc, From acd757d6602637afcfba27ae7664c56c40fe0fbc Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 18 Jun 2018 12:32:02 +0200 Subject: [PATCH 097/160] Fix tests in ethcore/light --- Cargo.lock | 1 + ethcore/light/Cargo.toml | 1 + ethcore/light/src/cht.rs | 4 ++-- ethcore/light/src/lib.rs | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index abba5df83e8..f4bf8f3a080 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -646,6 +646,7 @@ dependencies = [ "memorydb 0.2.0", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", + "patricia-trie-ethereum 0.1.0", "plain_hasher 0.2.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index 98d304011f5..7cdc8841068 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -14,6 +14,7 @@ ethcore-transaction = { path = "../transaction" } ethereum-types = "0.3" memorydb = { path = "../../util/memorydb" } patricia-trie = { path = "../../util/patricia_trie" } +patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } ethcore-network = { path = "../../util/network" } ethcore-io = { path = "../../util/io" } hashdb = { path = "../../util/hashdb" } diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index b510089195b..4d18240c7ff 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -29,8 +29,8 @@ use hashdb::HashDB; use keccak_hasher::KeccakHasher; use memorydb::MemoryDB; use bytes::Bytes; -use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder}; -use ethtrie::RlpCodec; +use trie::{TrieMut, TrieDBMut, Trie, TrieDB, Recorder}; +use ethtrie::{self, RlpCodec}; use rlp::{RlpStream, Rlp}; use stream_encoder::Stream; diff --git a/ethcore/light/src/lib.rs b/ethcore/light/src/lib.rs index 6e8f8e2c887..990b071777b 100644 --- a/ethcore/light/src/lib.rs +++ b/ethcore/light/src/lib.rs @@ -67,6 +67,7 @@ extern crate itertools; extern crate keccak_hasher; extern crate memorydb; extern crate patricia_trie as trie; +extern crate patricia_trie_ethereum as ethtrie; extern crate plain_hasher; extern crate rand; extern crate rlp; From 3376c7d4b7c9a050634817c3dedffdb1fdac1285 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 18 Jun 2018 14:17:26 +0200 Subject: [PATCH 098/160] Fix broken iterator Item type --- util/patricia_trie/src/fatdb.rs | 4 ++-- util/patricia_trie/src/lib.rs | 4 ++-- util/patricia_trie/src/sectriedb.rs | 2 +- util/patricia_trie/src/triedb.rs | 25 ++++--------------------- 4 files changed, 9 insertions(+), 26 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index 5e0852c8c73..f53cde35a09 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -65,7 +65,7 @@ where self.raw.get_with(Self::H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { + fn iter<'a>(&'a self) -> Result::Out>> + 'a>, ::Out> { FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) } } @@ -110,7 +110,7 @@ where H: Hasher, C: NodeCodec { - type Item = TrieItem<'db, H>; + type Item = TrieItem<'db, H::Out>; fn next(&mut self) -> Option { self.trie_iterator.next() diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 1007783a650..52209431aae 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -184,7 +184,7 @@ pub trait Trie { fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> where 'a: 'key; /// Returns a depth-first iterator over the elements of trie. - fn iter<'a>(&'a self) -> Result> + 'a>, ::Out>; + fn iter<'a>(&'a self) -> Result::Out>> + 'a>, ::Out>; } /// A key-value datastore implemented as a database-backed modified Merkle tree. @@ -286,7 +286,7 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { wrapper!(self, get_with, key, query) } - fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { + fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { wrapper!(self, iter,) } } diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index 4c9308a6602..daa3dbc8831 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -74,7 +74,7 @@ where self.raw.get_with(Self::H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { + fn iter<'a>(&'a self) -> Result::Out>> + 'a>, ::Out> { TrieDB::iter(&self.raw) } } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 15318757ea9..eff26e196e9 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -127,7 +127,7 @@ where }.look_up(NibbleSlice::new(key)) } - fn iter<'a>(&'a self) -> Result> + 'a>, H::Out> { + fn iter<'a>(&'a self) -> Result::Out>> + 'a>, H::Out> { TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>) } } @@ -347,10 +347,9 @@ impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C } impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { - type Item = TrieItem<'a, H>; + type Item = TrieItem<'a, H::Out>; fn next(&mut self) -> Option { - // enum IterStep { enum IterStep { Continue, PopTrail, @@ -406,24 +405,8 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { let node = C::decode(&d).expect("rlp read from db; qed"); self.descend_into_node(node.into()) }, - IterStep::Descend::(Err(_e)) => { -// return Some(Err(e)) // <–– REVIEW: This causes a compiler error I can't figure out: - /* - error[E0308]: mismatched types - --> util/patricia_trie/src/triedb.rs:404:22 - | - 404 | return Some(Err(e)) - | ^ expected type parameter, found associated type - | - = note: expected type `std::boxed::Box>` - found type `std::boxed::Box::Out>>` - */ - /* - The only way we end up here is when `self.db.get_raw_or_lookup()` fails. When it does it - returns `Box::new(TrieError::IncompleteDatabase(key))`, i.e. a proper `Box>`. - I'm missing something. - */ - panic!("FIXME: this causes `expected type parameter, found associated type`") + IterStep::Descend::(Err(e)) => { + return Some(Err(e)) } IterStep::Continue => {}, } From 46609fcc04d94c43c44441999ff5c7afa73ab765 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 18 Jun 2018 14:19:00 +0200 Subject: [PATCH 099/160] Export an ethereum TrieError --- util/patricia-trie-ethereum/Cargo.toml | 1 + util/patricia-trie-ethereum/src/lib.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml index 8be0cd69663..e6cef15da53 100644 --- a/util/patricia-trie-ethereum/Cargo.toml +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -12,3 +12,4 @@ license = "GPL-3.0" rlp = { path = "../rlp" } ethcore-bytes = { path = "../bytes" } stream-encoder = { path = "../stream-encoder" } + ethereum-types = "0.3" diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index d331c63b8da..cdff98ad3a1 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -22,8 +22,10 @@ extern crate hashdb; extern crate keccak_hasher; extern crate rlp; extern crate stream_encoder; +extern crate ethereum_types; use ethcore_bytes::Bytes; +use ethereum_types::H256; use hashdb::Hasher; use keccak_hasher::KeccakHasher; use rlp::{Decodable, RlpStream, DecoderError, Rlp, Prototype}; @@ -34,6 +36,7 @@ pub use trie::{Lookup, NibbleSlice, node_codec::NodeCodec}; pub type KeccakTrieResult = trie::Result::Out>; pub type RlpCodec = RlpNodeCodec; +pub type TrieError = trie::TrieError; #[derive(Default, Clone)] pub struct RlpNodeCodec {mark: PhantomData} From 40c4bebe3fda70f7204d7e657a45fb03007dea1b Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 18 Jun 2018 14:32:41 +0200 Subject: [PATCH 100/160] Fix trie error handling in ethcore --- Cargo.lock | 1 + ethcore/light/src/on_demand/request.rs | 14 ++++++-------- ethcore/private-tx/src/error.rs | 4 ++-- ethcore/src/client/error.rs | 14 +++++--------- ethcore/src/client/evm_test_client.rs | 6 ++---- ethcore/src/error.rs | 4 ++-- ethcore/src/snapshot/error.rs | 14 +++++--------- 7 files changed, 23 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4bf8f3a080..aaac68f662f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2454,6 +2454,7 @@ name = "patricia-trie-ethereum" version = "0.1.0" dependencies = [ "ethcore-bytes 0.1.0", + "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", "keccak-hasher 0.1.0", "patricia-trie 0.2.0", diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 7ebffe93b89..9259c1356e3 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -39,8 +39,8 @@ use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::Bytes; use memorydb::MemoryDB; -use trie::{Trie, TrieDB, TrieError}; -use ethtrie::RlpCodec; +use trie::{Trie, TrieDB}; +use ethtrie::{TrieError, RlpCodec}; const SUPPLIED_MATCHES: &'static str = "supplied responses always match produced requests; enforced by `check_response`; qed"; @@ -630,8 +630,7 @@ pub enum Error { /// Empty response. Empty, /// Trie lookup error (result of bad proof) - // Trie(TrieError), - Trie(TrieError<::Out>), // REVIEW: how do I fix this without making `Error` generic also? + Trie(TrieError), /// Bad inclusion proof BadProof, /// Header by number instead of hash. @@ -654,10 +653,9 @@ impl From<::rlp::DecoderError> for Error { } } -impl From>> for Error { - fn from(err: Box>) -> Self { - // Error::Trie(*err) - Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // REVIEW: how do I fix this without making `Error` generic also? +impl From> for Error { + fn from(err: Box) -> Self { + Error::Trie(*err) } } diff --git a/ethcore/private-tx/src/error.rs b/ethcore/private-tx/src/error.rs index abcf1142d49..b0b1e732965 100644 --- a/ethcore/private-tx/src/error.rs +++ b/ethcore/private-tx/src/error.rs @@ -16,7 +16,7 @@ use ethereum_types::{Address, H256}; use rlp::DecoderError; -use trie::TrieError; +use ethtrie::TrieError; use ethcore::account_provider::SignError; use ethcore::error::{Error as EthcoreError, ExecutionError}; use transaction::Error as TransactionError; @@ -26,7 +26,7 @@ error_chain! { foreign_links { Io(::std::io::Error) #[doc = "Error concerning the Rust standard library's IO subsystem."]; Decoder(DecoderError) #[doc = "RLP decoding error."]; - Trie(TrieError) #[doc = "Error concerning TrieDBs."]; // REVIEW: error_chain requires concrete types, but this probably is not ok. What's a better way? + Trie(TrieError) #[doc = "Error concerning TrieDBs."]; } errors { diff --git a/ethcore/src/client/error.rs b/ethcore/src/client/error.rs index a9f753db653..9ab4cddd028 100644 --- a/ethcore/src/client/error.rs +++ b/ethcore/src/client/error.rs @@ -17,26 +17,22 @@ use std::fmt::{Display, Formatter, Error as FmtError}; use util_error::UtilError; use kvdb; -use trie::TrieError; - -use hashdb::Hasher; -use keccak_hasher::KeccakHasher; +use ethtrie::TrieError; /// Client configuration errors. #[derive(Debug)] pub enum Error { /// TrieDB-related error. - Trie(TrieError<::Out>), // REVIEW: this doesn't look right – need to make `Error` generic too? + Trie(TrieError), /// Database error Database(kvdb::Error), /// Util error Util(UtilError), } -impl From> for Error { - fn from(err: TrieError) -> Self { - // Error::Trie(err) - Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // REVIEW: how do I fix this without making `Error` generic also? +impl From for Error { + fn from(err: TrieError) -> Self { + Error::Trie(err) } } diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index d4ed9a52063..5ab00491df8 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -25,15 +25,13 @@ use {state, state_db, client, executive, trace, transaction, db, spec, pod_state use factory::Factories; use evm::{VMType, FinalizationResult}; use vm::{self, ActionParams}; -use hashdb::Hasher; -use keccak_hasher::KeccakHasher; +use ethtrie; /// EVM test Error. #[derive(Debug)] pub enum EvmTestError { /// Trie integrity error. - // Trie(trie::TrieError), - Trie(trie::TrieError<::Out>), // REVIEW: how do I fix this without making `EvmTestError` generic also? + Trie(ethtrie::TrieError), /// EVM error. Evm(vm::Error), /// Initialization error. diff --git a/ethcore/src/error.rs b/ethcore/src/error.rs index 94c768dedd2..a83b6fb99b3 100644 --- a/ethcore/src/error.rs +++ b/ethcore/src/error.rs @@ -23,7 +23,7 @@ use ethereum_types::{H256, U256, Address, Bloom}; use util_error::{self, UtilError}; use snappy::InvalidInput; use unexpected::{Mismatch, OutOfBounds}; -use trie::TrieError; +use ethtrie::TrieError; use io::*; use header::BlockNumber; use client::Error as ClientError; @@ -245,7 +245,7 @@ error_chain! { foreign_links { Io(IoError) #[doc = "Io create error"]; StdIo(::std::io::Error) #[doc = "Error concerning the Rust standard library's IO subsystem."]; - Trie(TrieError) #[doc = "Error concerning TrieDBs."]; // REVIEW: error_chain requires concrete types, but this probably is not ok. What's a better way? + Trie(TrieError) #[doc = "Error concerning TrieDBs."]; Execution(ExecutionError) #[doc = "Error concerning EVM code execution."]; Block(BlockError) #[doc = "Error concerning block processing."]; Transaction(TransactionError) #[doc = "Error concerning transaction processing."]; diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index 201f6c30a2c..527b4e2882c 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -21,10 +21,8 @@ use std::fmt; use ids::BlockId; use ethereum_types::H256; -use trie::TrieError; +use ethtrie::TrieError; use rlp::DecoderError; -use hashdb::Hasher; -use keccak_hasher::KeccakHasher; /// Snapshot-related errors. #[derive(Debug)] @@ -50,8 +48,7 @@ pub enum Error { /// Restoration aborted. RestorationAborted, /// Trie error. - // Trie(TrieError), - Trie(TrieError<::Out>), // REVIEW: how do I fix this without making snapshot `Error` generic also? + Trie(TrieError), /// Decoder error. Decoder(DecoderError), /// Io error. @@ -104,10 +101,9 @@ impl From<::std::io::Error> for Error { } } -impl From> for Error { - fn from(err: TrieError) -> Self { - // Error::Trie(err) - Error::Trie(TrieError::InvalidStateRoot(::Out::new())) // REVIEW: how do I fix this without making snapshot `Error` generic also? +impl From for Error { + fn from(err: TrieError) -> Self { + Error::Trie(err) } } From 9cd81451a4abe5b71f40e31089c733030f401e39 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 19 Jun 2018 08:35:04 +0200 Subject: [PATCH 101/160] Add stream_encoder and patricia-trie-ethereum where needed --- ethcore/private-tx/Cargo.toml | 1 + ethcore/private-tx/src/lib.rs | 1 + ethcore/vm/Cargo.toml | 1 + ethcore/vm/src/call_type.rs | 1 + ethcore/vm/src/lib.rs | 2 ++ 5 files changed, 6 insertions(+) diff --git a/ethcore/private-tx/Cargo.toml b/ethcore/private-tx/Cargo.toml index 441c9882e29..9e403fe30da 100644 --- a/ethcore/private-tx/Cargo.toml +++ b/ethcore/private-tx/Cargo.toml @@ -26,6 +26,7 @@ keccak-hash = { path = "../../util/hash" } log = "0.3" parking_lot = "0.5" patricia-trie = { path = "../../util/patricia_trie" } +patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } rand = "0.3" rlp = { path = "../../util/rlp" } rlp_derive = { path = "../../util/rlp_derive" } diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index 31abdb1eca5..a8ee99d49d5 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -40,6 +40,7 @@ extern crate futures; extern crate keccak_hash as hash; extern crate parking_lot; extern crate patricia_trie as trie; +extern crate patricia_trie_ethereum as ethtrie; extern crate rlp; extern crate url; extern crate rustc_hex; diff --git a/ethcore/vm/Cargo.toml b/ethcore/vm/Cargo.toml index c5d31f58e61..f5519f2c45c 100644 --- a/ethcore/vm/Cargo.toml +++ b/ethcore/vm/Cargo.toml @@ -12,4 +12,5 @@ log = "0.3" common-types = { path = "../types" } ethjson = { path = "../../json" } rlp = { path = "../../util/rlp" } +stream-encoder = { path = "../../util/stream-encoder" } keccak-hash = { path = "../../util/hash" } diff --git a/ethcore/vm/src/call_type.rs b/ethcore/vm/src/call_type.rs index 0e58d76bbd2..0af40c334fd 100644 --- a/ethcore/vm/src/call_type.rs +++ b/ethcore/vm/src/call_type.rs @@ -63,6 +63,7 @@ impl Decodable for CallType { mod tests { use rlp::*; use super::CallType; + use stream_encoder::Stream; #[test] fn encode_call_type() { diff --git a/ethcore/vm/src/lib.rs b/ethcore/vm/src/lib.rs index 0dc1b799549..7ae0f947f3f 100644 --- a/ethcore/vm/src/lib.rs +++ b/ethcore/vm/src/lib.rs @@ -23,6 +23,8 @@ extern crate ethjson; extern crate rlp; extern crate keccak_hash as hash; extern crate patricia_trie as trie; +#[cfg(test)] +extern crate stream_encoder; mod action_params; mod call_type; From 1eeaa1fcd564cb6e225ee129b5bc360690a1e401 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 19 Jun 2018 08:36:08 +0200 Subject: [PATCH 102/160] lockfile --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index aaac68f662f..08e11e9cb8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,6 +782,7 @@ dependencies = [ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", + "patricia-trie-ethereum 0.1.0", "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", "rlp_derive 0.1.0", @@ -3688,6 +3689,7 @@ dependencies = [ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", "rlp 0.2.1", + "stream-encoder 0.1.0", ] [[package]] From bc5435812879d4b515df3a23d71a1c2f95b8b072 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 19 Jun 2018 08:38:04 +0200 Subject: [PATCH 103/160] fix docs --- util/patricia_trie/src/node_codec.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 694028b0896..d34a4f60f7f 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -25,13 +25,13 @@ use stream_encoder::Stream; /// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { - type E: ::std::error::Error; + type E: ::std::error::Error + 'static; type S: Stream; - /// Encode a Node to bytes (`Vec`). + /// Encode a Node to bytes (aka `Vec`). fn encode(&Node) -> Bytes; - /// Decode bytes to a `Node`. Returns `Result` on failure. + /// Decode bytes to a `Node`. Returns `Self::E` on failure. fn decode(data: &[u8]) -> Result; /// Decode bytes to the `Hasher`s output type. Assumes 32 bytes long hashes! Returns `None` on failure. From f5c04405e890236e036e8184936fe554cc24896d Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 11:28:31 +0200 Subject: [PATCH 104/160] TrieError takes two type params Implies changes to the result type which in turn implies adding a NodeCodec everywhere it wasn't already present, which in turn means the Hasher and the NodeCodec have to be regular generics and not associated types anymore. --- util/patricia_trie/src/fatdb.rs | 28 +++++----- util/patricia_trie/src/fatdbmut.rs | 25 +++++---- util/patricia_trie/src/lib.rs | 75 ++++++++++++-------------- util/patricia_trie/src/lookup.rs | 22 +++++--- util/patricia_trie/src/node_codec.rs | 2 +- util/patricia_trie/src/sectriedb.rs | 18 +++---- util/patricia_trie/src/sectriedbmut.rs | 23 ++++---- util/patricia_trie/src/triedb.rs | 48 +++++++++-------- util/patricia_trie/src/triedbmut.rs | 58 ++++++++++---------- 9 files changed, 150 insertions(+), 149 deletions(-) diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index f53cde35a09..9d11db6e0f9 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -38,7 +38,7 @@ where /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { Ok(FatDB { raw: TrieDB::new(db, root)? }) } @@ -46,27 +46,25 @@ where pub fn db(&self) -> &HashDB { self.raw.db() } } -impl<'db, H, C> Trie for FatDB<'db, H, C> +impl<'db, H, C> Trie for FatDB<'db, H, C> where H: Hasher, C: NodeCodec { - type H = H; + fn root(&self) -> &H::Out { self.raw.root() } - fn root(&self) -> &::Out { self.raw.root() } - - fn contains(&self, key: &[u8]) -> Result::Out> { - self.raw.contains(Self::H::hash(key).as_ref()) + fn contains(&self, key: &[u8]) -> Result { + self.raw.contains(H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> where 'a: 'key { - self.raw.get_with(Self::H::hash(key).as_ref(), query) + self.raw.get_with(H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> Result::Out>> + 'a>, ::Out> { - FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) + fn iter<'a>(&'a self) -> Result> + 'a>, ::Out, C::E> { + FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) } } @@ -86,7 +84,7 @@ where C: NodeCodec { /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> Result { + pub fn new(trie: &'db TrieDB) -> Result { Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie: trie, @@ -94,12 +92,12 @@ where } } -impl<'db, H, C> TrieIterator for FatDBIterator<'db, H, C> +impl<'db, H, C> TrieIterator for FatDBIterator<'db, H, C> where H: Hasher, C: NodeCodec { - fn seek(&mut self, key: &[u8]) -> Result<(), H::Out> { + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, C::E> { let hashed_key = H::hash(key); self.trie_iterator.seek(hashed_key.as_ref()) } @@ -110,7 +108,7 @@ where H: Hasher, C: NodeCodec { - type Item = TrieItem<'db, H::Out>; + type Item = TrieItem<'db, H::Out, C::E>; fn next(&mut self) -> Option { self.trie_iterator.next() diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index f9d2ba141ec..a9f2c155372 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -45,7 +45,7 @@ where /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { Ok(FatDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -60,41 +60,40 @@ where } } -impl<'db, H, C> TrieMut for FatDBMut<'db, H, C> +impl<'db, H, C> TrieMut for FatDBMut<'db, H, C> where H: Hasher, C: NodeCodec { - type H = H; - fn root(&mut self) -> &::Out { self.raw.root() } + fn root(&mut self) -> &H::Out { self.raw.root() } fn is_empty(&self) -> bool { self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> Result::Out> { - self.raw.contains(Self::H::hash(key).as_ref()) + fn contains(&self, key: &[u8]) -> Result { + self.raw.contains(H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> where 'a: 'key { - self.raw.get(Self::H::hash(key).as_ref()) + self.raw.get(H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out> { - let hash = Self::H::hash(key); + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E> { + let hash = H::hash(key); let out = self.raw.insert(hash.as_ref(), value)?; let db = self.raw.db_mut(); // don't insert if it doesn't exist. if out.is_none() { - let aux_hash = Self::H::hash(hash.as_ref()); + let aux_hash = H::hash(hash.as_ref()); db.emplace(aux_hash, DBValue::from_slice(key)); } Ok(out) } - fn remove(&mut self, key: &[u8]) -> Result, ::Out> { - let hash = Self::H::hash(key); + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E> { + let hash = H::hash(key); let out = self.raw.remove(hash.as_ref())?; // don't remove if it already exists. diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 52209431aae..9def63ab657 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -47,7 +47,6 @@ use hashdb::{HashDB, DBValue, Hasher}; use std::marker::PhantomData; pub mod node; -pub mod node_codec; pub mod triedb; pub mod triedbmut; pub mod sectriedb; @@ -59,6 +58,7 @@ mod fatdbmut; mod lookup; mod nibblevec; mod nibbleslice; +mod node_codec; pub use self::triedb::{TrieDB, TrieDBIterator}; pub use self::triedbmut::TrieDBMut; @@ -69,28 +69,23 @@ pub use self::fatdbmut::FatDBMut; pub use self::recorder::Recorder; pub use self::lookup::Lookup; pub use self::nibbleslice::NibbleSlice; -use node_codec::NodeCodec; +pub use node_codec::NodeCodec; /// Trie Errors. /// /// These borrow the data within them to avoid excessive copying on every /// trie operation. #[derive(Debug, PartialEq, Eq, Clone)] -pub enum TrieError { +pub enum TrieError { /// Attempted to create a trie with a state root not in the DB. InvalidStateRoot(T), /// Trie item not found in the database, IncompleteDatabase(T), /// Corrupt Trie item - // REVIEW: what we'd really like to do here is include the `rlp::DecoderError` in the `TrieError` - // but if we add a `Box` here we run into issues in the `ethcore` crate: - // "the trait bound `std::error::Error + 'static: std::marker::Send` is not satisfied" - // Investigate if using `Box` would help here (it does compile). - // Another potential problem is that the PartialEq, Eq and Clone derives do not work. - DecoderError(T, String), + DecoderError(T, E), } -impl fmt::Display for TrieError where T: std::fmt::Debug { +impl fmt::Display for TrieError where T: std::fmt::Debug, E: std::fmt::Debug { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { TrieError::InvalidStateRoot(ref root) => write!(f, "Invalid state root: {:?}", root), @@ -100,21 +95,23 @@ impl fmt::Display for TrieError where T: std::fmt::Debug { } } -impl error::Error for TrieError where T: std::fmt::Debug { +impl error::Error for TrieError where T: std::fmt::Debug, E: std::error::Error { fn description(&self) -> &str { match *self { TrieError::InvalidStateRoot(_) => "Invalid state root", TrieError::IncompleteDatabase(_) => "Incomplete database", - TrieError::DecoderError(_, ref err_string) => err_string, + // TrieError::DecoderError(_, ref err_string) => err_string, + TrieError::DecoderError(_, ref err) => err.description(), } } } /// Trie result type. Boxed to avoid copying around extra space for the `Hasher`s `Out`s on successful queries. -pub type Result = ::std::result::Result>>; +pub type Result = ::std::result::Result>; + /// Trie-Item type used for iterators over trie data. -pub type TrieItem<'a, U> = Result<(Vec, DBValue), U>; +pub type TrieItem<'a, U, E> = Result<(Vec, DBValue), U, E>; /// Description of what kind of query will be made to the trie. /// @@ -154,10 +151,9 @@ impl<'a, F, T, H: Hasher> Query for (&'a mut Recorder, F) where F: FnOnce( } /// A key-value datastore implemented as a database-backed modified Merkle tree. -pub trait Trie { - type H: Hasher; +pub trait Trie> { /// Return the root of the trie. - fn root(&self) -> &::Out; + fn root(&self) -> &H::Out; /// Is the trie empty? // TODO: The `Hasher` should not carry RLP-specifics; the null node should live in `NodeCodec`. Not sure what the best way to deal with this is. @@ -167,56 +163,55 @@ pub trait Trie { // make `impl NodeCodec for MyGenericCodec` impossible without better const // function evaluation, so we should limit ourselves only to // `impl NodeCodec for MyGenericCodec` or similar - fn is_empty(&self) -> bool { *self.root() == Self::H::HASHED_NULL_RLP } + fn is_empty(&self) -> bool { *self.root() == H::HASHED_NULL_RLP } /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result::Out> { + fn contains(&self, key: &[u8]) -> Result { self.get(key).map(|x|x.is_some() ) } /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> where 'a: 'key { + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> where 'a: 'key { self.get_with(key, DBValue::from_slice) } /// Search for the key with the given query parameter. See the docs of the `Query` /// trait for more details. - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> where 'a: 'key; + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> where 'a: 'key; /// Returns a depth-first iterator over the elements of trie. - fn iter<'a>(&'a self) -> Result::Out>> + 'a>, ::Out>; + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E>; } /// A key-value datastore implemented as a database-backed modified Merkle tree. -pub trait TrieMut { - type H: Hasher; +pub trait TrieMut> { /// Return the root of the trie. - fn root(&mut self) -> &::Out; + fn root(&mut self) -> &H::Out; /// Is the trie empty? fn is_empty(&self) -> bool; /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result::Out> { + fn contains(&self, key: &[u8]) -> Result { self.get(key).map(|x| x.is_some()) } /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> where 'a: 'key; + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> where 'a: 'key; /// Insert a `key`/`value` pair into the trie. An empty value is equivalent to removing /// `key` from the trie. Returns the old value associated with this key, if it existed. - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out>; + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E>; /// Remove a `key` from the trie. Equivalent to making it equal to the empty /// value. Returns the old value associated with this key, if it existed. - fn remove(&mut self, key: &[u8]) -> Result, ::Out>; + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E>; } /// A trie iterator that also supports random access (`seek()`). -pub trait TrieIterator: Iterator { +pub trait TrieIterator>: Iterator { /// Position the iterator on the first element with key > `key` - fn seek(&mut self, key: &[u8]) -> Result<(), H::Out>; + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, >::E>; } /// Trie types @@ -266,9 +261,8 @@ macro_rules! wrapper { } } -impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { - type H = H; - fn root(&self) -> &::Out { +impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { + fn root(&self) -> &H::Out { wrapper!(self, root,) } @@ -276,17 +270,17 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { wrapper!(self, is_empty,) } - fn contains(&self, key: &[u8]) -> Result::Out> { + fn contains(&self, key: &[u8]) -> Result { wrapper!(self, contains, key) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> where 'a: 'key { wrapper!(self, get_with, key, query) } - fn iter<'a>(&'a self) -> Result> + 'a>, ::Out> { + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E> { wrapper!(self, iter,) } } @@ -294,7 +288,6 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { impl<'db, H, C> TrieFactory where H: Hasher, - H::Out:, C: NodeCodec + 'db { /// Creates new factory. @@ -303,7 +296,7 @@ where } /// Create new immutable instance of Trie. - pub fn readonly(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out> { + pub fn readonly(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out, >::E> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), @@ -312,7 +305,7 @@ where } /// Create new mutable instance of Trie. - pub fn create(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { + pub fn create(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Box + 'db> { match self.spec { TrieSpec::Generic => Box::new(TrieDBMut::<_, C>::new(db, root)), TrieSpec::Secure => Box::new(SecTrieDBMut::<_, C>::new(db, root)), @@ -321,7 +314,7 @@ where } /// Create new mutable instance of trie and check for errors. - pub fn from_existing(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out> { + pub fn from_existing(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out, >::E> { match self.spec { TrieSpec::Generic => Ok(Box::new(TrieDBMut::<_, C>::from_existing(db, root)?)), TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::<_, C>::from_existing(db, root)?)), diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index b5430dc8eaf..3a75021947e 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -37,22 +37,26 @@ pub struct Lookup<'a, H: Hasher + 'a, C: NodeCodec, Q: Query> { impl<'a, H, C, Q> Lookup<'a, H, C, Q> where H: Hasher + 'a, - C: NodeCodec, + C: NodeCodec + 'a, Q: Query, { /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. - pub fn look_up(mut self, mut key: NibbleSlice) -> Result, H::Out> { + pub fn look_up(mut self, mut key: NibbleSlice) -> Result, H::Out, C::E> { let mut hash = self.hash; // this loop iterates through non-inline nodes. for depth in 0.. { let node_data = match self.db.get(&hash) { Some(value) => value, - None => return Err(Box::new(match depth { + None => return Err(match depth { 0 => TrieError::InvalidStateRoot(hash), _ => TrieError::IncompleteDatabase(hash), - })), + }), + // None => return Err(Box::new(match depth { + // 0 => TrieError::<_, C::E>::InvalidStateRoot(hash), + // _ => TrieError::<_, C::E>::IncompleteDatabase(hash), + // })), }; self.query.record(&hash, &node_data, depth); @@ -61,8 +65,14 @@ where // without incrementing the depth. let mut node_data = &node_data[..]; loop { - // TODO: see comment on `TrieError::DecoderError` for what to do here - match C::decode(node_data).map_err(|e| TrieError::DecoderError(hash, format!("{:?}",e)))? { + let decoded = match C::decode(node_data) { + Ok(node) => node, + Err(e) => { + return Err(TrieError::DecoderError(hash, e)) + // return Err(Box::new(TrieError::DecoderError(hash, Box::new(e)))) + } + }; + match decoded { Node::Leaf(slice, value) => { return Ok(match slice == key { true => Some(self.query.decode(value)), diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index d34a4f60f7f..469e7a66632 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -25,7 +25,7 @@ use stream_encoder::Stream; /// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { - type E: ::std::error::Error + 'static; + type E: ::std::error::Error; type S: Stream; /// Encode a Node to bytes (aka `Vec`). diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index daa3dbc8831..c87564e0dbc 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -40,7 +40,7 @@ where /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. /// Returns an error if root does not exist. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { Ok(SecTrieDB { raw: TrieDB::new(db, root)? }) } @@ -55,26 +55,24 @@ where } } -impl<'db, H, C> Trie for SecTrieDB<'db, H, C> +impl<'db, H, C> Trie for SecTrieDB<'db, H, C> where H: Hasher, C: NodeCodec { - type H = H; + fn root(&self) -> &H::Out { self.raw.root() } - fn root(&self) -> &::Out { self.raw.root() } - - fn contains(&self, key: &[u8]) -> Result::Out> { - self.raw.contains(Self::H::hash(key).as_ref()) + fn contains(&self, key: &[u8]) -> Result { + self.raw.contains(H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, ::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> where 'a: 'key { - self.raw.get_with(Self::H::hash(key).as_ref(), query) + self.raw.get_with(H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> Result::Out>> + 'a>, ::Out> { + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E> { TrieDB::iter(&self.raw) } } diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 9ba1005ce61..56dbeb1929e 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -44,7 +44,7 @@ where /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { Ok(SecTrieDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -55,13 +55,12 @@ where pub fn db_mut(&mut self) -> &mut HashDB { self.raw.db_mut() } } -impl<'db, H, C> TrieMut for SecTrieDBMut<'db, H, C> +impl<'db, H, C> TrieMut for SecTrieDBMut<'db, H, C> where H: Hasher, C: NodeCodec { - type H = H; - fn root(&mut self) -> &::Out { + fn root(&mut self) -> &H::Out { self.raw.root() } @@ -69,22 +68,22 @@ where self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> Result::Out> { - self.raw.contains(&Self::H::hash(key).as_ref()) + fn contains(&self, key: &[u8]) -> Result { + self.raw.contains(&H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, ::Out> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> where 'a: 'key { - self.raw.get(&Self::H::hash(key).as_ref()) + self.raw.get(&H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out> { - self.raw.insert(&Self::H::hash(key).as_ref(), value) + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E> { + self.raw.insert(&H::hash(key).as_ref(), value) } - fn remove(&mut self, key: &[u8]) -> Result, ::Out> { - self.raw.remove(&Self::H::hash(key).as_ref()) + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E> { + self.raw.remove(&H::hash(key).as_ref()) } } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index eff26e196e9..db9dd6e3405 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -77,9 +77,10 @@ where { /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist - pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { if !db.contains(root) { - Err(Box::new(TrieError::InvalidStateRoot(*root))) + Err(TrieError::InvalidStateRoot(*root)) + // Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { Ok(TrieDB {db, root, hash_count: 0, codec_marker: PhantomData}) } @@ -89,34 +90,35 @@ where pub fn db(&'db self) -> &'db HashDB { self.db } /// Get the data of the root node. - fn root_data(&self) -> Result { + fn root_data(&self) -> Result { self.db .get(self.root) - .ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) + .ok_or_else(|| TrieError::InvalidStateRoot(*self.root)) + // .ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) } /// Given some node-describing data `node`, return the actual node RLP. /// This could be a simple identity operation in the case that the node is sufficiently small, but /// may require a database lookup. - fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> Result { + fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> Result { match C::try_decode_hash(node) { Some(key) => { - self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) + self.db.get(&key).ok_or_else(|| TrieError::IncompleteDatabase(key)) + // self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) } None => Ok(DBValue::from_slice(node)) } } } -impl<'db, H, C> Trie for TrieDB<'db, H, C> +impl<'db, H, C> Trie for TrieDB<'db, H, C> where H: Hasher, C: NodeCodec { - type H = H; - fn root(&self) -> &::Out { self.root } + fn root(&self) -> &H::Out { self.root } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> where 'a: 'key { Lookup { @@ -127,7 +129,7 @@ where }.look_up(NibbleSlice::new(key)) } - fn iter<'a>(&'a self) -> Result::Out>> + 'a>, H::Out> { + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E> { TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>) } } @@ -234,13 +236,13 @@ pub struct TrieDBIterator<'a, H: Hasher + 'a, C: NodeCodec + 'a> { impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> Result, H::Out> { + pub fn new(db: &'a TrieDB) -> Result, H::Out, C::E> { let mut r = TrieDBIterator { db, trail: vec![], key_nibbles: Vec::new() }; db.root_data().and_then(|root| r.descend(&root))?; Ok(r) } - fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> Result<(), H::Out> { + fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> Result<(), H::Out, C::E> { loop { let (data, mid) = { let node = C::decode(&node_data).expect("rlp read from db; qed"); @@ -304,7 +306,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { } /// Descend into a payload. - fn descend(&mut self, d: &[u8]) -> Result<(), H::Out> { + fn descend(&mut self, d: &[u8]) -> Result<(), H::Out, C::E> { let node_data = &self.db.get_raw_or_lookup(d)?; let node = C::decode(&node_data).expect("encoded node read from db; qed"); Ok(self.descend_into_node(node.into())) @@ -336,9 +338,9 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { } } -impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C> { +impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C> { /// Position the iterator on the first element with key >= `key` - fn seek(&mut self, key: &[u8]) -> Result<(), H::Out> { + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, C::E> { self.trail.clear(); self.key_nibbles.clear(); let root_rlp = self.db.root_data()?; @@ -347,13 +349,13 @@ impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C } impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { - type Item = TrieItem<'a, H::Out>; + type Item = TrieItem<'a, H::Out, C::E>; fn next(&mut self) -> Option { - enum IterStep { + enum IterStep { Continue, PopTrail, - Descend(Result), + Descend(Result), } loop { let iter_step = { @@ -376,7 +378,7 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { return Some(Ok((self.key(), v.clone()))); }, (Status::At, &OwnedNode::Extension(_, ref d)) => { - IterStep::Descend::(self.db.get_raw_or_lookup(&*d)) + IterStep::Descend::(self.db.get_raw_or_lookup(&*d)) }, (Status::At, &OwnedNode::Branch(_, _)) => IterStep::Continue, (Status::AtChild(i), &OwnedNode::Branch(ref children, _)) if children[i].len() > 0 => { @@ -385,7 +387,7 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { i => *self.key_nibbles.last_mut() .expect("pushed as 0; moves sequentially; removed afterwards; qed") = i as u8, } - IterStep::Descend::(self.db.get_raw_or_lookup(&*children[i])) + IterStep::Descend::(self.db.get_raw_or_lookup(&*children[i])) }, (Status::AtChild(i), &OwnedNode::Branch(_, _)) => { if i == 0 { @@ -401,11 +403,11 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { IterStep::PopTrail => { self.trail.pop(); }, - IterStep::Descend::(Ok(d)) => { + IterStep::Descend::(Ok(d)) => { let node = C::decode(&d).expect("rlp read from db; qed"); self.descend_into_node(node.into()) }, - IterStep::Descend::(Err(e)) => { + IterStep::Descend::(Err(e)) => { return Some(Err(e)) } IterStep::Continue => {}, diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 48ebe0e5d52..17826280ed2 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -315,8 +315,9 @@ where } impl<'a, H, C> TrieDBMut<'a, H, C> -where H: Hasher, - C: NodeCodec +where + H: Hasher, + C: NodeCodec { /// Create a new trie with backing database `db` and empty `root`. pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { @@ -336,9 +337,10 @@ where H: Hasher, /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. - pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> Result { + pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> Result { if !db.contains(root) { - return Err(Box::new(TrieError::InvalidStateRoot(*root))); + return Err(TrieError::InvalidStateRoot(*root)); + // return Err(Box::new(TrieError::InvalidStateRoot(*root))); } let root_handle = NodeHandle::Hash(*root); @@ -363,8 +365,9 @@ where H: Hasher, } // cache a node by hash - fn cache(&mut self, hash: H::Out) -> Result { - let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; + fn cache(&mut self, hash: H::Out) -> Result { + let node_encoded = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; + // let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; let node = Node::from_encoded::( &node_encoded, &*self.db, @@ -375,8 +378,8 @@ where H: Hasher, // inspect a node, choosing either to replace, restore, or delete it. // if restored or replaced, returns the new node along with a flag of whether it was changed. - fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out> - where F: FnOnce(&mut Self, Node) -> Result, H::Out> { + fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::E> + where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::E> { Ok(match stored { Stored::New(node) => match inspector(self, node)? { Action::Restore(node) => Some((Stored::New(node), false)), @@ -398,7 +401,7 @@ where H: Hasher, } // walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out> + fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::E> where 'x: 'key { let mut handle = handle; @@ -447,7 +450,7 @@ where H: Hasher, } /// insert a key, value pair into the trie, creating new nodes if necessary. - fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out> { + fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::E> { let h = match handle { NodeHandle::InMemory(h) => h, NodeHandle::Hash(h) => self.cache(h)?, @@ -461,7 +464,7 @@ where H: Hasher, } /// the insertion inspector. - fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out> { + fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::E> { trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty()); Ok(match node { @@ -622,7 +625,7 @@ where H: Hasher, } /// Remove a node from the trie based on key. - fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out> { + fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::E> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), NodeHandle::Hash(h) => { @@ -637,7 +640,7 @@ where H: Hasher, } /// the removal inspector - fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out> { + fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::E> { Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete, (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), @@ -723,7 +726,7 @@ where H: Hasher, /// _invalid state_ means: /// - Branch node where there is only a single entry; /// - Extension node followed by anything other than a Branch node. - fn fix(&mut self, node: Node) -> Result, H::Out> { + fn fix(&mut self, node: Node) -> Result, H::Out, C::E> { match node { Node::Branch(mut children, value) => { // if only a single value, transmute to leaf/extension and feed through fixed. @@ -892,20 +895,19 @@ where H: Hasher, } } -impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> -where H: Hasher, - C: NodeCodec +impl<'a, H, C> TrieMut for TrieDBMut<'a, H, C> +where + H: Hasher, + C: NodeCodec { - type H = H; - - fn root(&mut self) -> &::Out { + fn root(&mut self) -> &H::Out { self.commit(); self.root } fn is_empty(&self) -> bool { match self.root_handle { - NodeHandle::Hash(h) => h == Self::H::HASHED_NULL_RLP, + NodeHandle::Hash(h) => h == H::HASHED_NULL_RLP, NodeHandle::InMemory(ref h) => match self.storage[h] { Node::Empty => true, _ => false, @@ -913,13 +915,13 @@ where H: Hasher, } } - fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, ::Out> + fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, H::Out, C::E> where 'x: 'key { self.lookup(NibbleSlice::new(key), &self.root_handle) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, ::Out> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E> { if value.is_empty() { return self.remove(key) } let mut old_val = None; @@ -940,7 +942,7 @@ where H: Hasher, Ok(old_val) } - fn remove(&mut self, key: &[u8]) -> Result, ::Out> { + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E> { trace!(target: "trie", "remove: key={:?}", key.pretty()); let root_handle = self.root_handle(); @@ -954,8 +956,8 @@ where H: Hasher, } None => { trace!(target: "trie", "remove: obliterated trie"); - self.root_handle = NodeHandle::Hash(Self::H::HASHED_NULL_RLP); - *self.root = Self::H::HASHED_NULL_RLP; + self.root_handle = NodeHandle::Hash(H::HASHED_NULL_RLP); + *self.root = H::HASHED_NULL_RLP; } } @@ -982,8 +984,8 @@ mod tests { use rlp::{Decodable, Encodable}; use triehash::trie_root; use standardmap::*; - use ethtrie::trie::{TrieMut, TrieDBMut}; - use ethtrie::{NodeCodec, RlpCodec}; + use ethtrie::trie::{TrieMut, TrieDBMut, NodeCodec}; + use ethtrie::RlpCodec; fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec From 7ccb3504b6d9caad6763647bcaf3504e85f24fa4 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 11:33:25 +0200 Subject: [PATCH 105/160] Export concrete impls of the Trie result/error --- util/patricia-trie-ethereum/src/lib.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index cdff98ad3a1..f51001afb7c 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -16,7 +16,7 @@ //! Façade crate for `patricia_trie` for Ethereum specific impls -pub extern crate patricia_trie as trie; +pub extern crate patricia_trie as trie; // `pub` because we need to import this crate for the tests in `patricia_trie` and there were issues: https://gist.github.com/dvdplm/869251ee557a1b4bd53adc7c971979aa extern crate ethcore_bytes; extern crate hashdb; extern crate keccak_hasher; @@ -28,15 +28,14 @@ use ethcore_bytes::Bytes; use ethereum_types::H256; use hashdb::Hasher; use keccak_hasher::KeccakHasher; -use rlp::{Decodable, RlpStream, DecoderError, Rlp, Prototype}; +use rlp::{DecoderError, Decodable, RlpStream, Rlp, Prototype}; use std::marker::PhantomData; use stream_encoder::Stream; -use trie::node::Node; // TODO: re-export from root? -pub use trie::{Lookup, NibbleSlice, node_codec::NodeCodec}; +use trie::{NibbleSlice, NodeCodec, node::Node}; -pub type KeccakTrieResult = trie::Result::Out>; pub type RlpCodec = RlpNodeCodec; -pub type TrieError = trie::TrieError; +pub type TrieError = trie::TrieError; +pub type Result = trie::Result; #[derive(Default, Clone)] pub struct RlpNodeCodec {mark: PhantomData} @@ -78,7 +77,7 @@ where H::Out: Decodable } } } - fn decode(data: &[u8]) -> Result { + fn decode(data: &[u8]) -> ::std::result::Result { let r = Rlp::new(data); match r.prototype()? { // either leaf or extension - decode first item with NibbleSlice::??? From 15a754b45ed2d144463d63108625de957b0e43ff Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 11:36:03 +0200 Subject: [PATCH 106/160] Use better result/error types in ethcore --- ethcore/light/src/cht.rs | 2 +- ethcore/light/src/on_demand/request.rs | 14 +++++++++----- ethcore/src/client/test_client.rs | 8 ++++---- ethcore/src/executed.rs | 16 +++++++++++++--- ethcore/src/json_tests/executive.rs | 2 +- ethcore/src/state/account.rs | 15 ++++++++------- ethcore/src/state/mod.rs | 7 +++---- ethcore/vm/Cargo.toml | 1 + ethcore/vm/src/error.rs | 12 ++++++++---- ethcore/vm/src/lib.rs | 1 + 10 files changed, 49 insertions(+), 29 deletions(-) diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index 4d18240c7ff..e996059e79d 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -69,7 +69,7 @@ impl> CHT { /// Generate an inclusion proof for the entry at a specific block. /// Nodes before level `from_level` will be omitted. /// Returns an error on an incomplete trie, and `Ok(None)` on an unprovable request. - pub fn prove(&self, num: u64, from_level: u32) -> ethtrie::KeccakTrieResult>> { + pub fn prove(&self, num: u64, from_level: u32) -> ethtrie::Result>> { if block_to_cht_number(num) != Some(self.number) { return Ok(None) } let mut recorder = Recorder::with_depth(from_level); diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 9259c1356e3..9fd4b02eaa3 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -34,8 +34,7 @@ use rlp::{RlpStream, Rlp}; use stream_encoder::Stream; use ethereum_types::{H256, U256, Address}; use parking_lot::Mutex; -use hashdb::{HashDB, Hasher}; -use keccak_hasher::KeccakHasher; +use hashdb::HashDB; use kvdb::DBValue; use bytes::Bytes; use memorydb::MemoryDB; @@ -653,11 +652,16 @@ impl From<::rlp::DecoderError> for Error { } } -impl From> for Error { - fn from(err: Box) -> Self { - Error::Trie(*err) +impl From for Error { + fn from(err: TrieError) -> Self { + Error::Trie(err) } } +// impl From> for Error { +// fn from(err: Box) -> Self { +// Error::Trie(*err) +// } +// } /// Request for header proof by number #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 5aacbe80bbb..c948754126d 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -582,10 +582,10 @@ impl Call for TestBlockChainClient { } impl StateInfo for () { - fn nonce(&self, _address: &Address) -> ethtrie::KeccakTrieResult { unimplemented!() } - fn balance(&self, _address: &Address) -> ethtrie::KeccakTrieResult { unimplemented!() } - fn storage_at(&self, _address: &Address, _key: &H256) -> ethtrie::KeccakTrieResult { unimplemented!() } - fn code(&self, _address: &Address) -> ethtrie::KeccakTrieResult>> { unimplemented!() } + fn nonce(&self, _address: &Address) -> ethtrie::Result { unimplemented!() } + fn balance(&self, _address: &Address) -> ethtrie::Result { unimplemented!() } + fn storage_at(&self, _address: &Address, _key: &H256) -> ethtrie::Result { unimplemented!() } + fn code(&self, _address: &Address) -> ethtrie::Result>> { unimplemented!() } } impl StateClient for TestBlockChainClient { diff --git a/ethcore/src/executed.rs b/ethcore/src/executed.rs index 8ed2bca28bf..7a3df7b8dea 100644 --- a/ethcore/src/executed.rs +++ b/ethcore/src/executed.rs @@ -18,7 +18,7 @@ use ethereum_types::{U256, U512, Address}; use bytes::Bytes; -use trie; +use ethtrie; use vm; use trace::{VMTrace, FlatTrace}; use log_entry::LogEntry; @@ -117,11 +117,21 @@ pub enum ExecutionError { TransactionMalformed(String), } -impl From>> for ExecutionError where T: fmt::Debug { - fn from(err: Box>) -> Self { +impl From> for ExecutionError { + fn from(err: Box) -> Self { ExecutionError::Internal(format!("{:?}", err)) } } +impl From for ExecutionError { + fn from(err: ethtrie::TrieError) -> Self { + ExecutionError::Internal(format!("{:?}", err)) + } +} +// impl From>> for ExecutionError where T: fmt::Debug { +// fn from(err: Box>) -> Self { +// ExecutionError::Internal(format!("{:?}", err)) +// } +// } impl fmt::Display for ExecutionError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 7ae2bb61c4f..4c4b722a39e 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -80,7 +80,7 @@ impl<'a, T: 'a, V: 'a, B: 'a> TestExt<'a, T, V, B> address: Address, tracer: &'a mut T, vm_tracer: &'a mut V, - ) -> ethtrie::KeccakTrieResult { + ) -> ethtrie::TrieResult { let static_call = false; Ok(TestExt { nonce: state.nonce(&address)?, diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index 511b94dcfc0..a9096f1ee7a 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -22,13 +22,12 @@ use std::collections::{HashMap, BTreeMap}; use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use ethereum_types::{H256, U256, Address}; use error::Error; -use hashdb::{Hasher, HashDB}; +use hashdb::HashDB; use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::{Bytes, ToPretty}; -use trie; -use trie::{SecTrieDB, Trie, TrieFactory, TrieError}; -use ethtrie::RlpCodec; +use trie::{SecTrieDB, Trie, TrieFactory}; +use ethtrie::{RlpCodec, Result as TrieResult}; use pod_account::*; use rlp::{RlpStream, encode}; use lru_cache::LruCache; @@ -202,7 +201,7 @@ impl Account { /// Get (and cache) the contents of the trie's storage at `key`. /// Takes modified storage into account. - pub fn storage_at(&self, db: &HashDB, key: &H256) -> trie::Result { + pub fn storage_at(&self, db: &HashDB, key: &H256) -> TrieResult { if let Some(value) = self.cached_storage_at(key) { return Ok(value); } @@ -377,7 +376,7 @@ impl Account { } /// Commit the `storage_changes` to the backing DB and update `storage_root`. - pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> trie::Result<(), ::Out> { + pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> TrieResult<()> { let mut t = trie_factory.from_existing(db, &mut self.storage_root)?; for (k, v) in self.storage_changes.drain() { // cast key and value to trait type, @@ -475,7 +474,9 @@ impl Account { /// trie. /// `storage_key` is the hash of the desired storage key, meaning /// this will only work correctly under a secure trie. - pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box::Out>>> { + // pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box::Out>>> { + // pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box> { + pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> TrieResult<(Vec, H256)> { use trie::{Trie, TrieDB}; use trie::recorder::Recorder; diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index cd47846f065..94eadc0e5f1 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -48,9 +48,8 @@ use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::Bytes; -use trie::{Trie, TrieError, TrieDB}; -use ethtrie::{RlpCodec, KeccakTrieResult as TrieResult}; -use trie::recorder::Recorder; +use trie::{Trie, TrieDB, TrieError, Recorder}; +use ethtrie::{RlpCodec, Result as TrieResult}; mod account; mod substate; @@ -380,7 +379,7 @@ impl State { } /// Creates new state with existing state root - pub fn from_existing(db: B, root: H256, account_start_nonce: U256, factories: Factories) -> Result, TrieError> { + pub fn from_existing(db: B, root: H256, account_start_nonce: U256, factories: Factories) -> TrieResult> { if !db.as_hashdb().contains(&root) { return Err(TrieError::InvalidStateRoot(root)); } diff --git a/ethcore/vm/Cargo.toml b/ethcore/vm/Cargo.toml index f5519f2c45c..2f1365e000c 100644 --- a/ethcore/vm/Cargo.toml +++ b/ethcore/vm/Cargo.toml @@ -8,6 +8,7 @@ byteorder = "1.0" ethcore-bytes = { path = "../../util/bytes" } ethereum-types = "0.3" patricia-trie = { path = "../../util/patricia_trie" } +patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } log = "0.3" common-types = { path = "../types" } ethjson = { path = "../../json" } diff --git a/ethcore/vm/src/error.rs b/ethcore/vm/src/error.rs index 4b90f6698d5..b5e337a75fb 100644 --- a/ethcore/vm/src/error.rs +++ b/ethcore/vm/src/error.rs @@ -16,8 +16,8 @@ //! VM errors module -use trie; use std::fmt; +use ethtrie; /// VM errors. #[derive(Debug, Clone, PartialEq)] @@ -71,9 +71,13 @@ pub enum Error { Reverted, } - -impl From>> for Error where T: fmt::Debug { - fn from(err: Box>) -> Self { +impl From> for Error { + fn from(err: Box) -> Self { + Error::Internal(format!("Internal error: {}", err)) + } +} +impl From for Error { + fn from(err: ethtrie::TrieError) -> Self { Error::Internal(format!("Internal error: {}", err)) } } diff --git a/ethcore/vm/src/lib.rs b/ethcore/vm/src/lib.rs index 7ae0f947f3f..924f1cf3e51 100644 --- a/ethcore/vm/src/lib.rs +++ b/ethcore/vm/src/lib.rs @@ -22,6 +22,7 @@ extern crate common_types as types; extern crate ethjson; extern crate rlp; extern crate keccak_hash as hash; +extern crate patricia_trie_ethereum as ethtrie; extern crate patricia_trie as trie; #[cfg(test)] extern crate stream_encoder; From f1be1a12bcfb83d40ffa492cbd530e1e53c73d89 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 11:36:22 +0200 Subject: [PATCH 107/160] lock file --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 08e11e9cb8d..4b7f34ac08b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3688,6 +3688,7 @@ dependencies = [ "keccak-hash 0.1.2", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", + "patricia-trie-ethereum 0.1.0", "rlp 0.2.1", "stream-encoder 0.1.0", ] From b48e2c8360b815d7717b08e90be270ae51ae643b Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 12:16:49 +0200 Subject: [PATCH 108/160] Do not export a Keccak impl of MemoryDB --- ethcore/src/json_tests/trie.rs | 6 +++--- ethcore/src/state/backend.rs | 18 +++++++----------- util/memorydb/Cargo.toml | 4 ++-- util/memorydb/src/lib.rs | 6 +----- util/patricia_trie/src/lib.rs | 1 - 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index cd340faf5ce..25ed6eddc6a 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -18,8 +18,8 @@ use ethjson; use trie::{TrieFactory, TrieSpec}; use ethtrie::RlpCodec; use ethereum_types::H256; -use memorydb::KeccakMemoryDB; - +use memorydb::MemoryDB; +use keccak_hasher::KeccakHasher; fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { let tests = ethjson::trie::Test::load(json).unwrap(); @@ -27,7 +27,7 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { let mut result = vec![]; for (name, test) in tests.into_iter() { - let mut memdb = KeccakMemoryDB::new(); + let mut memdb = MemoryDB::new(); let mut root = H256::default(); let mut t = factory.create(&mut memdb, &mut root); diff --git a/ethcore/src/state/backend.rs b/ethcore/src/state/backend.rs index 6ec4f77ac0e..d07124d8d70 100644 --- a/ethcore/src/state/backend.rs +++ b/ethcore/src/state/backend.rs @@ -27,7 +27,7 @@ use std::sync::Arc; use state::Account; use parking_lot::Mutex; use ethereum_types::{Address, H256}; -use memorydb::KeccakMemoryDB; +use memorydb::MemoryDB; use hashdb::{AsHashDB, HashDB, DBValue}; use keccak_hasher::KeccakHasher; @@ -76,12 +76,12 @@ pub trait Backend: Send { // TODO: when account lookup moved into backends, this won't rely as tenuously on intended // usage. #[derive(Clone, PartialEq)] -pub struct ProofCheck(KeccakMemoryDB); +pub struct ProofCheck(MemoryDB); impl ProofCheck { /// Create a new `ProofCheck` backend from the given state items. pub fn new(proof: &[DBValue]) -> Self { - let mut db = KeccakMemoryDB::new(); + let mut db = MemoryDB::::new(); for item in proof { db.insert(item); } ProofCheck(db) } @@ -136,7 +136,7 @@ impl Backend for ProofCheck { /// This doesn't cache anything or rely on the canonical state caches. pub struct Proving> { base: H, // state we're proving values from. - changed: KeccakMemoryDB, // changed state via insertions. + changed: MemoryDB, // changed state via insertions. proof: Mutex>, } @@ -183,13 +183,9 @@ impl + Send + Sync> HashDB for Proving + Send + Sync> Backend for Proving { - fn as_hashdb(&self) -> &HashDB { - self - } + fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { - self - } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } fn add_to_account_cache(&mut self, _: Address, _: Option, _: bool) { } @@ -214,7 +210,7 @@ impl> Proving { pub fn new(base: H) -> Self { Proving { base: base, - changed: KeccakMemoryDB::new(), + changed: MemoryDB::::new(), proof: Mutex::new(HashSet::new()), } } diff --git a/util/memorydb/Cargo.toml b/util/memorydb/Cargo.toml index aa27a0a7218..5e539fad48f 100644 --- a/util/memorydb/Cargo.toml +++ b/util/memorydb/Cargo.toml @@ -11,8 +11,8 @@ heapsize = "0.4" hashdb = { version = "0.2.0", path = "../hashdb" } plain_hasher = { path = "../plain_hasher" } rlp = { version = "0.2.1", path = "../rlp" } -keccak-hasher = { path = "../keccak-hasher" } [dev-dependencies] tiny-keccak = "1.4.2" -ethereum-types = "0.3" \ No newline at end of file +ethereum-types = "0.3" +keccak-hasher = { path = "../keccak-hasher" } \ No newline at end of file diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index d5ee1030cc3..5d07bc14623 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -18,14 +18,13 @@ extern crate elastic_array; extern crate hashdb; extern crate heapsize; -extern crate keccak_hasher; extern crate plain_hasher; extern crate rlp; +#[cfg(test)] extern crate keccak_hasher; #[cfg(test)] extern crate tiny_keccak; #[cfg(test)] extern crate ethereum_types; use hashdb::{HashDB, Hasher, DBValue, AsHashDB}; -use keccak_hasher::KeccakHasher; use heapsize::HeapSizeOf; use plain_hasher::H256FastMap; use rlp::NULL_RLP; @@ -85,9 +84,6 @@ pub struct MemoryDB { data: H256FastMap, } -/// Convenience type for crates that need a `MemoryDB` with Keccak hashes -pub type KeccakMemoryDB = MemoryDB; - impl MemoryDB { /// Create a new instance of the memory DB. pub fn new() -> MemoryDB { diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 9def63ab657..8e140602ccb 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -100,7 +100,6 @@ impl error::Error for TrieError where T: std::fmt::Debug, E: std::er match *self { TrieError::InvalidStateRoot(_) => "Invalid state root", TrieError::IncompleteDatabase(_) => "Incomplete database", - // TrieError::DecoderError(_, ref err_string) => err_string, TrieError::DecoderError(_, ref err) => err.description(), } } From 4febf0db5e93996faa77d24fa107c75667ce1310 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 13:49:08 +0200 Subject: [PATCH 109/160] Cleanup rlp Stream and add test for append_bytes --- util/rlp/src/stream.rs | 96 ++++++++++------------------------------- util/rlp/tests/tests.rs | 10 +++++ 2 files changed, 33 insertions(+), 73 deletions(-) diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index eeccff42df4..e07e38b6a40 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -42,16 +42,8 @@ impl Default for RlpStream { } } -// pub trait Stream { -// fn new() -> Self; -// fn new_list(len: usize) -> Self; -// fn append_empty_data(&mut self) -> &mut Self; -// fn drain(self) -> ElasticArray1024; -// fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; -// fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; -// } - impl Stream for RlpStream { + /// Initializes instance of empty `Stream`. fn new() -> Self { RlpStream { unfinished_lists: ElasticArray16::new(), @@ -59,11 +51,29 @@ impl Stream for RlpStream { finished_list: false, } } + + /// Initializes the `Stream` as a list. fn new_list(len: usize) -> Self { let mut stream = RlpStream::new(); stream.begin_list(len); stream } + + /// Apends null to the end of stream, chainable. + /// + /// ```rust + /// extern crate rlp; + /// extern crate stream_encoder; + /// use rlp::*; + /// use stream_encoder::Stream; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_empty_data().append_empty_data(); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); + /// } + /// ``` fn append_empty_data(&mut self) -> &mut Self { // self push raw item self.buffer.push(0x80); @@ -75,6 +85,7 @@ impl Stream for RlpStream { self } + /// Drain the object and return the underlying ElasticArray. Panics if it is not finished. fn drain(self) -> ElasticArray1024 { match self.is_finished() { true => self.buffer, @@ -82,13 +93,14 @@ impl Stream for RlpStream { } } + /// Append bytes to the end of stream as a single item, chainable. fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self { - self.finished_list = false; self.encoder().encode_value(bytes); - if !self.finished_list { self.note_appended(1) } + self.note_appended(1); self } + /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self { // push raw items self.buffer.append_slice(bytes); @@ -102,22 +114,6 @@ impl Stream for RlpStream { } impl RlpStream { - /// Initializes instance of empty `Stream`. - // pub fn new() -> Self { - // RlpStream { - // unfinished_lists: ElasticArray16::new(), - // buffer: ElasticArray1024::new(), - // finished_list: false, - // } - // } - - /// Initializes the `Stream` as a list. - // pub fn new_list(len: usize) -> Self { - // let mut stream = RlpStream::new(); - // stream.begin_list(len); - // stream - // } - /// Appends value to the end of stream, chainable. /// /// ```rust @@ -209,44 +205,6 @@ impl RlpStream { self } - /// Apends null to the end of stream, chainable. - /// - /// ```rust - /// extern crate rlp; - /// extern crate stream_encoder; - /// use rlp::*; - /// use stream_encoder::Stream; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append_empty_data().append_empty_data(); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); - /// } - /// ``` - // pub fn append_empty_data(&mut self) -> &mut RlpStream { - // // self push raw item - // self.buffer.push(0x80); - - // // try to finish and prepend the length - // self.note_appended(1); - - // // return chainable self - // self - // } - - /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. - // pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { - // // push raw items - // self.buffer.append_slice(bytes); - - // // try to finish and prepend the length - // self.note_appended(item_count); - - // // return chainable self - // self - // } - /// Appends raw (pre-serialised) RLP data. Checks for size oveflow. pub fn append_raw_checked<'a>(&'a mut self, bytes: &[u8], item_count: usize, max_size: usize) -> bool { if self.estimate_size(bytes.len()) > max_size { @@ -370,14 +328,6 @@ impl RlpStream { BasicEncoder::new(self) } - /// Drain the object and return the underlying ElasticArray. - // pub fn drain(self) -> ElasticArray1024 { - // match self.is_finished() { - // true => self.buffer, - // false => panic!() - // } - // } - /// Finalize current ubnbound list. Panics if no unbounded list has been opened. pub fn complete_unbounded_list(&mut self) { let list = self.unfinished_lists.pop().expect("No open list."); diff --git a/util/rlp/tests/tests.rs b/util/rlp/tests/tests.rs index e2ce9dfeb67..d172176a1f0 100644 --- a/util/rlp/tests/tests.rs +++ b/util/rlp/tests/tests.rs @@ -426,3 +426,13 @@ fn test_rlp_stream_unbounded_list() { assert!(stream.is_finished()); } +#[test] +fn test_rlp_stream_append_bytes() { + let mut stream = RlpStream::new(); + stream.begin_unbounded_list(); + stream.append_bytes(&[0x0, 0x1, 0x2]); + stream.append_bytes(&[0x8D]); + stream.complete_unbounded_list(); + let out = stream.out(); + assert_eq!(out, vec![0xC6, 0x83, 0x0, 0x1, 0x2, 0x81, 0x8D]); +} \ No newline at end of file From 5f3ba135b498e22650cbe7a316b2a5103d133a82 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 14:07:41 +0200 Subject: [PATCH 110/160] Fix json_tests --- ethcore/src/json_tests/executive.rs | 5 +++-- ethcore/src/json_tests/trie.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 4c4b722a39e..579604cc2ad 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -30,8 +30,9 @@ use ethjson; use trace::{Tracer, NoopTracer}; use trace::{VMTracer, NoopVMTracer}; use bytes::{Bytes, BytesRef}; -use trie; +use ethtrie; use rlp::RlpStream; +use stream_encoder::Stream; use hash::keccak; use machine::EthereumMachine as Machine; @@ -80,7 +81,7 @@ impl<'a, T: 'a, V: 'a, B: 'a> TestExt<'a, T, V, B> address: Address, tracer: &'a mut T, vm_tracer: &'a mut V, - ) -> ethtrie::TrieResult { + ) -> ethtrie::Result { let static_call = false; Ok(TestExt { nonce: state.nonce(&address)?, diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index 25ed6eddc6a..192d6678757 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -27,7 +27,7 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { let mut result = vec![]; for (name, test) in tests.into_iter() { - let mut memdb = MemoryDB::new(); + let mut memdb = MemoryDB::::new(); let mut root = H256::default(); let mut t = factory.create(&mut memdb, &mut root); From bcd1ad28e6eb11886c9f2215a65ec862e081d4e2 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 20:14:45 +0200 Subject: [PATCH 111/160] Fix benches --- util/patricia_trie/benches/trie.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/util/patricia_trie/benches/trie.rs b/util/patricia_trie/benches/trie.rs index 3744930d13c..9eb38d606c6 100644 --- a/util/patricia_trie/benches/trie.rs +++ b/util/patricia_trie/benches/trie.rs @@ -21,6 +21,7 @@ extern crate ethcore_bytes; extern crate ethereum_types; extern crate memorydb; extern crate patricia_trie as trie; +extern crate patricia_trie_ethereum as ethtrie; extern crate keccak_hasher; extern crate keccak_hash; extern crate trie_standardmap; @@ -35,9 +36,7 @@ use trie::{TrieDBMut, TrieDB, TrieMut, Trie}; use trie_standardmap::{Alphabet, ValueMode, StandardMap}; use hashdb::Hasher; use keccak_hasher::KeccakHasher; -use trie::node_codec::RlpNodeCodec; - -type RlpCodec = RlpNodeCodec; +use ethtrie::RlpCodec; fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { assert!(min_count + diff_count <= 32); From ce5d5b9905b96965716e8337661d5b0a53f45978 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 20 Jun 2018 20:23:03 +0200 Subject: [PATCH 112/160] Minor cleanup --- ethcore/src/engines/tendermint/mod.rs | 2 +- util/patricia-trie-ethereum/src/lib.rs | 3 +-- util/patricia_trie/src/triedbmut.rs | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index 1fe8b4640f9..2d88b32423a 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -146,7 +146,7 @@ impl super::EpochVerifier for EpochVerifier fn check_finality_proof(&self, proof: &[u8]) -> Option> { match ::rlp::decode(proof) { Ok(header) => self.verify_light(&header).ok().map(|_| vec![header.hash()]), - Err(_) => None // REVIEW: log perhaps? Not sure what the policy is. + Err(_) => None } } } diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index f51001afb7c..183a9f1300d 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -113,7 +113,6 @@ where H::Out: Decodable } fn is_empty_node(data: &[u8]) -> bool { - // REVIEW: Could also be `Rlp::new(data).is_empty()` – better? - data.len() != 0 && (data[0] == 0xC0 || data[0] == 0x80) + Rlp::new(data).is_empty() } } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 17826280ed2..2cf6450d8d7 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -1303,7 +1303,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // REVIEW: `t` was immutable – how could this pass? + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } @@ -1331,7 +1331,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); // REVIEW: `t` was immutable – how could this pass? + let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap().is_none()); assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); From 1d5c2b329fd712d154b78026d12dc8760271afb5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 07:51:22 +0200 Subject: [PATCH 113/160] Cleanup --- util/keccak-hasher/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/util/keccak-hasher/src/lib.rs b/util/keccak-hasher/src/lib.rs index 5b255671814..aa0341c9cd0 100644 --- a/util/keccak-hasher/src/lib.rs +++ b/util/keccak-hasher/src/lib.rs @@ -36,7 +36,3 @@ impl Hasher for KeccakHasher { out.into() } } - -// Move concrete impl to own "façade" crate? -// /// Convenience type for crates that need a `HashDB` with Keccak hashes -// pub type KeccakHashDB = HashDB; From 438e21c48566d1af93fe007c0e58c50ea7f5b3b6 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 10:31:00 +0200 Subject: [PATCH 114/160] Move the null node hash from the Hasher trait to NodeCodec --- util/hashdb/src/lib.rs | 1 - util/keccak-hasher/src/lib.rs | 2 -- util/memorydb/src/lib.rs | 19 ++++++++++--------- util/patricia-trie-ethereum/src/lib.rs | 14 +++++++++----- util/patricia_trie/src/lib.rs | 2 +- util/patricia_trie/src/node_codec.rs | 6 ++++++ util/patricia_trie/src/triedbmut.rs | 18 +++++++++--------- 7 files changed, 35 insertions(+), 27 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 47b97bb1cc0..5376cbec46e 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -25,7 +25,6 @@ use std::{fmt::Debug, hash::Hash}; pub trait Hasher: Sync + Send { type Out: AsRef<[u8]> + Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? --> */ + HeapSizeOf; - const HASHED_NULL_RLP: Self::Out; const LENGTH: usize; fn hash(x: &[u8]) -> Self::Out; } diff --git a/util/keccak-hasher/src/lib.rs b/util/keccak-hasher/src/lib.rs index aa0341c9cd0..9f9a96ca22a 100644 --- a/util/keccak-hasher/src/lib.rs +++ b/util/keccak-hasher/src/lib.rs @@ -27,8 +27,6 @@ use tiny_keccak::Keccak; pub struct KeccakHasher; impl Hasher for KeccakHasher { type Out = H256; - // TODO: this is a problem: we do not want the Hasher to make assumptions about the encoding scheme. - const HASHED_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); const LENGTH: usize = 32; fn hash(x: &[u8]) -> Self::Out { let mut out = [0;32]; diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index 5d07bc14623..b0571a997e0 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -82,13 +82,15 @@ use std::mem; #[derive(Default, Clone, PartialEq)] pub struct MemoryDB { data: H256FastMap, + hashed_null_node: H::Out, } impl MemoryDB { /// Create a new instance of the memory DB. pub fn new() -> MemoryDB { MemoryDB { - data: H256FastMap::::default() + data: H256FastMap::::default(), + hashed_null_node: H::hash(&NULL_RLP) } } @@ -133,7 +135,7 @@ impl MemoryDB { /// Even when Some is returned, the data is only guaranteed to be useful /// when the refs > 0. pub fn raw(&self, key: &::Out) -> Option<(DBValue, i32)> { - if key == &H::HASHED_NULL_RLP { + if key == &self.hashed_null_node { return Some((DBValue::from_slice(&NULL_RLP), 1)); } self.data.get(key).cloned() @@ -147,7 +149,7 @@ impl MemoryDB { /// Remove an element and delete it from storage if reference count reaches zero. /// If the value was purged, return the old value. pub fn remove_and_purge(&mut self, key: &::Out) -> Option { - if key == &H::HASHED_NULL_RLP { + if key == &self.hashed_null_node { return None; } match self.data.entry(key.clone()) { @@ -188,8 +190,7 @@ impl HashDB for MemoryDB { // REVIEW: this method is what made it necessary to add a type param to H256FastMap, which I'd rather have avoided. // The problem is that the keys returned are `H256` and type inference fails on the `collect()` call. - // I could not make it work without parameterizing H256FastMap too. It all sort of adds up as I could - // avoid adding PhantomData to MemoryDB, but still quite annoying. What's a better way? + // I could not make it work without parameterizing H256FastMap too. fn keys(&self) -> HashMap { self.data.iter() .filter_map(|(k, v)| if v.1 != 0 { @@ -201,7 +202,7 @@ impl HashDB for MemoryDB { } fn get(&self, key: &H::Out) -> Option { - if key == &H::HASHED_NULL_RLP { + if key == &self.hashed_null_node { return Some(DBValue::from_slice(&NULL_RLP)); } @@ -212,7 +213,7 @@ impl HashDB for MemoryDB { } fn contains(&self, key: &H::Out) -> bool { - if key == &H::HASHED_NULL_RLP { + if key == &self.hashed_null_node { return true; } @@ -224,7 +225,7 @@ impl HashDB for MemoryDB { fn insert(&mut self, value: &[u8]) -> H::Out { if value == &NULL_RLP { - return H::HASHED_NULL_RLP.clone(); + return self.hashed_null_node.clone(); } let key = H::hash(value); match self.data.entry(key) { @@ -262,7 +263,7 @@ impl HashDB for MemoryDB { } fn remove(&mut self, key: &H::Out) { - if key == &H::HASHED_NULL_RLP { + if key == &self.hashed_null_node { return; } diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index 183a9f1300d..d0022261a68 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -40,11 +40,15 @@ pub type Result = trie::Result; #[derive(Default, Clone)] pub struct RlpNodeCodec {mark: PhantomData} -impl NodeCodec for RlpNodeCodec -where H::Out: Decodable -{ +// NOTE: what we'd really like here is: +// `impl NodeCodec for RlpNodeCodec where H::Out: Decodable` +// but due to the current limitations of Rust const evaluation we can't +// do `const HASHED_NULL_NODE: H::Out = H::Out( … … )`. Perhaps one day soon? +impl NodeCodec for RlpNodeCodec { type E = DecoderError; type S = RlpStream; + const HASHED_NULL_NODE : H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); + fn encode(node: &Node) -> Bytes { match *node { Node::Leaf(ref slice, ref value) => { @@ -103,9 +107,9 @@ where H::Out: Decodable _ => Err(DecoderError::Custom("Rlp is not valid.")) } } - fn try_decode_hash(data: &[u8]) -> Option { + fn try_decode_hash(data: &[u8]) -> Option<::Out> { let r = Rlp::new(data); - if r.is_data() && r.size() == H::LENGTH { + if r.is_data() && r.size() == KeccakHasher::LENGTH { Some(r.as_val().expect("Hash is the correct size; qed")) } else { None diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 8e140602ccb..51ef35a2fe8 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -162,7 +162,7 @@ pub trait Trie> { // make `impl NodeCodec for MyGenericCodec` impossible without better const // function evaluation, so we should limit ourselves only to // `impl NodeCodec for MyGenericCodec` or similar - fn is_empty(&self) -> bool { *self.root() == H::HASHED_NULL_RLP } + fn is_empty(&self) -> bool { *self.root() == C::HASHED_NULL_NODE } /// Does the trie contain a given key? fn contains(&self, key: &[u8]) -> Result { diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 469e7a66632..a222a8180c6 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -25,9 +25,15 @@ use stream_encoder::Stream; /// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { + /// Encoding error type type E: ::std::error::Error; + + /// Encoded stream type type S: Stream; + /// Null node type + const HASHED_NULL_NODE: H::Out; + /// Encode a Node to bytes (aka `Vec`). fn encode(&Node) -> Bytes; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 2cf6450d8d7..4e1e5ccf8b1 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -321,8 +321,8 @@ where { /// Create a new trie with backing database `db` and empty `root`. pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { - *root = H::HASHED_NULL_RLP; - let root_handle = NodeHandle::Hash(H::HASHED_NULL_RLP); + *root = C::HASHED_NULL_NODE; + let root_handle = NodeHandle::Hash(C::HASHED_NULL_NODE); TrieDBMut { storage: NodeStorage::empty(), @@ -907,7 +907,7 @@ where fn is_empty(&self) -> bool { match self.root_handle { - NodeHandle::Hash(h) => h == H::HASHED_NULL_RLP, + NodeHandle::Hash(h) => h == C::HASHED_NULL_NODE, NodeHandle::InMemory(ref h) => match self.storage[h] { Node::Empty => true, _ => false, @@ -956,8 +956,8 @@ where } None => { trace!(target: "trie", "remove: obliterated trie"); - self.root_handle = NodeHandle::Hash(H::HASHED_NULL_RLP); - *self.root = H::HASHED_NULL_RLP; + self.root_handle = NodeHandle::Hash(C::HASHED_NULL_NODE); + *self.root = C::HASHED_NULL_NODE; } } @@ -1042,7 +1042,7 @@ mod tests { assert_eq!(*memtrie.root(), real); unpopulate_trie(&mut memtrie, &x); memtrie.commit(); - if *memtrie.root() != KeccakHasher::HASHED_NULL_RLP { + if *memtrie.root() != RlpCodec::HASHED_NULL_NODE { println!("- TRIE MISMATCH"); println!(""); println!("{:?} vs {:?}", memtrie.root(), real); @@ -1050,7 +1050,7 @@ mod tests { println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); } } - assert_eq!(*memtrie.root(), KeccakHasher::HASHED_NULL_RLP); + assert_eq!(*memtrie.root(), RlpCodec::HASHED_NULL_NODE); } } @@ -1059,7 +1059,7 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); - assert_eq!(*t.root(), KeccakHasher::HASHED_NULL_RLP); + assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); } #[test] @@ -1315,7 +1315,7 @@ mod tests { } assert!(t.is_empty()); - assert_eq!(*t.root(), KeccakHasher::HASHED_NULL_RLP); + assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); } #[test] From 27fbbb32c9305955d9cbe317546c3856299cd257 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 10:33:23 +0200 Subject: [PATCH 115/160] Remove TODO --- util/patricia_trie/src/lib.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 51ef35a2fe8..584df78ffe6 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -155,13 +155,6 @@ pub trait Trie> { fn root(&self) -> &H::Out; /// Is the trie empty? - // TODO: The `Hasher` should not carry RLP-specifics; the null node should live in `NodeCodec`. Not sure what the best way to deal with this is. - // rpheimer: specifically, this should draw from the node codec, because non-rlp-encoded tries will have - // non-rlp-encoded null nodes. so the hash will be different. this is an architectural issue: - // `trait NodeCodec` should have the `const HASH_NULL_NODE: H::Out`. This will - // make `impl NodeCodec for MyGenericCodec` impossible without better const - // function evaluation, so we should limit ourselves only to - // `impl NodeCodec for MyGenericCodec` or similar fn is_empty(&self) -> bool { *self.root() == C::HASHED_NULL_NODE } /// Does the trie contain a given key? From 9c8445307107383b774052358d4c70e0041bd71d Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 10:52:06 +0200 Subject: [PATCH 116/160] cleanup --- util/hashdb/src/lib.rs | 2 +- util/patricia-trie-ethereum/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 5376cbec46e..490f35c28c7 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -24,7 +24,7 @@ use std::collections::HashMap; use std::{fmt::Debug, hash::Hash}; pub trait Hasher: Sync + Send { - type Out: AsRef<[u8]> + Debug + PartialEq + Eq + Clone + Copy + Hash + Send + Sync /* REVIEW: how do I get around this? --> */ + HeapSizeOf; + type Out: AsRef<[u8]> + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy; const LENGTH: usize; fn hash(x: &[u8]) -> Self::Out; } diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index d0022261a68..b8935769861 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -28,7 +28,7 @@ use ethcore_bytes::Bytes; use ethereum_types::H256; use hashdb::Hasher; use keccak_hasher::KeccakHasher; -use rlp::{DecoderError, Decodable, RlpStream, Rlp, Prototype}; +use rlp::{DecoderError, RlpStream, Rlp, Prototype}; use std::marker::PhantomData; use stream_encoder::Stream; use trie::{NibbleSlice, NodeCodec, node::Node}; From 0142e70bdda8a450b1e34f5e80253d4e64810660 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 11:36:36 +0200 Subject: [PATCH 117/160] Box TrieErrors --- ethcore/light/src/on_demand/request.rs | 11 +++-------- ethcore/src/client/evm_test_client.rs | 2 +- ethcore/src/state/mod.rs | 2 +- util/patricia_trie/src/lib.rs | 4 ++-- util/patricia_trie/src/lookup.rs | 11 +++-------- util/patricia_trie/src/triedb.rs | 9 +++------ util/patricia_trie/src/triedbmut.rs | 3 +-- 7 files changed, 14 insertions(+), 28 deletions(-) diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 9fd4b02eaa3..f3df8b62655 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -652,16 +652,11 @@ impl From<::rlp::DecoderError> for Error { } } -impl From for Error { - fn from(err: TrieError) -> Self { - Error::Trie(err) +impl From> for Error { + fn from(err: Box) -> Self { + Error::Trie(*err) } } -// impl From> for Error { -// fn from(err: Box) -> Self { -// Error::Trie(*err) -// } -// } /// Request for header proof by number #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index 5ab00491df8..23c081782e4 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -31,7 +31,7 @@ use ethtrie; #[derive(Debug)] pub enum EvmTestError { /// Trie integrity error. - Trie(ethtrie::TrieError), + Trie(Box), /// EVM error. Evm(vm::Error), /// Initialization error. diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 94eadc0e5f1..32308a444af 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -381,7 +381,7 @@ impl State { /// Creates new state with existing state root pub fn from_existing(db: B, root: H256, account_start_nonce: U256, factories: Factories) -> TrieResult> { if !db.as_hashdb().contains(&root) { - return Err(TrieError::InvalidStateRoot(root)); + return Err(Box::new(TrieError::InvalidStateRoot(root))); } let state = State { diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 584df78ffe6..ccdf0365831 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -105,8 +105,8 @@ impl error::Error for TrieError where T: std::fmt::Debug, E: std::er } } -/// Trie result type. Boxed to avoid copying around extra space for the `Hasher`s `Out`s on successful queries. -pub type Result = ::std::result::Result>; +/// Trie result type. Boxed to avoid copying around extra space for the `Hasher`s `Out` on successful queries. +pub type Result = ::std::result::Result>>; /// Trie-Item type used for iterators over trie data. diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 3a75021947e..22fd3a2bb21 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -49,14 +49,10 @@ where for depth in 0.. { let node_data = match self.db.get(&hash) { Some(value) => value, - None => return Err(match depth { + None => return Err(Box::new(match depth { 0 => TrieError::InvalidStateRoot(hash), _ => TrieError::IncompleteDatabase(hash), - }), - // None => return Err(Box::new(match depth { - // 0 => TrieError::<_, C::E>::InvalidStateRoot(hash), - // _ => TrieError::<_, C::E>::IncompleteDatabase(hash), - // })), + })), }; self.query.record(&hash, &node_data, depth); @@ -68,8 +64,7 @@ where let decoded = match C::decode(node_data) { Ok(node) => node, Err(e) => { - return Err(TrieError::DecoderError(hash, e)) - // return Err(Box::new(TrieError::DecoderError(hash, Box::new(e)))) + return Err(Box::new(TrieError::DecoderError(hash, e))) } }; match decoded { diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index db9dd6e3405..b74f5f01cb6 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -79,8 +79,7 @@ where /// Returns an error if `root` does not exist pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { if !db.contains(root) { - Err(TrieError::InvalidStateRoot(*root)) - // Err(Box::new(TrieError::InvalidStateRoot(*root))) + Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { Ok(TrieDB {db, root, hash_count: 0, codec_marker: PhantomData}) } @@ -93,8 +92,7 @@ where fn root_data(&self) -> Result { self.db .get(self.root) - .ok_or_else(|| TrieError::InvalidStateRoot(*self.root)) - // .ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) + .ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) } /// Given some node-describing data `node`, return the actual node RLP. @@ -103,8 +101,7 @@ where fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> Result { match C::try_decode_hash(node) { Some(key) => { - self.db.get(&key).ok_or_else(|| TrieError::IncompleteDatabase(key)) - // self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) + self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) } None => Ok(DBValue::from_slice(node)) } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 4e1e5ccf8b1..f0b592625d6 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -339,8 +339,7 @@ where /// Returns an error if `root` does not exist. pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> Result { if !db.contains(root) { - return Err(TrieError::InvalidStateRoot(*root)); - // return Err(Box::new(TrieError::InvalidStateRoot(*root))); + return Err(Box::new(TrieError::InvalidStateRoot(*root))); } let root_handle = NodeHandle::Hash(*root); From 56668738d1b9a5323c537f3f11bf43222c6b6764 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 16:59:34 +0200 Subject: [PATCH 118/160] whitespace --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 05de9eed8e3..997f2ccd436 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,8 +137,8 @@ members = [ "whisper", "whisper/cli", "util/stream-encoder", - "util/keccak-hasher", - "util/patricia-trie-ethereum", + "util/keccak-hasher", + "util/patricia-trie-ethereum", ] [patch.crates-io] From 8bd884777f57a6b20e6bd7c6ddc0cce799d307f4 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 20:01:54 +0200 Subject: [PATCH 119/160] Colesce notes on why the AsHashDB trait looks the way it does --- util/hashdb/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 490f35c28c7..035a193c340 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -57,7 +57,6 @@ pub trait HashDB: Send + Sync + AsHashDB { fn remove(&mut self, key: &H::Out); } -//TODO: investigate if we can/should get rid of this trait altogether. There used to be a `impl AsHashDB for T` but that does not work with generics so we currently have concrete impls in `ethcore` for this – perhaps we can just impl the needed methods where needed and get rid of this? /// Upcast trait. pub trait AsHashDB { /// Perform upcast to HashDB for anything that derives from HashDB. @@ -66,7 +65,8 @@ pub trait AsHashDB { fn as_hashdb_mut(&mut self) -> &mut HashDB; } -// TODO: This conflicts with an impl `for T`, see https://stackoverflow.com/questions/48432842/implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im +// NOTE: There used to be a `impl AsHashDB for T` but that does not work with generics. See https://stackoverflow.com/questions/48432842/implementing-a-trait-for-reference-and-non-reference-types-causes-conflicting-im +// This means we need concrete impls of AsHashDB in several places, which somewhat defeats the point of the trait. impl<'a, H: Hasher> AsHashDB for &'a mut HashDB { fn as_hashdb(&self) -> &HashDB { &**self } fn as_hashdb_mut(&mut self) -> &mut HashDB { &mut **self } From e54ff1fc8eaf034e92be4440ed19a6664c4061a1 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 22 Jun 2018 20:06:59 +0200 Subject: [PATCH 120/160] Fix Rlp benches --- util/rlp/benches/rlp.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/util/rlp/benches/rlp.rs b/util/rlp/benches/rlp.rs index 6aeabaf5de1..6191b39f270 100644 --- a/util/rlp/benches/rlp.rs +++ b/util/rlp/benches/rlp.rs @@ -14,13 +14,15 @@ #![feature(test)] -extern crate test; -extern crate ethcore_bigint as bigint; +extern crate ethereum_types; extern crate rlp; +extern crate stream_encoder; +extern crate test; -use test::Bencher; -use bigint::prelude::U256; +use ethereum_types::U256; use rlp::{RlpStream, Rlp}; +use stream_encoder::Stream; +use test::Bencher; #[bench] fn bench_stream_u64_value(b: &mut Bencher) { @@ -38,7 +40,7 @@ fn bench_decode_u64_value(b: &mut Bencher) { // u64 let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; let rlp = Rlp::new(&data); - let _: u64 = rlp.as_val(); + let _: u64 = rlp.as_val().unwrap(); }); } @@ -61,7 +63,7 @@ fn bench_decode_u256_value(b: &mut Bencher) { 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]; let rlp = Rlp::new(&data); - let _ : U256 = rlp.as_val(); + let _ : U256 = rlp.as_val().unwrap(); }); } @@ -83,11 +85,11 @@ fn bench_decode_nested_empty_lists(b: &mut Bencher) { // [ [], [[]], [ [], [[]] ] ] let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; let rlp = Rlp::new(&data); - let _v0: Vec = rlp.at(0).as_list(); - let _v1: Vec = rlp.at(1).at(0).as_list(); - let nested_rlp = rlp.at(2); - let _v2a: Vec = nested_rlp.at(0).as_list(); - let _v2b: Vec = nested_rlp.at(1).at(0).as_list(); + let _v0: Vec = rlp.at(0).unwrap().as_list().unwrap(); + let _v1: Vec = rlp.at(1).unwrap().at(0).unwrap().as_list().unwrap(); + let nested_rlp = rlp.at(2).unwrap(); + let _v2a: Vec = nested_rlp.at(0).unwrap().as_list().unwrap(); + let _v2b: Vec = nested_rlp.at(1).unwrap().at(0).unwrap().as_list().unwrap(); }); } From 0aab7e21c19e906ab2ba3d5685864ca6e6245872 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 11:24:26 +0200 Subject: [PATCH 121/160] Address docs grumble --- ethcore/src/executed.rs | 5 ----- util/hashdb/src/lib.rs | 7 +++++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ethcore/src/executed.rs b/ethcore/src/executed.rs index 7a3df7b8dea..2dc0c041be1 100644 --- a/ethcore/src/executed.rs +++ b/ethcore/src/executed.rs @@ -127,11 +127,6 @@ impl From for ExecutionError { ExecutionError::Internal(format!("{:?}", err)) } } -// impl From>> for ExecutionError where T: fmt::Debug { -// fn from(err: Box>) -> Self { -// ExecutionError::Internal(format!("{:?}", err)) -// } -// } impl fmt::Display for ExecutionError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 035a193c340..f6c02e6fb97 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -23,9 +23,16 @@ use heapsize::HeapSizeOf; use std::collections::HashMap; use std::{fmt::Debug, hash::Hash}; +/// Trait describing an object that can hash a slice of bytes. Used to abstract +/// other types over the hashing algorithm. Defines a single `hash` method and an +/// `Out` associated type with the necessary bounds. pub trait Hasher: Sync + Send { + /// The output type of the `Hasher` type Out: AsRef<[u8]> + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy; + /// The length in bytes of the `Hasher` output const LENGTH: usize; + + /// Compute the hash of the provided slice of bytes returning the `Out` type of the `Hasher` fn hash(x: &[u8]) -> Self::Out; } From b5e44859e3a9f5be993c40ded5fac92fded4fbee Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 11:40:19 +0200 Subject: [PATCH 122/160] Move AsHashDB impls out of the way --- util/journaldb/src/lib.rs | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index c5c1f90fd68..652d8903931 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -40,8 +40,6 @@ extern crate kvdb_memorydb; use std::{fmt, str}; use std::sync::Arc; -use hashdb::{AsHashDB, HashDB}; -use keccak_hasher::KeccakHasher; /// Export the journaldb module. mod traits; @@ -50,38 +48,13 @@ mod earlymergedb; mod overlayrecentdb; mod refcounteddb; mod util; +mod as_hash_db_impls; pub mod overlaydb; /// Export the `JournalDB` trait. pub use self::traits::JournalDB; -use archivedb::ArchiveDB; -impl AsHashDB for ArchiveDB { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } -} -use earlymergedb::EarlyMergeDB; -impl AsHashDB for EarlyMergeDB { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } -} -use overlayrecentdb::OverlayRecentDB; -impl AsHashDB for OverlayRecentDB { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } -} -use refcounteddb::RefCountedDB; -impl AsHashDB for RefCountedDB { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } -} -use overlaydb::OverlayDB; -impl AsHashDB for OverlayDB { - fn as_hashdb(&self) -> &HashDB { self } - fn as_hashdb_mut(&mut self) -> &mut HashDB { self } -} - /// Journal database operating strategy. #[derive(Debug, PartialEq, Clone, Copy)] pub enum Algorithm { From ae5346e6298abf2cd75d5bbe4cc6723e42801f74 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 11:51:35 +0200 Subject: [PATCH 123/160] Add missing docs --- util/keccak-hasher/src/lib.rs | 1 + util/patricia_trie/src/triedbmut.rs | 1 - util/stream-encoder/src/lib.rs | 9 +++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/util/keccak-hasher/src/lib.rs b/util/keccak-hasher/src/lib.rs index 9f9a96ca22a..2156660637c 100644 --- a/util/keccak-hasher/src/lib.rs +++ b/util/keccak-hasher/src/lib.rs @@ -23,6 +23,7 @@ use hashdb::Hasher; use ethereum_types::H256; use tiny_keccak::Keccak; +/// Concrete `Hasher` impl for the Keccak-256 hash #[derive(Default, Debug, Clone, PartialEq)] pub struct KeccakHasher; impl Hasher for KeccakHasher { diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index f0b592625d6..27d229082b3 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -125,7 +125,6 @@ impl Node { } } - // // TODO: parallelize fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where diff --git a/util/stream-encoder/src/lib.rs b/util/stream-encoder/src/lib.rs index 709be0b8d7a..93456e3f813 100644 --- a/util/stream-encoder/src/lib.rs +++ b/util/stream-encoder/src/lib.rs @@ -19,13 +19,18 @@ extern crate elastic_array; use elastic_array::ElasticArray1024; -/// Trait with common encoding operations -// TODO: document this +/// Trait with common stream encoding operations pub trait Stream { + /// New instance of an empty stream fn new() -> Self; + /// New instance of an empty stream, initialized as a "list" fn new_list(len: usize) -> Self; + /// Append the encoding schemes "null" value. Chainable. fn append_empty_data(&mut self) -> &mut Self; + /// Drain the object and return the underlying data fn drain(self) -> ElasticArray1024; // TODO: add as assoc type? Makes the types kind of hairy and requires some extra trait bounds, but not sure if it's worth it. Needs AsRef I think. + /// Append the provided bytes to the end of the stream as a single item. Chainable. fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; + /// Append pre-serialized data. Use with caution. Chainable. fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; } \ No newline at end of file From 66d3b76234f2c11cca7b84244f140e18e10a1b42 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 11:55:32 +0200 Subject: [PATCH 124/160] Missing docs --- util/patricia-trie-ethereum/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index b8935769861..99f2a4f8e9c 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -33,10 +33,14 @@ use std::marker::PhantomData; use stream_encoder::Stream; use trie::{NibbleSlice, NodeCodec, node::Node}; +/// Convenience type alias to instantiate a Keccak-flavoured `RlpNodeCodec` pub type RlpCodec = RlpNodeCodec; +/// Convenience type alias for Keccak/Rlp flavoured trie errors pub type TrieError = trie::TrieError; +/// Convenience type alias for Keccak/Rlp flavoured trie results pub type Result = trie::Result; +/// Concrete implementation of a `NodeCodec` with Rlp encoding, generic over the `Hasher` #[derive(Default, Clone)] pub struct RlpNodeCodec {mark: PhantomData} From 06fd57eeecd828c60709b66e61d508c51e4949ee Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 12:08:34 +0200 Subject: [PATCH 125/160] Add missing file --- util/journaldb/src/as_hash_db_impls.rs | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 util/journaldb/src/as_hash_db_impls.rs diff --git a/util/journaldb/src/as_hash_db_impls.rs b/util/journaldb/src/as_hash_db_impls.rs new file mode 100644 index 00000000000..bd3b0e2d76f --- /dev/null +++ b/util/journaldb/src/as_hash_db_impls.rs @@ -0,0 +1,49 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Impls of the `AsHashDB` upcast trait for all different variants of DB +use hashdb::{HashDB, AsHashDB}; +use keccak_hasher::KeccakHasher; +use archivedb::ArchiveDB; +use earlymergedb::EarlyMergeDB; +use overlayrecentdb::OverlayRecentDB; +use refcounteddb::RefCountedDB; +use overlaydb::OverlayDB; + +impl AsHashDB for ArchiveDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +impl AsHashDB for EarlyMergeDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +impl AsHashDB for OverlayRecentDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +impl AsHashDB for RefCountedDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} + +impl AsHashDB for OverlayDB { + fn as_hashdb(&self) -> &HashDB { self } + fn as_hashdb_mut(&mut self) -> &mut HashDB { self } +} \ No newline at end of file From 7f004a9b114ebdc8988001c801ddd4715719f8da Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 13:38:39 +0200 Subject: [PATCH 126/160] Fix deps --- ethcore/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 56d058e96c8..fcef86c6b03 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -70,10 +70,11 @@ unexpected = { path = "../util/unexpected" } journaldb = { path = "../util/journaldb" } stream-encoder = { path = "../util/stream-encoder" } keccak-hasher = { path = "../util/keccak-hasher" } -tempdir = "0.3" kvdb-rocksdb = { path = "../util/kvdb-rocksdb" } +tempdir = {version="0.3", optional = true} [dev-dependencies] +tempdir = "0.3" trie-standardmap = { path = "../util/trie-standardmap" } [features] From 75d2d231dd0657daccb031f575f97fe54da5c36f Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 14:17:33 +0200 Subject: [PATCH 127/160] Add some simple benches for memorydb --- util/memorydb/Cargo.toml | 3 +- util/memorydb/benches/memdb.rs | 79 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 util/memorydb/benches/memdb.rs diff --git a/util/memorydb/Cargo.toml b/util/memorydb/Cargo.toml index 5e539fad48f..57d7439cfeb 100644 --- a/util/memorydb/Cargo.toml +++ b/util/memorydb/Cargo.toml @@ -15,4 +15,5 @@ rlp = { version = "0.2.1", path = "../rlp" } [dev-dependencies] tiny-keccak = "1.4.2" ethereum-types = "0.3" -keccak-hasher = { path = "../keccak-hasher" } \ No newline at end of file +keccak-hasher = { path = "../keccak-hasher" } +keccak-hash = { path = "../hash" } \ No newline at end of file diff --git a/util/memorydb/benches/memdb.rs b/util/memorydb/benches/memdb.rs new file mode 100644 index 00000000000..cfc676ccd50 --- /dev/null +++ b/util/memorydb/benches/memdb.rs @@ -0,0 +1,79 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +#![feature(test)] + +extern crate hashdb; +extern crate memorydb; +extern crate keccak_hasher; +extern crate keccak_hash; +extern crate rlp; +extern crate test; + +use memorydb::MemoryDB; +use keccak_hasher::KeccakHasher; +use hashdb::{HashDB, Hasher}; +use keccak_hash::KECCAK_NULL_RLP; +use rlp::NULL_RLP; +use test::{Bencher, black_box}; + + +#[bench] +fn instantiation(b: &mut Bencher) { + b.iter(|| { + MemoryDB::::new(); + }) +} + +#[bench] +fn compare_to_null_embedded_in_struct(b: &mut Bencher) { + struct X {a_hash: ::Out}; + let x = X {a_hash: KeccakHasher::hash(&NULL_RLP)}; + let key = KeccakHasher::hash(b"abc"); + + b.iter(|| { + black_box(key == x.a_hash); + }) +} + +#[bench] +fn compare_to_null_in_const(b: &mut Bencher) { + let key = KeccakHasher::hash(b"abc"); + + b.iter(|| { + black_box(key == KECCAK_NULL_RLP); + }) +} + +#[bench] +fn contains_with_non_null_key(b: &mut Bencher) { + let mut m = MemoryDB::::new(); + let key = KeccakHasher::hash(b"abc"); + m.insert(b"abcefghijklmnopqrstuvxyz"); + b.iter(|| { + m.contains(&key); + }) +} + +#[bench] +fn contains_with_null_key(b: &mut Bencher) { + let mut m = MemoryDB::::new(); + let null_key = KeccakHasher::hash(&NULL_RLP); + m.insert(b"abcefghijklmnopqrstuvxyz"); + b.iter(|| { + m.contains(&null_key); + }) +} \ No newline at end of file From c061f382c22ea2279f9379c31d089495e9b43231 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 25 Jun 2018 14:18:18 +0200 Subject: [PATCH 128/160] Add dep for memorydb benches --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index daa88bebee5..eb27a8fcbe7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1691,6 +1691,7 @@ dependencies = [ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hash 0.1.2", "keccak-hasher 0.1.0", "plain_hasher 0.2.0", "rlp 0.2.1", From 5d03e716dccf847ca43e17544e0ad1e670f9b8b5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 26 Jun 2018 07:38:48 +0200 Subject: [PATCH 129/160] Move RlpNodeCodec impl to own file --- util/patricia-trie-ethereum/src/lib.rs | 96 +-------------- .../src/rlp_node_codec.rs | 111 ++++++++++++++++++ 2 files changed, 116 insertions(+), 91 deletions(-) create mode 100644 util/patricia-trie-ethereum/src/rlp_node_codec.rs diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index 99f2a4f8e9c..edbd6098c93 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -24,14 +24,13 @@ extern crate rlp; extern crate stream_encoder; extern crate ethereum_types; -use ethcore_bytes::Bytes; +mod rlp_node_codec; + +pub use rlp_node_codec::RlpNodeCodec; + use ethereum_types::H256; -use hashdb::Hasher; use keccak_hasher::KeccakHasher; -use rlp::{DecoderError, RlpStream, Rlp, Prototype}; -use std::marker::PhantomData; -use stream_encoder::Stream; -use trie::{NibbleSlice, NodeCodec, node::Node}; +use rlp::DecoderError; /// Convenience type alias to instantiate a Keccak-flavoured `RlpNodeCodec` pub type RlpCodec = RlpNodeCodec; @@ -39,88 +38,3 @@ pub type RlpCodec = RlpNodeCodec; pub type TrieError = trie::TrieError; /// Convenience type alias for Keccak/Rlp flavoured trie results pub type Result = trie::Result; - -/// Concrete implementation of a `NodeCodec` with Rlp encoding, generic over the `Hasher` -#[derive(Default, Clone)] -pub struct RlpNodeCodec {mark: PhantomData} - -// NOTE: what we'd really like here is: -// `impl NodeCodec for RlpNodeCodec where H::Out: Decodable` -// but due to the current limitations of Rust const evaluation we can't -// do `const HASHED_NULL_NODE: H::Out = H::Out( … … )`. Perhaps one day soon? -impl NodeCodec for RlpNodeCodec { - type E = DecoderError; - type S = RlpStream; - const HASHED_NULL_NODE : H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); - - fn encode(node: &Node) -> Bytes { - match *node { - Node::Leaf(ref slice, ref value) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(true)); - stream.append(value); - stream.out() - }, - Node::Extension(ref slice, ref raw_rlp) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(false)); - stream.append_raw(raw_rlp, 1); - stream.out() - }, - Node::Branch(ref nodes, ref value) => { - let mut stream = RlpStream::new_list(17); - for i in 0..16 { - stream.append_raw(nodes[i], 1); - } - match *value { - Some(ref n) => { stream.append(n); }, - None => { stream.append_empty_data(); }, - } - stream.out() - }, - Node::Empty => { - let mut stream = RlpStream::new(); - stream.append_empty_data(); - stream.out() - } - } - } - fn decode(data: &[u8]) -> ::std::result::Result { - let r = Rlp::new(data); - match r.prototype()? { - // either leaf or extension - decode first item with NibbleSlice::??? - // and use is_leaf return to figure out which. - // if leaf, second item is a value (is_data()) - // if extension, second item is a node (either SHA3 to be looked up and - // fed back into this function or inline RLP which can be fed back into this function). - Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { - (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), - (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), - }, - // branch - first 16 are nodes, 17th is a value (or empty). - Prototype::List(17) => { - let mut nodes = [&[] as &[u8]; 16]; - for i in 0..16 { - nodes[i] = r.at(i)?.as_raw(); - } - Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) - }, - // an empty branch index. - Prototype::Data(0) => Ok(Node::Empty), - // something went wrong. - _ => Err(DecoderError::Custom("Rlp is not valid.")) - } - } - fn try_decode_hash(data: &[u8]) -> Option<::Out> { - let r = Rlp::new(data); - if r.is_data() && r.size() == KeccakHasher::LENGTH { - Some(r.as_val().expect("Hash is the correct size; qed")) - } else { - None - } - } - - fn is_empty_node(data: &[u8]) -> bool { - Rlp::new(data).is_empty() - } -} diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs new file mode 100644 index 00000000000..53f852d9f6d --- /dev/null +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -0,0 +1,111 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! `NodeCodec` implementation for Rlp + +use ethcore_bytes::Bytes; +use ethereum_types::H256; +use hashdb::Hasher; +use keccak_hasher::KeccakHasher; +use rlp::{DecoderError, RlpStream, Rlp, Prototype}; +use std::marker::PhantomData; +use stream_encoder::Stream; +use trie::{NibbleSlice, NodeCodec, node::Node}; + +/// Concrete implementation of a `NodeCodec` with Rlp encoding, generic over the `Hasher` +#[derive(Default, Clone)] +pub struct RlpNodeCodec {mark: PhantomData} + +// NOTE: what we'd really like here is: +// `impl NodeCodec for RlpNodeCodec where H::Out: Decodable` +// but due to the current limitations of Rust const evaluation we can't +// do `const HASHED_NULL_NODE: H::Out = H::Out( … … )`. Perhaps one day soon? +impl NodeCodec for RlpNodeCodec { + type E = DecoderError; + type S = RlpStream; + const HASHED_NULL_NODE : H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); + + fn encode(node: &Node) -> Bytes { + match *node { + Node::Leaf(ref slice, ref value) => { + let mut stream = RlpStream::new_list(2); + stream.append(&&*slice.encoded(true)); + stream.append(value); + stream.out() + }, + Node::Extension(ref slice, ref raw_rlp) => { + let mut stream = RlpStream::new_list(2); + stream.append(&&*slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + stream.out() + }, + Node::Branch(ref nodes, ref value) => { + let mut stream = RlpStream::new_list(17); + for i in 0..16 { + stream.append_raw(nodes[i], 1); + } + match *value { + Some(ref n) => { stream.append(n); }, + None => { stream.append_empty_data(); }, + } + stream.out() + }, + Node::Empty => { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.out() + } + } + } + fn decode(data: &[u8]) -> ::std::result::Result { + let r = Rlp::new(data); + match r.prototype()? { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0)?.data()?) { + (slice, true) => Ok(Node::Leaf(slice, r.at(1)?.data()?)), + (slice, false) => Ok(Node::Extension(slice, r.at(1)?.as_raw())), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes = [&[] as &[u8]; 16]; + for i in 0..16 { + nodes[i] = r.at(i)?.as_raw(); + } + Ok(Node::Branch(nodes, if r.at(16)?.is_empty() { None } else { Some(r.at(16)?.data()?) })) + }, + // an empty branch index. + Prototype::Data(0) => Ok(Node::Empty), + // something went wrong. + _ => Err(DecoderError::Custom("Rlp is not valid.")) + } + } + fn try_decode_hash(data: &[u8]) -> Option<::Out> { + let r = Rlp::new(data); + if r.is_data() && r.size() == KeccakHasher::LENGTH { + Some(r.as_val().expect("Hash is the correct size; qed")) + } else { + None + } + } + + fn is_empty_node(data: &[u8]) -> bool { + Rlp::new(data).is_empty() + } +} From 54ab6360ca034b78719160a80a347ad23339c6e0 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 26 Jun 2018 16:49:51 +0200 Subject: [PATCH 130/160] WIP typechecks, failing tests --- Cargo.lock | 2 + util/hashdb/src/lib.rs | 2 +- util/patricia-trie-ethereum/Cargo.toml | 1 + util/patricia-trie-ethereum/src/lib.rs | 3 +- .../src/rlp_node_codec.rs | 53 +++++++- util/patricia_trie/Cargo.toml | 3 +- util/patricia_trie/src/lib.rs | 2 + util/patricia_trie/src/node_codec.rs | 14 ++ util/patricia_trie/src/triedbmut.rs | 125 ++++++++++++++---- 9 files changed, 176 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb27a8fcbe7..da699e11f13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2450,6 +2450,7 @@ name = "patricia-trie" version = "0.2.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", "ethcore-logger 1.12.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2470,6 +2471,7 @@ dependencies = [ name = "patricia-trie-ethereum" version = "0.1.0" dependencies = [ + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index f6c02e6fb97..830e41309cd 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -28,7 +28,7 @@ use std::{fmt::Debug, hash::Hash}; /// `Out` associated type with the necessary bounds. pub trait Hasher: Sync + Send { /// The output type of the `Hasher` - type Out: AsRef<[u8]> + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy; + type Out: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy; /// The length in bytes of the `Hasher` output const LENGTH: usize; diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml index e6cef15da53..80ecd8fcf59 100644 --- a/util/patricia-trie-ethereum/Cargo.toml +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -13,3 +13,4 @@ license = "GPL-3.0" ethcore-bytes = { path = "../bytes" } stream-encoder = { path = "../stream-encoder" } ethereum-types = "0.3" +elastic-array = "0.10" diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index edbd6098c93..c3df8fed7b7 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -17,12 +17,13 @@ //! Façade crate for `patricia_trie` for Ethereum specific impls pub extern crate patricia_trie as trie; // `pub` because we need to import this crate for the tests in `patricia_trie` and there were issues: https://gist.github.com/dvdplm/869251ee557a1b4bd53adc7c971979aa +extern crate elastic_array; extern crate ethcore_bytes; +extern crate ethereum_types; extern crate hashdb; extern crate keccak_hasher; extern crate rlp; extern crate stream_encoder; -extern crate ethereum_types; mod rlp_node_codec; diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs index 53f852d9f6d..7e32a620434 100644 --- a/util/patricia-trie-ethereum/src/rlp_node_codec.rs +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -16,6 +16,7 @@ //! `NodeCodec` implementation for Rlp +use elastic_array::{ElasticArray1024, ElasticArray128}; use ethcore_bytes::Bytes; use ethereum_types::H256; use hashdb::Hasher; @@ -24,6 +25,7 @@ use rlp::{DecoderError, RlpStream, Rlp, Prototype}; use std::marker::PhantomData; use stream_encoder::Stream; use trie::{NibbleSlice, NodeCodec, node::Node}; +use trie::triedbmut::{ChildReference, NodeHandle}; /// Concrete implementation of a `NodeCodec` with Rlp encoding, generic over the `Hasher` #[derive(Default, Clone)] @@ -98,14 +100,61 @@ impl NodeCodec for RlpNodeCodec { } fn try_decode_hash(data: &[u8]) -> Option<::Out> { let r = Rlp::new(data); - if r.is_data() && r.size() == KeccakHasher::LENGTH { + if r.is_data() && r.size() == KeccakHasher::LENGTH { Some(r.as_val().expect("Hash is the correct size; qed")) } else { None } } - fn is_empty_node(data: &[u8]) -> bool { Rlp::new(data).is_empty() } + fn empty_node() -> ElasticArray1024 { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.drain() + } + + fn leaf_node(partial: &[u8], value: &[u8]) -> ElasticArray1024 { + let mut stream = RlpStream::new_list(2); + stream.append_bytes(partial); + stream.append_bytes(value); + stream.drain() + } + + fn ext_node(partial: &[u8], child: NodeHandle, mut cb: F) -> ElasticArray1024 + where F: FnMut(NodeHandle) -> ChildReference<::Out>, + { + let mut stream = RlpStream::new_list(2); + stream.append_bytes(partial); + match cb(child) { + ChildReference::Hash(h) => stream.append_bytes(&h.as_ref()), + ChildReference::Inline(inline_data, len) => stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ), + }; + stream.drain() + } + + fn branch_node(children: I, value: Option>) -> ElasticArray1024 + where + I: IntoIterator::Out>>> + { + let mut stream = RlpStream::new_list(17); + for child_ref in children { + match child_ref { + Some(c) => match c { + ChildReference::Hash(h) => stream.append_bytes(&h.as_ref()), + ChildReference::Inline(inline_data, len) => stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ), + }, + None => stream.append_empty_data() + }; + } + + if let Some(value) = value { + stream.append_bytes(&value); + } else { + stream.append_empty_data(); + } + stream.drain() + } + } diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index f25a2586311..56aedc33ece 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -22,4 +22,5 @@ keccak-hash = { version = "0.1.0", path = "../hash" } keccak-hasher = { path = "../keccak-hasher" } rlp = { version = "0.2.1", path = "../rlp" } trie-standardmap = { path = "../trie-standardmap" } -patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } \ No newline at end of file +patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } +env_logger = "*" \ No newline at end of file diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index ccdf0365831..5dbe54dd016 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -41,6 +41,8 @@ extern crate keccak_hash as keccak; extern crate keccak_hasher; #[cfg(test)] extern crate triehash; +#[cfg(test)] +extern crate env_logger; use std::{fmt, error}; use hashdb::{HashDB, DBValue, Hasher}; diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index a222a8180c6..5d92fa4311b 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -22,6 +22,9 @@ use bytes::Bytes; use hashdb::Hasher; use node::Node; use stream_encoder::Stream; +use super::triedbmut::{ChildReference, NodeHandle}; // TODO: tidy this up + +use elastic_array::{ElasticArray1024, ElasticArray128}; /// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { @@ -45,4 +48,15 @@ pub trait NodeCodec: Sized { // Check if the provided bytes correspond to the codecs "empty" node. fn is_empty_node(data: &[u8]) -> bool; + + + fn empty_node() -> ElasticArray1024; + fn leaf_node(partial: &[u8], value: &[u8]) -> ElasticArray1024; + + fn ext_node(partial: &[u8], child: NodeHandle, cb: F) -> ElasticArray1024 + where F: FnMut(NodeHandle) -> ChildReference; + + fn branch_node(children: I, value: Option>) -> ElasticArray1024 + where + I: IntoIterator>>; } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 27d229082b3..7aee35fcdaa 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -18,7 +18,7 @@ use super::{Result, TrieError, TrieMut}; use super::lookup::Lookup; -use super::node::Node as RlpNode; +use super::node::Node as EncodedNode; use node_codec::NodeCodec; use super::node::NodeKey; @@ -36,11 +36,11 @@ use stream_encoder::Stream; // For lookups into the Node storage buffer. // This is deliberately non-copyable. #[derive(Debug)] -struct StorageHandle(usize); +pub struct StorageHandle(usize); // TODO: must be pub? // Handles to nodes in the trie. #[derive(Debug)] -enum NodeHandle { +pub enum NodeHandle { // TODO: must be pub? /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node @@ -96,14 +96,14 @@ impl Node { where C: NodeCodec { match C::decode(data).expect("encoded bytes read from db; qed") { - RlpNode::Empty => Node::Empty, - RlpNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)), - RlpNode::Extension(key, cb) => { + EncodedNode::Empty => Node::Empty, + EncodedNode::Leaf(k, v) => Node::Leaf(k.encoded(true), DBValue::from_slice(&v)), + EncodedNode::Extension(key, cb) => { Node::Extension( key.encoded(false), Self::inline_or_hash::(cb, db, storage)) } - RlpNode::Branch(ref encoded_children, val) => { + EncodedNode::Branch(ref encoded_children, val) => { let mut child = |i:usize| { let raw = encoded_children[i]; if !C::is_empty_node(raw) { @@ -125,7 +125,33 @@ impl Node { } } + fn into_encoded2(self, mut child_cb: F) -> ElasticArray1024 + where + C: NodeCodec, + F: FnMut(NodeHandle) -> ChildReference + { + match self { + Node::Empty => C::empty_node(), + Node::Leaf(partial, value) => C::leaf_node(&partial, &value), + Node::Extension(partial, child) => C::ext_node(&partial, child, child_cb), + Node::Branch(mut children, value) => { + // Branch(Box<[Option>; 16]>, Option) + // C::empty_node() // dummy + // robs suggestion: C::branch_node(children.iter().map(|maybe_child| maybe_child.map(|child| child_cb(child))), value) + C::branch_node( + children.iter_mut() + .map(Option::take) + .map(|maybe_child| + maybe_child.map(|child| child_cb(child)) + ), + value + ) + } + } + } + // TODO: parallelize + #[allow(dead_code)] fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where F: FnMut(NodeHandle, &mut >::S), @@ -133,17 +159,31 @@ impl Node { { match self { Node::Empty => { - let mut stream = >::S::new(); - stream.append_empty_data(); - stream.drain() + C::empty_node() + // // C::empty_buf() // sort of ElasticArray1024::new().push(0x80) + // let mut stream = >::S::new(); + // stream.append_empty_data(); + // stream.drain() } Node::Leaf(partial, value) => { - let mut stream = >::S::new_list(2); - stream.append_bytes(&&*partial); - stream.append_bytes(&&*value); - stream.drain() + C::leaf_node(&partial, &value) + // // let buf = C::padded_buf(); // sort of let buf = ElasticArray1024::new().push(0); + // // C::begin_list_of_length(2); // sort of unfinished_lists.push(ListInfo{position: buf.len(), 0, Some(2)}); + // // C::encode_value(buf, &&*partial); + // // C::encode_value(buf, &&*value); + // // buf + // let mut stream = >::S::new_list(2); + // stream.append_bytes(&&*partial); + // stream.append_bytes(&&*value); + // stream.drain() } Node::Extension(partial, child) => { + // let buf = C::padded_buf(); + // C::begin_list_of_length(2); + // C::encode_value(buf, &&*partial); + // child_cb(child, &mut buf); + // buf + // C::ext_node(partial, child, child_cb); let mut stream = >::S::new_list(2); stream.append_bytes(&&*partial); child_cb(child, &mut stream); @@ -212,6 +252,12 @@ enum Stored { Cached(Node, H::Out), } +// TODO: must be pub? +pub enum ChildReference { // `HO` is e.g. `H256`, i.e. the output of a `Hasher` + Inline(HO, usize), // usize is the length of the node data we store in the `H::Out` + Hash(HO), +} + /// Compact and cache-friendly storage for Trie nodes. struct NodeStorage { nodes: Vec>, @@ -280,9 +326,9 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; -/// use ethtrie::RlpNodeCodec; +/// use ethtrie::EncodedNodeCodec; /// -/// type RlpCodec = RlpNodeCodec; +/// type RlpCodec = EncodedNodeCodec; /// /// fn main() { /// let mut memdb = MemoryDB::::new(); @@ -447,7 +493,7 @@ where } } - /// insert a key, value pair into the trie, creating new nodes if necessary. + /// insert a key-value pair into the trie, creating new nodes if necessary. fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::E> { let h = match handle { NodeHandle::InMemory(h) => h, @@ -844,7 +890,8 @@ where match self.storage.destroy(handle) { Stored::New(node) => { - let encoded_root = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); + // let encoded_root = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); + let encoded_root = node.into_encoded2::<_, C>(|child| self.commit_child(child) ); *self.root = self.db.insert(&encoded_root[..]); self.hash_count += 1; @@ -859,6 +906,31 @@ where } } + fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { + match handle { + NodeHandle::Hash(hash) => ChildReference::Hash(hash), + NodeHandle::InMemory(storage_handle) => { + match self.storage.destroy(storage_handle) { + Stored::Cached(_, hash) => ChildReference::Hash(hash), + Stored::New(node) => { + let encoded = node.into_encoded2::<_, C>(|node_handle| self.commit_child(node_handle) ); + if encoded.len() >= 32 { + let hash = self.db.insert(&encoded[..]); + self.hash_count +=1; + ChildReference::Hash(hash) + } else { + // it's a small value, so we cram it into a `H::Out` and pretend like nothing were + let mut h = H::Out::default(); + let len = encoded.len(); + h.as_mut()[..len].copy_from_slice(&encoded[..len]); + ChildReference::Inline(h, encoded.len()) + } + } + } + } + } + } + /// commit a node, hashing it, committing it to the db, /// and writing it to the encoded stream as necessary. fn commit_node(&mut self, handle: NodeHandle, stream: &mut >::S) { @@ -899,6 +971,7 @@ where C: NodeCodec { fn root(&mut self) -> &H::Out { + trace!(target: "trie", "root, TrieMut, calling commit()"); self.commit(); self.root } @@ -984,6 +1057,7 @@ mod tests { use standardmap::*; use ethtrie::trie::{TrieMut, TrieDBMut, NodeCodec}; use ethtrie::RlpCodec; + use env_logger; fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec @@ -1290,30 +1364,33 @@ mod tests { #[test] fn insert_empty() { + let _ = env_logger::init(); let mut seed = ::Out::new(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 5, journal_key: 0, value_mode: ValueMode::Index, - count: 4, + count: 2, }.make_with(&mut seed); let mut db = MemoryDB::::new(); let mut root = ::Out::new(); let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); for &(ref key, ref value) in &x { + println!("[insert_empty] inserting key {:?} value {:?}", key, value); + trace!(target: "trie", "test, insert_empty: key={:?}, value={:?}", key.pretty(), value.pretty()); t.insert(key, value).unwrap(); } assert_eq!(*t.root(), trie_root(x.clone())); - for &(ref key, _) in &x { - t.insert(key, &[]).unwrap(); - } + // for &(ref key, _) in &x { + // t.insert(key, &[]).unwrap(); + // } - assert!(t.is_empty()); - assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); + // assert!(t.is_empty()); + // assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); } #[test] From 7cbe20f600ddeb4e9ad529a31ff9fbae2be71656 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 26 Jun 2018 17:07:37 +0200 Subject: [PATCH 131/160] WIP instrument into_encoded --- util/patricia_trie/Cargo.toml | 3 ++- util/patricia_trie/src/lib.rs | 3 +++ util/patricia_trie/src/triedbmut.rs | 9 ++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index f25a2586311..efcf38ea544 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -22,4 +22,5 @@ keccak-hash = { version = "0.1.0", path = "../hash" } keccak-hasher = { path = "../keccak-hasher" } rlp = { version = "0.2.1", path = "../rlp" } trie-standardmap = { path = "../trie-standardmap" } -patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } \ No newline at end of file +patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } +env_logger = "*" # TODO: remove \ No newline at end of file diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index ccdf0365831..848c2b7becb 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -42,6 +42,9 @@ extern crate keccak_hasher; #[cfg(test)] extern crate triehash; +#[cfg(test)] +extern crate env_logger; // TODO: remove + use std::{fmt, error}; use hashdb::{HashDB, DBValue, Hasher}; use std::marker::PhantomData; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 27d229082b3..554bba535e5 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -131,25 +131,30 @@ impl Node { F: FnMut(NodeHandle, &mut >::S), C: NodeCodec, { + trace!(target: "trie", "into_encoded"); match self { Node::Empty => { + trace!(target: "trie", "into_encoded, Node::Empty"); let mut stream = >::S::new(); stream.append_empty_data(); stream.drain() } Node::Leaf(partial, value) => { + trace!(target: "trie", "into_encoded, Node::Leaf, partial={:?}, value={:?}", partial, value); let mut stream = >::S::new_list(2); stream.append_bytes(&&*partial); stream.append_bytes(&&*value); stream.drain() } Node::Extension(partial, child) => { + trace!(target: "trie", "into_encoded, Node::Extension, partial={:?}", partial); let mut stream = >::S::new_list(2); stream.append_bytes(&&*partial); child_cb(child, &mut stream); stream.drain() } Node::Branch(mut children, value) => { + trace!(target: "trie", "into_encoded, Node::Branch, value={:?}", value); let mut stream = >::S::new_list(17); for child in children.iter_mut().map(Option::take) { if let Some(handle) = child { @@ -984,6 +989,7 @@ mod tests { use standardmap::*; use ethtrie::trie::{TrieMut, TrieDBMut, NodeCodec}; use ethtrie::RlpCodec; + use env_logger; fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec @@ -1290,13 +1296,14 @@ mod tests { #[test] fn insert_empty() { + let _ = env_logger::init(); let mut seed = ::Out::new(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 5, journal_key: 0, value_mode: ValueMode::Index, - count: 4, + count: 2, // TODO: set back to 4 }.make_with(&mut seed); let mut db = MemoryDB::::new(); From db2346daf50d2719ff947ae7c4d19c6d07a4f29a Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 26 Jun 2018 17:08:21 +0200 Subject: [PATCH 132/160] lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index eb27a8fcbe7..2e20067005a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2450,6 +2450,7 @@ name = "patricia-trie" version = "0.2.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", "ethcore-logger 1.12.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", From 31c4cb49b8328777352171663ded4e435e4e83ff Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 26 Jun 2018 17:07:37 +0200 Subject: [PATCH 133/160] WIP instrument into_encoded --- util/patricia_trie/Cargo.toml | 2 +- util/patricia_trie/src/lib.rs | 3 +++ util/patricia_trie/src/triedbmut.rs | 42 +++++++++++------------------ 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 56aedc33ece..efcf38ea544 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -23,4 +23,4 @@ keccak-hasher = { path = "../keccak-hasher" } rlp = { version = "0.2.1", path = "../rlp" } trie-standardmap = { path = "../trie-standardmap" } patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } -env_logger = "*" \ No newline at end of file +env_logger = "*" # TODO: remove \ No newline at end of file diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 5dbe54dd016..6392c327d7e 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -44,6 +44,9 @@ extern crate triehash; #[cfg(test)] extern crate env_logger; +#[cfg(test)] +extern crate env_logger; // TODO: remove + use std::{fmt, error}; use hashdb::{HashDB, DBValue, Hasher}; use std::marker::PhantomData; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 7aee35fcdaa..6594d676011 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -157,39 +157,30 @@ impl Node { F: FnMut(NodeHandle, &mut >::S), C: NodeCodec, { + trace!(target: "trie", "into_encoded"); match self { Node::Empty => { - C::empty_node() - // // C::empty_buf() // sort of ElasticArray1024::new().push(0x80) - // let mut stream = >::S::new(); - // stream.append_empty_data(); - // stream.drain() + trace!(target: "trie", "into_encoded, Node::Empty"); + let mut stream = >::S::new(); + stream.append_empty_data(); + stream.drain() } Node::Leaf(partial, value) => { - C::leaf_node(&partial, &value) - // // let buf = C::padded_buf(); // sort of let buf = ElasticArray1024::new().push(0); - // // C::begin_list_of_length(2); // sort of unfinished_lists.push(ListInfo{position: buf.len(), 0, Some(2)}); - // // C::encode_value(buf, &&*partial); - // // C::encode_value(buf, &&*value); - // // buf - // let mut stream = >::S::new_list(2); - // stream.append_bytes(&&*partial); - // stream.append_bytes(&&*value); - // stream.drain() + trace!(target: "trie", "into_encoded, Node::Leaf, partial={:?}, value={:?}", partial, value); + let mut stream = >::S::new_list(2); + stream.append_bytes(&&*partial); + stream.append_bytes(&&*value); + stream.drain() } Node::Extension(partial, child) => { - // let buf = C::padded_buf(); - // C::begin_list_of_length(2); - // C::encode_value(buf, &&*partial); - // child_cb(child, &mut buf); - // buf - // C::ext_node(partial, child, child_cb); + trace!(target: "trie", "into_encoded, Node::Extension, partial={:?}", partial); let mut stream = >::S::new_list(2); stream.append_bytes(&&*partial); child_cb(child, &mut stream); stream.drain() } Node::Branch(mut children, value) => { + trace!(target: "trie", "into_encoded, Node::Branch, value={:?}", value); let mut stream = >::S::new_list(17); for child in children.iter_mut().map(Option::take) { if let Some(handle) = child { @@ -326,9 +317,9 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; -/// use ethtrie::EncodedNodeCodec; +/// use ethtrie::RlpNodeCodec; /// -/// type RlpCodec = EncodedNodeCodec; +/// type RlpCodec = RlpNodeCodec; /// /// fn main() { /// let mut memdb = MemoryDB::::new(); @@ -971,7 +962,6 @@ where C: NodeCodec { fn root(&mut self) -> &H::Out { - trace!(target: "trie", "root, TrieMut, calling commit()"); self.commit(); self.root } @@ -1371,15 +1361,13 @@ mod tests { min_key: 5, journal_key: 0, value_mode: ValueMode::Index, - count: 2, + count: 2, // TODO: set back to 4 }.make_with(&mut seed); let mut db = MemoryDB::::new(); let mut root = ::Out::new(); let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); for &(ref key, ref value) in &x { - println!("[insert_empty] inserting key {:?} value {:?}", key, value); - trace!(target: "trie", "test, insert_empty: key={:?}, value={:?}", key.pretty(), value.pretty()); t.insert(key, value).unwrap(); } From 4fcd004d6240f33209dd936261b4d40ce5e3d0e7 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 26 Jun 2018 17:08:21 +0200 Subject: [PATCH 134/160] Heavily instrumented test --- Cargo.lock | 2 + util/patricia-trie-ethereum/Cargo.toml | 18 +++-- util/patricia-trie-ethereum/src/lib.rs | 4 + .../src/rlp_node_codec.rs | 37 ++++++--- util/patricia_trie/src/lib.rs | 2 - util/patricia_trie/src/node_codec.rs | 5 +- util/patricia_trie/src/triedbmut.rs | 76 +++++++++++++------ 7 files changed, 100 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da699e11f13..cbd6decffd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2472,10 +2472,12 @@ name = "patricia-trie-ethereum" version = "0.1.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", "keccak-hasher 0.1.0", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", "rlp 0.2.1", "stream-encoder 0.1.0", diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml index 80ecd8fcf59..e7ca283bdc7 100644 --- a/util/patricia-trie-ethereum/Cargo.toml +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -6,11 +6,15 @@ description = "Merkle-Patricia Trie (Ethereum Style)" license = "GPL-3.0" [dependencies] - patricia-trie = { path = "../patricia_trie" } - keccak-hasher = { path = "../keccak-hasher" } - hashdb = { path = "../hashdb" } - rlp = { path = "../rlp" } - ethcore-bytes = { path = "../bytes" } - stream-encoder = { path = "../stream-encoder" } - ethereum-types = "0.3" +patricia-trie = { path = "../patricia_trie" } +keccak-hasher = { path = "../keccak-hasher" } +hashdb = { path = "../hashdb" } +rlp = { path = "../rlp" } +ethcore-bytes = { path = "../bytes" } +stream-encoder = { path = "../stream-encoder" } +ethereum-types = "0.3" elastic-array = "0.10" +log = "0.3" # TODO: remove + +[dev-dependencies] +env_logger = "*" # TODO: remove \ No newline at end of file diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index c3df8fed7b7..df8b0420e6f 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -24,6 +24,10 @@ extern crate hashdb; extern crate keccak_hasher; extern crate rlp; extern crate stream_encoder; +#[cfg(test)] +extern crate env_logger; // TODO: remove +#[macro_use] +extern crate log; // TODO: remove mod rlp_node_codec; diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs index 7e32a620434..1e55f0d1202 100644 --- a/util/patricia-trie-ethereum/src/rlp_node_codec.rs +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -16,7 +16,7 @@ //! `NodeCodec` implementation for Rlp -use elastic_array::{ElasticArray1024, ElasticArray128}; +use elastic_array::{ElasticArray1024, ElasticArray128, ElasticArray36}; use ethcore_bytes::Bytes; use ethereum_types::H256; use hashdb::Hasher; @@ -119,19 +119,30 @@ impl NodeCodec for RlpNodeCodec { let mut stream = RlpStream::new_list(2); stream.append_bytes(partial); stream.append_bytes(value); - stream.drain() + let out = stream.drain(); + trace!(target: "ethtrie", "leaf_node, partial={:?}, \n out={:X?}", partial, out); + out } - fn ext_node(partial: &[u8], child: NodeHandle, mut cb: F) -> ElasticArray1024 + // fn ext_node(partial: &[u8], child: NodeHandle, mut cb: F) -> ElasticArray1024 + fn ext_node(partial: ElasticArray36, child: NodeHandle, mut cb: F) -> ElasticArray1024 where F: FnMut(NodeHandle) -> ChildReference<::Out>, { let mut stream = RlpStream::new_list(2); - stream.append_bytes(partial); + stream.append_bytes(&&*partial); match cb(child) { ChildReference::Hash(h) => stream.append_bytes(&h.as_ref()), - ChildReference::Inline(inline_data, len) => stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ), + ChildReference::Inline(inline_data, len) => { + let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; + trace!(target: "ethtrie", "ext_node, unpacked bytes={:X?}", bytes); + // stream.append_bytes( bytes ) + stream.append_raw(bytes, 1) + // stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ) + }, }; - stream.drain() + let out = stream.drain(); + trace!(target: "ethtrie", "ext_node, partial={:?}, \n out={:X?}", partial, out); + out } fn branch_node(children: I, value: Option>) -> ElasticArray1024 @@ -143,18 +154,26 @@ impl NodeCodec for RlpNodeCodec { match child_ref { Some(c) => match c { ChildReference::Hash(h) => stream.append_bytes(&h.as_ref()), - ChildReference::Inline(inline_data, len) => stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ), + ChildReference::Inline(inline_data, len) => { + let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; + trace!(target: "ethtrie", "branch_node, unpacked bytes={:X?}", bytes); + // stream.append_bytes( bytes ) + stream.append_raw(bytes, 1) + // stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ) + }, }, None => stream.append_empty_data() }; } - + let v = value.clone(); // TODO: remove if let Some(value) = value { stream.append_bytes(&value); } else { stream.append_empty_data(); } - stream.drain() + let out = stream.drain(); + trace!(target: "ethtrie", "branch_node, value={:?}, \n out={:X?}", &v, out); + out } } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 6392c327d7e..848c2b7becb 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -41,8 +41,6 @@ extern crate keccak_hash as keccak; extern crate keccak_hasher; #[cfg(test)] extern crate triehash; -#[cfg(test)] -extern crate env_logger; #[cfg(test)] extern crate env_logger; // TODO: remove diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 5d92fa4311b..c43d8c5db55 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -24,7 +24,7 @@ use node::Node; use stream_encoder::Stream; use super::triedbmut::{ChildReference, NodeHandle}; // TODO: tidy this up -use elastic_array::{ElasticArray1024, ElasticArray128}; +use elastic_array::{ElasticArray1024, ElasticArray128, ElasticArray36}; /// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { @@ -53,7 +53,8 @@ pub trait NodeCodec: Sized { fn empty_node() -> ElasticArray1024; fn leaf_node(partial: &[u8], value: &[u8]) -> ElasticArray1024; - fn ext_node(partial: &[u8], child: NodeHandle, cb: F) -> ElasticArray1024 + // fn ext_node(partial: &[u8], child: NodeHandle, cb: F) -> ElasticArray1024 + fn ext_node(partial: ElasticArray36, child: NodeHandle, cb: F) -> ElasticArray1024 where F: FnMut(NodeHandle) -> ChildReference; fn branch_node(children: I, value: Option>) -> ElasticArray1024 diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 6594d676011..7a35e364243 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -125,19 +125,26 @@ impl Node { } } + #[allow(dead_code)] fn into_encoded2(self, mut child_cb: F) -> ElasticArray1024 where C: NodeCodec, F: FnMut(NodeHandle) -> ChildReference { + trace!(target: "trie", "into_encoded2"); match self { - Node::Empty => C::empty_node(), - Node::Leaf(partial, value) => C::leaf_node(&partial, &value), - Node::Extension(partial, child) => C::ext_node(&partial, child, child_cb), + Node::Empty => { + trace!(target: "trie", "into_encoded2, Node::Empty"); + C::empty_node() + }, + Node::Leaf(partial, value) => { + trace!(target: "trie", "into_encoded2, Node::Leaf, partial={:?}, value={:?}", partial, value); + C::leaf_node(&partial, &value)}, + Node::Extension(partial, child) => { + trace!(target: "trie", "into_encoded2, Node::Extension, partial={:?}", partial); + C::ext_node(partial, child, child_cb)}, Node::Branch(mut children, value) => { - // Branch(Box<[Option>; 16]>, Option) - // C::empty_node() // dummy - // robs suggestion: C::branch_node(children.iter().map(|maybe_child| maybe_child.map(|child| child_cb(child))), value) + trace!(target: "trie", "into_encoded2, Node::Branch, value={:?}", value); C::branch_node( children.iter_mut() .map(Option::take) @@ -157,30 +164,34 @@ impl Node { F: FnMut(NodeHandle, &mut >::S), C: NodeCodec, { - trace!(target: "trie", "into_encoded"); + trace!(target: "trie", ""); match self { Node::Empty => { - trace!(target: "trie", "into_encoded, Node::Empty"); + trace!(target: "trie", ", Node::Empty"); let mut stream = >::S::new(); stream.append_empty_data(); stream.drain() } Node::Leaf(partial, value) => { - trace!(target: "trie", "into_encoded, Node::Leaf, partial={:?}, value={:?}", partial, value); + // trace!(target: "trie", ", Node::Leaf, partial={:?}, value={:?}", partial, value); let mut stream = >::S::new_list(2); stream.append_bytes(&&*partial); stream.append_bytes(&&*value); - stream.drain() + let out = stream.drain(); + trace!(target: "trie", "leaf_node, partial={:?}, \n out={:X?}", partial, out); + out } Node::Extension(partial, child) => { - trace!(target: "trie", "into_encoded, Node::Extension, partial={:?}", partial); + // trace!(target: "trie", ", Node::Extension, partial={:?}", partial); let mut stream = >::S::new_list(2); stream.append_bytes(&&*partial); child_cb(child, &mut stream); - stream.drain() + let out = stream.drain(); + trace!(target: "trie", "ext_node, partial={:?}, \n out={:X?}", partial, out); + out } Node::Branch(mut children, value) => { - trace!(target: "trie", "into_encoded, Node::Branch, value={:?}", value); + // trace!(target: "trie", ", Node::Branch, value={:?}", value); let mut stream = >::S::new_list(17); for child in children.iter_mut().map(Option::take) { if let Some(handle) = child { @@ -189,13 +200,16 @@ impl Node { stream.append_empty_data(); } } + let v = value.clone(); // TODO: remove if let Some(value) = value { stream.append_bytes(&&*value); } else { stream.append_empty_data(); } - stream.drain() + let out = stream.drain(); + trace!(target: "trie", "branch_node, value={:?}, \n out={:X?}", &v, out); + out } } } @@ -899,22 +913,30 @@ where fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { match handle { - NodeHandle::Hash(hash) => ChildReference::Hash(hash), + NodeHandle::Hash(hash) => { + trace!(target: "trie", "commit_child, NodeHandle::Hash, h={:X?}", hash.pretty()); + ChildReference::Hash(hash)}, NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { - Stored::Cached(_, hash) => ChildReference::Hash(hash), + Stored::Cached(_, hash) => { + trace!(target: "trie", "commit_child, NodeHandle::InMemory, Cached, h={:X?}", hash.pretty()); + ChildReference::Hash(hash)}, Stored::New(node) => { + trace!(target: "trie", "commit_child, NodeHandle::InMemory, New"); let encoded = node.into_encoded2::<_, C>(|node_handle| self.commit_child(node_handle) ); + trace!(target: "trie", "commit_child, NodeHandle::InMemory, New – back after recursive call to into_encoded2"); if encoded.len() >= 32 { let hash = self.db.insert(&encoded[..]); self.hash_count +=1; + trace!(target: "trie", "commit_child, NodeHandle::InMemory, New, long node, hash={:X?}", &hash.pretty()); ChildReference::Hash(hash) } else { - // it's a small value, so we cram it into a `H::Out` and pretend like nothing were + // it's a small value, so we cram it into a `H::Out` and tag with length let mut h = H::Out::default(); let len = encoded.len(); h.as_mut()[..len].copy_from_slice(&encoded[..len]); - ChildReference::Inline(h, encoded.len()) + trace!(target: "trie", "commit_child, NodeHandle::InMemory, New, short node, len={}, encoded_node={:X?}", len, h.pretty()); + ChildReference::Inline(h, len) } } } @@ -927,19 +949,25 @@ where fn commit_node(&mut self, handle: NodeHandle, stream: &mut >::S) { match handle { NodeHandle::Hash(h) => { + trace!(target: "trie", "commit_node, NodeHandle::Hash, h={:X?}", &h.pretty()); stream.append_bytes(&h.as_ref()) }, NodeHandle::InMemory(h) => match self.storage.destroy(h) { Stored::Cached(_, h) => { + trace!(target: "trie", "commit_node, NodeHandle::InMemory, Cached, h={:X?}", &h.pretty()); stream.append_bytes(&h.as_ref()) }, Stored::New(node) => { + trace!(target: "trie", "commit_node, NodeHandle::InMemory, New"); let encoded_node = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); + trace!(target: "trie", "commit_node, NodeHandle::InMemory, New – back after recursive call to into_encoded"); if encoded_node.len() >= 32 { let hash = self.db.insert(&encoded_node[..]); self.hash_count += 1; + trace!(target: "trie", "commit_node, NodeHandle::InMemory, New, long node, hash={:X?}", &hash.pretty()); stream.append_bytes(&hash.as_ref()) } else { + trace!(target: "trie", "commit_node, NodeHandle::InMemory, New, short node, encoded_node={:X?}", &encoded_node.pretty()); stream.append_raw(&encoded_node, 1) } } @@ -1361,7 +1389,7 @@ mod tests { min_key: 5, journal_key: 0, value_mode: ValueMode::Index, - count: 2, // TODO: set back to 4 + count: 4, }.make_with(&mut seed); let mut db = MemoryDB::::new(); @@ -1373,12 +1401,12 @@ mod tests { assert_eq!(*t.root(), trie_root(x.clone())); - // for &(ref key, _) in &x { - // t.insert(key, &[]).unwrap(); - // } + for &(ref key, _) in &x { + t.insert(key, &[]).unwrap(); + } - // assert!(t.is_empty()); - // assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); + assert!(t.is_empty()); + assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); } #[test] From aab40776025c4a42d96e52b777c1386a3ccee796 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 09:05:06 +0200 Subject: [PATCH 135/160] Remove instrumentation and cleanup --- .../src/rlp_node_codec.rs | 31 ++-- util/patricia_trie/src/lib.rs | 2 +- util/patricia_trie/src/node_codec.rs | 21 +-- util/patricia_trie/src/triedbmut.rs | 144 +++--------------- 4 files changed, 43 insertions(+), 155 deletions(-) diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs index 1e55f0d1202..00f544b7961 100644 --- a/util/patricia-trie-ethereum/src/rlp_node_codec.rs +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -16,7 +16,7 @@ //! `NodeCodec` implementation for Rlp -use elastic_array::{ElasticArray1024, ElasticArray128, ElasticArray36}; +use elastic_array::{ElasticArray1024, ElasticArray128}; use ethcore_bytes::Bytes; use ethereum_types::H256; use hashdb::Hasher; @@ -24,8 +24,7 @@ use keccak_hasher::KeccakHasher; use rlp::{DecoderError, RlpStream, Rlp, Prototype}; use std::marker::PhantomData; use stream_encoder::Stream; -use trie::{NibbleSlice, NodeCodec, node::Node}; -use trie::triedbmut::{ChildReference, NodeHandle}; +use trie::{NibbleSlice, NodeCodec, node::Node, ChildReference}; /// Concrete implementation of a `NodeCodec` with Rlp encoding, generic over the `Hasher` #[derive(Default, Clone)] @@ -120,34 +119,28 @@ impl NodeCodec for RlpNodeCodec { stream.append_bytes(partial); stream.append_bytes(value); let out = stream.drain(); - trace!(target: "ethtrie", "leaf_node, partial={:?}, \n out={:X?}", partial, out); + trace!(target: "ethtrie", "leaf_node, partial={:?}, out={:X?}", partial, out); out } - // fn ext_node(partial: &[u8], child: NodeHandle, mut cb: F) -> ElasticArray1024 - fn ext_node(partial: ElasticArray36, child: NodeHandle, mut cb: F) -> ElasticArray1024 - where F: FnMut(NodeHandle) -> ChildReference<::Out>, - { + fn ext_node(partial: &[u8], child_ref: ChildReference<::Out>) -> ElasticArray1024 { let mut stream = RlpStream::new_list(2); - stream.append_bytes(&&*partial); - match cb(child) { + stream.append_bytes(partial); + match child_ref { ChildReference::Hash(h) => stream.append_bytes(&h.as_ref()), ChildReference::Inline(inline_data, len) => { let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; trace!(target: "ethtrie", "ext_node, unpacked bytes={:X?}", bytes); - // stream.append_bytes( bytes ) stream.append_raw(bytes, 1) - // stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ) }, }; let out = stream.drain(); - trace!(target: "ethtrie", "ext_node, partial={:?}, \n out={:X?}", partial, out); + trace!(target: "ethtrie", "ext_node, partial={:?}, out={:X?}", partial, out); out - } + } fn branch_node(children: I, value: Option>) -> ElasticArray1024 - where - I: IntoIterator::Out>>> + where I: IntoIterator::Out>>> { let mut stream = RlpStream::new_list(17); for child_ref in children { @@ -157,23 +150,19 @@ impl NodeCodec for RlpNodeCodec { ChildReference::Inline(inline_data, len) => { let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; trace!(target: "ethtrie", "branch_node, unpacked bytes={:X?}", bytes); - // stream.append_bytes( bytes ) stream.append_raw(bytes, 1) - // stream.append_bytes( &AsRef::<[u8]>::as_ref(&inline_data)[..len] ) }, }, None => stream.append_empty_data() }; } - let v = value.clone(); // TODO: remove if let Some(value) = value { stream.append_bytes(&value); } else { stream.append_empty_data(); } let out = stream.drain(); - trace!(target: "ethtrie", "branch_node, value={:?}, \n out={:X?}", &v, out); + trace!(target: "ethtrie", "branch_node, out={:X?}", out); out } - } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 848c2b7becb..ad3f3a4712b 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -64,7 +64,7 @@ mod nibbleslice; mod node_codec; pub use self::triedb::{TrieDB, TrieDBIterator}; -pub use self::triedbmut::TrieDBMut; +pub use self::triedbmut::{TrieDBMut, ChildReference}; pub use self::sectriedbmut::SecTrieDBMut; pub use self::sectriedb::SecTrieDB; pub use self::fatdb::{FatDB, FatDBIterator}; diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index c43d8c5db55..ac8d75c471d 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -22,9 +22,9 @@ use bytes::Bytes; use hashdb::Hasher; use node::Node; use stream_encoder::Stream; -use super::triedbmut::{ChildReference, NodeHandle}; // TODO: tidy this up +use ChildReference; -use elastic_array::{ElasticArray1024, ElasticArray128, ElasticArray36}; +use elastic_array::{ElasticArray1024, ElasticArray128}; /// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { @@ -43,21 +43,22 @@ pub trait NodeCodec: Sized { /// Decode bytes to a `Node`. Returns `Self::E` on failure. fn decode(data: &[u8]) -> Result; - /// Decode bytes to the `Hasher`s output type. Assumes 32 bytes long hashes! Returns `None` on failure. + /// Decode bytes to the `Hasher`s output type. Returns `None` on failure. fn try_decode_hash(data: &[u8]) -> Option; - // Check if the provided bytes correspond to the codecs "empty" node. + /// Check if the provided bytes correspond to the codecs "empty" node. fn is_empty_node(data: &[u8]) -> bool; - + /// Returns an empty node fn empty_node() -> ElasticArray1024; + + /// Returns an encoded leaft node fn leaf_node(partial: &[u8], value: &[u8]) -> ElasticArray1024; - // fn ext_node(partial: &[u8], child: NodeHandle, cb: F) -> ElasticArray1024 - fn ext_node(partial: ElasticArray36, child: NodeHandle, cb: F) -> ElasticArray1024 - where F: FnMut(NodeHandle) -> ChildReference; + /// Returns an encoded extension node + fn ext_node(partial: &[u8], child_ref: ChildReference) -> ElasticArray1024; + /// Returns an encoded branch node. Takes an iterator yielding `ChildReference` and an optional value fn branch_node(children: I, value: Option>) -> ElasticArray1024 - where - I: IntoIterator>>; + where I: IntoIterator>>; } diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 7a35e364243..ed341e0f14b 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -31,16 +31,15 @@ use std::collections::{HashSet, VecDeque}; use std::marker::PhantomData; use std::mem; use std::ops::Index; -use stream_encoder::Stream; // For lookups into the Node storage buffer. // This is deliberately non-copyable. #[derive(Debug)] -pub struct StorageHandle(usize); // TODO: must be pub? +struct StorageHandle(usize); // Handles to nodes in the trie. #[derive(Debug)] -pub enum NodeHandle { // TODO: must be pub? +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node @@ -125,27 +124,19 @@ impl Node { } } - #[allow(dead_code)] - fn into_encoded2(self, mut child_cb: F) -> ElasticArray1024 + fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 where C: NodeCodec, F: FnMut(NodeHandle) -> ChildReference { - trace!(target: "trie", "into_encoded2"); match self { - Node::Empty => { - trace!(target: "trie", "into_encoded2, Node::Empty"); - C::empty_node() - }, - Node::Leaf(partial, value) => { - trace!(target: "trie", "into_encoded2, Node::Leaf, partial={:?}, value={:?}", partial, value); - C::leaf_node(&partial, &value)}, - Node::Extension(partial, child) => { - trace!(target: "trie", "into_encoded2, Node::Extension, partial={:?}", partial); - C::ext_node(partial, child, child_cb)}, + Node::Empty => C::empty_node(), + Node::Leaf(partial, value) => C::leaf_node(&partial, &value), + Node::Extension(partial, child) => C::ext_node(&partial, child_cb(child)), + // Node::Extension(partial, child) => C::ext_node(&partial, child, child_cb), Node::Branch(mut children, value) => { - trace!(target: "trie", "into_encoded2, Node::Branch, value={:?}", value); C::branch_node( + // map the `NodeHandle`s from the Branch to `ChildReferences` children.iter_mut() .map(Option::take) .map(|maybe_child| @@ -157,62 +148,6 @@ impl Node { } } - // TODO: parallelize - #[allow(dead_code)] - fn into_encoded(self, mut child_cb: F) -> ElasticArray1024 - where - F: FnMut(NodeHandle, &mut >::S), - C: NodeCodec, - { - trace!(target: "trie", ""); - match self { - Node::Empty => { - trace!(target: "trie", ", Node::Empty"); - let mut stream = >::S::new(); - stream.append_empty_data(); - stream.drain() - } - Node::Leaf(partial, value) => { - // trace!(target: "trie", ", Node::Leaf, partial={:?}, value={:?}", partial, value); - let mut stream = >::S::new_list(2); - stream.append_bytes(&&*partial); - stream.append_bytes(&&*value); - let out = stream.drain(); - trace!(target: "trie", "leaf_node, partial={:?}, \n out={:X?}", partial, out); - out - } - Node::Extension(partial, child) => { - // trace!(target: "trie", ", Node::Extension, partial={:?}", partial); - let mut stream = >::S::new_list(2); - stream.append_bytes(&&*partial); - child_cb(child, &mut stream); - let out = stream.drain(); - trace!(target: "trie", "ext_node, partial={:?}, \n out={:X?}", partial, out); - out - } - Node::Branch(mut children, value) => { - // trace!(target: "trie", ", Node::Branch, value={:?}", value); - let mut stream = >::S::new_list(17); - for child in children.iter_mut().map(Option::take) { - if let Some(handle) = child { - child_cb(handle, &mut stream); - } else { - stream.append_empty_data(); - } - } - let v = value.clone(); // TODO: remove - if let Some(value) = value { - stream.append_bytes(&&*value); - } else { - stream.append_empty_data(); - } - - let out = stream.drain(); - trace!(target: "trie", "branch_node, value={:?}, \n out={:X?}", &v, out); - out - } - } - } } // post-inspect action. @@ -257,10 +192,10 @@ enum Stored { Cached(Node, H::Out), } -// TODO: must be pub? +/// Used to build a collection of child nodes from a collection of `NodeHandle`s pub enum ChildReference { // `HO` is e.g. `H256`, i.e. the output of a `Hasher` - Inline(HO, usize), // usize is the length of the node data we store in the `H::Out` Hash(HO), + Inline(HO, usize), // usize is the length of the node data we store in the `H::Out` } /// Compact and cache-friendly storage for Trie nodes. @@ -367,7 +302,7 @@ where impl<'a, H, C> TrieDBMut<'a, H, C> where H: Hasher, - C: NodeCodec + C: NodeCodec, { /// Create a new trie with backing database `db` and empty `root`. pub fn new(db: &'a mut HashDB, root: &'a mut H::Out) -> Self { @@ -895,8 +830,7 @@ where match self.storage.destroy(handle) { Stored::New(node) => { - // let encoded_root = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); - let encoded_root = node.into_encoded2::<_, C>(|child| self.commit_child(child) ); + let encoded_root = node.into_encoded::<_, C>(|child| self.commit_child(child) ); *self.root = self.db.insert(&encoded_root[..]); self.hash_count += 1; @@ -911,31 +845,28 @@ where } } + /// Commit a node by hashing it and writing it to the db. Returns a + /// `ChildReference` which in most cases carries a normal hash but for the + /// case where we can fit the actual data in the `Hasher`s output type, we + /// store the data inline. This function is used as the callback to the + /// `into_encoded` method of `Node`. fn commit_child(&mut self, handle: NodeHandle) -> ChildReference { match handle { - NodeHandle::Hash(hash) => { - trace!(target: "trie", "commit_child, NodeHandle::Hash, h={:X?}", hash.pretty()); - ChildReference::Hash(hash)}, + NodeHandle::Hash(hash) => ChildReference::Hash(hash), NodeHandle::InMemory(storage_handle) => { match self.storage.destroy(storage_handle) { - Stored::Cached(_, hash) => { - trace!(target: "trie", "commit_child, NodeHandle::InMemory, Cached, h={:X?}", hash.pretty()); - ChildReference::Hash(hash)}, + Stored::Cached(_, hash) => ChildReference::Hash(hash), Stored::New(node) => { - trace!(target: "trie", "commit_child, NodeHandle::InMemory, New"); - let encoded = node.into_encoded2::<_, C>(|node_handle| self.commit_child(node_handle) ); - trace!(target: "trie", "commit_child, NodeHandle::InMemory, New – back after recursive call to into_encoded2"); - if encoded.len() >= 32 { + let encoded = node.into_encoded::<_, C>(|node_handle| self.commit_child(node_handle) ); + if encoded.len() >= 32 { // TODO: this should use the `Hasher::LENGTH` const let hash = self.db.insert(&encoded[..]); self.hash_count +=1; - trace!(target: "trie", "commit_child, NodeHandle::InMemory, New, long node, hash={:X?}", &hash.pretty()); ChildReference::Hash(hash) } else { // it's a small value, so we cram it into a `H::Out` and tag with length let mut h = H::Out::default(); let len = encoded.len(); h.as_mut()[..len].copy_from_slice(&encoded[..len]); - trace!(target: "trie", "commit_child, NodeHandle::InMemory, New, short node, len={}, encoded_node={:X?}", len, h.pretty()); ChildReference::Inline(h, len) } } @@ -944,37 +875,6 @@ where } } - /// commit a node, hashing it, committing it to the db, - /// and writing it to the encoded stream as necessary. - fn commit_node(&mut self, handle: NodeHandle, stream: &mut >::S) { - match handle { - NodeHandle::Hash(h) => { - trace!(target: "trie", "commit_node, NodeHandle::Hash, h={:X?}", &h.pretty()); - stream.append_bytes(&h.as_ref()) - }, - NodeHandle::InMemory(h) => match self.storage.destroy(h) { - Stored::Cached(_, h) => { - trace!(target: "trie", "commit_node, NodeHandle::InMemory, Cached, h={:X?}", &h.pretty()); - stream.append_bytes(&h.as_ref()) - }, - Stored::New(node) => { - trace!(target: "trie", "commit_node, NodeHandle::InMemory, New"); - let encoded_node = node.into_encoded::<_, C>(|child, stream| self.commit_node(child, stream)); - trace!(target: "trie", "commit_node, NodeHandle::InMemory, New – back after recursive call to into_encoded"); - if encoded_node.len() >= 32 { - let hash = self.db.insert(&encoded_node[..]); - self.hash_count += 1; - trace!(target: "trie", "commit_node, NodeHandle::InMemory, New, long node, hash={:X?}", &hash.pretty()); - stream.append_bytes(&hash.as_ref()) - } else { - trace!(target: "trie", "commit_node, NodeHandle::InMemory, New, short node, encoded_node={:X?}", &encoded_node.pretty()); - stream.append_raw(&encoded_node, 1) - } - } - } - }; - } - // a hack to get the root node's handle fn root_handle(&self) -> NodeHandle { match self.root_handle { @@ -1075,7 +975,6 @@ mod tests { use standardmap::*; use ethtrie::trie::{TrieMut, TrieDBMut, NodeCodec}; use ethtrie::RlpCodec; - use env_logger; fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec @@ -1382,7 +1281,6 @@ mod tests { #[test] fn insert_empty() { - let _ = env_logger::init(); let mut seed = ::Out::new(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), From a88616f51ec1aa0c9704225c11abde86f4783197 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 11:32:44 +0200 Subject: [PATCH 136/160] Remove stream_encoder Fix `expect` messages Use Hasher::LENGTH to check if there's space to store a node inline --- util/patricia_trie/Cargo.toml | 2 -- util/patricia_trie/src/lib.rs | 1 - util/patricia_trie/src/node_codec.rs | 7 +------ util/patricia_trie/src/triedb.rs | 4 ++-- util/patricia_trie/src/triedbmut.rs | 4 ++-- 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index efcf38ea544..96a4495d3ea 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -12,7 +12,6 @@ rand = "0.4" ethcore-bytes = { version = "0.1.0", path = "../bytes" } hashdb = { version = "0.2", path = "../hashdb" } ethcore-logger = { version = "1.9.0", path = "../../logger" } -stream-encoder = { path = "../stream-encoder" } [dev-dependencies] triehash = { version = "0.1.0", path = "../triehash" } @@ -23,4 +22,3 @@ keccak-hasher = { path = "../keccak-hasher" } rlp = { version = "0.2.1", path = "../rlp" } trie-standardmap = { path = "../trie-standardmap" } patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } -env_logger = "*" # TODO: remove \ No newline at end of file diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index ad3f3a4712b..3df5080d020 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -20,7 +20,6 @@ extern crate ethcore_bytes as bytes; extern crate ethcore_logger; extern crate hashdb; extern crate rand; -extern crate stream_encoder; #[macro_use] extern crate log; diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index ac8d75c471d..86043b20b25 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -15,13 +15,11 @@ // along with Parity. If not, see . //! Generic trait for trie node encoding/decoding. Takes a `hashdb::Hasher` -//! to parametrize the hashes used in the codec; takes a `stream_encoder::Stream` -//! implementation to do streaming encoding. +//! to parametrize the hashes used in the codec. use bytes::Bytes; use hashdb::Hasher; use node::Node; -use stream_encoder::Stream; use ChildReference; use elastic_array::{ElasticArray1024, ElasticArray128}; @@ -31,9 +29,6 @@ pub trait NodeCodec: Sized { /// Encoding error type type E: ::std::error::Error; - /// Encoded stream type - type S: Stream; - /// Null node type const HASHED_NULL_NODE: H::Out; diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index b74f5f01cb6..f342ed9656d 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -242,7 +242,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> Result<(), H::Out, C::E> { loop { let (data, mid) = { - let node = C::decode(&node_data).expect("rlp read from db; qed"); + let node = C::decode(&node_data).expect("encoded data read from db; qed"); match node { Node::Leaf(slice, _) => { if slice == key { @@ -401,7 +401,7 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { self.trail.pop(); }, IterStep::Descend::(Ok(d)) => { - let node = C::decode(&d).expect("rlp read from db; qed"); + let node = C::decode(&d).expect("encoded data read from db; qed"); self.descend_into_node(node.into()) }, IterStep::Descend::(Err(e)) => { diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index ed341e0f14b..393eb629af1 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -39,7 +39,7 @@ struct StorageHandle(usize); // Handles to nodes in the trie. #[derive(Debug)] -enum NodeHandle { +enum NodeHandle { /// Loaded into memory. InMemory(StorageHandle), /// Either a hash or an inline node @@ -858,7 +858,7 @@ where Stored::Cached(_, hash) => ChildReference::Hash(hash), Stored::New(node) => { let encoded = node.into_encoded::<_, C>(|node_handle| self.commit_child(node_handle) ); - if encoded.len() >= 32 { // TODO: this should use the `Hasher::LENGTH` const + if encoded.len() >= H::LENGTH { let hash = self.db.insert(&encoded[..]); self.hash_count +=1; ChildReference::Hash(hash) From 17e6c928a60bf64b23b8ffc911e1c17047048dfa Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 11:34:48 +0200 Subject: [PATCH 137/160] Remove stream_encoder usage --- Cargo.lock | 23 --------------- Cargo.toml | 1 - ethcore/Cargo.toml | 1 - ethcore/light/Cargo.toml | 1 - ethcore/light/src/cht.rs | 2 +- ethcore/light/src/client/header_chain.rs | 2 +- ethcore/light/src/lib.rs | 1 - ethcore/light/src/net/mod.rs | 2 +- ethcore/light/src/net/status.rs | 2 +- ethcore/light/src/net/tests/mod.rs | 4 +-- ethcore/light/src/on_demand/request.rs | 2 +- ethcore/light/src/types/request/mod.rs | 6 ++-- ethcore/src/block.rs | 2 +- ethcore/src/blockchain/blockchain.rs | 2 +- ethcore/src/blockchain/extras.rs | 2 +- ethcore/src/client/test_client.rs | 2 +- ethcore/src/encoded.rs | 2 +- ethcore/src/engines/authority_round/mod.rs | 2 +- ethcore/src/engines/tendermint/message.rs | 2 +- ethcore/src/engines/tendermint/mod.rs | 2 +- .../engines/validator_set/safe_contract.rs | 2 +- ethcore/src/executive.rs | 2 +- ethcore/src/header.rs | 2 +- ethcore/src/json_tests/executive.rs | 2 +- ethcore/src/lib.rs | 1 - ethcore/src/pod_account.rs | 2 +- ethcore/src/snapshot/account.rs | 2 +- ethcore/src/snapshot/block.rs | 2 +- ethcore/src/snapshot/consensus/authority.rs | 2 +- ethcore/src/snapshot/consensus/work.rs | 2 +- ethcore/src/snapshot/io.rs | 2 +- ethcore/src/snapshot/mod.rs | 2 +- ethcore/src/snapshot/tests/proof_of_work.rs | 2 +- ethcore/src/snapshot/tests/state.rs | 2 +- ethcore/src/spec/seal.rs | 2 +- ethcore/src/spec/spec.rs | 2 +- ethcore/src/state/account.rs | 2 +- ethcore/src/test_helpers.rs | 2 +- ethcore/src/tests/client.rs | 2 +- ethcore/src/trace/types/error.rs | 2 +- ethcore/src/trace/types/flat.rs | 2 +- ethcore/src/verification/verification.rs | 4 +-- ethcore/sync/Cargo.toml | 1 - ethcore/sync/src/blocks.rs | 2 +- ethcore/sync/src/chain/mod.rs | 2 +- ethcore/sync/src/chain/propagator.rs | 2 +- ethcore/sync/src/chain/requester.rs | 2 +- ethcore/sync/src/chain/supplier.rs | 2 +- ethcore/sync/src/lib.rs | 1 - ethcore/transaction/Cargo.toml | 1 - ethcore/transaction/src/lib.rs | 1 - ethcore/transaction/src/transaction.rs | 2 +- ethcore/types/Cargo.toml | 1 - ethcore/types/src/lib.rs | 1 - ethcore/types/src/snapshot_manifest.rs | 2 +- ethcore/vm/Cargo.toml | 1 - ethcore/vm/src/call_type.rs | 2 +- ethcore/vm/src/lib.rs | 2 -- rpc/Cargo.toml | 1 - rpc/src/lib.rs | 1 - rpc/src/v1/helpers/dispatch.rs | 2 +- util/network-devp2p/Cargo.toml | 1 - util/network-devp2p/src/connection.rs | 2 +- util/network-devp2p/src/discovery.rs | 2 +- util/network-devp2p/src/handshake.rs | 2 +- util/network-devp2p/src/lib.rs | 1 - util/network-devp2p/src/session.rs | 2 +- util/patricia-trie-ethereum/Cargo.toml | 1 - util/patricia-trie-ethereum/src/lib.rs | 1 - .../src/rlp_node_codec.rs | 2 -- util/rlp/Cargo.toml | 1 - util/rlp/benches/rlp.rs | 2 -- util/rlp/src/lib.rs | 2 -- util/rlp/src/stream.rs | 29 +++++-------------- util/rlp/src/traits.rs | 1 - util/rlp/tests/tests.rs | 2 -- util/rlp_compress/Cargo.toml | 1 - util/rlp_compress/src/lib.rs | 3 +- util/triehash/Cargo.toml | 1 - util/triehash/src/lib.rs | 2 -- util/version/Cargo.toml | 1 - util/version/src/lib.rs | 3 +- whisper/Cargo.toml | 1 - whisper/src/lib.rs | 1 - whisper/src/message.rs | 2 +- whisper/src/net/mod.rs | 2 +- 86 files changed, 65 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbd6decffd0..c1c945a7c9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,7 +257,6 @@ dependencies = [ "rlp 0.2.1", "rlp_derive 0.1.0", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "stream-encoder 0.1.0", ] [[package]] @@ -570,7 +569,6 @@ dependencies = [ "snappy 0.1.0 (git+https://github.com/paritytech/rust-snappy)", "stats 0.1.0", "stop-guard 0.1.0", - "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "trace-time 0.1.0", "trie-standardmap 0.1.0", @@ -657,7 +655,6 @@ dependencies = [ "serde_derive 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", - "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "triehash 0.1.0", "vm 0.1.0", @@ -755,7 +752,6 @@ dependencies = [ "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "snappy 0.1.0 (git+https://github.com/paritytech/rust-snappy)", - "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -901,7 +897,6 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "stream-encoder 0.1.0", "trace-time 0.1.0", "triehash 0.1.0", ] @@ -918,7 +913,6 @@ dependencies = [ "keccak-hash 0.1.2", "rlp 0.2.1", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "stream-encoder 0.1.0", "unexpected 0.1.0", ] @@ -2287,7 +2281,6 @@ dependencies = [ "serde_derive 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", - "stream-encoder 0.1.0", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2362,7 +2355,6 @@ dependencies = [ "ethcore-bytes 0.1.0", "rlp 0.2.1", "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "stream-encoder 0.1.0", "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "vergen 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2403,7 +2395,6 @@ dependencies = [ "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "stream-encoder 0.1.0", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2450,7 +2441,6 @@ name = "patricia-trie" version = "0.2.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", "ethcore-logger 1.12.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2462,7 +2452,6 @@ dependencies = [ "patricia-trie-ethereum 0.1.0", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", - "stream-encoder 0.1.0", "trie-standardmap 0.1.0", "triehash 0.1.0", ] @@ -2480,7 +2469,6 @@ dependencies = [ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", "rlp 0.2.1", - "stream-encoder 0.1.0", ] [[package]] @@ -2812,7 +2800,6 @@ dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "stream-encoder 0.1.0", ] [[package]] @@ -2822,7 +2809,6 @@ dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", - "stream-encoder 0.1.0", ] [[package]] @@ -3116,13 +3102,6 @@ dependencies = [ name = "stop-guard" version = "0.1.0" -[[package]] -name = "stream-encoder" -version = "0.1.0" -dependencies = [ - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "strsim" version = "0.6.0" @@ -3558,7 +3537,6 @@ dependencies = [ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2", "rlp 0.2.1", - "stream-encoder 0.1.0", "trie-standardmap 0.1.0", ] @@ -3710,7 +3688,6 @@ dependencies = [ "patricia-trie 0.2.0", "patricia-trie-ethereum 0.1.0", "rlp 0.2.1", - "stream-encoder 0.1.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 997f2ccd436..7fde04a07f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,7 +136,6 @@ members = [ "transaction-pool", "whisper", "whisper/cli", - "util/stream-encoder", "util/keccak-hasher", "util/patricia-trie-ethereum", ] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index fcef86c6b03..63680129c32 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -68,7 +68,6 @@ keccak-hash = { path = "../util/hash" } triehash = { path = "../util/triehash" } unexpected = { path = "../util/unexpected" } journaldb = { path = "../util/journaldb" } -stream-encoder = { path = "../util/stream-encoder" } keccak-hasher = { path = "../util/keccak-hasher" } kvdb-rocksdb = { path = "../util/kvdb-rocksdb" } tempdir = {version="0.3", optional = true} diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index 7cdc8841068..a9506ee3f9f 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -38,7 +38,6 @@ triehash = { path = "../../util/triehash" } kvdb = { path = "../../util/kvdb" } memory-cache = { path = "../../util/memory_cache" } error-chain = { version = "0.11", default-features = false } -stream-encoder = { path = "../../util/stream-encoder" } [dev-dependencies] ethcore = { path = "..", features = ["test-helpers"] } diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index e996059e79d..de2ba01681a 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -32,7 +32,7 @@ use bytes::Bytes; use trie::{TrieMut, TrieDBMut, Trie, TrieDB, Recorder}; use ethtrie::{self, RlpCodec}; use rlp::{RlpStream, Rlp}; -use stream_encoder::Stream; + // encode a key. macro_rules! key { diff --git a/ethcore/light/src/client/header_chain.rs b/ethcore/light/src/client/header_chain.rs index 945a84f95a5..0824b55760a 100644 --- a/ethcore/light/src/client/header_chain.rs +++ b/ethcore/light/src/client/header_chain.rs @@ -47,7 +47,7 @@ use heapsize::HeapSizeOf; use kvdb::{DBTransaction, KeyValueDB}; use plain_hasher::H256FastMap; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; -use stream_encoder::Stream; + use cache::Cache; use parking_lot::{Mutex, RwLock}; diff --git a/ethcore/light/src/lib.rs b/ethcore/light/src/lib.rs index 990b071777b..2000131a696 100644 --- a/ethcore/light/src/lib.rs +++ b/ethcore/light/src/lib.rs @@ -71,7 +71,6 @@ extern crate patricia_trie_ethereum as ethtrie; extern crate plain_hasher; extern crate rand; extern crate rlp; -extern crate stream_encoder; extern crate parking_lot; #[macro_use] extern crate rlp_derive; diff --git a/ethcore/light/src/net/mod.rs b/ethcore/light/src/net/mod.rs index 5a3a12ec4a6..749ab82c1e4 100644 --- a/ethcore/light/src/net/mod.rs +++ b/ethcore/light/src/net/mod.rs @@ -23,7 +23,7 @@ use transaction::UnverifiedTransaction; use io::TimerToken; use network::{NetworkProtocolHandler, NetworkContext, PeerId}; use rlp::{RlpStream, Rlp}; -use stream_encoder::Stream; + use ethereum_types::{H256, U256}; use kvdb::DBValue; use parking_lot::{Mutex, RwLock}; diff --git a/ethcore/light/src/net/status.rs b/ethcore/light/src/net/status.rs index dcc14f87c0e..e07045095cf 100644 --- a/ethcore/light/src/net/status.rs +++ b/ethcore/light/src/net/status.rs @@ -17,7 +17,7 @@ //! Peer status and capabilities. use rlp::{DecoderError, Encodable, Decodable, RlpStream, Rlp}; -use stream_encoder::Stream; + use ethereum_types::{H256, U256}; use super::request_credits::FlowParams; diff --git a/ethcore/light/src/net/tests/mod.rs b/ethcore/light/src/net/tests/mod.rs index b2f159932a4..7ea014d2871 100644 --- a/ethcore/light/src/net/tests/mod.rs +++ b/ethcore/light/src/net/tests/mod.rs @@ -32,7 +32,7 @@ use request; use request::*; use rlp::{Rlp, RlpStream}; -use stream_encoder::Stream; + use ethereum_types::{H256, U256, Address}; use std::sync::Arc; @@ -643,7 +643,7 @@ fn proof_of_execution() { fn id_guard() { use super::request_set::RequestSet; use super::ReqId; - use stream_encoder::Stream; + let capabilities = capabilities(); diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index f3df8b62655..24239b20b29 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -31,7 +31,7 @@ use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY, KECCAK_EMPTY_LIST_RLP, keccak}; use request::{self as net_request, IncompleteRequest, CompleteRequest, Output, OutputKind, Field}; use rlp::{RlpStream, Rlp}; -use stream_encoder::Stream; + use ethereum_types::{H256, U256, Address}; use parking_lot::Mutex; use hashdb::HashDB; diff --git a/ethcore/light/src/types/request/mod.rs b/ethcore/light/src/types/request/mod.rs index 96a7227cb22..5f269741ec4 100644 --- a/ethcore/light/src/types/request/mod.rs +++ b/ethcore/light/src/types/request/mod.rs @@ -672,7 +672,7 @@ pub mod header { use super::{Field, HashOrNumber, NoSuchOutput, OutputKind, Output}; use ethcore::encoded; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; - use stream_encoder::Stream; + /// Potentially incomplete headers request. #[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] @@ -1027,7 +1027,7 @@ pub mod block_body { use super::{Field, NoSuchOutput, OutputKind, Output}; use ethcore::encoded; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; - use stream_encoder::Stream; + use ethereum_types::H256; /// Potentially incomplete block body request. @@ -1636,7 +1636,7 @@ pub mod epoch_signal { mod tests { use super::*; use ethcore::header::Header; - use stream_encoder::Stream; + fn check_roundtrip(val: T) where T: ::rlp::Encodable + ::rlp::Decodable + PartialEq + ::std::fmt::Debug diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 90dc28db8f0..a883278f5ee 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -23,7 +23,7 @@ use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP}; use triehash::ordered_trie_root; use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError, encode_list}; -use stream_encoder::Stream; + use ethereum_types::{H256, U256, Address, Bloom}; use bytes::Bytes; use unexpected::{Mismatch, OutOfBounds}; diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index e7c660293ff..f134f567f5f 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -47,7 +47,7 @@ use engines::epoch::{Transition as EpochTransition, PendingTransition as Pending use rayon::prelude::*; use ansi_term::Colour; use kvdb::{DBTransaction, KeyValueDB}; -use stream_encoder::Stream; + use error::Error; use std::path::Path; diff --git a/ethcore/src/blockchain/extras.rs b/ethcore/src/blockchain/extras.rs index 45a91b35a9e..57b43146090 100644 --- a/ethcore/src/blockchain/extras.rs +++ b/ethcore/src/blockchain/extras.rs @@ -252,7 +252,7 @@ pub struct EpochTransitions { #[cfg(test)] mod tests { use rlp::*; - use stream_encoder::Stream; + use super::BlockReceipts; #[test] diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index c948754126d..505c1e940ef 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -30,7 +30,7 @@ use kvdb::DBValue; use kvdb_memorydb; use bytes::Bytes; use rlp::{Rlp, RlpStream}; -use stream_encoder::Stream; + use ethkey::{Generator, Random}; use ethcore_miner::pool::VerifiedTransaction; use transaction::{self, Transaction, LocalizedTransaction, SignedTransaction, Action}; diff --git a/ethcore/src/encoded.rs b/ethcore/src/encoded.rs index 19436dfd8be..4b1a139eaf0 100644 --- a/ethcore/src/encoded.rs +++ b/ethcore/src/encoded.rs @@ -29,7 +29,7 @@ use hash::keccak; use header::{BlockNumber, Header as FullHeader}; use heapsize::HeapSizeOf; use rlp::{self, Rlp, RlpStream}; -use stream_encoder::Stream; + use transaction::UnverifiedTransaction; use views::{self, BlockView, HeaderView, BodyView}; diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index 2b8070c8de7..260c8727844 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -44,7 +44,7 @@ use ethkey::{self, Password, Signature}; use io::{IoContext, IoHandler, TimerToken, IoService}; use itertools::{self, Itertools}; use rlp::{encode, Decodable, DecoderError, Encodable, RlpStream, Rlp}; -use stream_encoder::Stream; + use ethereum_types::{H256, H520, Address, U128, U256}; use parking_lot::{Mutex, RwLock}; use unexpected::{Mismatch, OutOfBounds}; diff --git a/ethcore/src/engines/tendermint/message.rs b/ethcore/src/engines/tendermint/message.rs index 956fc69e5fc..073b04188fd 100644 --- a/ethcore/src/engines/tendermint/message.rs +++ b/ethcore/src/engines/tendermint/message.rs @@ -26,7 +26,7 @@ use header::Header; use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError}; use ethkey::{recover, public_to_address}; use super::super::vote_collector::Message; -use stream_encoder::Stream; + /// Message transmitted between consensus participants. #[derive(Debug, PartialEq, Eq, Clone, Hash, Default)] diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index b05536c17a7..f2158b75815 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -50,7 +50,7 @@ use super::vote_collector::VoteCollector; use self::message::*; use self::params::TendermintParams; use machine::{AuxiliaryData, EthereumMachine}; -use stream_encoder::Stream; + #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] pub enum Step { diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 4996b874879..4b7277fde8b 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -34,7 +34,7 @@ use header::Header; use ids::BlockId; use log_entry::LogEntry; use receipt::Receipt; -use stream_encoder::Stream; + use super::{SystemCall, ValidatorSet}; use super::simple_list::SimpleList; diff --git a/ethcore/src/executive.rs b/ethcore/src/executive.rs index 491d126afec..c16be01609a 100644 --- a/ethcore/src/executive.rs +++ b/ethcore/src/executive.rs @@ -33,7 +33,7 @@ use trace::{self, Tracer, VMTracer}; use transaction::{Action, SignedTransaction}; use crossbeam; pub use executed::{Executed, ExecutionResult}; -use stream_encoder::Stream; + #[cfg(debug_assertions)] /// Roughly estimate what stack size each level of evm depth will use. (Debug build) diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index ad383f1ab85..eb86277d938 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -22,7 +22,7 @@ use heapsize::HeapSizeOf; use ethereum_types::{H256, U256, Address, Bloom}; use bytes::Bytes; use rlp::{Rlp, RlpStream, Encodable, DecoderError, Decodable}; -use stream_encoder::Stream; + pub use types::BlockNumber; diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 031bdfad11b..0a4e9fdb55d 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -33,7 +33,7 @@ use trace::{VMTracer, NoopVMTracer}; use bytes::{Bytes, BytesRef}; use ethtrie; use rlp::RlpStream; -use stream_encoder::Stream; + use hash::keccak; use machine::EthereumMachine as Machine; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index fef0c068a35..053a6b7367f 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -101,7 +101,6 @@ extern crate ansi_term; extern crate unexpected; extern crate util_error; extern crate snappy; -extern crate stream_encoder; extern crate ethabi; extern crate rustc_hex; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 0915628abc0..9875611de81 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -29,7 +29,7 @@ use state::Account; use ethjson; use types::account_diff::*; use rlp::{self, RlpStream}; -use stream_encoder::Stream; + #[derive(Debug, Clone, PartialEq, Eq)] /// An account, expressed as Plain-Old-Data (hence the name). diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 3985e1cec02..594c921beca 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -27,7 +27,7 @@ use bytes::Bytes; use trie::{TrieDB, Trie}; use ethtrie::RlpCodec; use rlp::{RlpStream, Rlp}; -use stream_encoder::Stream; + use std::collections::HashSet; diff --git a/ethcore/src/snapshot/block.rs b/ethcore/src/snapshot/block.rs index 77944eed99d..18b72671607 100644 --- a/ethcore/src/snapshot/block.rs +++ b/ethcore/src/snapshot/block.rs @@ -25,7 +25,7 @@ use rlp::{DecoderError, RlpStream, Rlp}; use ethereum_types::H256; use bytes::Bytes; use triehash::ordered_trie_root; -use stream_encoder::Stream; + const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; diff --git a/ethcore/src/snapshot/consensus/authority.rs b/ethcore/src/snapshot/consensus/authority.rs index 58e4cecab4d..8ae4cc33c0c 100644 --- a/ethcore/src/snapshot/consensus/authority.rs +++ b/ethcore/src/snapshot/consensus/authority.rs @@ -37,7 +37,7 @@ use rlp::{RlpStream, Rlp}; use ethereum_types::{H256, U256}; use kvdb::KeyValueDB; use bytes::Bytes; -use stream_encoder::Stream; + /// Snapshot creation and restoration for PoA chains. /// Chunk format: diff --git a/ethcore/src/snapshot/consensus/work.rs b/ethcore/src/snapshot/consensus/work.rs index 25d573f8f6f..10c5c512df5 100644 --- a/ethcore/src/snapshot/consensus/work.rs +++ b/ethcore/src/snapshot/consensus/work.rs @@ -35,7 +35,7 @@ use kvdb::KeyValueDB; use bytes::Bytes; use rlp::{RlpStream, Rlp}; use rand::OsRng; -use stream_encoder::Stream; + /// Snapshot creation and restoration for PoW chains. /// This includes blocks from the head of the chain as a diff --git a/ethcore/src/snapshot/io.rs b/ethcore/src/snapshot/io.rs index cb9c9105abb..6852309980a 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/src/snapshot/io.rs @@ -28,7 +28,7 @@ use std::path::{Path, PathBuf}; use bytes::Bytes; use ethereum_types::H256; use rlp::{RlpStream, Rlp}; -use stream_encoder::Stream; + use super::ManifestData; diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 4b4d6e0e2fe..a18c8e4253f 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -43,7 +43,7 @@ use trie::{TrieDB, TrieDBMut, Trie, TrieMut}; use ethtrie::RlpCodec; use rlp::{RlpStream, Rlp}; use bloom_journal::Bloom; -use stream_encoder::Stream; + use self::io::SnapshotWriter; diff --git a/ethcore/src/snapshot/tests/proof_of_work.rs b/ethcore/src/snapshot/tests/proof_of_work.rs index 2479993ea9d..bb19ebe3d0d 100644 --- a/ethcore/src/snapshot/tests/proof_of_work.rs +++ b/ethcore/src/snapshot/tests/proof_of_work.rs @@ -115,7 +115,7 @@ fn chunk_and_restore_4k() { #[test] fn checks_flag() { use rlp::RlpStream; - use stream_encoder::Stream; + use ethereum_types::H256; let mut stream = RlpStream::new_list(5); diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index fa7d5047615..b49de24d22b 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -95,7 +95,7 @@ fn snap_and_restore() { fn get_code_from_prev_chunk() { use std::collections::HashSet; use rlp::RlpStream; - use stream_encoder::Stream; + use ethereum_types::{H256, U256}; use hashdb::HashDB; diff --git a/ethcore/src/spec/seal.rs b/ethcore/src/spec/seal.rs index c9f875b5292..0da9aceef56 100644 --- a/ethcore/src/spec/seal.rs +++ b/ethcore/src/spec/seal.rs @@ -19,7 +19,7 @@ use rlp::RlpStream; use ethereum_types::{H64, H256, H520}; use ethjson; -use stream_encoder::Stream; + /// Classic ethereum seal. pub struct Ethereum { diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index 8a1035d20d2..ef1c8e4b77c 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -45,7 +45,7 @@ use spec::seal::Generic as GenericSeal; use state::backend::Basic as BasicBackend; use state::{Backend, State, Substate}; use trace::{NoopTracer, NoopVMTracer}; -use stream_encoder::Stream; + pub use ethash::OptimizeFor; diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index a9096f1ee7a..a2418900fea 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -32,7 +32,7 @@ use pod_account::*; use rlp::{RlpStream, encode}; use lru_cache::LruCache; use basic_account::BasicAccount; -use stream_encoder::Stream; + use std::cell::{RefCell, Cell}; diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index 0d5305406b4..2fb90badafc 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -40,7 +40,7 @@ use state::*; use std::sync::Arc; use transaction::{Action, Transaction, SignedTransaction}; use views::BlockView; -use stream_encoder::Stream; + use blooms_db; use kvdb::KeyValueDB; use kvdb_rocksdb; diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index b00eb1b6c45..a0d2b648684 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -36,7 +36,7 @@ use ethkey::KeyPair; use transaction::{PendingTransaction, Transaction, Action, Condition}; use miner::MinerService; use rlp::{RlpStream, EMPTY_LIST_RLP}; -use stream_encoder::Stream; + use tempdir::TempDir; use test_helpers; diff --git a/ethcore/src/trace/types/error.rs b/ethcore/src/trace/types/error.rs index f449fc62c29..9b72e832afc 100644 --- a/ethcore/src/trace/types/error.rs +++ b/ethcore/src/trace/types/error.rs @@ -138,7 +138,7 @@ impl Decodable for Error { #[cfg(test)] mod tests { use rlp::*; - use stream_encoder::Stream; + use super::Error; #[test] diff --git a/ethcore/src/trace/types/flat.rs b/ethcore/src/trace/types/flat.rs index ab1a73886c5..5de8326accc 100644 --- a/ethcore/src/trace/types/flat.rs +++ b/ethcore/src/trace/types/flat.rs @@ -137,7 +137,7 @@ impl Into> for FlatBlockTraces { #[cfg(test)] mod tests { use rlp::*; - use stream_encoder::Stream; + use super::{FlatBlockTraces, FlatTransactionTraces, FlatTrace}; use trace::trace::{Action, Res, CallResult, Call, Suicide, Reward}; use evm::CallType; diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 547e7c73f7c..35ee2fb9daf 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -365,7 +365,7 @@ mod tests { use transaction::{SignedTransaction, Transaction, UnverifiedTransaction, Action}; use types::log_entry::{LogEntry, LocalizedLogEntry}; use rlp; - use stream_encoder::Stream; + use triehash::ordered_trie_root; fn check_ok(result: Result<(), Error>) { @@ -547,7 +547,7 @@ mod tests { #[test] fn test_verify_block() { use rlp::RlpStream; - use stream_encoder::Stream; + // Test against morden let mut good = Header::new(); diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index f9c75fa3d5c..5ca13c3b228 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -32,7 +32,6 @@ heapsize = "0.4" semver = "0.9" smallvec = { version = "0.4", features = ["heapsizeof"] } parking_lot = "0.5" -stream-encoder = { path = "../../util/stream-encoder" } trace-time = { path = "../../util/trace-time" } ipnetwork = "0.12.6" diff --git a/ethcore/sync/src/blocks.rs b/ethcore/sync/src/blocks.rs index cd0fe72a0e0..d4b39b1d7c8 100644 --- a/ethcore/sync/src/blocks.rs +++ b/ethcore/sync/src/blocks.rs @@ -23,7 +23,7 @@ use ethereum_types::H256; use triehash::ordered_trie_root; use bytes::Bytes; use rlp::{Rlp, RlpStream, DecoderError}; -use stream_encoder::Stream; + use network; use ethcore::encoded::Block; use ethcore::views::{HeaderView, BodyView}; diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 97d3ff5df00..9ebba28d4b1 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -104,7 +104,7 @@ use plain_hasher::H256FastMap; use parking_lot::RwLock; use bytes::Bytes; use rlp::{Rlp, RlpStream, DecoderError}; -use stream_encoder::Stream; + use network::{self, PeerId, PacketId}; use ethcore::header::{BlockNumber}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockQueueInfo}; diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index 222a2dceaf8..f05c628c6ca 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -24,7 +24,7 @@ use rlp::{Encodable, RlpStream}; use sync_io::SyncIo; use std::cmp; use std::collections::HashSet; -use stream_encoder::Stream; + use transaction::SignedTransaction; use super::{ diff --git a/ethcore/sync/src/chain/requester.rs b/ethcore/sync/src/chain/requester.rs index 105cf4a76d8..a71b583ddc9 100644 --- a/ethcore/sync/src/chain/requester.rs +++ b/ethcore/sync/src/chain/requester.rs @@ -22,7 +22,7 @@ use ethereum_types::H256; use network::{PeerId, PacketId}; use rlp::RlpStream; use std::time::Instant; -use stream_encoder::Stream; + use sync_io::SyncIo; use super::{ diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 8bcf1b17081..e8a5c93ea3d 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -22,7 +22,7 @@ use network::{self, PeerId}; use parking_lot::RwLock; use rlp::{Rlp, RlpStream}; use std::cmp; -use stream_encoder::Stream; + use sync_io::SyncIo; use super::{ diff --git a/ethcore/sync/src/lib.rs b/ethcore/sync/src/lib.rs index 708c3b89a92..6e49b1ce78d 100644 --- a/ethcore/sync/src/lib.rs +++ b/ethcore/sync/src/lib.rs @@ -42,7 +42,6 @@ extern crate keccak_hash as hash; extern crate keccak_hasher; extern crate triehash; extern crate kvdb; -extern crate stream_encoder; extern crate ethcore_light as light; diff --git a/ethcore/transaction/Cargo.toml b/ethcore/transaction/Cargo.toml index 317030dbfbd..79e7282c388 100644 --- a/ethcore/transaction/Cargo.toml +++ b/ethcore/transaction/Cargo.toml @@ -13,7 +13,6 @@ keccak-hash = { path = "../../util/hash" } rlp = { path = "../../util/rlp" } unexpected = { path = "../../util/unexpected" } ethereum-types = "0.3" -stream-encoder = { path = "../../util/stream-encoder" } [dev-dependencies] rustc-hex= "1.0" diff --git a/ethcore/transaction/src/lib.rs b/ethcore/transaction/src/lib.rs index 48d1c23d462..829613cf9c7 100644 --- a/ethcore/transaction/src/lib.rs +++ b/ethcore/transaction/src/lib.rs @@ -24,7 +24,6 @@ extern crate heapsize; extern crate keccak_hash as hash; extern crate rlp; extern crate unexpected; -extern crate stream_encoder; #[cfg(test)] extern crate rustc_hex; diff --git a/ethcore/transaction/src/transaction.rs b/ethcore/transaction/src/transaction.rs index add4525cb90..0a1248b3e83 100644 --- a/ethcore/transaction/src/transaction.rs +++ b/ethcore/transaction/src/transaction.rs @@ -25,7 +25,7 @@ use evm::Schedule; use hash::keccak; use heapsize::HeapSizeOf; use rlp::{self, RlpStream, Rlp, DecoderError, Encodable}; -use stream_encoder::Stream; + type Bytes = Vec; type BlockNumber = u64; diff --git a/ethcore/types/Cargo.toml b/ethcore/types/Cargo.toml index 29ca4317a09..92cc74551d3 100644 --- a/ethcore/types/Cargo.toml +++ b/ethcore/types/Cargo.toml @@ -12,7 +12,6 @@ ethereum-types = "0.3" ethjson = { path = "../../json" } keccak-hash = { path = "../../util/hash" } heapsize = "0.4" -stream-encoder = { path = "../../util/stream-encoder" } [dev-dependencies] rustc-hex= "1.0" diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index de23cdfa2a1..c88199ad2f0 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -20,7 +20,6 @@ extern crate ethereum_types; extern crate ethcore_bytes as bytes; extern crate ethjson; extern crate rlp; -extern crate stream_encoder; #[macro_use] extern crate rlp_derive; diff --git a/ethcore/types/src/snapshot_manifest.rs b/ethcore/types/src/snapshot_manifest.rs index 0709c74230b..bb6d730ac2a 100644 --- a/ethcore/types/src/snapshot_manifest.rs +++ b/ethcore/types/src/snapshot_manifest.rs @@ -19,7 +19,7 @@ use ethereum_types::H256; use rlp::{Rlp, RlpStream, DecoderError}; use bytes::Bytes; -use stream_encoder::Stream; + /// Manifest data. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/ethcore/vm/Cargo.toml b/ethcore/vm/Cargo.toml index 2f1365e000c..7348951d790 100644 --- a/ethcore/vm/Cargo.toml +++ b/ethcore/vm/Cargo.toml @@ -13,5 +13,4 @@ log = "0.3" common-types = { path = "../types" } ethjson = { path = "../../json" } rlp = { path = "../../util/rlp" } -stream-encoder = { path = "../../util/stream-encoder" } keccak-hash = { path = "../../util/hash" } diff --git a/ethcore/vm/src/call_type.rs b/ethcore/vm/src/call_type.rs index 0af40c334fd..5a9716ea34d 100644 --- a/ethcore/vm/src/call_type.rs +++ b/ethcore/vm/src/call_type.rs @@ -63,7 +63,7 @@ impl Decodable for CallType { mod tests { use rlp::*; use super::CallType; - use stream_encoder::Stream; + #[test] fn encode_call_type() { diff --git a/ethcore/vm/src/lib.rs b/ethcore/vm/src/lib.rs index 924f1cf3e51..658420f74e3 100644 --- a/ethcore/vm/src/lib.rs +++ b/ethcore/vm/src/lib.rs @@ -24,8 +24,6 @@ extern crate rlp; extern crate keccak_hash as hash; extern crate patricia_trie_ethereum as ethtrie; extern crate patricia_trie as trie; -#[cfg(test)] -extern crate stream_encoder; mod action_params; mod call_type; diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 455df7600a2..d227f45f535 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -62,7 +62,6 @@ parity-version = { path = "../util/version" } patricia-trie = { path = "../util/patricia_trie" } rlp = { path = "../util/rlp" } stats = { path = "../util/stats" } -stream-encoder = { path = "../util/stream-encoder" } vm = { path = "../ethcore/vm" } [dev-dependencies] diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 276f5d81c56..2d49a8c7717 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -65,7 +65,6 @@ extern crate parity_reactor; extern crate parity_updater as updater; extern crate parity_version as version; extern crate rlp; -extern crate stream_encoder; extern crate stats; extern crate keccak_hash as hash; extern crate hardware_wallet; diff --git a/rpc/src/v1/helpers/dispatch.rs b/rpc/src/v1/helpers/dispatch.rs index c02e890a026..f6a1f03e95d 100644 --- a/rpc/src/v1/helpers/dispatch.rs +++ b/rpc/src/v1/helpers/dispatch.rs @@ -25,7 +25,7 @@ use light::client::LightChainClient; use light::on_demand::{request, OnDemand}; use light::TransactionQueue as LightTransactionQueue; use rlp; -use stream_encoder::Stream; + use hash::keccak; use ethereum_types::{H256, H520, Address, U256}; use bytes::Bytes; diff --git a/util/network-devp2p/Cargo.toml b/util/network-devp2p/Cargo.toml index e185ca83029..4a5d2d942ec 100644 --- a/util/network-devp2p/Cargo.toml +++ b/util/network-devp2p/Cargo.toml @@ -35,7 +35,6 @@ serde = "1.0" serde_json = "1.0" serde_derive = "1.0" error-chain = { version = "0.11", default-features = false } -stream-encoder = { path = "../stream-encoder" } [dev-dependencies] tempdir = "0.3" diff --git a/util/network-devp2p/src/connection.rs b/util/network-devp2p/src/connection.rs index b08dccc99e2..4a7ed864996 100644 --- a/util/network-devp2p/src/connection.rs +++ b/util/network-devp2p/src/connection.rs @@ -36,7 +36,7 @@ use tiny_keccak::Keccak; use bytes::{Buf, BufMut}; use ethkey::crypto; use network::{Error, ErrorKind}; -use stream_encoder::Stream; + const ENCRYPTED_HEADER_LEN: usize = 32; const RECEIVE_PAYLOAD: Duration = Duration::from_secs(30); diff --git a/util/network-devp2p/src/discovery.rs b/util/network-devp2p/src/discovery.rs index ea170fe50a8..39807719067 100644 --- a/util/network-devp2p/src/discovery.rs +++ b/util/network-devp2p/src/discovery.rs @@ -26,7 +26,7 @@ use node_table::*; use network::{Error, ErrorKind}; use ethkey::{Secret, KeyPair, sign, recover}; use network::IpFilter; -use stream_encoder::Stream; + use PROTOCOL_VERSION; diff --git a/util/network-devp2p/src/handshake.rs b/util/network-devp2p/src/handshake.rs index 64948a11028..e0075f94111 100644 --- a/util/network-devp2p/src/handshake.rs +++ b/util/network-devp2p/src/handshake.rs @@ -28,7 +28,7 @@ use ethkey::{KeyPair, Public, Secret, recover, sign, Generator, Random}; use ethkey::crypto::{ecdh, ecies}; use network::{Error, ErrorKind}; use host::HostInfo; -use stream_encoder::Stream; + #[derive(PartialEq, Eq, Debug)] enum HandshakeState { diff --git a/util/network-devp2p/src/lib.rs b/util/network-devp2p/src/lib.rs index a09b12d6b43..01fc1fe25f4 100644 --- a/util/network-devp2p/src/lib.rs +++ b/util/network-devp2p/src/lib.rs @@ -85,7 +85,6 @@ extern crate keccak_hash as hash; extern crate serde; extern crate serde_json; extern crate snappy; -extern crate stream_encoder; #[macro_use] extern crate error_chain; diff --git a/util/network-devp2p/src/session.rs b/util/network-devp2p/src/session.rs index 3d9fc91700d..3fe6b15aaf5 100644 --- a/util/network-devp2p/src/session.rs +++ b/util/network-devp2p/src/session.rs @@ -32,7 +32,7 @@ use network::SessionCapabilityInfo; use host::*; use node_table::NodeId; use snappy; -use stream_encoder::Stream; + // Timeout must be less than (interval - 1). const PING_TIMEOUT: Duration = Duration::from_secs(60); diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml index e7ca283bdc7..5d8cc191a3c 100644 --- a/util/patricia-trie-ethereum/Cargo.toml +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -11,7 +11,6 @@ keccak-hasher = { path = "../keccak-hasher" } hashdb = { path = "../hashdb" } rlp = { path = "../rlp" } ethcore-bytes = { path = "../bytes" } -stream-encoder = { path = "../stream-encoder" } ethereum-types = "0.3" elastic-array = "0.10" log = "0.3" # TODO: remove diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index df8b0420e6f..78cb85aaa5f 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -23,7 +23,6 @@ extern crate ethereum_types; extern crate hashdb; extern crate keccak_hasher; extern crate rlp; -extern crate stream_encoder; #[cfg(test)] extern crate env_logger; // TODO: remove #[macro_use] diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs index 00f544b7961..90425ad809e 100644 --- a/util/patricia-trie-ethereum/src/rlp_node_codec.rs +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -23,7 +23,6 @@ use hashdb::Hasher; use keccak_hasher::KeccakHasher; use rlp::{DecoderError, RlpStream, Rlp, Prototype}; use std::marker::PhantomData; -use stream_encoder::Stream; use trie::{NibbleSlice, NodeCodec, node::Node, ChildReference}; /// Concrete implementation of a `NodeCodec` with Rlp encoding, generic over the `Hasher` @@ -36,7 +35,6 @@ pub struct RlpNodeCodec {mark: PhantomData} // do `const HASHED_NULL_NODE: H::Out = H::Out( … … )`. Perhaps one day soon? impl NodeCodec for RlpNodeCodec { type E = DecoderError; - type S = RlpStream; const HASHED_NULL_NODE : H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); fn encode(node: &Node) -> Bytes { diff --git a/util/rlp/Cargo.toml b/util/rlp/Cargo.toml index 38afd24d9b2..3bde7206ff8 100644 --- a/util/rlp/Cargo.toml +++ b/util/rlp/Cargo.toml @@ -7,7 +7,6 @@ version = "0.2.1" authors = ["Parity Technologies "] [dependencies] -stream-encoder = { path = "../stream-encoder"} elastic-array = "0.10" ethereum-types = "0.3" rustc-hex = "1.0" diff --git a/util/rlp/benches/rlp.rs b/util/rlp/benches/rlp.rs index 6191b39f270..7f2e9f64591 100644 --- a/util/rlp/benches/rlp.rs +++ b/util/rlp/benches/rlp.rs @@ -16,12 +16,10 @@ extern crate ethereum_types; extern crate rlp; -extern crate stream_encoder; extern crate test; use ethereum_types::U256; use rlp::{RlpStream, Rlp}; -use stream_encoder::Stream; use test::Bencher; #[bench] diff --git a/util/rlp/src/lib.rs b/util/rlp/src/lib.rs index b084acd0773..08c36522dab 100644 --- a/util/rlp/src/lib.rs +++ b/util/rlp/src/lib.rs @@ -36,7 +36,6 @@ extern crate byteorder; extern crate ethereum_types as bigint; extern crate elastic_array; extern crate rustc_hex; -extern crate stream_encoder; mod traits; mod error; @@ -46,7 +45,6 @@ mod impls; use elastic_array::ElasticArray1024; use std::borrow::Borrow; -use stream_encoder::Stream; pub use error::DecoderError; pub use traits::{Decodable, Encodable}; diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index e07e38b6a40..89306fbef58 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -10,7 +10,6 @@ use std::borrow::Borrow; use byteorder::{ByteOrder, BigEndian}; use elastic_array::{ElasticArray16, ElasticArray1024}; use traits::Encodable; -use stream_encoder::Stream; #[derive(Debug, Copy, Clone)] struct ListInfo { @@ -42,9 +41,9 @@ impl Default for RlpStream { } } -impl Stream for RlpStream { +impl RlpStream { /// Initializes instance of empty `Stream`. - fn new() -> Self { + pub fn new() -> Self { RlpStream { unfinished_lists: ElasticArray16::new(), buffer: ElasticArray1024::new(), @@ -53,7 +52,7 @@ impl Stream for RlpStream { } /// Initializes the `Stream` as a list. - fn new_list(len: usize) -> Self { + pub fn new_list(len: usize) -> Self { let mut stream = RlpStream::new(); stream.begin_list(len); stream @@ -63,9 +62,7 @@ impl Stream for RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate stream_encoder; /// use rlp::*; - /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -74,7 +71,7 @@ impl Stream for RlpStream { /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); /// } /// ``` - fn append_empty_data(&mut self) -> &mut Self { + pub fn append_empty_data(&mut self) -> &mut Self { // self push raw item self.buffer.push(0x80); @@ -86,7 +83,7 @@ impl Stream for RlpStream { } /// Drain the object and return the underlying ElasticArray. Panics if it is not finished. - fn drain(self) -> ElasticArray1024 { + pub fn drain(self) -> ElasticArray1024 { match self.is_finished() { true => self.buffer, false => panic!() @@ -94,14 +91,14 @@ impl Stream for RlpStream { } /// Append bytes to the end of stream as a single item, chainable. - fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self { + pub fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self { self.encoder().encode_value(bytes); self.note_appended(1); self } /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. - fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self { + pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self { // push raw items self.buffer.append_slice(bytes); @@ -110,17 +107,13 @@ impl Stream for RlpStream { // return chainable self self - } -} + } -impl RlpStream { /// Appends value to the end of stream, chainable. /// /// ```rust /// extern crate rlp; - /// extern crate stream_encoder; /// use rlp::*; - /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -158,9 +151,7 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate stream_encoder; /// use rlp::*; - /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); @@ -238,9 +229,7 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate stream_encoder; /// use rlp::*; - /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(3); @@ -262,9 +251,7 @@ impl RlpStream { /// /// ```rust /// extern crate rlp; - /// extern crate stream_encoder; /// use rlp::*; - /// use stream_encoder::Stream; /// /// fn main () { /// let mut stream = RlpStream::new_list(2); diff --git a/util/rlp/src/traits.rs b/util/rlp/src/traits.rs index 7f0dee76cf0..1596009e75f 100644 --- a/util/rlp/src/traits.rs +++ b/util/rlp/src/traits.rs @@ -9,7 +9,6 @@ //! Common RLP traits use elastic_array::ElasticArray1024; use {DecoderError, Rlp, RlpStream}; -use stream_encoder::Stream; /// RLP decodable trait pub trait Decodable: Sized { diff --git a/util/rlp/tests/tests.rs b/util/rlp/tests/tests.rs index d172176a1f0..c37e51a25dc 100644 --- a/util/rlp/tests/tests.rs +++ b/util/rlp/tests/tests.rs @@ -8,12 +8,10 @@ extern crate ethereum_types as bigint; extern crate rlp; -extern crate stream_encoder; use std::{fmt, cmp}; use bigint::{U256, H160}; use rlp::{Encodable, Decodable, Rlp, RlpStream, DecoderError}; -use stream_encoder::Stream; #[test] fn rlp_at() { diff --git a/util/rlp_compress/Cargo.toml b/util/rlp_compress/Cargo.toml index 847354ccbec..d5f85425f23 100644 --- a/util/rlp_compress/Cargo.toml +++ b/util/rlp_compress/Cargo.toml @@ -7,4 +7,3 @@ authors = ["Parity Technologies "] rlp = { path = "../rlp" } elastic-array = "0.10" lazy_static = "1.0" -stream-encoder = { path = "../stream-encoder" } diff --git a/util/rlp_compress/src/lib.rs b/util/rlp_compress/src/lib.rs index e600c7b9521..20338439278 100644 --- a/util/rlp_compress/src/lib.rs +++ b/util/rlp_compress/src/lib.rs @@ -10,7 +10,6 @@ extern crate elastic_array; #[macro_use] extern crate lazy_static; extern crate rlp; -extern crate stream_encoder; mod common; @@ -19,7 +18,7 @@ use std::collections::HashMap; use elastic_array::ElasticArray1024; use rlp::{Rlp, RlpStream}; use common::{SNAPSHOT_SWAPPER, BLOCKS_SWAPPER}; -use stream_encoder::Stream; + pub fn snapshot_swapper() -> &'static Swapper<'static> { &SNAPSHOT_SWAPPER as &Swapper diff --git a/util/triehash/Cargo.toml b/util/triehash/Cargo.toml index 7468ef6acb5..ee42b9d8253 100644 --- a/util/triehash/Cargo.toml +++ b/util/triehash/Cargo.toml @@ -10,7 +10,6 @@ elastic-array = "0.10" rlp = { version = "0.2.1", path = "../rlp" } ethereum-types = "0.3" keccak-hash = { version = "0.1", path = "../hash" } -stream-encoder = { path = "../stream-encoder" } [dev-dependencies] trie-standardmap = { path = "../trie-standardmap" } diff --git a/util/triehash/src/lib.rs b/util/triehash/src/lib.rs index e6fefb09378..c78ed0ca1ba 100644 --- a/util/triehash/src/lib.rs +++ b/util/triehash/src/lib.rs @@ -22,7 +22,6 @@ extern crate elastic_array; extern crate ethereum_types; extern crate keccak_hash as hash; extern crate rlp; -extern crate stream_encoder; use std::collections::BTreeMap; use std::cmp; @@ -30,7 +29,6 @@ use elastic_array::{ElasticArray4, ElasticArray8}; use ethereum_types::H256; use hash::keccak; use rlp::RlpStream; -use stream_encoder::Stream; fn shared_prefix_len(first: &[T], second: &[T]) -> usize { let len = cmp::min(first.len(), second.len()); diff --git a/util/version/Cargo.toml b/util/version/Cargo.toml index 9273d84db26..c7bd4f581e3 100644 --- a/util/version/Cargo.toml +++ b/util/version/Cargo.toml @@ -23,7 +23,6 @@ kovan = { forkBlock = 6600000, critical = false } [dependencies] ethcore-bytes = { path = "../bytes" } rlp = { path = "../rlp" } -stream-encoder = { path = "../stream-encoder" } target_info = "0.1" [build-dependencies] diff --git a/util/version/src/lib.rs b/util/version/src/lib.rs index 01ffd8b360f..4d5757ffce5 100644 --- a/util/version/src/lib.rs +++ b/util/version/src/lib.rs @@ -19,12 +19,11 @@ extern crate target_info; extern crate ethcore_bytes as bytes; extern crate rlp; -extern crate stream_encoder; use target_info::Target; use bytes::Bytes; use rlp::RlpStream; -use stream_encoder::Stream; + mod vergen { #![allow(unused)] diff --git a/whisper/Cargo.toml b/whisper/Cargo.toml index 0cfab7ae7cd..e503a74fd6d 100644 --- a/whisper/Cargo.toml +++ b/whisper/Cargo.toml @@ -24,7 +24,6 @@ serde_json = "1.0" slab = "0.3" smallvec = "0.4" tiny-keccak = "1.4" -stream-encoder = { path = "../util/stream-encoder" } jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.11" } jsonrpc-macros = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.11" } diff --git a/whisper/src/lib.rs b/whisper/src/lib.rs index bb72d54a2f8..190169b2c21 100644 --- a/whisper/src/lib.rs +++ b/whisper/src/lib.rs @@ -32,7 +32,6 @@ extern crate serde; extern crate slab; extern crate smallvec; extern crate tiny_keccak; -extern crate stream_encoder; extern crate jsonrpc_core; extern crate jsonrpc_pubsub; diff --git a/whisper/src/message.rs b/whisper/src/message.rs index 9560a430397..7d29ee767b1 100644 --- a/whisper/src/message.rs +++ b/whisper/src/message.rs @@ -21,7 +21,7 @@ use std::time::{self, SystemTime, Duration, Instant}; use ethereum_types::{H256, H512}; use rlp::{self, DecoderError, RlpStream, Rlp}; -use stream_encoder::Stream; + use smallvec::SmallVec; use tiny_keccak::{keccak256, Keccak}; diff --git a/whisper/src/net/mod.rs b/whisper/src/net/mod.rs index 766d622e39f..6a4b220b090 100644 --- a/whisper/src/net/mod.rs +++ b/whisper/src/net/mod.rs @@ -27,7 +27,7 @@ use network::{self, NetworkContext, NodeId, PeerId, ProtocolId, TimerToken}; use ordered_float::OrderedFloat; use parking_lot::{Mutex, RwLock}; use rlp::{DecoderError, RlpStream, Rlp}; -use stream_encoder::Stream; + use message::{Message, Error as MessageError}; From e1c4bea912d9213547f2e2f09c1f971ac9cf6a38 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 11:36:20 +0200 Subject: [PATCH 138/160] Remove stream_encoder crate --- util/stream-encoder/Cargo.toml | 9 --------- util/stream-encoder/src/lib.rs | 36 ---------------------------------- 2 files changed, 45 deletions(-) delete mode 100644 util/stream-encoder/Cargo.toml delete mode 100644 util/stream-encoder/src/lib.rs diff --git a/util/stream-encoder/Cargo.toml b/util/stream-encoder/Cargo.toml deleted file mode 100644 index aab4a4a33fb..00000000000 --- a/util/stream-encoder/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "stream-encoder" -version = "0.1.0" -authors = ["Parity Technologies "] -description = "trait for encoded streams" -license = "GPL-3.0" - -[dependencies] -elastic-array = "0.10" diff --git a/util/stream-encoder/src/lib.rs b/util/stream-encoder/src/lib.rs deleted file mode 100644 index 93456e3f813..00000000000 --- a/util/stream-encoder/src/lib.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2015-2018 Parity Technologies (UK) Ltd. -// This file is part of Parity. - -// Parity is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity. If not, see . - -//! Trait for encoded streams of bytes, used by RLP and other encoding schemes -extern crate elastic_array; - -use elastic_array::ElasticArray1024; - -/// Trait with common stream encoding operations -pub trait Stream { - /// New instance of an empty stream - fn new() -> Self; - /// New instance of an empty stream, initialized as a "list" - fn new_list(len: usize) -> Self; - /// Append the encoding schemes "null" value. Chainable. - fn append_empty_data(&mut self) -> &mut Self; - /// Drain the object and return the underlying data - fn drain(self) -> ElasticArray1024; // TODO: add as assoc type? Makes the types kind of hairy and requires some extra trait bounds, but not sure if it's worth it. Needs AsRef I think. - /// Append the provided bytes to the end of the stream as a single item. Chainable. - fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self; - /// Append pre-serialized data. Use with caution. Chainable. - fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; -} \ No newline at end of file From bfcf114774ba2acca86e5f2ad10c36ad810d12ce Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 11:50:40 +0200 Subject: [PATCH 139/160] Remove unused extern crate --- util/patricia_trie/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 3df5080d020..6fc832598bd 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -41,9 +41,6 @@ extern crate keccak_hasher; #[cfg(test)] extern crate triehash; -#[cfg(test)] -extern crate env_logger; // TODO: remove - use std::{fmt, error}; use hashdb::{HashDB, DBValue, Hasher}; use std::marker::PhantomData; From c45cad9792f5119c59f2ceda756e659e8ac4ccb9 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 13:56:25 +0200 Subject: [PATCH 140/160] Preallocate space for trail and key_nibbles (minor speedup) --- util/patricia_trie/src/triedb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index f342ed9656d..eade13e56cf 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -234,7 +234,7 @@ pub struct TrieDBIterator<'a, H: Hasher + 'a, C: NodeCodec + 'a> { impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { /// Create a new iterator. pub fn new(db: &'a TrieDB) -> Result, H::Out, C::E> { - let mut r = TrieDBIterator { db, trail: vec![], key_nibbles: Vec::new() }; + let mut r = TrieDBIterator { db, trail: Vec::with_capacity(8), key_nibbles: Vec::with_capacity(64) }; db.root_data().and_then(|root| r.descend(&root))?; Ok(r) } From 627567d77eb6a7ce76280c6b62fdefb6c737bfa5 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 13:58:44 +0200 Subject: [PATCH 141/160] Ignore build artifacts --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3bc041c9081..b22c1b842bb 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ node_modules # Build artifacts out/ +parity-clib-example/build/ .vscode rls/ From f55d96a84f55152f6dc9df9863f928f3ec96c63b Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 14:12:37 +0200 Subject: [PATCH 142/160] Use RlpStream::append Remove RlpStream::append_bytes Cleanup --- .../src/rlp_node_codec.rs | 28 +++++++------------ util/rlp/src/stream.rs | 7 ----- util/rlp/tests/tests.rs | 11 -------- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs index 90425ad809e..8b20e57cb0a 100644 --- a/util/patricia-trie-ethereum/src/rlp_node_codec.rs +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -114,27 +114,22 @@ impl NodeCodec for RlpNodeCodec { fn leaf_node(partial: &[u8], value: &[u8]) -> ElasticArray1024 { let mut stream = RlpStream::new_list(2); - stream.append_bytes(partial); - stream.append_bytes(value); - let out = stream.drain(); - trace!(target: "ethtrie", "leaf_node, partial={:?}, out={:X?}", partial, out); - out + stream.append(&partial); + stream.append(&value); + stream.drain() } fn ext_node(partial: &[u8], child_ref: ChildReference<::Out>) -> ElasticArray1024 { let mut stream = RlpStream::new_list(2); - stream.append_bytes(partial); + stream.append(&partial); match child_ref { - ChildReference::Hash(h) => stream.append_bytes(&h.as_ref()), + ChildReference::Hash(h) => stream.append(&h), ChildReference::Inline(inline_data, len) => { let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; - trace!(target: "ethtrie", "ext_node, unpacked bytes={:X?}", bytes); stream.append_raw(bytes, 1) - }, + }, }; - let out = stream.drain(); - trace!(target: "ethtrie", "ext_node, partial={:?}, out={:X?}", partial, out); - out + stream.drain() } fn branch_node(children: I, value: Option>) -> ElasticArray1024 @@ -144,10 +139,9 @@ impl NodeCodec for RlpNodeCodec { for child_ref in children { match child_ref { Some(c) => match c { - ChildReference::Hash(h) => stream.append_bytes(&h.as_ref()), + ChildReference::Hash(h) => stream.append(&h), ChildReference::Inline(inline_data, len) => { let bytes = &AsRef::<[u8]>::as_ref(&inline_data)[..len]; - trace!(target: "ethtrie", "branch_node, unpacked bytes={:X?}", bytes); stream.append_raw(bytes, 1) }, }, @@ -155,12 +149,10 @@ impl NodeCodec for RlpNodeCodec { }; } if let Some(value) = value { - stream.append_bytes(&value); + stream.append(&&*value); } else { stream.append_empty_data(); } - let out = stream.drain(); - trace!(target: "ethtrie", "branch_node, out={:X?}", out); - out + stream.drain() } } diff --git a/util/rlp/src/stream.rs b/util/rlp/src/stream.rs index 89306fbef58..13ccddaa721 100644 --- a/util/rlp/src/stream.rs +++ b/util/rlp/src/stream.rs @@ -90,13 +90,6 @@ impl RlpStream { } } - /// Append bytes to the end of stream as a single item, chainable. - pub fn append_bytes<'a>(&'a mut self, bytes: &[u8]) -> &'a mut Self { - self.encoder().encode_value(bytes); - self.note_appended(1); - self - } - /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self { // push raw items diff --git a/util/rlp/tests/tests.rs b/util/rlp/tests/tests.rs index c37e51a25dc..7aa2920c60c 100644 --- a/util/rlp/tests/tests.rs +++ b/util/rlp/tests/tests.rs @@ -423,14 +423,3 @@ fn test_rlp_stream_unbounded_list() { stream.complete_unbounded_list(); assert!(stream.is_finished()); } - -#[test] -fn test_rlp_stream_append_bytes() { - let mut stream = RlpStream::new(); - stream.begin_unbounded_list(); - stream.append_bytes(&[0x0, 0x1, 0x2]); - stream.append_bytes(&[0x8D]); - stream.complete_unbounded_list(); - let out = stream.out(); - assert_eq!(out, vec![0xC6, 0x83, 0x0, 0x1, 0x2, 0x81, 0x8D]); -} \ No newline at end of file From 6bd549aa2839e98d56df486546cf72c8cc6b01c2 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 14:46:20 +0200 Subject: [PATCH 143/160] Inline NibbleVec::len and NibbleVec::at --- util/patricia_trie/src/nibblevec.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/patricia_trie/src/nibblevec.rs b/util/patricia_trie/src/nibblevec.rs index 4398dbc6f7e..3fe8c9fb75c 100644 --- a/util/patricia_trie/src/nibblevec.rs +++ b/util/patricia_trie/src/nibblevec.rs @@ -41,12 +41,14 @@ impl NibbleVec { } /// Length of the `NibbleVec` + #[inline(always)] pub fn len(&self) -> usize { self.len } /// Retrurns true if `NibbleVec` has zero length pub fn is_empty(&self) -> bool { self.len == 0 } /// Try to get the nibble at the given offset. + #[inline] pub fn at(&self, idx: usize) -> u8 { if idx % 2 == 0 { self.inner[idx / 2] >> 4 From e4cb2ac02ee1cfd385fbd0b742568e97d77ead43 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 15:32:35 +0200 Subject: [PATCH 144/160] Aggressive inlining of NibbleSlice::at and NibbleSlice::encoded --- Cargo.lock | 2 -- Cargo.toml | 3 +++ util/patricia-trie-ethereum/Cargo.toml | 6 +----- util/patricia_trie/src/nibbleslice.rs | 3 +++ 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 21ca5636a90..277cf3024e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2465,12 +2465,10 @@ name = "patricia-trie-ethereum" version = "0.1.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", "keccak-hasher 0.1.0", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.2.0", "rlp 0.2.1", ] diff --git a/Cargo.toml b/Cargo.toml index a141f6dc5a5..feaf16bb3c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,9 @@ name = "parity" [profile.release] debug = false +[profile.bench] +debug = true + [workspace] members = [ "chainspec", diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml index 5d8cc191a3c..239c7c99f03 100644 --- a/util/patricia-trie-ethereum/Cargo.toml +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -12,8 +12,4 @@ hashdb = { path = "../hashdb" } rlp = { path = "../rlp" } ethcore-bytes = { path = "../bytes" } ethereum-types = "0.3" -elastic-array = "0.10" -log = "0.3" # TODO: remove - -[dev-dependencies] -env_logger = "*" # TODO: remove \ No newline at end of file +elastic-array = "0.10" \ No newline at end of file diff --git a/util/patricia_trie/src/nibbleslice.rs b/util/patricia_trie/src/nibbleslice.rs index 8cb03e53dd6..e8f7b32ffbb 100644 --- a/util/patricia_trie/src/nibbleslice.rs +++ b/util/patricia_trie/src/nibbleslice.rs @@ -105,9 +105,11 @@ impl<'a> NibbleSlice<'a> { pub fn is_empty(&self) -> bool { self.len() == 0 } /// Get the length (in nibbles, naturally) of this slice. + #[inline] pub fn len(&self) -> usize { (self.data.len() + self.data_encode_suffix.len()) * 2 - self.offset - self.offset_encode_suffix } /// Get the nibble at position `i`. + #[inline(always)] pub fn at(&self, i: usize) -> u8 { let l = self.data.len() * 2 - self.offset; if i < l { @@ -154,6 +156,7 @@ impl<'a> NibbleSlice<'a> { } /// Encode while nibble slice in prefixed hex notation, noting whether it `is_leaf`. + #[inline(always)] pub fn encoded(&self, is_leaf: bool) -> ElasticArray36 { let l = self.len(); let mut r = ElasticArray36::new(); From fc3d48c5d811f8d4be300b4c087ec124934e8fde Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 15:37:59 +0200 Subject: [PATCH 145/160] Remove bench profile --- Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index feaf16bb3c6..a141f6dc5a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,9 +119,6 @@ name = "parity" [profile.release] debug = false -[profile.bench] -debug = true - [workspace] members = [ "chainspec", From 1cf7c28371d1d103f1ddd365787a59cbfe7afca8 Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 15:52:37 +0200 Subject: [PATCH 146/160] Fix tests --- util/patricia-trie-ethereum/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index 78cb85aaa5f..145164607e4 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -23,10 +23,6 @@ extern crate ethereum_types; extern crate hashdb; extern crate keccak_hasher; extern crate rlp; -#[cfg(test)] -extern crate env_logger; // TODO: remove -#[macro_use] -extern crate log; // TODO: remove mod rlp_node_codec; From 44bb8b6390bd3fe2adb91fe1e2d8a421374f519d Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 15:59:49 +0200 Subject: [PATCH 147/160] Rename NodeCodec::E to Error --- .../src/rlp_node_codec.rs | 4 +-- util/patricia_trie/src/fatdb.rs | 14 +++++----- util/patricia_trie/src/fatdbmut.rs | 10 +++---- util/patricia_trie/src/lib.rs | 28 +++++++++---------- util/patricia_trie/src/lookup.rs | 2 +- util/patricia_trie/src/node_codec.rs | 4 +-- util/patricia_trie/src/sectriedb.rs | 8 +++--- util/patricia_trie/src/sectriedbmut.rs | 10 +++---- util/patricia_trie/src/triedb.rs | 28 +++++++++---------- util/patricia_trie/src/triedbmut.rs | 26 ++++++++--------- 10 files changed, 67 insertions(+), 67 deletions(-) diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs index 8b20e57cb0a..8b1d3c0b593 100644 --- a/util/patricia-trie-ethereum/src/rlp_node_codec.rs +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -34,7 +34,7 @@ pub struct RlpNodeCodec {mark: PhantomData} // but due to the current limitations of Rust const evaluation we can't // do `const HASHED_NULL_NODE: H::Out = H::Out( … … )`. Perhaps one day soon? impl NodeCodec for RlpNodeCodec { - type E = DecoderError; + type Error = DecoderError; const HASHED_NULL_NODE : H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); fn encode(node: &Node) -> Bytes { @@ -69,7 +69,7 @@ impl NodeCodec for RlpNodeCodec { } } } - fn decode(data: &[u8]) -> ::std::result::Result { + fn decode(data: &[u8]) -> ::std::result::Result { let r = Rlp::new(data); match r.prototype()? { // either leaf or extension - decode first item with NibbleSlice::??? diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index 9d11db6e0f9..2ae886fdf0d 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -38,7 +38,7 @@ where /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { Ok(FatDB { raw: TrieDB::new(db, root)? }) } @@ -53,17 +53,17 @@ where { fn root(&self) -> &H::Out { self.raw.root() } - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result { self.raw.contains(H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::Error> where 'a: 'key { self.raw.get_with(H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> Result> + 'a>, ::Out, C::E> { + fn iter<'a>(&'a self) -> Result> + 'a>, ::Out, C::Error> { FatDBIterator::::new(&self.raw).map(|iter| Box::new(iter) as Box<_>) } } @@ -84,7 +84,7 @@ where C: NodeCodec { /// Creates new iterator. - pub fn new(trie: &'db TrieDB) -> Result { + pub fn new(trie: &'db TrieDB) -> Result { Ok(FatDBIterator { trie_iterator: TrieDBIterator::new(trie)?, trie: trie, @@ -97,7 +97,7 @@ where H: Hasher, C: NodeCodec { - fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, C::E> { + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, C::Error> { let hashed_key = H::hash(key); self.trie_iterator.seek(hashed_key.as_ref()) } @@ -108,7 +108,7 @@ where H: Hasher, C: NodeCodec { - type Item = TrieItem<'db, H::Out, C::E>; + type Item = TrieItem<'db, H::Out, C::Error>; fn next(&mut self) -> Option { self.trie_iterator.next() diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index a9f2c155372..467dfd9eb62 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -45,7 +45,7 @@ where /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { Ok(FatDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -69,17 +69,17 @@ where fn is_empty(&self) -> bool { self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result { self.raw.contains(H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::Error> where 'a: 'key { self.raw.get(H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::Error> { let hash = H::hash(key); let out = self.raw.insert(hash.as_ref(), value)?; let db = self.raw.db_mut(); @@ -92,7 +92,7 @@ where Ok(out) } - fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E> { + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::Error> { let hash = H::hash(key); let out = self.raw.remove(hash.as_ref())?; diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 6fc832598bd..d21ef38e382 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -157,21 +157,21 @@ pub trait Trie> { fn is_empty(&self) -> bool { *self.root() == C::HASHED_NULL_NODE } /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result { self.get(key).map(|x|x.is_some() ) } /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> where 'a: 'key { + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::Error> where 'a: 'key { self.get_with(key, DBValue::from_slice) } /// Search for the key with the given query parameter. See the docs of the `Query` /// trait for more details. - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> where 'a: 'key; + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::Error> where 'a: 'key; /// Returns a depth-first iterator over the elements of trie. - fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E>; + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::Error>; } /// A key-value datastore implemented as a database-backed modified Merkle tree. @@ -183,26 +183,26 @@ pub trait TrieMut> { fn is_empty(&self) -> bool; /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result { self.get(key).map(|x| x.is_some()) } /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> where 'a: 'key; + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::Error> where 'a: 'key; /// Insert a `key`/`value` pair into the trie. An empty value is equivalent to removing /// `key` from the trie. Returns the old value associated with this key, if it existed. - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E>; + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::Error>; /// Remove a `key` from the trie. Equivalent to making it equal to the empty /// value. Returns the old value associated with this key, if it existed. - fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E>; + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::Error>; } /// A trie iterator that also supports random access (`seek()`). pub trait TrieIterator>: Iterator { /// Position the iterator on the first element with key > `key` - fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, >::E>; + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, >::Error>; } /// Trie types @@ -261,17 +261,17 @@ impl<'db, H: Hasher, C: NodeCodec> Trie for TrieKinds<'db, H, C> { wrapper!(self, is_empty,) } - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result { wrapper!(self, contains, key) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::Error> where 'a: 'key { wrapper!(self, get_with, key, query) } - fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E> { + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::Error> { wrapper!(self, iter,) } } @@ -287,7 +287,7 @@ where } /// Create new immutable instance of Trie. - pub fn readonly(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out, >::E> { + pub fn readonly(&self, db: &'db HashDB, root: &'db H::Out) -> Result, H::Out, >::Error> { match self.spec { TrieSpec::Generic => Ok(TrieKinds::Generic(TrieDB::new(db, root)?)), TrieSpec::Secure => Ok(TrieKinds::Secure(SecTrieDB::new(db, root)?)), @@ -305,7 +305,7 @@ where } /// Create new mutable instance of trie and check for errors. - pub fn from_existing(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out, >::E> { + pub fn from_existing(&self, db: &'db mut HashDB, root: &'db mut H::Out) -> Result + 'db>, H::Out, >::Error> { match self.spec { TrieSpec::Generic => Ok(Box::new(TrieDBMut::<_, C>::from_existing(db, root)?)), TrieSpec::Secure => Ok(Box::new(SecTrieDBMut::<_, C>::from_existing(db, root)?)), diff --git a/util/patricia_trie/src/lookup.rs b/util/patricia_trie/src/lookup.rs index 22fd3a2bb21..6e9c7ff75fb 100644 --- a/util/patricia_trie/src/lookup.rs +++ b/util/patricia_trie/src/lookup.rs @@ -42,7 +42,7 @@ where { /// Look up the given key. If the value is found, it will be passed to the given /// function to decode or copy. - pub fn look_up(mut self, mut key: NibbleSlice) -> Result, H::Out, C::E> { + pub fn look_up(mut self, mut key: NibbleSlice) -> Result, H::Out, C::Error> { let mut hash = self.hash; // this loop iterates through non-inline nodes. diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index 86043b20b25..c5890b7d47b 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -27,7 +27,7 @@ use elastic_array::{ElasticArray1024, ElasticArray128}; /// Trait for trie node encoding/decoding pub trait NodeCodec: Sized { /// Encoding error type - type E: ::std::error::Error; + type Error: ::std::error::Error; /// Null node type const HASHED_NULL_NODE: H::Out; @@ -36,7 +36,7 @@ pub trait NodeCodec: Sized { fn encode(&Node) -> Bytes; /// Decode bytes to a `Node`. Returns `Self::E` on failure. - fn decode(data: &[u8]) -> Result; + fn decode(data: &[u8]) -> Result; /// Decode bytes to the `Hasher`s output type. Returns `None` on failure. fn try_decode_hash(data: &[u8]) -> Option; diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index c87564e0dbc..a163ae04b96 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -40,7 +40,7 @@ where /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. /// Returns an error if root does not exist. - pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { Ok(SecTrieDB { raw: TrieDB::new(db, root)? }) } @@ -62,17 +62,17 @@ where { fn root(&self) -> &H::Out { self.raw.root() } - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result { self.raw.contains(H::hash(key).as_ref()) } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::Error> where 'a: 'key { self.raw.get_with(H::hash(key).as_ref(), query) } - fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E> { + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::Error> { TrieDB::iter(&self.raw) } } diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 56dbeb1929e..0ad0bfccc26 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -44,7 +44,7 @@ where /// Create a new trie with the backing database `db` and `root`. /// /// Returns an error if root does not exist. - pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H::Out) -> Result { Ok(SecTrieDBMut { raw: TrieDBMut::from_existing(db, root)? }) } @@ -68,21 +68,21 @@ where self.raw.is_empty() } - fn contains(&self, key: &[u8]) -> Result { + fn contains(&self, key: &[u8]) -> Result { self.raw.contains(&H::hash(key).as_ref()) } - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::E> + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Result, H::Out, C::Error> where 'a: 'key { self.raw.get(&H::hash(key).as_ref()) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::Error> { self.raw.insert(&H::hash(key).as_ref(), value) } - fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E> { + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::Error> { self.raw.remove(&H::hash(key).as_ref()) } } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index eade13e56cf..484032269be 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -77,7 +77,7 @@ where { /// Create a new trie with the backing database `db` and `root` /// Returns an error if `root` does not exist - pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { + pub fn new(db: &'db HashDB, root: &'db H::Out) -> Result { if !db.contains(root) { Err(Box::new(TrieError::InvalidStateRoot(*root))) } else { @@ -89,7 +89,7 @@ where pub fn db(&'db self) -> &'db HashDB { self.db } /// Get the data of the root node. - fn root_data(&self) -> Result { + fn root_data(&self) -> Result { self.db .get(self.root) .ok_or_else(|| Box::new(TrieError::InvalidStateRoot(*self.root))) @@ -98,7 +98,7 @@ where /// Given some node-describing data `node`, return the actual node RLP. /// This could be a simple identity operation in the case that the node is sufficiently small, but /// may require a database lookup. - fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> Result { + fn get_raw_or_lookup(&'db self, node: &'db [u8]) -> Result { match C::try_decode_hash(node) { Some(key) => { self.db.get(&key).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(key))) @@ -115,7 +115,7 @@ where { fn root(&self) -> &H::Out { self.root } - fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::E> + fn get_with<'a, 'key, Q: Query>(&'a self, key: &'key [u8], query: Q) -> Result, H::Out, C::Error> where 'a: 'key { Lookup { @@ -126,7 +126,7 @@ where }.look_up(NibbleSlice::new(key)) } - fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::E> { + fn iter<'a>(&'a self) -> Result> + 'a>, H::Out, C::Error> { TrieDBIterator::new(self).map(|iter| Box::new(iter) as Box<_>) } } @@ -233,13 +233,13 @@ pub struct TrieDBIterator<'a, H: Hasher + 'a, C: NodeCodec + 'a> { impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { /// Create a new iterator. - pub fn new(db: &'a TrieDB) -> Result, H::Out, C::E> { + pub fn new(db: &'a TrieDB) -> Result, H::Out, C::Error> { let mut r = TrieDBIterator { db, trail: Vec::with_capacity(8), key_nibbles: Vec::with_capacity(64) }; db.root_data().and_then(|root| r.descend(&root))?; Ok(r) } - fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> Result<(), H::Out, C::E> { + fn seek<'key>(&mut self, mut node_data: DBValue, mut key: NibbleSlice<'key>) -> Result<(), H::Out, C::Error> { loop { let (data, mid) = { let node = C::decode(&node_data).expect("encoded data read from db; qed"); @@ -303,7 +303,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { } /// Descend into a payload. - fn descend(&mut self, d: &[u8]) -> Result<(), H::Out, C::E> { + fn descend(&mut self, d: &[u8]) -> Result<(), H::Out, C::Error> { let node_data = &self.db.get_raw_or_lookup(d)?; let node = C::decode(&node_data).expect("encoded node read from db; qed"); Ok(self.descend_into_node(node.into())) @@ -337,7 +337,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieDBIterator<'a, H, C> { impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H, C> { /// Position the iterator on the first element with key >= `key` - fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, C::E> { + fn seek(&mut self, key: &[u8]) -> Result<(), H::Out, C::Error> { self.trail.clear(); self.key_nibbles.clear(); let root_rlp = self.db.root_data()?; @@ -346,7 +346,7 @@ impl<'a, H: Hasher, C: NodeCodec> TrieIterator for TrieDBIterator<'a, H } impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { - type Item = TrieItem<'a, H::Out, C::E>; + type Item = TrieItem<'a, H::Out, C::Error>; fn next(&mut self) -> Option { enum IterStep { @@ -375,7 +375,7 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { return Some(Ok((self.key(), v.clone()))); }, (Status::At, &OwnedNode::Extension(_, ref d)) => { - IterStep::Descend::(self.db.get_raw_or_lookup(&*d)) + IterStep::Descend::(self.db.get_raw_or_lookup(&*d)) }, (Status::At, &OwnedNode::Branch(_, _)) => IterStep::Continue, (Status::AtChild(i), &OwnedNode::Branch(ref children, _)) if children[i].len() > 0 => { @@ -384,7 +384,7 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { i => *self.key_nibbles.last_mut() .expect("pushed as 0; moves sequentially; removed afterwards; qed") = i as u8, } - IterStep::Descend::(self.db.get_raw_or_lookup(&*children[i])) + IterStep::Descend::(self.db.get_raw_or_lookup(&*children[i])) }, (Status::AtChild(i), &OwnedNode::Branch(_, _)) => { if i == 0 { @@ -400,11 +400,11 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { IterStep::PopTrail => { self.trail.pop(); }, - IterStep::Descend::(Ok(d)) => { + IterStep::Descend::(Ok(d)) => { let node = C::decode(&d).expect("encoded data read from db; qed"); self.descend_into_node(node.into()) }, - IterStep::Descend::(Err(e)) => { + IterStep::Descend::(Err(e)) => { return Some(Err(e)) } IterStep::Continue => {}, diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 42657bebd03..0d367d66dc0 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -321,7 +321,7 @@ where /// Create a new trie with the backing database `db` and `root. /// Returns an error if `root` does not exist. - pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> Result { + pub fn from_existing(db: &'a mut HashDB, root: &'a mut H::Out) -> Result { if !db.contains(root) { return Err(Box::new(TrieError::InvalidStateRoot(*root))); } @@ -348,7 +348,7 @@ where } // cache a node by hash - fn cache(&mut self, hash: H::Out) -> Result { + fn cache(&mut self, hash: H::Out) -> Result { let node_encoded = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; // let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; let node = Node::from_encoded::( @@ -361,8 +361,8 @@ where // inspect a node, choosing either to replace, restore, or delete it. // if restored or replaced, returns the new node along with a flag of whether it was changed. - fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::E> - where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::E> { + fn inspect(&mut self, stored: Stored, inspector: F) -> Result, bool)>, H::Out, C::Error> + where F: FnOnce(&mut Self, Node) -> Result, H::Out, C::Error> { Ok(match stored { Stored::New(node) => match inspector(self, node)? { Action::Restore(node) => Some((Stored::New(node), false)), @@ -384,7 +384,7 @@ where } // walk the trie, attempting to find the key's node. - fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::E> + fn lookup<'x, 'key>(&'x self, mut partial: NibbleSlice<'key>, handle: &NodeHandle) -> Result, H::Out, C::Error> where 'x: 'key { let mut handle = handle; @@ -433,7 +433,7 @@ where } /// insert a key-value pair into the trie, creating new nodes if necessary. - fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::E> { + fn insert_at(&mut self, handle: NodeHandle, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result<(StorageHandle, bool), H::Out, C::Error> { let h = match handle { NodeHandle::InMemory(h) => h, NodeHandle::Hash(h) => self.cache(h)?, @@ -447,7 +447,7 @@ where } /// the insertion inspector. - fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::E> { + fn insert_inspector(&mut self, node: Node, partial: NibbleSlice, value: DBValue, old_val: &mut Option) -> Result, H::Out, C::Error> { trace!(target: "trie", "augmented (partial: {:?}, value: {:?})", partial, value.pretty()); Ok(match node { @@ -608,7 +608,7 @@ where } /// Remove a node from the trie based on key. - fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::E> { + fn remove_at(&mut self, handle: NodeHandle, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { let stored = match handle { NodeHandle::InMemory(h) => self.storage.destroy(h), NodeHandle::Hash(h) => { @@ -623,7 +623,7 @@ where } /// the removal inspector - fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::E> { + fn remove_inspector(&mut self, node: Node, partial: NibbleSlice, old_val: &mut Option) -> Result, H::Out, C::Error> { Ok(match (node, partial.is_empty()) { (Node::Empty, _) => Action::Delete, (Node::Branch(c, None), true) => Action::Restore(Node::Branch(c, None)), @@ -709,7 +709,7 @@ where /// _invalid state_ means: /// - Branch node where there is only a single entry; /// - Extension node followed by anything other than a Branch node. - fn fix(&mut self, node: Node) -> Result, H::Out, C::E> { + fn fix(&mut self, node: Node) -> Result, H::Out, C::Error> { match node { Node::Branch(mut children, value) => { // if only a single value, transmute to leaf/extension and feed through fixed. @@ -903,13 +903,13 @@ where } } - fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, H::Out, C::E> + fn get<'x, 'key>(&'x self, key: &'key [u8]) -> Result, H::Out, C::Error> where 'x: 'key { self.lookup(NibbleSlice::new(key), &self.root_handle) } - fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::E> { + fn insert(&mut self, key: &[u8], value: &[u8]) -> Result, H::Out, C::Error> { if value.is_empty() { return self.remove(key) } let mut old_val = None; @@ -930,7 +930,7 @@ where Ok(old_val) } - fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::E> { + fn remove(&mut self, key: &[u8]) -> Result, H::Out, C::Error> { trace!(target: "trie", "remove: key={:?}", key.pretty()); let root_handle = self.root_handle(); From 29bf049c8c08cf226dd62eb7718e372ff53caa7c Mon Sep 17 00:00:00 2001 From: David Palm Date: Wed, 27 Jun 2018 16:37:56 +0200 Subject: [PATCH 148/160] Make Recorder and Record generic over the Hasher output rather than the Hasher itself --- util/patricia_trie/src/lib.rs | 4 ++-- util/patricia_trie/src/recorder.rs | 36 +++++++++++------------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index d21ef38e382..a7cb215090e 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -127,7 +127,7 @@ pub trait Query { fn record(&mut self, _hash: &H::Out, _data: &[u8], _depth: u32) {} } -impl<'a, H: Hasher> Query for &'a mut Recorder { +impl<'a, H: Hasher> Query for &'a mut Recorder { type Item = DBValue; fn decode(self, value: &[u8]) -> DBValue { DBValue::from_slice(value) } fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) { @@ -140,7 +140,7 @@ impl Query for F where F: for<'a> FnOnce(&'a [u8]) -> T { fn decode(self, value: &[u8]) -> T { (self)(value) } } -impl<'a, F, T, H: Hasher> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T { +impl<'a, F, T, H: Hasher> Query for (&'a mut Recorder, F) where F: FnOnce(&[u8]) -> T { type Item = T; fn decode(self, value: &[u8]) -> T { (self.1)(value) } fn record(&mut self, hash: &H::Out, data: &[u8], depth: u32) { diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 2c6f203976d..8a8e3705789 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -17,12 +17,10 @@ //! Trie query recorder. use bytes::Bytes; -use hashdb::Hasher; -use std::marker::PhantomData; /// A record of a visited node. #[derive(PartialEq, Eq, Debug, Clone)] -pub struct Record { +pub struct Record { /// The depth of this node. pub depth: u32, @@ -30,25 +28,23 @@ pub struct Record { pub data: Bytes, /// The hash of the data. - pub hash: H::Out, - marker: PhantomData, + pub hash: HO, } /// Records trie nodes as they pass it. #[derive(Debug)] -pub struct Recorder { - nodes: Vec>, +pub struct Recorder { + nodes: Vec>, min_depth: u32, - marker: PhantomData, } -impl Default for Recorder { +impl Default for Recorder { fn default() -> Self { Recorder::new() } } -impl Recorder { +impl Recorder { /// Create a new `Recorder` which records all given nodes. #[inline] pub fn new() -> Self { @@ -60,26 +56,22 @@ impl Recorder { Recorder { nodes: Vec::new(), min_depth: depth, - marker: PhantomData } } /// Record a visited node, given its hash, data, and depth. - pub fn record(&mut self, hash: &::Out, data: &[u8], depth: u32) { - debug_assert_eq!(H::hash(data), *hash); - + pub fn record(&mut self, hash: &HO, data: &[u8], depth: u32) { if depth >= self.min_depth { self.nodes.push(Record { depth: depth, data: data.into(), hash: *hash, - marker: PhantomData }) } } /// Drain all visited records. - pub fn drain(&mut self) -> Vec> { + pub fn drain(&mut self) -> Vec> { ::std::mem::replace(&mut self.nodes, Vec::new()) } } @@ -89,10 +81,11 @@ mod tests { use super::*; use keccak::keccak; use keccak_hasher::KeccakHasher; + use ethereum_types::H256; #[test] fn basic_recorder() { - let mut basic = Recorder::::new(); + let mut basic = Recorder::::new(); let node1 = vec![1, 2, 3, 4]; let node2 = vec![4, 5, 6, 7, 8, 9, 10]; @@ -105,14 +98,12 @@ mod tests { data: node1, hash: hash1, depth: 0, - marker: PhantomData, }; let record2 = Record { data: node2, hash: hash2, depth: 456, - marker: PhantomData, }; @@ -121,7 +112,7 @@ mod tests { #[test] fn basic_recorder_min_depth() { - let mut basic = Recorder::::with_depth(400); + let mut basic = Recorder::::with_depth(400); let node1 = vec![1, 2, 3, 4]; let node2 = vec![4, 5, 6, 7, 8, 9, 10]; @@ -139,7 +130,6 @@ mod tests { data: node2, hash: hash2, depth: 456, - marker: PhantomData, }); } @@ -151,7 +141,7 @@ mod tests { let mut db = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::default(); { let mut x = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); @@ -167,7 +157,7 @@ mod tests { } let trie = TrieDB::<_, RlpCodec>::new(&db, &root).unwrap(); - let mut recorder = Recorder::::new(); + let mut recorder = Recorder::::new(); trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); From 7be2120971a5d7bfccb91850fa213e8ee14a0f75 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 28 Jun 2018 12:34:01 +0200 Subject: [PATCH 149/160] Fix trat descr --- util/hashdb/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 830e41309cd..4aae65ca3d5 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -39,7 +39,7 @@ pub trait Hasher: Sync + Send { /// `HashDB` value type. pub type DBValue = ElasticArray128; -/// Trait modelling datastore keyed by a 32-byte Keccak hash. +/// Trait modelling datastore keyed by a hash defined by the `Hasher`. pub trait HashDB: Send + Sync + AsHashDB { /// Get the keys in the database together with number of underlying references. fn keys(&self) -> HashMap; From 211882e5f76db70a35c264b622ecae1511a2a240 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 28 Jun 2018 13:16:05 +0200 Subject: [PATCH 150/160] Remove dead code --- .../src/rlp_node_codec.rs | 34 ------------------- util/patricia_trie/src/lib.rs | 1 - util/patricia_trie/src/node_codec.rs | 4 --- 3 files changed, 39 deletions(-) diff --git a/util/patricia-trie-ethereum/src/rlp_node_codec.rs b/util/patricia-trie-ethereum/src/rlp_node_codec.rs index 8b1d3c0b593..414a129efe8 100644 --- a/util/patricia-trie-ethereum/src/rlp_node_codec.rs +++ b/util/patricia-trie-ethereum/src/rlp_node_codec.rs @@ -17,7 +17,6 @@ //! `NodeCodec` implementation for Rlp use elastic_array::{ElasticArray1024, ElasticArray128}; -use ethcore_bytes::Bytes; use ethereum_types::H256; use hashdb::Hasher; use keccak_hasher::KeccakHasher; @@ -36,39 +35,6 @@ pub struct RlpNodeCodec {mark: PhantomData} impl NodeCodec for RlpNodeCodec { type Error = DecoderError; const HASHED_NULL_NODE : H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); - - fn encode(node: &Node) -> Bytes { - match *node { - Node::Leaf(ref slice, ref value) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(true)); - stream.append(value); - stream.out() - }, - Node::Extension(ref slice, ref raw_rlp) => { - let mut stream = RlpStream::new_list(2); - stream.append(&&*slice.encoded(false)); - stream.append_raw(raw_rlp, 1); - stream.out() - }, - Node::Branch(ref nodes, ref value) => { - let mut stream = RlpStream::new_list(17); - for i in 0..16 { - stream.append_raw(nodes[i], 1); - } - match *value { - Some(ref n) => { stream.append(n); }, - None => { stream.append_empty_data(); }, - } - stream.out() - }, - Node::Empty => { - let mut stream = RlpStream::new(); - stream.append_empty_data(); - stream.out() - } - } - } fn decode(data: &[u8]) -> ::std::result::Result { let r = Rlp::new(data); match r.prototype()? { diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index a7cb215090e..1745cec7bce 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -20,7 +20,6 @@ extern crate ethcore_bytes as bytes; extern crate ethcore_logger; extern crate hashdb; extern crate rand; - #[macro_use] extern crate log; diff --git a/util/patricia_trie/src/node_codec.rs b/util/patricia_trie/src/node_codec.rs index c5890b7d47b..1dec20f90f5 100644 --- a/util/patricia_trie/src/node_codec.rs +++ b/util/patricia_trie/src/node_codec.rs @@ -17,7 +17,6 @@ //! Generic trait for trie node encoding/decoding. Takes a `hashdb::Hasher` //! to parametrize the hashes used in the codec. -use bytes::Bytes; use hashdb::Hasher; use node::Node; use ChildReference; @@ -32,9 +31,6 @@ pub trait NodeCodec: Sized { /// Null node type const HASHED_NULL_NODE: H::Out; - /// Encode a Node to bytes (aka `Vec`). - fn encode(&Node) -> Bytes; - /// Decode bytes to a `Node`. Returns `Self::E` on failure. fn decode(data: &[u8]) -> Result; From 609ebde94429fab065f80e08d9e896dce0b6715e Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 28 Jun 2018 13:37:13 +0200 Subject: [PATCH 151/160] Ditch ethcore-logger for env-logger --- Cargo.lock | 70 ++++++++++++++++++++++++++++- util/patricia_trie/Cargo.toml | 10 ++--- util/patricia_trie/src/lib.rs | 3 +- util/patricia_trie/src/triedbmut.rs | 4 +- 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 277cf3024e2..2f2b5c66561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -435,6 +435,18 @@ dependencies = [ "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "env_logger" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "error-chain" version = "0.11.0" @@ -1276,6 +1288,14 @@ name = "httparse" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "humantime" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hyper" version = "0.11.24" @@ -2445,8 +2465,8 @@ name = "patricia-trie" version = "0.2.0" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-bytes 0.1.0", - "ethcore-logger 1.12.0", "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", "keccak-hash 0.1.2", @@ -2757,11 +2777,31 @@ dependencies = [ "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "regex" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex-syntax" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "regex-syntax" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "registrar" version = "0.0.1" @@ -3210,6 +3250,14 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "termcolor" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "termion" version = "1.5.1" @@ -3542,6 +3590,11 @@ dependencies = [ "trie-standardmap 0.1.0", ] +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "uint" version = "0.2.1" @@ -3788,6 +3841,14 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "wincolor" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ws" version = "0.7.5" @@ -3897,6 +3958,7 @@ dependencies = [ "checksum either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "740178ddf48b1a9e878e6d6509a1442a2d42fd2928aae8e7a6f8a36fb01981b3" "checksum elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88d4851b005ef16de812ea9acdb7bece2f0a40dd86c07b85631d7dafa54537bb" "checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" +"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" "checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" "checksum error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07e791d3be96241c77c43846b665ef1384606da2cd2a48730abe606a12906e02" "checksum eth-secp256k1 0.5.7 (git+https://github.com/paritytech/rust-secp256k1)" = "" @@ -3927,6 +3989,7 @@ dependencies = [ "checksum hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" "checksum hidapi 0.3.1 (git+https://github.com/paritytech/hidapi-rs)" = "" "checksum httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "af2f2dd97457e8fb1ae7c5a420db346af389926e36f43768b96f101546b04a07" +"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" "checksum hyper 0.11.24 (registry+https://github.com/rust-lang/crates.io-index)" = "df4dd5dae401458087396b6db7fabc4d6760aa456a5fa8e92bda549f39cae661" "checksum hyper-rustls 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d6cdc1751771a14b8175764394f025e309a28c825ed9eaf97fa62bb831dc8c5" "checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" @@ -4027,7 +4090,9 @@ dependencies = [ "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "744554e01ccbd98fff8c457c3b092cd67af62a555a43bfe97ae8a0451f7799fa" +"checksum regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13c93d55961981ba9226a213b385216f83ab43bd6ac53ab16b2eeb47e337cf4e" "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" +"checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54" "checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" "checksum ring 0.12.1 (git+https://github.com/paritytech/ring)" = "" "checksum rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb)" = "" @@ -4077,6 +4142,7 @@ dependencies = [ "checksum tempfile 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "11ce2fe9db64b842314052e2421ac61a73ce41b898dc8e3750398b219c5fc1e0" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327" +"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" "checksum thread-id 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" @@ -4104,6 +4170,7 @@ dependencies = [ "checksum toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a7540f4ffc193e0d3c94121edb19b055670d369f77d5804db11ae053a45b6e7e" "checksum transient-hashmap 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "715254c8f0811be1a79ad3ea5e6fa3c8eddec2b03d7f5ba78cf093e56d79c24f" "checksum trezor-sys 1.0.0 (git+https://github.com/paritytech/trezor-sys)" = "" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum uint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "38051a96565903d81c9a9210ce11076b2218f3b352926baa1f5f6abbdfce8273" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" "checksum unicase 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284b6d3db520d67fbe88fd778c21510d1b0ba4a551e5d0fbb023d33405f6de8a" @@ -4130,6 +4197,7 @@ dependencies = [ "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" "checksum ws 0.7.5 (git+https://github.com/tomusdrw/ws-rs)" = "" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" "checksum xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a66b7c2281ebde13cf4391d70d4c7e5946c3c25e72a7b859ca8f677dcd0b0c61" diff --git a/util/patricia_trie/Cargo.toml b/util/patricia_trie/Cargo.toml index 96a4495d3ea..5324656785b 100644 --- a/util/patricia_trie/Cargo.toml +++ b/util/patricia_trie/Cargo.toml @@ -9,16 +9,16 @@ license = "GPL-3.0" elastic-array = "0.10" log = "0.3" rand = "0.4" -ethcore-bytes = { version = "0.1.0", path = "../bytes" } hashdb = { version = "0.2", path = "../hashdb" } -ethcore-logger = { version = "1.9.0", path = "../../logger" } +ethcore-bytes = { version = "0.1.0", path = "../bytes" } [dev-dependencies] -triehash = { version = "0.1.0", path = "../triehash" } -memorydb = { version = "0.2", path = "../memorydb" } +env_logger = "0.5" ethereum-types = "0.3" keccak-hash = { version = "0.1.0", path = "../hash" } keccak-hasher = { path = "../keccak-hasher" } +memorydb = { version = "0.2", path = "../memorydb" } +patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } rlp = { version = "0.2.1", path = "../rlp" } trie-standardmap = { path = "../trie-standardmap" } -patricia-trie-ethereum = { path = "../patricia-trie-ethereum" } +triehash = { version = "0.1.0", path = "../triehash" } diff --git a/util/patricia_trie/src/lib.rs b/util/patricia_trie/src/lib.rs index 1745cec7bce..7cc623b1a14 100644 --- a/util/patricia_trie/src/lib.rs +++ b/util/patricia_trie/src/lib.rs @@ -17,12 +17,13 @@ //! Trie interface and implementation. extern crate elastic_array; extern crate ethcore_bytes as bytes; -extern crate ethcore_logger; extern crate hashdb; extern crate rand; #[macro_use] extern crate log; +#[cfg(test)] +extern crate env_logger; #[cfg(test)] extern crate ethereum_types; #[cfg(test)] diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 0d367d66dc0..790e1cf17d6 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -974,6 +974,7 @@ mod tests { use standardmap::*; use ethtrie::trie::{TrieMut, TrieDBMut, NodeCodec}; use ethtrie::RlpCodec; + use env_logger; fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec @@ -998,8 +999,7 @@ mod tests { #[test] fn playpen() { - ::ethcore_logger::init_log(); - + env_logger::init(); let mut seed = ::Out::new(); for test_i in 0..10 { if test_i % 50 == 0 { From 308f281acbd9b8fe87912aa04acfb24f8dc80b4c Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 28 Jun 2018 15:20:03 +0200 Subject: [PATCH 152/160] Parametrize the HashMap builder from the Hasher --- Cargo.lock | 1 + ethcore/light/src/client/header_chain.rs | 3 +-- ethcore/light/src/transaction_queue.rs | 3 +-- ethcore/sync/src/chain/mod.rs | 2 +- ethcore/sync/src/transactions_stats.rs | 4 +-- util/hashdb/src/lib.rs | 2 ++ util/journaldb/src/overlayrecentdb.rs | 2 +- util/keccak-hasher/Cargo.toml | 5 ++-- util/keccak-hasher/src/lib.rs | 4 ++- util/memorydb/src/lib.rs | 31 ++++++++++++------------ util/plain_hasher/src/lib.rs | 4 +-- 11 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f2b5c66561..59820000c36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1532,6 +1532,7 @@ version = "0.1.0" dependencies = [ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.2.0", + "plain_hasher 0.2.0", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/ethcore/light/src/client/header_chain.rs b/ethcore/light/src/client/header_chain.rs index 0824b55760a..c286fa82f60 100644 --- a/ethcore/light/src/client/header_chain.rs +++ b/ethcore/light/src/client/header_chain.rs @@ -42,7 +42,6 @@ use ethcore::engines::epoch::{ }; use ethereum_types::{H256, H264, U256}; -use keccak_hasher::KeccakHasher; use heapsize::HeapSizeOf; use kvdb::{DBTransaction, KeyValueDB}; use plain_hasher::H256FastMap; @@ -209,7 +208,7 @@ pub struct HeaderChain { genesis_header: encoded::Header, // special-case the genesis. candidates: RwLock>, best_block: RwLock, - live_epoch_proofs: RwLock>, + live_epoch_proofs: RwLock>, db: Arc, col: Option, cache: Arc>, diff --git a/ethcore/light/src/transaction_queue.rs b/ethcore/light/src/transaction_queue.rs index 290dacc5386..e8880037a17 100644 --- a/ethcore/light/src/transaction_queue.rs +++ b/ethcore/light/src/transaction_queue.rs @@ -30,7 +30,6 @@ use std::collections::hash_map::Entry; use transaction::{self, Condition, PendingTransaction, SignedTransaction}; use ethereum_types::{H256, U256, Address}; use plain_hasher::H256FastMap; -use keccak_hasher::KeccakHasher; // Knowledge of an account's current nonce. #[derive(Debug, Clone, PartialEq, Eq)] @@ -139,7 +138,7 @@ type Listener = Box; #[derive(Default)] pub struct TransactionQueue { by_account: HashMap, - by_hash: H256FastMap, + by_hash: H256FastMap, listeners: Vec, } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 9ebba28d4b1..6da21a9671d 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -491,7 +491,7 @@ impl ChainSync { } /// Returns transactions propagation statistics - pub fn transactions_stats(&self) -> &H256FastMap { + pub fn transactions_stats(&self) -> &H256FastMap { self.transactions_stats.stats() } diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index ad2954a5df7..2875d495fbc 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -52,7 +52,7 @@ impl<'a> From<&'a Stats> for TransactionStats { #[derive(Debug, Default)] pub struct TransactionsStats { - pending_transactions: H256FastMap, + pending_transactions: H256FastMap, } impl TransactionsStats { @@ -70,7 +70,7 @@ impl TransactionsStats { self.pending_transactions.get(hash) } - pub fn stats(&self) -> &H256FastMap { + pub fn stats(&self) -> &H256FastMap { &self.pending_transactions } diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 4aae65ca3d5..5961d90f98a 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -29,6 +29,8 @@ use std::{fmt::Debug, hash::Hash}; pub trait Hasher: Sync + Send { /// The output type of the `Hasher` type Out: AsRef<[u8]> + AsMut<[u8]> + Default + HeapSizeOf + Debug + PartialEq + Eq + Hash + Send + Sync + Clone + Copy; + /// What to use to build `HashMap`s with this `Hasher` + type StdHasher: Sync + Send + Default + std::hash::Hasher; /// The length in bytes of the `Hasher` output const LENGTH: usize; diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index c4027e29c3c..d38f91c7cb3 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -122,7 +122,7 @@ impl<'a> Encodable for DatabaseValueRef<'a> { #[derive(PartialEq)] struct JournalOverlay { backing_overlay: MemoryDB, // Nodes added in the history period - pending_overlay: H256FastMap, // Nodes being transfered from backing_overlay to backing db + pending_overlay: H256FastMap, // Nodes being transfered from backing_overlay to backing db journal: HashMap>, latest_era: Option, earliest_era: Option, diff --git a/util/keccak-hasher/Cargo.toml b/util/keccak-hasher/Cargo.toml index 7b5d0175889..e7c8e950ec9 100644 --- a/util/keccak-hasher/Cargo.toml +++ b/util/keccak-hasher/Cargo.toml @@ -6,6 +6,7 @@ description = "Keccak-256 implementation of the Hasher trait" license = "GPL-3.0" [dependencies] -hashdb = { path = "../hashdb" } ethereum-types = "0.3" -tiny-keccak = "1.4.2" \ No newline at end of file +tiny-keccak = "1.4.2" +hashdb = { path = "../hashdb" } +plain_hasher = { path = "../plain_hasher" } \ No newline at end of file diff --git a/util/keccak-hasher/src/lib.rs b/util/keccak-hasher/src/lib.rs index 2156660637c..bb2b5b45ffe 100644 --- a/util/keccak-hasher/src/lib.rs +++ b/util/keccak-hasher/src/lib.rs @@ -18,16 +18,18 @@ extern crate hashdb; extern crate ethereum_types; extern crate tiny_keccak; +extern crate plain_hasher; use hashdb::Hasher; use ethereum_types::H256; use tiny_keccak::Keccak; - +use plain_hasher::PlainHasher; /// Concrete `Hasher` impl for the Keccak-256 hash #[derive(Default, Debug, Clone, PartialEq)] pub struct KeccakHasher; impl Hasher for KeccakHasher { type Out = H256; + type StdHasher = PlainHasher; const LENGTH: usize = 32; fn hash(x: &[u8]) -> Self::Out { let mut out = [0;32]; diff --git a/util/memorydb/src/lib.rs b/util/memorydb/src/lib.rs index b0571a997e0..538738995c1 100644 --- a/util/memorydb/src/lib.rs +++ b/util/memorydb/src/lib.rs @@ -18,20 +18,22 @@ extern crate elastic_array; extern crate hashdb; extern crate heapsize; -extern crate plain_hasher; extern crate rlp; #[cfg(test)] extern crate keccak_hasher; #[cfg(test)] extern crate tiny_keccak; #[cfg(test)] extern crate ethereum_types; -use hashdb::{HashDB, Hasher, DBValue, AsHashDB}; +use hashdb::{HashDB, Hasher as KeyHasher, DBValue, AsHashDB}; use heapsize::HeapSizeOf; -use plain_hasher::H256FastMap; use rlp::NULL_RLP; use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::hash; use std::mem; +// Backing `HashMap` parametrized with a `Hasher` for the keys `Hasher::Out` and the `Hasher::StdHasher` as hash map builder. +type FastMap = HashMap<::Out, T, hash::BuildHasherDefault<::StdHasher>>; + /// Reference-counted memory-based `HashDB` implementation. /// /// Use `new()` to create a new database. Insert items with `insert()`, remove items @@ -80,16 +82,16 @@ use std::mem; /// } /// ``` #[derive(Default, Clone, PartialEq)] -pub struct MemoryDB { - data: H256FastMap, +pub struct MemoryDB { + data: FastMap, hashed_null_node: H::Out, } -impl MemoryDB { +impl MemoryDB { /// Create a new instance of the memory DB. pub fn new() -> MemoryDB { MemoryDB { - data: H256FastMap::::default(), + data: FastMap::::default(), hashed_null_node: H::hash(&NULL_RLP) } } @@ -125,8 +127,8 @@ impl MemoryDB { } /// Return the internal map of hashes to data, clearing the current state. - pub fn drain(&mut self) -> H256FastMap { - mem::replace(&mut self.data, H256FastMap::::default()) + pub fn drain(&mut self) -> FastMap { + mem::replace(&mut self.data, FastMap::::default()) } /// Grab the raw information associated with a key. Returns None if the key @@ -134,7 +136,7 @@ impl MemoryDB { /// /// Even when Some is returned, the data is only guaranteed to be useful /// when the refs > 0. - pub fn raw(&self, key: &::Out) -> Option<(DBValue, i32)> { + pub fn raw(&self, key: &::Out) -> Option<(DBValue, i32)> { if key == &self.hashed_null_node { return Some((DBValue::from_slice(&NULL_RLP), 1)); } @@ -148,7 +150,7 @@ impl MemoryDB { /// Remove an element and delete it from storage if reference count reaches zero. /// If the value was purged, return the old value. - pub fn remove_and_purge(&mut self, key: &::Out) -> Option { + pub fn remove_and_purge(&mut self, key: &::Out) -> Option { if key == &self.hashed_null_node { return None; } @@ -186,11 +188,8 @@ impl MemoryDB { } } -impl HashDB for MemoryDB { +impl HashDB for MemoryDB { - // REVIEW: this method is what made it necessary to add a type param to H256FastMap, which I'd rather have avoided. - // The problem is that the keys returned are `H256` and type inference fails on the `collect()` call. - // I could not make it work without parameterizing H256FastMap too. fn keys(&self) -> HashMap { self.data.iter() .filter_map(|(k, v)| if v.1 != 0 { @@ -279,7 +278,7 @@ impl HashDB for MemoryDB { } } -impl AsHashDB for MemoryDB { +impl AsHashDB for MemoryDB { fn as_hashdb(&self) -> &HashDB { self } fn as_hashdb_mut(&mut self) -> &mut HashDB { self } } diff --git a/util/plain_hasher/src/lib.rs b/util/plain_hasher/src/lib.rs index ef081543e55..4a8a1044162 100644 --- a/util/plain_hasher/src/lib.rs +++ b/util/plain_hasher/src/lib.rs @@ -20,11 +20,11 @@ extern crate ethereum_types; extern crate hashdb; use ethereum_types::H256; -use hashdb::Hasher; +// use hashdb::Hasher; use std::hash; use std::collections::{HashMap, HashSet}; /// Specialized version of `HashMap` with H256 keys and fast hashing function. -pub type H256FastMap = HashMap<::Out, T, hash::BuildHasherDefault>; +pub type H256FastMap = HashMap>; /// Specialized version of `HashSet` with H256 keys and fast hashing function. pub type H256FastSet = HashSet>; From 1aaaa63092657ef1fb96cd1888a1656e0f472657 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 28 Jun 2018 15:34:28 +0200 Subject: [PATCH 153/160] Less aggressive inlining --- util/patricia_trie/src/nibbleslice.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/patricia_trie/src/nibbleslice.rs b/util/patricia_trie/src/nibbleslice.rs index e8f7b32ffbb..b87a6636974 100644 --- a/util/patricia_trie/src/nibbleslice.rs +++ b/util/patricia_trie/src/nibbleslice.rs @@ -156,7 +156,7 @@ impl<'a> NibbleSlice<'a> { } /// Encode while nibble slice in prefixed hex notation, noting whether it `is_leaf`. - #[inline(always)] + #[inline] pub fn encoded(&self, is_leaf: bool) -> ElasticArray36 { let l = self.len(); let mut r = ElasticArray36::new(); From b76fffa263d4a4c4db2db9bb4bc77b76097bdf30 Mon Sep 17 00:00:00 2001 From: David Palm Date: Thu, 28 Jun 2018 16:43:08 +0200 Subject: [PATCH 154/160] Add convenience types and use them for less line noise --- ethcore/light/src/cht.rs | 10 ++--- ethcore/light/src/on_demand/request.rs | 15 ++++--- ethcore/src/snapshot/account.rs | 9 ++-- ethcore/src/snapshot/mod.rs | 8 ++-- ethcore/src/snapshot/tests/helpers.rs | 12 ++--- ethcore/src/state/account.rs | 13 +++--- ethcore/src/state/mod.rs | 8 ++-- util/patricia-trie-ethereum/src/lib.rs | 22 ++++++++++ util/patricia_trie/benches/trie.rs | 20 ++++----- util/patricia_trie/src/fatdb.rs | 8 ++-- util/patricia_trie/src/fatdbmut.rs | 8 ++-- util/patricia_trie/src/recorder.rs | 8 ++-- util/patricia_trie/src/sectriedb.rs | 7 ++- util/patricia_trie/src/sectriedbmut.rs | 7 ++- util/patricia_trie/src/triedb.rs | 31 ++++++------- util/patricia_trie/src/triedbmut.rs | 61 ++++++++++++-------------- 16 files changed, 128 insertions(+), 119 deletions(-) diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index de2ba01681a..b90e103101b 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -29,8 +29,8 @@ use hashdb::HashDB; use keccak_hasher::KeccakHasher; use memorydb::MemoryDB; use bytes::Bytes; -use trie::{TrieMut, TrieDBMut, Trie, TrieDB, Recorder}; -use ethtrie::{self, RlpCodec}; +use trie::{TrieMut, Trie, Recorder}; +use ethtrie::{self, TrieDB, TrieDBMut}; use rlp::{RlpStream, Rlp}; @@ -73,7 +73,7 @@ impl> CHT { if block_to_cht_number(num) != Some(self.number) { return Ok(None) } let mut recorder = Recorder::with_depth(from_level); - let t = TrieDB::<_, RlpCodec>::new(&self.db, &self.root)?; + let t = TrieDB::new(&self.db, &self.root)?; t.get_with(&key!(num), &mut recorder)?; Ok(Some(recorder.drain().into_iter().map(|x| x.data).collect())) @@ -105,7 +105,7 @@ pub fn build(cht_num: u64, mut fetcher: F) -> Option::new(&mut db, &mut root); + let mut t = TrieDBMut::new(&mut db, &mut root); for blk_num in (0..SIZE).map(|n| last_num - n) { let info = match fetcher(id) { Some(info) => info, @@ -153,7 +153,7 @@ pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256) let mut db = MemoryDB::::new(); for node in proof { db.insert(&node[..]); } - let res = match TrieDB::<_, RlpCodec>::new(&db, &root) { + let res = match TrieDB::new(&db, &root) { Err(_) => return None, Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| { let rlp = Rlp::new(val); diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 24239b20b29..9deaee659f3 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -38,8 +38,8 @@ use hashdb::HashDB; use kvdb::DBValue; use bytes::Bytes; use memorydb::MemoryDB; -use trie::{Trie, TrieDB}; -use ethtrie::{TrieError, RlpCodec}; +use trie::Trie; +use ethtrie::{TrieError, TrieDB}; const SUPPLIED_MATCHES: &'static str = "supplied responses always match produced requests; enforced by `check_response`; qed"; @@ -830,7 +830,7 @@ impl Account { let mut db = MemoryDB::new(); for node in proof { db.insert(&node[..]); } - match TrieDB::<_, RlpCodec>::new(&db, &state_root).and_then(|t| t.get(&keccak(&self.address)))? { + match TrieDB::new(&db, &state_root).and_then(|t| t.get(&keccak(&self.address)))? { Some(val) => { let rlp = Rlp::new(&val); Ok(Some(BasicAccount { @@ -937,8 +937,9 @@ mod tests { use ethereum_types::{H256, Address}; use memorydb::MemoryDB; use parking_lot::Mutex; - use trie::{Trie, TrieMut, SecTrieDB, SecTrieDBMut}; - use trie::recorder::Recorder; + use trie::{Trie, TrieMut}; + use ethtrie::{SecTrieDB, SecTrieDBMut}; + use trie::Recorder; use hash::keccak; use ::ethcore::client::{BlockChainClient, BlockInfo, TestBlockChainClient, EachBlockWith}; @@ -1053,7 +1054,7 @@ mod tests { stream.out() }; { - let mut trie = SecTrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); + let mut trie = SecTrieDBMut::new(&mut db, &mut root); for _ in 0..100 { let address = Address::random(); trie.insert(&*address, &rand_acc()).unwrap(); @@ -1063,7 +1064,7 @@ mod tests { } let proof = { - let trie = SecTrieDB::<_, RlpCodec>::new(&db, &root).unwrap(); + let trie = SecTrieDB::new(&db, &root).unwrap(); let mut recorder = Recorder::new(); trie.get_with(&*addr, &mut recorder).unwrap().unwrap(); diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 594c921beca..b62eb415c53 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -24,8 +24,8 @@ use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; use ethereum_types::{H256, U256}; use hashdb::HashDB; use bytes::Bytes; -use trie::{TrieDB, Trie}; -use ethtrie::RlpCodec; +use trie::{Trie, TrieMut}; +use ethtrie::{TrieDB, TrieDBMut}; use rlp::{RlpStream, Rlp}; @@ -69,7 +69,7 @@ impl CodeState { // account address hash, account properties and the storage. Each item contains at most `max_storage_items` // storage records split according to snapshot format definition. pub fn to_fat_rlps(account_hash: &H256, acc: &BasicAccount, acct_db: &AccountDB, used_code: &mut HashSet, first_chunk_size: usize, max_chunk_size: usize) -> Result, Error> { - let db = TrieDB::<_, RlpCodec>::new(acct_db, &acc.storage_root)?; + let db = TrieDB::new(acct_db, &acc.storage_root)?; let mut chunks = Vec::new(); let mut db_iter = db.iter()?; let mut target_chunk_size = first_chunk_size; @@ -153,7 +153,6 @@ pub fn from_fat_rlp( rlp: Rlp, mut storage_root: H256, ) -> Result<(BasicAccount, Option), Error> { - use trie::{TrieDBMut, TrieMut}; // check for special case of empty account. if rlp.is_empty() { @@ -185,7 +184,7 @@ pub fn from_fat_rlp( { let mut storage_trie = if storage_root.is_zero() { - TrieDBMut::<_, RlpCodec>::new(acct_db, &mut storage_root) + TrieDBMut::new(acct_db, &mut storage_root) } else { TrieDBMut::from_existing(acct_db, &mut storage_root)? }; diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index a18c8e4253f..9c78085f9db 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -39,8 +39,8 @@ use bytes::Bytes; use parking_lot::Mutex; use journaldb::{self, Algorithm, JournalDB}; use kvdb::KeyValueDB; -use trie::{TrieDB, TrieDBMut, Trie, TrieMut}; -use ethtrie::RlpCodec; +use trie::{Trie, TrieMut}; +use ethtrie::{TrieDB, TrieDBMut}; use rlp::{RlpStream, Rlp}; use bloom_journal::Bloom; @@ -268,7 +268,7 @@ impl<'a> StateChunker<'a> { /// Returns a list of hashes of chunks created, or any error it may /// have encountered. pub fn chunk_state<'a>(db: &HashDB, root: &H256, writer: &Mutex, progress: &'a Progress) -> Result, Error> { - let account_trie = TrieDB::::new(db, &root)?; + let account_trie = TrieDB::new(db, &root)?; let mut chunker = StateChunker { hashes: Vec::new(), @@ -365,7 +365,7 @@ impl StateRebuilder { // batch trie writes { let mut account_trie = if self.state_root != KECCAK_NULL_RLP { - TrieDBMut::<_, RlpCodec>::from_existing(self.db.as_hashdb_mut(), &mut self.state_root)? + TrieDBMut::from_existing(self.db.as_hashdb_mut(), &mut self.state_root)? } else { TrieDBMut::new(self.db.as_hashdb_mut(), &mut self.state_root) }; diff --git a/ethcore/src/snapshot/tests/helpers.rs b/ethcore/src/snapshot/tests/helpers.rs index 7098f6e220a..19f50e94690 100644 --- a/ethcore/src/snapshot/tests/helpers.rs +++ b/ethcore/src/snapshot/tests/helpers.rs @@ -38,8 +38,8 @@ use ethereum_types::H256; use hashdb::HashDB; use keccak_hasher::KeccakHasher; use journaldb; -use trie::{SecTrieDBMut, TrieMut, TrieDB, TrieDBMut, Trie}; -use ethtrie::RlpCodec; +use trie::{TrieMut, Trie}; +use ethtrie::{SecTrieDBMut, TrieDB, TrieDBMut}; use self::trie_standardmap::{Alphabet, StandardMap, ValueMode}; // the proportion of accounts we will alter each tick. @@ -65,7 +65,7 @@ impl StateProducer { pub fn tick(&mut self, rng: &mut R, db: &mut HashDB) { // modify existing accounts. let mut accounts_to_modify: Vec<_> = { - let trie = TrieDB::<_, RlpCodec>::new(&*db, &self.state_root).unwrap(); + let trie = TrieDB::new(&*db, &self.state_root).unwrap(); let temp = trie.iter().unwrap() // binding required due to complicated lifetime stuff .filter(|_| rng.gen::() < ACCOUNT_CHURN) .map(Result::unwrap) @@ -84,7 +84,7 @@ impl StateProducer { } // sweep again to alter account trie. - let mut trie = TrieDBMut::<_, RlpCodec>::from_existing(db, &mut self.state_root).unwrap(); + let mut trie = TrieDBMut::from_existing(db, &mut self.state_root).unwrap(); for (address_hash, account_data) in accounts_to_modify { trie.insert(&address_hash[..], &account_data).unwrap(); @@ -119,9 +119,9 @@ pub fn fill_storage(mut db: AccountDBMut, root: &mut H256, seed: &mut H256) { }; { let mut trie = if *root == KECCAK_NULL_RLP { - SecTrieDBMut::<_, RlpCodec>::new(&mut db, root) + SecTrieDBMut::new(&mut db, root) } else { - SecTrieDBMut::<_, RlpCodec>::from_existing(&mut db, root).unwrap() + SecTrieDBMut::from_existing(&mut db, root).unwrap() }; for (k, v) in map.make_with(seed) { diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index a2418900fea..ab4cd67d78d 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -26,8 +26,8 @@ use hashdb::HashDB; use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::{Bytes, ToPretty}; -use trie::{SecTrieDB, Trie, TrieFactory}; -use ethtrie::{RlpCodec, Result as TrieResult}; +use trie::{Trie, Recorder}; +use ethtrie::{TrieFactory, TrieDB, SecTrieDB, Result as TrieResult}; use pod_account::*; use rlp::{RlpStream, encode}; use lru_cache::LruCache; @@ -205,7 +205,7 @@ impl Account { if let Some(value) = self.cached_storage_at(key) { return Ok(value); } - let db = SecTrieDB::<_, RlpCodec>::new(db, &self.storage_root)?; + let db = SecTrieDB::new(db, &self.storage_root)?; let panicky_decoder = |bytes:&[u8]| ::rlp::decode(&bytes).expect("decoding db value failed"); let item: U256 = db.get_with(key, panicky_decoder)?.unwrap_or_else(U256::zero); let value: H256 = item.into(); @@ -376,7 +376,7 @@ impl Account { } /// Commit the `storage_changes` to the backing DB and update `storage_root`. - pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> TrieResult<()> { + pub fn commit_storage(&mut self, trie_factory: &TrieFactory, db: &mut HashDB) -> TrieResult<()> { let mut t = trie_factory.from_existing(db, &mut self.storage_root)?; for (k, v) in self.storage_changes.drain() { // cast key and value to trait type, @@ -477,12 +477,9 @@ impl Account { // pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box::Out>>> { // pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box> { pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> TrieResult<(Vec, H256)> { - use trie::{Trie, TrieDB}; - use trie::recorder::Recorder; - let mut recorder = Recorder::new(); - let trie = TrieDB::<_, RlpCodec>::new(db, &self.storage_root)?; + let trie = TrieDB::new(db, &self.storage_root)?; let item: U256 = { let panicky_decoder = |bytes:&[u8]| ::rlp::decode(bytes).expect("decoding db value failed"); let query = (&mut recorder, panicky_decoder); diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 32308a444af..f785e24a335 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -48,8 +48,8 @@ use keccak_hasher::KeccakHasher; use kvdb::DBValue; use bytes::Bytes; -use trie::{Trie, TrieDB, TrieError, Recorder}; -use ethtrie::{RlpCodec, Result as TrieResult}; +use trie::{Trie, TrieError, Recorder}; +use ethtrie::{TrieDB, Result as TrieResult}; mod account; mod substate; @@ -1051,7 +1051,7 @@ impl State { /// `account_key` == keccak(address) pub fn prove_account(&self, account_key: H256) -> TrieResult<(Vec, BasicAccount)> { let mut recorder = Recorder::new(); - let trie = TrieDB::<_, RlpCodec>::new(self.db.as_hashdb(), &self.root)?; + let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; let maybe_account: Option = { let panicky_decoder = |bytes: &[u8]| { ::rlp::decode(bytes).expect(&format!("prove_account, could not query trie for account key={}", &account_key)) @@ -1077,7 +1077,7 @@ impl State { pub fn prove_storage(&self, account_key: H256, storage_key: H256) -> TrieResult<(Vec, H256)> { // TODO: probably could look into cache somehow but it's keyed by // address, not keccak(address). - let trie = TrieDB::<_, RlpCodec>::new(self.db.as_hashdb(), &self.root)?; + let trie = TrieDB::new(self.db.as_hashdb(), &self.root)?; let from_rlp = |b: &[u8]| Account::from_rlp(b).expect("decoding db value failed"); let acc = match trie.get_with(&account_key, from_rlp)? { Some(acc) => acc, diff --git a/util/patricia-trie-ethereum/src/lib.rs b/util/patricia-trie-ethereum/src/lib.rs index 145164607e4..a252152c776 100644 --- a/util/patricia-trie-ethereum/src/lib.rs +++ b/util/patricia-trie-ethereum/src/lib.rs @@ -34,6 +34,28 @@ use rlp::DecoderError; /// Convenience type alias to instantiate a Keccak-flavoured `RlpNodeCodec` pub type RlpCodec = RlpNodeCodec; + +/// Convenience type alias to instantiate a Keccak/Rlp-flavoured `TrieDB` +pub type TrieDB<'db> = trie::TrieDB<'db, KeccakHasher, RlpCodec>; + +/// Convenience type alias to instantiate a Keccak/Rlp-flavoured `SecTrieDB` +pub type SecTrieDB<'db> = trie::SecTrieDB<'db, KeccakHasher, RlpCodec>; + +/// Convenience type alias to instantiate a Keccak/Rlp-flavoured `FatDB` +pub type FatDB<'db> = trie::FatDB<'db, KeccakHasher, RlpCodec>; + +/// Convenience type alias to instantiate a Keccak/Rlp-flavoured `TrieDBMut` +pub type TrieDBMut<'db> = trie::TrieDBMut<'db, KeccakHasher, RlpCodec>; + +/// Convenience type alias to instantiate a Keccak/Rlp-flavoured `SecTrieDBMut` +pub type SecTrieDBMut<'db> = trie::SecTrieDBMut<'db, KeccakHasher, RlpCodec>; + +/// Convenience type alias to instantiate a Keccak/Rlp-flavoured `FatDBMut` +pub type FatDBMut<'db> = trie::FatDBMut<'db, KeccakHasher, RlpCodec>; + +/// Convenience type alias to instantiate a Keccak/Rlp-flavoured `TrieFactory` +pub type TrieFactory = trie::TrieFactory; + /// Convenience type alias for Keccak/Rlp flavoured trie errors pub type TrieError = trie::TrieError; /// Convenience type alias for Keccak/Rlp flavoured trie results diff --git a/util/patricia_trie/benches/trie.rs b/util/patricia_trie/benches/trie.rs index 9eb38d606c6..e01e4200a5b 100644 --- a/util/patricia_trie/benches/trie.rs +++ b/util/patricia_trie/benches/trie.rs @@ -32,11 +32,11 @@ use ethereum_types::H256; use keccak_hash::keccak; use memorydb::MemoryDB; use test::{Bencher, black_box}; -use trie::{TrieDBMut, TrieDB, TrieMut, Trie}; +use trie::{TrieMut, Trie}; use trie_standardmap::{Alphabet, ValueMode, StandardMap}; use hashdb::Hasher; use keccak_hasher::KeccakHasher; -use ethtrie::RlpCodec; +use ethtrie::{TrieDB, TrieDBMut}; fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { assert!(min_count + diff_count <= 32); @@ -77,7 +77,7 @@ fn trie_insertions_32_mir_1k(b: &mut Bencher) { b.iter(&mut ||{ let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -96,14 +96,14 @@ fn trie_iter(b: &mut Bencher) { let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } } b.iter(&mut ||{ - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); for n in t.iter().unwrap() { black_box(n).unwrap(); } @@ -124,7 +124,7 @@ fn trie_insertions_32_ran_1k(b: &mut Bencher) { b.iter(&mut ||{ let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -145,7 +145,7 @@ fn trie_insertions_six_high(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -165,7 +165,7 @@ fn trie_insertions_six_mid(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -186,7 +186,7 @@ fn trie_insertions_random_mid(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } @@ -207,7 +207,7 @@ fn trie_insertions_six_low(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); } diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index 2ae886fdf0d..a3799077668 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -126,18 +126,18 @@ mod test { use memorydb::MemoryDB; use hashdb::{Hasher, DBValue}; use keccak_hasher::KeccakHasher; - use ethtrie::RlpCodec; - use ethtrie::trie::{Trie, TrieMut, FatDB, FatDBMut}; + use ethtrie::trie::{Trie, TrieMut}; + use ethtrie::{FatDB, FatDBMut}; #[test] fn fatdb_to_trie() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = FatDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = FatDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = FatDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = FatDB::new(&memdb, &root).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); assert_eq!( t.iter().unwrap().map(Result::unwrap).collect::>(), diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 467dfd9eb62..41c22467d50 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -109,8 +109,8 @@ where mod test { use hashdb::{Hasher, DBValue}; use memorydb::MemoryDB; - use ethtrie::trie::{Trie, TrieMut, TrieDB, FatDBMut}; - use ethtrie::RlpCodec; + use ethtrie::trie::{Trie, TrieMut}; + use ethtrie::{TrieDB, FatDBMut}; use keccak_hasher::KeccakHasher; use keccak; @@ -119,10 +119,10 @@ mod test { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = FatDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = FatDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } } \ No newline at end of file diff --git a/util/patricia_trie/src/recorder.rs b/util/patricia_trie/src/recorder.rs index 8a8e3705789..9ad7d8c3309 100644 --- a/util/patricia_trie/src/recorder.rs +++ b/util/patricia_trie/src/recorder.rs @@ -135,16 +135,16 @@ mod tests { #[test] fn trie_record() { - use ethtrie::trie::{TrieDB, TrieDBMut, Trie, TrieMut, Recorder}; + use ethtrie::trie::{Trie, TrieMut, Recorder}; use memorydb::MemoryDB; - use ethtrie::RlpCodec; + use ethtrie::{TrieDB, TrieDBMut}; let mut db = MemoryDB::::new(); let mut root = H256::default(); { - let mut x = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); + let mut x = TrieDBMut::new(&mut db, &mut root); x.insert(b"dog", b"cat").unwrap(); x.insert(b"lunch", b"time").unwrap(); @@ -156,7 +156,7 @@ mod tests { x.insert(b"yo ho ho", b"and a bottle of rum").unwrap(); } - let trie = TrieDB::<_, RlpCodec>::new(&db, &root).unwrap(); + let trie = TrieDB::new(&db, &root).unwrap(); let mut recorder = Recorder::::new(); trie.get_with(b"pirate", &mut recorder).unwrap().unwrap(); diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index a163ae04b96..9494f8c0c5a 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -83,18 +83,17 @@ mod test { use hashdb::{Hasher, DBValue}; use keccak; use keccak_hasher::KeccakHasher; - use ethtrie::RlpCodec; - use ethtrie::trie::{Trie, TrieMut, TrieDBMut, SecTrieDB}; + use ethtrie::{TrieDBMut, SecTrieDB, trie::{Trie, TrieMut}}; #[test] fn trie_to_sectrie() { let mut db = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); + let mut t = TrieDBMut::new(&mut db, &mut root); t.insert(&keccak::keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); } - let t = SecTrieDB::<_, RlpCodec>::new(&db, &root).unwrap(); + let t = SecTrieDB::new(&db, &root).unwrap(); assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } } \ No newline at end of file diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index 0ad0bfccc26..aae7caadb91 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -93,18 +93,17 @@ mod test { use hashdb::{Hasher, DBValue}; use keccak; use keccak_hasher::KeccakHasher; - use ethtrie::RlpCodec; - use ethtrie::trie::{Trie, TrieMut, TrieDB, SecTrieDBMut}; + use ethtrie::{TrieDB, SecTrieDBMut, trie::{Trie, TrieMut}}; #[test] fn sectrie_to_trie() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = SecTrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = SecTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); assert_eq!(t.get(&keccak::keccak(&[0x01u8, 0x23])).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); } } diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 484032269be..0f5b031b2a0 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -45,15 +45,14 @@ use std::marker::PhantomData; /// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; -/// use ethtrie::RlpNodeCodec; +/// use ethtrie::{TrieDB, TrieDBMut}; /// -/// type RlpCodec = RlpNodeCodec; /// /// fn main() { /// let mut memdb = MemoryDB::::new(); /// let mut root = H256::new(); -/// TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap(); -/// let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); +/// TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap(); +/// let t = TrieDB::new(&memdb, &root).unwrap(); /// assert!(t.contains(b"foo").unwrap()); /// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar")); /// } @@ -418,9 +417,7 @@ mod tests { use hashdb::{Hasher, DBValue}; use keccak_hasher::KeccakHasher; use memorydb::*; - use ethtrie::trie::{Trie, TrieMut, TrieDB, Lookup}; - use ethtrie::trie::triedbmut::*; - use ethtrie::RlpCodec; + use ethtrie::{TrieDB, TrieDBMut, RlpCodec, trie::{Trie, TrieMut, Lookup}}; #[test] fn iterator() { @@ -429,13 +426,13 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { t.insert(x, x).unwrap(); } } - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); assert_eq!(d.iter().map(|i| i.clone().into_vec()).collect::>(), t.iter().unwrap().map(|x| x.unwrap().0).collect::>()); assert_eq!(d, t.iter().unwrap().map(|x| x.unwrap().1).collect::>()); } @@ -447,13 +444,13 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { t.insert(x, x).unwrap(); } } - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); let mut iter = t.iter().unwrap(); assert_eq!(iter.next().unwrap().unwrap(), (b"A".to_vec(), DBValue::from_slice(b"A"))); iter.seek(b"!").unwrap(); @@ -486,12 +483,12 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBA").unwrap(); } - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); assert_eq!(t.get_with(b"A", |x: &[u8]| x.len()).unwrap(), Some(3)); assert_eq!(t.get_with(b"B", |x: &[u8]| x.len()).unwrap(), Some(5)); assert_eq!(t.get_with(b"C", |x: &[u8]| x.len()).unwrap(), None); @@ -504,13 +501,13 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); let root = { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { t.insert(x, x).unwrap(); } t.root().clone() }; - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); assert_eq!(format!("{:?}", t), "TrieDB { hash_count: 0, root: Node::Extension { slice: 4, item: Node::Branch { nodes: [Node::Empty, Node::Branch { nodes: [Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Branch { nodes: [Node::Empty, Node::Leaf { slice: , value: [65, 65] }, Node::Leaf { slice: , value: [65, 66] }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: None }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: Some([65]) }, Node::Leaf { slice: , value: [66] }, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty, Node::Empty], value: None } } }"); assert_eq!(format!("{:#?}", t), @@ -614,12 +611,12 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::new(); { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); t.insert(b"B", b"ABCBA").unwrap(); } - let t = TrieDB::<_, RlpCodec>::new(&memdb, &root).unwrap(); + let t = TrieDB::new(&memdb, &root).unwrap(); // query for an invalid data type to trigger an error let q = rlp::decode::; diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 790e1cf17d6..d0d30751166 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -260,19 +260,16 @@ impl<'a, H: Hasher> Index<&'a StorageHandle> for NodeStorage { /// extern crate ethereum_types; /// /// use keccak_hash::KECCAK_NULL_RLP; -/// use trie::*; -/// use hashdb::*; +/// use ethtrie::{TrieDBMut, trie::TrieMut}; +/// use hashdb::DBValue; /// use keccak_hasher::KeccakHasher; /// use memorydb::*; /// use ethereum_types::H256; -/// use ethtrie::RlpNodeCodec; -/// -/// type RlpCodec = RlpNodeCodec; /// /// fn main() { /// let mut memdb = MemoryDB::::new(); /// let mut root = H256::new(); -/// let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); +/// let mut t = TrieDBMut::new(&mut memdb, &mut root); /// assert!(t.is_empty()); /// assert_eq!(*t.root(), KECCAK_NULL_RLP); /// t.insert(b"foo", b"bar").unwrap(); @@ -972,11 +969,11 @@ mod tests { use rlp::{Decodable, Encodable}; use triehash::trie_root; use standardmap::*; - use ethtrie::trie::{TrieMut, TrieDBMut, NodeCodec}; - use ethtrie::RlpCodec; + use ethtrie::{TrieDBMut, RlpCodec, trie::{TrieMut, NodeCodec}}; use env_logger; + use ethereum_types::H256; - fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H::Out, v: &[(Vec, Vec)]) -> TrieDBMut<'db, H, C> + fn populate_trie<'db, H, C>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec, Vec)]) -> TrieDBMut<'db> where H: Hasher, H::Out: Decodable + Encodable, C: NodeCodec { let mut t = TrieDBMut::new(db, root); @@ -988,9 +985,7 @@ mod tests { t } - fn unpopulate_trie<'db, H: Hasher, C: NodeCodec>(t: &mut TrieDBMut<'db, H, C>, v: &[(Vec, Vec)]) - where H::Out: Decodable + Encodable - { + fn unpopulate_trie<'db>(t: &mut TrieDBMut<'db>, v: &[(Vec, Vec)]) { for i in v { let key: &[u8]= &i.0; t.remove(key).unwrap(); @@ -1000,7 +995,7 @@ mod tests { #[test] fn playpen() { env_logger::init(); - let mut seed = ::Out::new(); + let mut seed = H256::new(); for test_i in 0..10 { if test_i % 50 == 0 { debug!("{:?} of 10000 stress tests done", test_i); @@ -1015,7 +1010,7 @@ mod tests { let real = trie_root(x.clone()); let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut memtrie = populate_trie::<_, RlpCodec>(&mut memdb, &mut root, &x); memtrie.commit(); @@ -1046,7 +1041,7 @@ mod tests { fn init() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); } @@ -1054,7 +1049,7 @@ mod tests { fn insert_on_empty() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } @@ -1065,12 +1060,12 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t1 = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t1 = TrieDBMut::new(&mut memdb, &mut root); t1.insert(&[0x01, 0x23], big_value).unwrap(); t1.insert(&[0x01, 0x34], big_value).unwrap(); let mut memdb2 = MemoryDB::::new(); let mut root2 = ::Out::default(); - let mut t2 = TrieDBMut::<_, RlpCodec>::new(&mut memdb2, &mut root2); + let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); t2.insert(&[0x01], big_value).unwrap(); t2.insert(&[0x01, 0x23], big_value).unwrap(); @@ -1082,7 +1077,7 @@ mod tests { fn insert_replace_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); @@ -1092,7 +1087,7 @@ mod tests { fn insert_make_branch_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1105,7 +1100,7 @@ mod tests { fn insert_into_branch_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -1120,7 +1115,7 @@ mod tests { fn insert_value_into_branch_root() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[], &[0x0]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1133,7 +1128,7 @@ mod tests { fn insert_split_leaf() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1146,7 +1141,7 @@ mod tests { fn insert_split_extenstion() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); @@ -1164,7 +1159,7 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1179,7 +1174,7 @@ mod tests { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value).unwrap(); t.insert(&[0x11u8, 0x23], big_value).unwrap(); assert_eq!(*t.root(), trie_root(vec![ @@ -1192,7 +1187,7 @@ mod tests { fn test_at_empty() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let t = TrieDBMut::new(&mut memdb, &mut root); assert_eq!(t.get(&[0x5]).unwrap(), None); } @@ -1200,7 +1195,7 @@ mod tests { fn test_at_one() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); t.commit(); @@ -1211,7 +1206,7 @@ mod tests { fn test_at_three() { let mut memdb = MemoryDB::::new(); let mut root = ::Out::default(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); @@ -1269,12 +1264,12 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::default(); { - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); + let mut t = TrieDBMut::new(&mut db, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); } { - let _ = TrieDBMut::<_, RlpCodec>::from_existing(&mut db, &mut root); + let _ = TrieDBMut::from_existing(&mut db, &mut root); } } @@ -1291,7 +1286,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); + let mut t = TrieDBMut::new(&mut db, &mut root); for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); } @@ -1319,7 +1314,7 @@ mod tests { let mut db = MemoryDB::::new(); let mut root = ::Out::new(); - let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root); + let mut t = TrieDBMut::new(&mut db, &mut root); for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap().is_none()); assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); From 92582c719ba32f6aa5d233758a79b4dd62ceb34b Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 29 Jun 2018 10:13:48 +0200 Subject: [PATCH 155/160] Remove unused deps --- ethcore/private-tx/src/error.rs | 2 +- ethcore/sync/src/chain/mod.rs | 1 - ethcore/sync/src/transactions_stats.rs | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ethcore/private-tx/src/error.rs b/ethcore/private-tx/src/error.rs index b0b1e732965..55a75d6d9ed 100644 --- a/ethcore/private-tx/src/error.rs +++ b/ethcore/private-tx/src/error.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use ethereum_types::{Address, H256}; +use ethereum_types::Address; use rlp::DecoderError; use ethtrie::TrieError; use ethcore::account_provider::SignError; diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 6da21a9671d..79bfb40d6c2 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -97,7 +97,6 @@ use std::collections::{HashSet, HashMap}; use std::cmp; use std::time::{Duration, Instant}; use hash::keccak; -use keccak_hasher::KeccakHasher; use heapsize::HeapSizeOf; use ethereum_types::{H256, U256}; use plain_hasher::H256FastMap; diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index 2875d495fbc..c45b1ad8b3c 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -18,7 +18,6 @@ use api::TransactionStats; use std::collections::{HashSet, HashMap}; use ethereum_types::{H256, H512}; use plain_hasher::H256FastMap; -use keccak_hasher::KeccakHasher; type NodeId = H512; type BlockNumber = u64; From 988bfafe8e7d6db34a0a72a8eb47e90ef737f892 Mon Sep 17 00:00:00 2001 From: David Palm Date: Fri, 29 Jun 2018 10:29:54 +0200 Subject: [PATCH 156/160] Clean up tests --- util/patricia_trie/benches/trie.rs | 21 ++++++------ util/patricia_trie/src/fatdb.rs | 5 +-- util/patricia_trie/src/fatdbmut.rs | 5 +-- util/patricia_trie/src/sectriedb.rs | 5 +-- util/patricia_trie/src/sectriedbmut.rs | 5 +-- util/patricia_trie/src/triedb.rs | 15 +++++---- util/patricia_trie/src/triedbmut.rs | 46 +++++++++++++------------- 7 files changed, 53 insertions(+), 49 deletions(-) diff --git a/util/patricia_trie/benches/trie.rs b/util/patricia_trie/benches/trie.rs index e01e4200a5b..114006ebe2e 100644 --- a/util/patricia_trie/benches/trie.rs +++ b/util/patricia_trie/benches/trie.rs @@ -34,7 +34,6 @@ use memorydb::MemoryDB; use test::{Bencher, black_box}; use trie::{TrieMut, Trie}; use trie_standardmap::{Alphabet, ValueMode, StandardMap}; -use hashdb::Hasher; use keccak_hasher::KeccakHasher; use ethtrie::{TrieDB, TrieDBMut}; @@ -76,7 +75,7 @@ fn trie_insertions_32_mir_1k(b: &mut Bencher) { let d = st.make(); b.iter(&mut ||{ let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); @@ -94,7 +93,7 @@ fn trie_iter(b: &mut Bencher) { }; let d = st.make(); let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { @@ -123,7 +122,7 @@ fn trie_insertions_32_ran_1k(b: &mut Bencher) { let mut r = H256::new(); b.iter(&mut ||{ let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); @@ -135,7 +134,7 @@ fn trie_insertions_32_ran_1k(b: &mut Bencher) { #[bench] fn trie_insertions_six_high(b: &mut Bencher) { let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = ::Out::new(); + let mut seed = H256::new(); for _ in 0..1000 { let k = random_bytes(6, 0, &mut seed); let v = random_value(&mut seed); @@ -144,7 +143,7 @@ fn trie_insertions_six_high(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); @@ -164,7 +163,7 @@ fn trie_insertions_six_mid(b: &mut Bencher) { } b.iter(||{ let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); @@ -176,7 +175,7 @@ fn trie_insertions_six_mid(b: &mut Bencher) { fn trie_insertions_random_mid(b: &mut Bencher) { let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = ::Out::new(); + let mut seed = H256::new(); for _ in 0..1000 { let k = random_word(alphabet, 1, 5, &mut seed); let v = random_value(&mut seed); @@ -185,7 +184,7 @@ fn trie_insertions_random_mid(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); @@ -197,7 +196,7 @@ fn trie_insertions_random_mid(b: &mut Bencher) { fn trie_insertions_six_low(b: &mut Bencher) { let alphabet = b"abcdef"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = ::Out::new(); + let mut seed = H256::new(); for _ in 0..1000 { let k = random_word(alphabet, 6, 0, &mut seed); let v = random_value(&mut seed); @@ -206,7 +205,7 @@ fn trie_insertions_six_low(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1).unwrap(); diff --git a/util/patricia_trie/src/fatdb.rs b/util/patricia_trie/src/fatdb.rs index a3799077668..34a49a5d0f6 100644 --- a/util/patricia_trie/src/fatdb.rs +++ b/util/patricia_trie/src/fatdb.rs @@ -124,15 +124,16 @@ where #[cfg(test)] mod test { use memorydb::MemoryDB; - use hashdb::{Hasher, DBValue}; + use hashdb::DBValue; use keccak_hasher::KeccakHasher; use ethtrie::trie::{Trie, TrieMut}; use ethtrie::{FatDB, FatDBMut}; + use ethereum_types::H256; #[test] fn fatdb_to_trie() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); { let mut t = FatDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); diff --git a/util/patricia_trie/src/fatdbmut.rs b/util/patricia_trie/src/fatdbmut.rs index 41c22467d50..67f4f14a75c 100644 --- a/util/patricia_trie/src/fatdbmut.rs +++ b/util/patricia_trie/src/fatdbmut.rs @@ -107,17 +107,18 @@ where #[cfg(test)] mod test { - use hashdb::{Hasher, DBValue}; + use hashdb::DBValue; use memorydb::MemoryDB; use ethtrie::trie::{Trie, TrieMut}; use ethtrie::{TrieDB, FatDBMut}; use keccak_hasher::KeccakHasher; use keccak; + use ethereum_types::H256; #[test] fn fatdbmut_to_trie() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); { let mut t = FatDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); diff --git a/util/patricia_trie/src/sectriedb.rs b/util/patricia_trie/src/sectriedb.rs index 9494f8c0c5a..a340de947be 100644 --- a/util/patricia_trie/src/sectriedb.rs +++ b/util/patricia_trie/src/sectriedb.rs @@ -80,15 +80,16 @@ where #[cfg(test)] mod test { use memorydb::MemoryDB; - use hashdb::{Hasher, DBValue}; + use hashdb::DBValue; use keccak; use keccak_hasher::KeccakHasher; use ethtrie::{TrieDBMut, SecTrieDB, trie::{Trie, TrieMut}}; + use ethereum_types::H256; #[test] fn trie_to_sectrie() { let mut db = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); { let mut t = TrieDBMut::new(&mut db, &mut root); t.insert(&keccak::keccak(&[0x01u8, 0x23]), &[0x01u8, 0x23]).unwrap(); diff --git a/util/patricia_trie/src/sectriedbmut.rs b/util/patricia_trie/src/sectriedbmut.rs index aae7caadb91..8750c2dd5c1 100644 --- a/util/patricia_trie/src/sectriedbmut.rs +++ b/util/patricia_trie/src/sectriedbmut.rs @@ -90,15 +90,16 @@ where #[cfg(test)] mod test { use memorydb::MemoryDB; - use hashdb::{Hasher, DBValue}; + use hashdb::DBValue; use keccak; use keccak_hasher::KeccakHasher; use ethtrie::{TrieDB, SecTrieDBMut, trie::{Trie, TrieMut}}; + use ethereum_types::H256; #[test] fn sectrie_to_trie() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); { let mut t = SecTrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); diff --git a/util/patricia_trie/src/triedb.rs b/util/patricia_trie/src/triedb.rs index 0f5b031b2a0..bf88ab2ec2f 100644 --- a/util/patricia_trie/src/triedb.rs +++ b/util/patricia_trie/src/triedb.rs @@ -414,17 +414,18 @@ impl<'a, H: Hasher, C: NodeCodec> Iterator for TrieDBIterator<'a, H, C> { #[cfg(test)] mod tests { - use hashdb::{Hasher, DBValue}; + use hashdb::DBValue; use keccak_hasher::KeccakHasher; - use memorydb::*; + use memorydb::MemoryDB; use ethtrie::{TrieDB, TrieDBMut, RlpCodec, trie::{Trie, TrieMut, Lookup}}; + use ethereum_types::H256; #[test] fn iterator() { let d = vec![DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B")]; let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { @@ -442,7 +443,7 @@ mod tests { let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { @@ -481,7 +482,7 @@ mod tests { #[test] fn get_len() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); @@ -499,7 +500,7 @@ mod tests { let d = vec![ DBValue::from_slice(b"A"), DBValue::from_slice(b"AA"), DBValue::from_slice(b"AB"), DBValue::from_slice(b"B") ]; let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let root = { let mut t = TrieDBMut::new(&mut memdb, &mut root); for x in &d { @@ -609,7 +610,7 @@ mod tests { use ethtrie::trie::NibbleSlice; let mut memdb = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); { let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(b"A", b"ABC").unwrap(); diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index d0d30751166..85d3e50ba7d 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -1040,7 +1040,7 @@ mod tests { #[test] fn init() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); assert_eq!(*t.root(), RlpCodec::HASHED_NULL_NODE); } @@ -1048,7 +1048,7 @@ mod tests { #[test] fn insert_on_empty() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); @@ -1059,12 +1059,12 @@ mod tests { let big_value = b"00000000000000000000000000000000"; let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t1 = TrieDBMut::new(&mut memdb, &mut root); t1.insert(&[0x01, 0x23], big_value).unwrap(); t1.insert(&[0x01, 0x34], big_value).unwrap(); let mut memdb2 = MemoryDB::::new(); - let mut root2 = ::Out::default(); + let mut root2 = H256::new(); let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); t2.insert(&[0x01], big_value).unwrap(); @@ -1076,7 +1076,7 @@ mod tests { #[test] fn insert_replace_root() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); @@ -1086,7 +1086,7 @@ mod tests { #[test] fn insert_make_branch_root() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); @@ -1099,7 +1099,7 @@ mod tests { #[test] fn insert_into_branch_root() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); @@ -1114,7 +1114,7 @@ mod tests { #[test] fn insert_value_into_branch_root() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[], &[0x0]).unwrap(); @@ -1127,7 +1127,7 @@ mod tests { #[test] fn insert_split_leaf() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); @@ -1140,7 +1140,7 @@ mod tests { #[test] fn insert_split_extenstion() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); @@ -1158,7 +1158,7 @@ mod tests { let big_value1 = b"11111111111111111111111111111111"; let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); @@ -1173,7 +1173,7 @@ mod tests { let big_value = b"00000000000000000000000000000000"; let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value).unwrap(); t.insert(&[0x11u8, 0x23], big_value).unwrap(); @@ -1186,7 +1186,7 @@ mod tests { #[test] fn test_at_empty() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let t = TrieDBMut::new(&mut memdb, &mut root); assert_eq!(t.get(&[0x5]).unwrap(), None); } @@ -1194,7 +1194,7 @@ mod tests { #[test] fn test_at_one() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); @@ -1205,7 +1205,7 @@ mod tests { #[test] fn test_at_three() { let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); @@ -1223,7 +1223,7 @@ mod tests { #[test] fn stress() { - let mut seed = ::Out::new(); + let mut seed = H256::new(); for _ in 0..50 { let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), @@ -1235,12 +1235,12 @@ mod tests { let real = trie_root(x.clone()); let mut memdb = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); let mut memtrie = populate_trie::<_, RlpCodec>(&mut memdb, &mut root, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); let mut memdb2 = MemoryDB::::new(); - let mut root2 = ::Out::default(); + let mut root2 = H256::new(); let mut memtrie_sorted = populate_trie::<_, RlpCodec>(&mut memdb2, &mut root2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); @@ -1262,7 +1262,7 @@ mod tests { #[test] fn test_trie_existing() { let mut db = MemoryDB::::new(); - let mut root = ::Out::default(); + let mut root = H256::new(); { let mut t = TrieDBMut::new(&mut db, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); @@ -1275,7 +1275,7 @@ mod tests { #[test] fn insert_empty() { - let mut seed = ::Out::new(); + let mut seed = H256::new(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 5, @@ -1285,7 +1285,7 @@ mod tests { }.make_with(&mut seed); let mut db = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut db, &mut root); for &(ref key, ref value) in &x { t.insert(key, value).unwrap(); @@ -1303,7 +1303,7 @@ mod tests { #[test] fn return_old_values() { - let mut seed = ::Out::new(); + let mut seed = H256::new(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 5, @@ -1313,7 +1313,7 @@ mod tests { }.make_with(&mut seed); let mut db = MemoryDB::::new(); - let mut root = ::Out::new(); + let mut root = H256::new(); let mut t = TrieDBMut::new(&mut db, &mut root); for &(ref key, ref value) in &x { assert!(t.insert(key, value).unwrap().is_none()); From d625d8e650e55241a8b651070e375f1a43cebdea Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 2 Jul 2018 09:32:21 +0200 Subject: [PATCH 157/160] Address grumbles --- ethcore/src/state/account.rs | 2 -- ethcore/src/state_db.rs | 2 -- 2 files changed, 4 deletions(-) diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index ab4cd67d78d..307a84cf7d6 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -474,8 +474,6 @@ impl Account { /// trie. /// `storage_key` is the hash of the desired storage key, meaning /// this will only work correctly under a secure trie. - // pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box::Out>>> { - // pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> Result<(Vec, H256), Box> { pub fn prove_storage(&self, db: &HashDB, storage_key: H256) -> TrieResult<(Vec, H256)> { let mut recorder = Recorder::new(); diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs index 80174d71723..ca5a42f7eab 100644 --- a/ethcore/src/state_db.rs +++ b/ethcore/src/state_db.rs @@ -311,13 +311,11 @@ impl StateDB { } } - // TODO: needed? /// Conversion method to interpret self as `HashDB` reference pub fn as_hashdb(&self) -> &HashDB { self.db.as_hashdb() } - // TODO: needed? /// Conversion method to interpret self as mutable `HashDB` reference pub fn as_hashdb_mut(&mut self) -> &mut HashDB { self.db.as_hashdb_mut() From 67003bea5195298812919b024e4715c3b9aba232 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 2 Jul 2018 09:38:38 +0200 Subject: [PATCH 158/160] Remove tempdir as test dep --- ethcore/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index d22652e064b..0e449f867ef 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -80,7 +80,6 @@ fake-hardware-wallet = { path = "../util/fake-hardware-wallet" } [dev-dependencies] tempdir = "0.3" trie-standardmap = { path = "../util/trie-standardmap" } -tempdir = "0.3" [features] # Display EVM debug traces. From 412ae9ca29ed3fc959cb778859eb432d70492758 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 2 Jul 2018 09:38:53 +0200 Subject: [PATCH 159/160] Boxed errors --- util/patricia_trie/src/triedbmut.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/patricia_trie/src/triedbmut.rs b/util/patricia_trie/src/triedbmut.rs index 85d3e50ba7d..f5c28bac32f 100644 --- a/util/patricia_trie/src/triedbmut.rs +++ b/util/patricia_trie/src/triedbmut.rs @@ -346,8 +346,7 @@ where // cache a node by hash fn cache(&mut self, hash: H::Out) -> Result { - let node_encoded = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; - // let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; + let node_encoded = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; let node = Node::from_encoded::( &node_encoded, &*self.db, From 07dfdc7b9c4a6a494eab84dfcc1e1697f91b0d06 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 2 Jul 2018 11:40:47 +0200 Subject: [PATCH 160/160] Remove spurious whitespace after use blocks --- ethcore/light/src/cht.rs | 1 - ethcore/light/src/client/header_chain.rs | 16 ++----- ethcore/light/src/net/mod.rs | 17 +++---- ethcore/light/src/net/status.rs | 3 +- ethcore/light/src/net/tests/mod.rs | 14 ++---- ethcore/light/src/on_demand/request.rs | 19 ++++---- ethcore/light/src/types/request/mod.rs | 3 -- ethcore/src/block.rs | 18 ++++--- ethcore/src/blockchain/blockchain.rs | 48 +++++++++---------- ethcore/src/blockchain/extras.rs | 10 ++-- ethcore/src/client/test_client.rs | 2 - ethcore/src/encoded.rs | 1 - ethcore/src/engines/authority_round/mod.rs | 4 -- ethcore/src/engines/tendermint/message.rs | 11 ++--- ethcore/src/engines/tendermint/mod.rs | 1 - .../engines/validator_set/safe_contract.rs | 23 ++++----- ethcore/src/executive.rs | 1 - ethcore/src/header.rs | 1 - ethcore/src/json_tests/executive.rs | 1 - ethcore/src/pod_account.rs | 1 - ethcore/src/snapshot/account.rs | 13 ++--- ethcore/src/snapshot/block.rs | 2 - ethcore/src/snapshot/consensus/work.rs | 1 - ethcore/src/snapshot/io.rs | 1 - ethcore/src/snapshot/mod.rs | 1 - ethcore/src/snapshot/tests/proof_of_work.rs | 1 - ethcore/src/snapshot/tests/state.rs | 1 - ethcore/src/spec/seal.rs | 1 - ethcore/src/spec/spec.rs | 1 - ethcore/src/state/account.rs | 1 - ethcore/src/state_db.rs | 1 - ethcore/src/test_helpers.rs | 1 - ethcore/src/tests/client.rs | 1 - ethcore/src/trace/types/error.rs | 1 - ethcore/src/trace/types/flat.rs | 1 - ethcore/src/verification/verification.rs | 2 - ethcore/sync/src/blocks.rs | 1 - ethcore/sync/src/chain/mod.rs | 1 - ethcore/sync/src/chain/propagator.rs | 1 - ethcore/sync/src/chain/requester.rs | 1 - ethcore/transaction/src/transaction.rs | 1 - ethcore/types/src/lib.rs | 1 - ethcore/types/src/snapshot_manifest.rs | 1 - ethcore/vm/src/call_type.rs | 1 - util/journaldb/src/earlymergedb.rs | 1 - util/network-devp2p/src/connection.rs | 1 - util/network-devp2p/src/discovery.rs | 1 - util/network-devp2p/src/handshake.rs | 1 - util/network-devp2p/src/session.rs | 1 - util/rlp_compress/src/lib.rs | 1 - util/version/src/lib.rs | 1 - whisper/src/message.rs | 1 - whisper/src/net/mod.rs | 1 - 53 files changed, 81 insertions(+), 161 deletions(-) diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index b90e103101b..18b0e5b06c9 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -33,7 +33,6 @@ use trie::{TrieMut, Trie, Recorder}; use ethtrie::{self, TrieDB, TrieDBMut}; use rlp::{RlpStream, Rlp}; - // encode a key. macro_rules! key { ($num: expr) => { ::rlp::encode(&$num) } diff --git a/ethcore/light/src/client/header_chain.rs b/ethcore/light/src/client/header_chain.rs index c286fa82f60..0c36232e67b 100644 --- a/ethcore/light/src/client/header_chain.rs +++ b/ethcore/light/src/client/header_chain.rs @@ -28,29 +28,21 @@ use std::collections::BTreeMap; use std::sync::Arc; +use cache::Cache; use cht; - use ethcore::block_status::BlockStatus; -use ethcore::error::{Error, ErrorKind, BlockImportError, BlockImportErrorKind, BlockError}; use ethcore::encoded; +use ethcore::engines::epoch::{Transition as EpochTransition, PendingTransition as PendingEpochTransition}; +use ethcore::error::{Error, ErrorKind, BlockImportError, BlockImportErrorKind, BlockError}; use ethcore::header::Header; use ethcore::ids::BlockId; use ethcore::spec::{Spec, SpecHardcodedSync}; -use ethcore::engines::epoch::{ - Transition as EpochTransition, - PendingTransition as PendingEpochTransition -}; - use ethereum_types::{H256, H264, U256}; use heapsize::HeapSizeOf; use kvdb::{DBTransaction, KeyValueDB}; +use parking_lot::{Mutex, RwLock}; use plain_hasher::H256FastMap; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; - - -use cache::Cache; -use parking_lot::{Mutex, RwLock}; - use smallvec::SmallVec; /// Store at least this many candidate headers at all times. diff --git a/ethcore/light/src/net/mod.rs b/ethcore/light/src/net/mod.rs index 749ab82c1e4..27d240feda9 100644 --- a/ethcore/light/src/net/mod.rs +++ b/ethcore/light/src/net/mod.rs @@ -20,23 +20,20 @@ use transaction::UnverifiedTransaction; -use io::TimerToken; -use network::{NetworkProtocolHandler, NetworkContext, PeerId}; -use rlp::{RlpStream, Rlp}; - use ethereum_types::{H256, U256}; +use io::TimerToken; use kvdb::DBValue; +use network::{NetworkProtocolHandler, NetworkContext, PeerId}; use parking_lot::{Mutex, RwLock}; -use std::time::{Duration, Instant}; - +use provider::Provider; +use request::{Request, NetworkRequests as Requests, Response}; +use rlp::{RlpStream, Rlp}; use std::collections::{HashMap, HashSet}; use std::fmt; +use std::ops::{BitOr, BitAnd, Not}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::ops::{BitOr, BitAnd, Not}; - -use provider::Provider; -use request::{Request, NetworkRequests as Requests, Response}; +use std::time::{Duration, Instant}; use self::request_credits::{Credits, FlowParams}; use self::context::{Ctx, TickCtx}; diff --git a/ethcore/light/src/net/status.rs b/ethcore/light/src/net/status.rs index e07045095cf..98b29f3e360 100644 --- a/ethcore/light/src/net/status.rs +++ b/ethcore/light/src/net/status.rs @@ -16,9 +16,8 @@ //! Peer status and capabilities. -use rlp::{DecoderError, Encodable, Decodable, RlpStream, Rlp}; - use ethereum_types::{H256, U256}; +use rlp::{DecoderError, Encodable, Decodable, RlpStream, Rlp}; use super::request_credits::FlowParams; diff --git a/ethcore/light/src/net/tests/mod.rs b/ethcore/light/src/net/tests/mod.rs index 7ea014d2871..0c9971d75bd 100644 --- a/ethcore/light/src/net/tests/mod.rs +++ b/ethcore/light/src/net/tests/mod.rs @@ -19,21 +19,18 @@ use ethcore::blockchain_info::BlockChainInfo; use ethcore::client::{EachBlockWith, TestBlockChainClient}; -use ethcore::ids::BlockId; use ethcore::encoded; -use network::{PeerId, NodeId}; -use transaction::{Action, PendingTransaction}; - +use ethcore::ids::BlockId; +use ethereum_types::{H256, U256, Address}; +use net::{LightProtocol, Params, packet, Peer}; use net::context::IoContext; use net::status::{Capabilities, Status}; -use net::{LightProtocol, Params, packet, Peer}; +use network::{PeerId, NodeId}; use provider::Provider; use request; use request::*; - use rlp::{Rlp, RlpStream}; - -use ethereum_types::{H256, U256, Address}; +use transaction::{Action, PendingTransaction}; use std::sync::Arc; use std::time::Instant; @@ -643,7 +640,6 @@ fn proof_of_execution() { fn id_guard() { use super::request_set::RequestSet; use super::ReqId; - let capabilities = capabilities(); diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index 9deaee659f3..9feb0a670ff 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -18,28 +18,25 @@ use std::sync::Arc; +use bytes::Bytes; use ethcore::basic_account::BasicAccount; use ethcore::encoded; use ethcore::engines::{EthEngine, StateDependentProof}; use ethcore::machine::EthereumMachine; use ethcore::receipt::Receipt; use ethcore::state::{self, ProvedExecution}; -use transaction::SignedTransaction; -use vm::EnvInfo; -use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY, KECCAK_EMPTY_LIST_RLP, keccak}; - -use request::{self as net_request, IncompleteRequest, CompleteRequest, Output, OutputKind, Field}; - -use rlp::{RlpStream, Rlp}; - use ethereum_types::{H256, U256, Address}; -use parking_lot::Mutex; +use ethtrie::{TrieError, TrieDB}; +use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY, KECCAK_EMPTY_LIST_RLP, keccak}; use hashdb::HashDB; use kvdb::DBValue; -use bytes::Bytes; use memorydb::MemoryDB; +use parking_lot::Mutex; +use request::{self as net_request, IncompleteRequest, CompleteRequest, Output, OutputKind, Field}; +use rlp::{RlpStream, Rlp}; +use transaction::SignedTransaction; use trie::Trie; -use ethtrie::{TrieError, TrieDB}; +use vm::EnvInfo; const SUPPLIED_MATCHES: &'static str = "supplied responses always match produced requests; enforced by `check_response`; qed"; diff --git a/ethcore/light/src/types/request/mod.rs b/ethcore/light/src/types/request/mod.rs index 5f269741ec4..538aa0c6bf4 100644 --- a/ethcore/light/src/types/request/mod.rs +++ b/ethcore/light/src/types/request/mod.rs @@ -672,7 +672,6 @@ pub mod header { use super::{Field, HashOrNumber, NoSuchOutput, OutputKind, Output}; use ethcore::encoded; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; - /// Potentially incomplete headers request. #[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] @@ -1027,7 +1026,6 @@ pub mod block_body { use super::{Field, NoSuchOutput, OutputKind, Output}; use ethcore::encoded; use rlp::{Encodable, Decodable, DecoderError, RlpStream, Rlp}; - use ethereum_types::H256; /// Potentially incomplete block body request. @@ -1636,7 +1634,6 @@ pub mod epoch_signal { mod tests { use super::*; use ethcore::header::Header; - fn check_roundtrip(val: T) where T: ::rlp::Encodable + ::rlp::Decodable + PartialEq + ::std::fmt::Debug diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index a883278f5ee..ba21cb41761 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -17,29 +17,27 @@ //! Blockchain block. use std::cmp; -use std::sync::Arc; use std::collections::HashSet; -use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP}; -use triehash::ordered_trie_root; - -use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError, encode_list}; +use std::sync::Arc; -use ethereum_types::{H256, U256, Address, Bloom}; use bytes::Bytes; -use unexpected::{Mismatch, OutOfBounds}; - -use vm::{EnvInfo, LastHashes}; use engines::EthEngine; use error::{Error, BlockError}; +use ethereum_types::{H256, U256, Address, Bloom}; use factory::Factories; +use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP}; use header::{Header, ExtendedHeader}; use receipt::{Receipt, TransactionOutcome}; -use state::State; +use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError, encode_list}; use state_db::StateDB; +use state::State; use trace::Tracing; use transaction::{UnverifiedTransaction, SignedTransaction, Error as TransactionError}; +use triehash::ordered_trie_root; +use unexpected::{Mismatch, OutOfBounds}; use verification::PreverifiedBlock; use views::BlockView; +use vm::{EnvInfo, LastHashes}; /// A block, encoded as it is on the block chain. #[derive(Default, Debug, Clone, PartialEq)] diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index fa761b4cc9d..5ff820701d8 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -17,39 +17,39 @@ //! Blockchain database. use std::collections::{HashMap, HashSet}; -use std::sync::Arc; use std::mem; -use itertools::Itertools; -use blooms_db; -use heapsize::HeapSizeOf; -use ethereum_types::{H256, Bloom, BloomRef, U256}; -use parking_lot::{Mutex, RwLock}; -use bytes::Bytes; -use rlp::RlpStream; -use rlp_compress::{compress, decompress, blocks_swapper}; -use header::*; -use transaction::*; -use views::{BlockView, HeaderView}; -use log_entry::{LogEntry, LocalizedLogEntry}; -use receipt::Receipt; +use std::path::Path; +use std::sync::Arc; + +use ansi_term::Colour; +use blockchain::{CacheSize, ImportRoute, Config}; use blockchain::best_block::{BestBlock, BestAncientBlock}; use blockchain::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; use blockchain::extras::{BlockReceipts, BlockDetails, TransactionAddress, EPOCH_KEY_PREFIX, EpochTransitions}; -use types::blockchain_info::BlockChainInfo; -use types::tree_route::TreeRoute; use blockchain::update::{ExtrasUpdate, ExtrasInsert}; -use blockchain::{CacheSize, ImportRoute, Config}; -use db::{self, Writable, Readable, CacheUpdatePolicy}; +use blooms_db; +use bytes::Bytes; use cache_manager::CacheManager; +use db::{self, Writable, Readable, CacheUpdatePolicy}; use encoded; -use engines::ForkChoice; use engines::epoch::{Transition as EpochTransition, PendingTransition as PendingEpochTransition}; -use rayon::prelude::*; -use ansi_term::Colour; -use kvdb::{DBTransaction, KeyValueDB}; - +use engines::ForkChoice; use error::Error; -use std::path::Path; +use ethereum_types::{H256, Bloom, BloomRef, U256}; +use header::*; +use heapsize::HeapSizeOf; +use itertools::Itertools; +use kvdb::{DBTransaction, KeyValueDB}; +use log_entry::{LogEntry, LocalizedLogEntry}; +use parking_lot::{Mutex, RwLock}; +use rayon::prelude::*; +use receipt::Receipt; +use rlp_compress::{compress, decompress, blocks_swapper}; +use rlp::RlpStream; +use transaction::*; +use types::blockchain_info::BlockChainInfo; +use types::tree_route::TreeRoute; +use views::{BlockView, HeaderView}; /// Database backing `BlockChain`. pub trait BlockChainDB: Send + Sync { diff --git a/ethcore/src/blockchain/extras.rs b/ethcore/src/blockchain/extras.rs index 57b43146090..bbf67374a1b 100644 --- a/ethcore/src/blockchain/extras.rs +++ b/ethcore/src/blockchain/extras.rs @@ -16,17 +16,17 @@ //! Blockchain DB extras. -use std::ops; use std::io::Write; +use std::ops; + use db::Key; use engines::epoch::{Transition as EpochTransition}; +use ethereum_types::{H256, H264, U256}; use header::BlockNumber; -use receipt::Receipt; -use rlp; - use heapsize::HeapSizeOf; -use ethereum_types::{H256, H264, U256}; use kvdb::PREFIX_LEN as DB_PREFIX_LEN; +use receipt::Receipt; +use rlp; /// Represents index of extra data in database #[derive(Copy, Debug, Hash, Eq, PartialEq, Clone)] diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 505c1e940ef..3cb4660fe79 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -30,7 +30,6 @@ use kvdb::DBValue; use kvdb_memorydb; use bytes::Bytes; use rlp::{Rlp, RlpStream}; - use ethkey::{Generator, Random}; use ethcore_miner::pool::VerifiedTransaction; use transaction::{self, Transaction, LocalizedTransaction, SignedTransaction, Action}; @@ -53,7 +52,6 @@ use miner::{self, Miner, MinerService}; use spec::Spec; use types::basic_account::BasicAccount; use types::pruning_info::PruningInfo; - use verification::queue::QueueInfo; use block::{OpenBlock, SealedBlock, ClosedBlock}; use executive::Executed; diff --git a/ethcore/src/encoded.rs b/ethcore/src/encoded.rs index 4b1a139eaf0..5bd723f0e24 100644 --- a/ethcore/src/encoded.rs +++ b/ethcore/src/encoded.rs @@ -29,7 +29,6 @@ use hash::keccak; use header::{BlockNumber, Header as FullHeader}; use heapsize::HeapSizeOf; use rlp::{self, Rlp, RlpStream}; - use transaction::UnverifiedTransaction; use views::{self, BlockView, HeaderView, BodyView}; diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index 260c8727844..378b86c3ece 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -34,17 +34,13 @@ use ethjson; use machine::{AuxiliaryData, Call, EthereumMachine}; use hash::keccak; use header::{Header, BlockNumber, ExtendedHeader}; - use super::signer::EngineSigner; use super::validator_set::{ValidatorSet, SimpleList, new_validator_set}; - use self::finality::RollingFinality; - use ethkey::{self, Password, Signature}; use io::{IoContext, IoHandler, TimerToken, IoService}; use itertools::{self, Itertools}; use rlp::{encode, Decodable, DecoderError, Encodable, RlpStream, Rlp}; - use ethereum_types::{H256, H520, Address, U128, U256}; use parking_lot::{Mutex, RwLock}; use unexpected::{Mismatch, OutOfBounds}; diff --git a/ethcore/src/engines/tendermint/message.rs b/ethcore/src/engines/tendermint/message.rs index 073b04188fd..7d76c32a275 100644 --- a/ethcore/src/engines/tendermint/message.rs +++ b/ethcore/src/engines/tendermint/message.rs @@ -16,18 +16,17 @@ //! Tendermint message handling. -use std::cmp; -use hash::keccak; -use ethereum_types::{H256, H520, Address}; use bytes::Bytes; -use super::{Height, View, BlockHash, Step}; use error::Error; +use ethereum_types::{H256, H520, Address}; +use ethkey::{recover, public_to_address}; +use hash::keccak; use header::Header; use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError}; -use ethkey::{recover, public_to_address}; +use std::cmp; +use super::{Height, View, BlockHash, Step}; use super::super::vote_collector::Message; - /// Message transmitted between consensus participants. #[derive(Debug, PartialEq, Eq, Clone, Hash, Default)] pub struct ConsensusMessage { diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index f2158b75815..c95d61aa783 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -51,7 +51,6 @@ use self::message::*; use self::params::TendermintParams; use machine::{AuxiliaryData, EthereumMachine}; - #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] pub enum Step { Propose, diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index 4b7277fde8b..a7f4f2c731a 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -16,28 +16,23 @@ /// Validator set maintained in a contract, updated using `getValidators` method. -use std::sync::{Weak, Arc}; -use hash::keccak; - -use ethereum_types::{H256, U256, Address, Bloom}; -use parking_lot::RwLock; - use bytes::Bytes; -use memory_cache::MemoryLruCache; -use unexpected::Mismatch; -use rlp::{Rlp, RlpStream}; -use kvdb::DBValue; - use client::EngineClient; -use machine::{AuxiliaryData, Call, EthereumMachine, AuxiliaryRequest}; +use ethereum_types::{H256, U256, Address, Bloom}; +use hash::keccak; use header::Header; use ids::BlockId; +use kvdb::DBValue; use log_entry::LogEntry; +use machine::{AuxiliaryData, Call, EthereumMachine, AuxiliaryRequest}; +use memory_cache::MemoryLruCache; +use parking_lot::RwLock; use receipt::Receipt; - - +use rlp::{Rlp, RlpStream}; +use std::sync::{Weak, Arc}; use super::{SystemCall, ValidatorSet}; use super::simple_list::SimpleList; +use unexpected::Mismatch; use_contract!(validator_set, "ValidatorSet", "res/contracts/validator_set.json"); diff --git a/ethcore/src/executive.rs b/ethcore/src/executive.rs index c16be01609a..f375f3c2e88 100644 --- a/ethcore/src/executive.rs +++ b/ethcore/src/executive.rs @@ -34,7 +34,6 @@ use transaction::{Action, SignedTransaction}; use crossbeam; pub use executed::{Executed, ExecutionResult}; - #[cfg(debug_assertions)] /// Roughly estimate what stack size each level of evm depth will use. (Debug build) const STACK_SIZE_PER_DEPTH: usize = 128 * 1024; diff --git a/ethcore/src/header.rs b/ethcore/src/header.rs index eb86277d938..5aa4be32375 100644 --- a/ethcore/src/header.rs +++ b/ethcore/src/header.rs @@ -23,7 +23,6 @@ use ethereum_types::{H256, U256, Address, Bloom}; use bytes::Bytes; use rlp::{Rlp, RlpStream, Encodable, DecoderError, Decodable}; - pub use types::BlockNumber; /// Semantic boolean for when a seal/signature is included. diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 0a4e9fdb55d..daeed18ebfb 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -33,7 +33,6 @@ use trace::{VMTracer, NoopVMTracer}; use bytes::{Bytes, BytesRef}; use ethtrie; use rlp::RlpStream; - use hash::keccak; use machine::EthereumMachine as Machine; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 9875611de81..f35f0517780 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -30,7 +30,6 @@ use ethjson; use types::account_diff::*; use rlp::{self, RlpStream}; - #[derive(Debug, Clone, PartialEq, Eq)] /// An account, expressed as Plain-Old-Data (hence the name). /// Does not have a DB overlay cache, code hash or anything like that. diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index b62eb415c53..13ab4b227ff 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -18,18 +18,15 @@ use account_db::{AccountDB, AccountDBMut}; use basic_account::BasicAccount; -use snapshot::Error; -use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; - -use ethereum_types::{H256, U256}; -use hashdb::HashDB; use bytes::Bytes; -use trie::{Trie, TrieMut}; +use ethereum_types::{H256, U256}; use ethtrie::{TrieDB, TrieDBMut}; +use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP}; +use hashdb::HashDB; use rlp::{RlpStream, Rlp}; - - +use snapshot::Error; use std::collections::HashSet; +use trie::{Trie, TrieMut}; // An empty account -- these were replaced with RLP null data for a space optimization in v1. const ACC_EMPTY: BasicAccount = BasicAccount { diff --git a/ethcore/src/snapshot/block.rs b/ethcore/src/snapshot/block.rs index 18b72671607..3a18015e0c5 100644 --- a/ethcore/src/snapshot/block.rs +++ b/ethcore/src/snapshot/block.rs @@ -19,14 +19,12 @@ use block::Block; use header::Header; use hash::keccak; - use views::BlockView; use rlp::{DecoderError, RlpStream, Rlp}; use ethereum_types::H256; use bytes::Bytes; use triehash::ordered_trie_root; - const HEADER_FIELDS: usize = 8; const BLOCK_FIELDS: usize = 2; diff --git a/ethcore/src/snapshot/consensus/work.rs b/ethcore/src/snapshot/consensus/work.rs index 10c5c512df5..963a96a7f90 100644 --- a/ethcore/src/snapshot/consensus/work.rs +++ b/ethcore/src/snapshot/consensus/work.rs @@ -36,7 +36,6 @@ use bytes::Bytes; use rlp::{RlpStream, Rlp}; use rand::OsRng; - /// Snapshot creation and restoration for PoW chains. /// This includes blocks from the head of the chain as a /// loose assurance that the chain is valid. diff --git a/ethcore/src/snapshot/io.rs b/ethcore/src/snapshot/io.rs index 6852309980a..7d2cbcf92cf 100644 --- a/ethcore/src/snapshot/io.rs +++ b/ethcore/src/snapshot/io.rs @@ -29,7 +29,6 @@ use bytes::Bytes; use ethereum_types::H256; use rlp::{RlpStream, Rlp}; - use super::ManifestData; const SNAPSHOT_VERSION: u64 = 2; diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 9c78085f9db..f4b9bf69970 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -44,7 +44,6 @@ use ethtrie::{TrieDB, TrieDBMut}; use rlp::{RlpStream, Rlp}; use bloom_journal::Bloom; - use self::io::SnapshotWriter; use super::state_db::StateDB; diff --git a/ethcore/src/snapshot/tests/proof_of_work.rs b/ethcore/src/snapshot/tests/proof_of_work.rs index bb19ebe3d0d..40733c3285f 100644 --- a/ethcore/src/snapshot/tests/proof_of_work.rs +++ b/ethcore/src/snapshot/tests/proof_of_work.rs @@ -115,7 +115,6 @@ fn chunk_and_restore_4k() { #[test] fn checks_flag() { use rlp::RlpStream; - use ethereum_types::H256; let mut stream = RlpStream::new_list(5); diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index b49de24d22b..12f19e8c27c 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -95,7 +95,6 @@ fn snap_and_restore() { fn get_code_from_prev_chunk() { use std::collections::HashSet; use rlp::RlpStream; - use ethereum_types::{H256, U256}; use hashdb::HashDB; diff --git a/ethcore/src/spec/seal.rs b/ethcore/src/spec/seal.rs index 0da9aceef56..0ed41acc846 100644 --- a/ethcore/src/spec/seal.rs +++ b/ethcore/src/spec/seal.rs @@ -20,7 +20,6 @@ use rlp::RlpStream; use ethereum_types::{H64, H256, H520}; use ethjson; - /// Classic ethereum seal. pub struct Ethereum { /// Seal nonce. diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index ef1c8e4b77c..6f785fe7fba 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -46,7 +46,6 @@ use state::backend::Basic as BasicBackend; use state::{Backend, State, Substate}; use trace::{NoopTracer, NoopVMTracer}; - pub use ethash::OptimizeFor; const MAX_TRANSACTION_SIZE: usize = 300 * 1024; diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index 307a84cf7d6..359b84b1eca 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -33,7 +33,6 @@ use rlp::{RlpStream, encode}; use lru_cache::LruCache; use basic_account::BasicAccount; - use std::cell::{RefCell, Cell}; const STORAGE_CACHE_ITEMS: usize = 8192; diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs index ca5a42f7eab..95cf7332861 100644 --- a/ethcore/src/state_db.rs +++ b/ethcore/src/state_db.rs @@ -34,7 +34,6 @@ use std::collections::{VecDeque, HashSet}; use std::sync::Arc; use util_error::UtilError; - /// Value used to initialize bloom bitmap size. /// /// Bitmap size is the size in bytes (not bits) that will be allocated in memory. diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index 2fb90badafc..438d7c82a19 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -40,7 +40,6 @@ use state::*; use std::sync::Arc; use transaction::{Action, Transaction, SignedTransaction}; use views::BlockView; - use blooms_db; use kvdb::KeyValueDB; use kvdb_rocksdb; diff --git a/ethcore/src/tests/client.rs b/ethcore/src/tests/client.rs index a0d2b648684..409248a2ea3 100644 --- a/ethcore/src/tests/client.rs +++ b/ethcore/src/tests/client.rs @@ -36,7 +36,6 @@ use ethkey::KeyPair; use transaction::{PendingTransaction, Transaction, Action, Condition}; use miner::MinerService; use rlp::{RlpStream, EMPTY_LIST_RLP}; - use tempdir::TempDir; use test_helpers; diff --git a/ethcore/src/trace/types/error.rs b/ethcore/src/trace/types/error.rs index 9b72e832afc..a934443c5db 100644 --- a/ethcore/src/trace/types/error.rs +++ b/ethcore/src/trace/types/error.rs @@ -138,7 +138,6 @@ impl Decodable for Error { #[cfg(test)] mod tests { use rlp::*; - use super::Error; #[test] diff --git a/ethcore/src/trace/types/flat.rs b/ethcore/src/trace/types/flat.rs index 5de8326accc..8610692200c 100644 --- a/ethcore/src/trace/types/flat.rs +++ b/ethcore/src/trace/types/flat.rs @@ -137,7 +137,6 @@ impl Into> for FlatBlockTraces { #[cfg(test)] mod tests { use rlp::*; - use super::{FlatBlockTraces, FlatTransactionTraces, FlatTrace}; use trace::trace::{Action, Res, CallResult, Call, Suicide, Reward}; use evm::CallType; diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 35ee2fb9daf..ebb8e79a704 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -365,7 +365,6 @@ mod tests { use transaction::{SignedTransaction, Transaction, UnverifiedTransaction, Action}; use types::log_entry::{LogEntry, LocalizedLogEntry}; use rlp; - use triehash::ordered_trie_root; fn check_ok(result: Result<(), Error>) { @@ -547,7 +546,6 @@ mod tests { #[test] fn test_verify_block() { use rlp::RlpStream; - // Test against morden let mut good = Header::new(); diff --git a/ethcore/sync/src/blocks.rs b/ethcore/sync/src/blocks.rs index d4b39b1d7c8..8485b1d75e6 100644 --- a/ethcore/sync/src/blocks.rs +++ b/ethcore/sync/src/blocks.rs @@ -23,7 +23,6 @@ use ethereum_types::H256; use triehash::ordered_trie_root; use bytes::Bytes; use rlp::{Rlp, RlpStream, DecoderError}; - use network; use ethcore::encoded::Block; use ethcore::views::{HeaderView, BodyView}; diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 79bfb40d6c2..84e6344e688 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -103,7 +103,6 @@ use plain_hasher::H256FastMap; use parking_lot::RwLock; use bytes::Bytes; use rlp::{Rlp, RlpStream, DecoderError}; - use network::{self, PeerId, PacketId}; use ethcore::header::{BlockNumber}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockQueueInfo}; diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index f05c628c6ca..75cf550f28b 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -24,7 +24,6 @@ use rlp::{Encodable, RlpStream}; use sync_io::SyncIo; use std::cmp; use std::collections::HashSet; - use transaction::SignedTransaction; use super::{ diff --git a/ethcore/sync/src/chain/requester.rs b/ethcore/sync/src/chain/requester.rs index a71b583ddc9..a85874d2927 100644 --- a/ethcore/sync/src/chain/requester.rs +++ b/ethcore/sync/src/chain/requester.rs @@ -22,7 +22,6 @@ use ethereum_types::H256; use network::{PeerId, PacketId}; use rlp::RlpStream; use std::time::Instant; - use sync_io::SyncIo; use super::{ diff --git a/ethcore/transaction/src/transaction.rs b/ethcore/transaction/src/transaction.rs index 0a1248b3e83..27b8b346cf2 100644 --- a/ethcore/transaction/src/transaction.rs +++ b/ethcore/transaction/src/transaction.rs @@ -26,7 +26,6 @@ use hash::keccak; use heapsize::HeapSizeOf; use rlp::{self, RlpStream, Rlp, DecoderError, Encodable}; - type Bytes = Vec; type BlockNumber = u64; diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index c88199ad2f0..5ac8ff12add 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -20,7 +20,6 @@ extern crate ethereum_types; extern crate ethcore_bytes as bytes; extern crate ethjson; extern crate rlp; - #[macro_use] extern crate rlp_derive; extern crate keccak_hash as hash; diff --git a/ethcore/types/src/snapshot_manifest.rs b/ethcore/types/src/snapshot_manifest.rs index bb6d730ac2a..40ff4c532f7 100644 --- a/ethcore/types/src/snapshot_manifest.rs +++ b/ethcore/types/src/snapshot_manifest.rs @@ -20,7 +20,6 @@ use ethereum_types::H256; use rlp::{Rlp, RlpStream, DecoderError}; use bytes::Bytes; - /// Manifest data. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ManifestData { diff --git a/ethcore/vm/src/call_type.rs b/ethcore/vm/src/call_type.rs index 5a9716ea34d..0e58d76bbd2 100644 --- a/ethcore/vm/src/call_type.rs +++ b/ethcore/vm/src/call_type.rs @@ -63,7 +63,6 @@ impl Decodable for CallType { mod tests { use rlp::*; use super::CallType; - #[test] fn encode_call_type() { diff --git a/util/journaldb/src/earlymergedb.rs b/util/journaldb/src/earlymergedb.rs index 77adeac7156..bee63ae7f87 100644 --- a/util/journaldb/src/earlymergedb.rs +++ b/util/journaldb/src/earlymergedb.rs @@ -106,7 +106,6 @@ enum RemoveFrom { /// ``` /// /// TODO: `store_reclaim_period` - pub struct EarlyMergeDB { overlay: MemoryDB, backing: Arc, diff --git a/util/network-devp2p/src/connection.rs b/util/network-devp2p/src/connection.rs index 4a7ed864996..37824ae5d75 100644 --- a/util/network-devp2p/src/connection.rs +++ b/util/network-devp2p/src/connection.rs @@ -37,7 +37,6 @@ use bytes::{Buf, BufMut}; use ethkey::crypto; use network::{Error, ErrorKind}; - const ENCRYPTED_HEADER_LEN: usize = 32; const RECEIVE_PAYLOAD: Duration = Duration::from_secs(30); pub const MAX_PAYLOAD_SIZE: usize = (1 << 24) - 1; diff --git a/util/network-devp2p/src/discovery.rs b/util/network-devp2p/src/discovery.rs index 39807719067..5f8f0cdbc2a 100644 --- a/util/network-devp2p/src/discovery.rs +++ b/util/network-devp2p/src/discovery.rs @@ -27,7 +27,6 @@ use network::{Error, ErrorKind}; use ethkey::{Secret, KeyPair, sign, recover}; use network::IpFilter; - use PROTOCOL_VERSION; const ADDRESS_BYTES_SIZE: usize = 32; // Size of address type in bytes. diff --git a/util/network-devp2p/src/handshake.rs b/util/network-devp2p/src/handshake.rs index e0075f94111..18869de55fc 100644 --- a/util/network-devp2p/src/handshake.rs +++ b/util/network-devp2p/src/handshake.rs @@ -29,7 +29,6 @@ use ethkey::crypto::{ecdh, ecies}; use network::{Error, ErrorKind}; use host::HostInfo; - #[derive(PartialEq, Eq, Debug)] enum HandshakeState { /// Just created diff --git a/util/network-devp2p/src/session.rs b/util/network-devp2p/src/session.rs index 3fe6b15aaf5..a405ad469dd 100644 --- a/util/network-devp2p/src/session.rs +++ b/util/network-devp2p/src/session.rs @@ -33,7 +33,6 @@ use host::*; use node_table::NodeId; use snappy; - // Timeout must be less than (interval - 1). const PING_TIMEOUT: Duration = Duration::from_secs(60); const PING_INTERVAL: Duration = Duration::from_secs(120); diff --git a/util/rlp_compress/src/lib.rs b/util/rlp_compress/src/lib.rs index 20338439278..af5b09aac79 100644 --- a/util/rlp_compress/src/lib.rs +++ b/util/rlp_compress/src/lib.rs @@ -19,7 +19,6 @@ use elastic_array::ElasticArray1024; use rlp::{Rlp, RlpStream}; use common::{SNAPSHOT_SWAPPER, BLOCKS_SWAPPER}; - pub fn snapshot_swapper() -> &'static Swapper<'static> { &SNAPSHOT_SWAPPER as &Swapper } diff --git a/util/version/src/lib.rs b/util/version/src/lib.rs index 4d5757ffce5..77fc71c70f3 100644 --- a/util/version/src/lib.rs +++ b/util/version/src/lib.rs @@ -24,7 +24,6 @@ use target_info::Target; use bytes::Bytes; use rlp::RlpStream; - mod vergen { #![allow(unused)] include!(concat!(env!("OUT_DIR"), "/version.rs")); diff --git a/whisper/src/message.rs b/whisper/src/message.rs index 7d29ee767b1..95c21125513 100644 --- a/whisper/src/message.rs +++ b/whisper/src/message.rs @@ -21,7 +21,6 @@ use std::time::{self, SystemTime, Duration, Instant}; use ethereum_types::{H256, H512}; use rlp::{self, DecoderError, RlpStream, Rlp}; - use smallvec::SmallVec; use tiny_keccak::{keccak256, Keccak}; diff --git a/whisper/src/net/mod.rs b/whisper/src/net/mod.rs index 6a4b220b090..6ec3b08a548 100644 --- a/whisper/src/net/mod.rs +++ b/whisper/src/net/mod.rs @@ -28,7 +28,6 @@ use ordered_float::OrderedFloat; use parking_lot::{Mutex, RwLock}; use rlp::{DecoderError, RlpStream, Rlp}; - use message::{Message, Error as MessageError}; #[cfg(test)]