Skip to content
This repository has been archived by the owner on Nov 6, 2020. It is now read-only.

Make HashDB generic #8739

Merged
merged 185 commits into from
Jul 2, 2018
Merged
Show file tree
Hide file tree
Changes from 167 commits
Commits
Show all changes
185 commits
Select commit Hold shift + click to select a range
e0c064a
WIP Make HashDB generic
dvdplm May 26, 2018
1cd7ded
Add HeapSizeOf bound to the output type of a Hasher
dvdplm May 27, 2018
4bf7f6d
Add comments and todos
dvdplm May 27, 2018
fd666d6
Cleanup and more todos
dvdplm May 27, 2018
5325a70
Move concrete hasher impl to hashdb – TODO: where do these go?
dvdplm May 28, 2018
041d6a2
Remove unused crates/uses
dvdplm May 28, 2018
f716105
Version bump – breaking change => 0.2.0
dvdplm May 28, 2018
8ce39eb
Version bump
dvdplm May 28, 2018
a7ff7c2
Derive Clone and PartialEq for KeccakHasher to satisfy the code in jo…
dvdplm May 28, 2018
ef1e48a
Thread `Hasher` throughout the crate – WIP
dvdplm May 28, 2018
3a7c490
Use v0.2 of hashdb and memorydb
dvdplm May 28, 2018
83342c0
Make journaldb use generic hashdb/memorydb
dvdplm May 28, 2018
aa268bd
Version bump: use newer hasdb
dvdplm May 28, 2018
bf01073
Additional bounds on Hasher output
dvdplm May 30, 2018
013666f
TrieDB, TrieDBMut, SecTrieDB, FatDB typechecks
dvdplm May 30, 2018
aabe7dd
Merge remote-tracking branch 'origin/master' into refactor/hashdb-gen…
dvdplm May 30, 2018
1f892fe
Fix tests
dvdplm May 30, 2018
f98e905
FatDBMut and test fixes
dvdplm May 30, 2018
b42f92e
FatDBMut test
dvdplm May 30, 2018
5e9fda0
SecTrieDB tests
dvdplm May 30, 2018
841abc4
Tests for TrieDBMut
dvdplm May 30, 2018
f29132c
Recorder tests
dvdplm May 30, 2018
201cdcb
SecTrieDBMut + test
dvdplm May 30, 2018
6bf7e5b
Remove scratch pad files
dvdplm May 30, 2018
fc058e6
Remove more traces of scratch pad files
dvdplm May 30, 2018
03ff20e
Add a test for using MemoryDB with a different hasher (TODO)
dvdplm May 30, 2018
8082d40
cleanup
dvdplm May 30, 2018
c9fc6a9
cleanup
dvdplm May 31, 2018
13d1de3
Comment out WIP test
dvdplm May 31, 2018
33e0531
cleanup
dvdplm May 31, 2018
ef707b0
TrieError<T> needs T: Debug
dvdplm May 31, 2018
ada0b7e
Cleanup
dvdplm May 31, 2018
56af2c4
Don't compare rc against i32::MIN
dvdplm May 31, 2018
cb61a01
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 1, 2018
3f1cfc5
missing use's
dvdplm Jun 1, 2018
4e2e81b
Remove Decodable bound
dvdplm Jun 4, 2018
c78037c
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 4, 2018
c149786
Comment out fatdb/sectriedb
dvdplm Jun 4, 2018
68fa728
Add `NodeCodec` with impl for `Node` (the Rlp-kind)
dvdplm Jun 4, 2018
c198d0f
WIP try to extract more rlp-dependent code into `NodeCodec` trait
dvdplm Jun 4, 2018
a7e9cbf
WIP Add NodeCodec in the proper place
dvdplm Jun 5, 2018
8bc455c
Fix FatDB, FatDBMut, SecTrieDB, SecTrieDBMut
dvdplm Jun 6, 2018
6d89eba
Make NodeCodec error type an associated type
dvdplm Jun 6, 2018
3d69e55
Minor doc changes
dvdplm Jun 6, 2018
e905494
Fix benches
dvdplm Jun 6, 2018
2a579a3
Handle DecoderError better
dvdplm Jun 7, 2018
12ea521
Include the DecoderError as a Box<Error>
dvdplm Jun 7, 2018
4fd7990
Merge remote-tracking branch 'origin/master' into refactor/hashdb-gen…
dvdplm Jun 7, 2018
829325e
Store a `String` in the TrieError::DecoderError variant instead of a …
dvdplm Jun 8, 2018
494e58f
WIP use generic hashdb in ethcore
dvdplm Jun 11, 2018
aaf5f4c
Switch JournalDB to use type param'd HashDB
dvdplm Jun 11, 2018
80f5545
Switch to use type param instead of assoc type
dvdplm Jun 11, 2018
89e22d1
Fix memorydb
dvdplm Jun 11, 2018
f374cc5
Fix patricia_trie
dvdplm Jun 11, 2018
e6e35b7
Fix ethcore/src with generic HashDB
dvdplm Jun 11, 2018
7d2a414
Fix running tests depending on the parent `test-helpers` feature
dvdplm Jun 11, 2018
ae55ca4
Fix ethcore/light
dvdplm Jun 11, 2018
6b4095e
"Handle" Trie errors
dvdplm Jun 11, 2018
4288d49
Add `version` key for new generic crates
dvdplm Jun 11, 2018
1d5f5e2
New cargo lockfile
dvdplm Jun 11, 2018
33f4e79
Fix tests
dvdplm Jun 11, 2018
a567462
Add review comment about error_chain and generic errors
dvdplm Jun 11, 2018
b146e3c
Bump version
dvdplm Jun 11, 2018
f3dd59b
Generic hasdb
dvdplm Jun 11, 2018
6f48370
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 11, 2018
1741d81
Merge branch 'refactor/hashdb-generic-hasher-as-type-param-not-assoc-…
dvdplm Jun 11, 2018
0374aa8
Fix merge issues
dvdplm Jun 11, 2018
1a1f98e
cleanup
dvdplm Jun 11, 2018
5d12612
Fix build error
dvdplm Jun 11, 2018
b9f1760
Fix json_tests
dvdplm Jun 11, 2018
4d733bc
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 12, 2018
fe91723
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 13, 2018
468525e
Minor cleanup
dvdplm Jun 13, 2018
071fa8f
cleanup
dvdplm Jun 13, 2018
03d266d
Add more questions to reviewers wrt error handling in ethcore
dvdplm Jun 13, 2018
9ce95cd
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 14, 2018
5db5de9
Remove `Decodable` bound and reduce the nr of `Encodable` bounds requ…
dvdplm Jun 14, 2018
b2e8b93
Add a rlp::Stream trait with the stuff we need in patricia_trie
dvdplm Jun 14, 2018
4f172f7
Get rid of more RLP dependent code
dvdplm Jun 14, 2018
a6fd313
Add special-cased `append_bytes` to `Stream` to work around the `Enco…
dvdplm Jun 14, 2018
073c213
Add trie-node-codec crate and move Stream trait there
dvdplm Jun 14, 2018
cf51e79
Rename trie-node-codec to stream-encoder
dvdplm Jun 15, 2018
574a22a
Address some grumbles
dvdplm Jun 15, 2018
42df94e
Add `LENGTH` const to `Hasher` and use it to remove the assumption of…
dvdplm Jun 15, 2018
f081b35
remove solved review comment
dvdplm Jun 15, 2018
c6333af
Add stream_encoder::Stream where needed
dvdplm Jun 15, 2018
c8a1c93
Fix tests in ethcore
dvdplm Jun 15, 2018
a66596d
A few grumbles and made more tests passing
dvdplm Jun 15, 2018
c5c6021
fix more tests
dvdplm Jun 15, 2018
8b219ad
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 16, 2018
f6cd87b
grumble: Idiomatic use of Result
dvdplm Jun 16, 2018
f57565c
Add keccak-hasher crate that implements Hasher with the Keccak-256 hash
dvdplm Jun 16, 2018
c2c7789
Remove KeccakHasher impl from hashdb
dvdplm Jun 16, 2018
046b5d6
memorydb: Use keccak_hasher::KeccakHasher
dvdplm Jun 16, 2018
91fd334
patricia_trie: Use keccak_hasher::KeccakHasher
dvdplm Jun 16, 2018
57b79b8
Add keccak-hasher crate to main Cargo.toml
dvdplm Jun 16, 2018
f3da9b1
journaldb: Use keccak-hasher
dvdplm Jun 16, 2018
5f47442
cargo lock changes
dvdplm Jun 16, 2018
b0c715b
rlp: Fix doc tests
dvdplm Jun 16, 2018
730099a
ethcore: Use keccak-hasher
dvdplm Jun 16, 2018
0e5fdfa
cargo lock
dvdplm Jun 16, 2018
5ac48b3
sync: Use keccak-hasher
dvdplm Jun 16, 2018
67bf644
Add patricia-trie-ethereum façade crate for keccak/rlp concrete impls…
dvdplm Jun 18, 2018
b8e9764
Remove all Rlp dependencies from patricia_trie
dvdplm Jun 18, 2018
c1eee0c
cleanup nibbleslice
debris Jun 18, 2018
60e7c90
Fix ethcore tests
dvdplm Jun 18, 2018
1d7dc83
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 18, 2018
50d66fd
Merge remote-tracking branch 'origin/nibbleslice_cleanup' into refact…
dvdplm Jun 18, 2018
28358da
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 18, 2018
acd757d
Fix tests in ethcore/light
dvdplm Jun 18, 2018
3376c7d
Fix broken iterator Item type
dvdplm Jun 18, 2018
46609fc
Export an ethereum TrieError
dvdplm Jun 18, 2018
40c4beb
Fix trie error handling in ethcore
dvdplm Jun 18, 2018
9cd8145
Add stream_encoder and patricia-trie-ethereum where needed
dvdplm Jun 19, 2018
1eeaa1f
lockfile
dvdplm Jun 19, 2018
bc54358
fix docs
dvdplm Jun 19, 2018
f5c0440
TrieError takes two type params
dvdplm Jun 20, 2018
7ccb350
Export concrete impls of the Trie result/error
dvdplm Jun 20, 2018
15a754b
Use better result/error types in ethcore
dvdplm Jun 20, 2018
f1be1a1
lock file
dvdplm Jun 20, 2018
b48e2c8
Do not export a Keccak impl of MemoryDB
dvdplm Jun 20, 2018
c366b2a
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 20, 2018
4febf0d
Cleanup rlp Stream and add test for append_bytes
dvdplm Jun 20, 2018
5f3ba13
Fix json_tests
dvdplm Jun 20, 2018
80af9db
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 20, 2018
bcd1ad2
Fix benches
dvdplm Jun 20, 2018
ce5d5b9
Minor cleanup
dvdplm Jun 20, 2018
1d5c2b3
Cleanup
dvdplm Jun 22, 2018
438e21c
Move the null node hash from the Hasher trait to NodeCodec
dvdplm Jun 22, 2018
27fbbb3
Remove TODO
dvdplm Jun 22, 2018
9c84453
cleanup
dvdplm Jun 22, 2018
0142e70
Box TrieErrors
dvdplm Jun 22, 2018
3e85645
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 22, 2018
deed06e
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 22, 2018
5666873
whitespace
dvdplm Jun 22, 2018
8bd8847
Colesce notes on why the AsHashDB trait looks the way it does
dvdplm Jun 22, 2018
e54ff1f
Fix Rlp benches
dvdplm Jun 22, 2018
027862a
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 25, 2018
37685ad
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 25, 2018
0aab7e2
Address docs grumble
dvdplm Jun 25, 2018
b5e4485
Move AsHashDB impls out of the way
dvdplm Jun 25, 2018
ae5346e
Add missing docs
dvdplm Jun 25, 2018
66d3b76
Missing docs
dvdplm Jun 25, 2018
06fd57e
Add missing file
dvdplm Jun 25, 2018
d1ae199
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 25, 2018
7f004a9
Fix deps
dvdplm Jun 25, 2018
75d2d23
Add some simple benches for memorydb
dvdplm Jun 25, 2018
c061f38
Add dep for memorydb benches
dvdplm Jun 25, 2018
5d03e71
Move RlpNodeCodec impl to own file
dvdplm Jun 26, 2018
54ab636
WIP typechecks, failing tests
dvdplm Jun 26, 2018
7cbe20f
WIP instrument into_encoded
dvdplm Jun 26, 2018
db2346d
lock
dvdplm Jun 26, 2018
31c4cb4
WIP instrument into_encoded
dvdplm Jun 26, 2018
4fcd004
Heavily instrumented test
dvdplm Jun 26, 2018
aab4077
Remove instrumentation and cleanup
dvdplm Jun 27, 2018
a88616f
Remove stream_encoder
dvdplm Jun 27, 2018
17e6c92
Remove stream_encoder usage
dvdplm Jun 27, 2018
e1c4bea
Remove stream_encoder crate
dvdplm Jun 27, 2018
03604e8
Merge branch 'refactor/hashdb-generic-without-stream-encoder' into re…
dvdplm Jun 27, 2018
bfcf114
Remove unused extern crate
dvdplm Jun 27, 2018
c45cad9
Preallocate space for trail and key_nibbles (minor speedup)
dvdplm Jun 27, 2018
627567d
Ignore build artifacts
dvdplm Jun 27, 2018
550f852
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jun 27, 2018
f55d96a
Use RlpStream::append
dvdplm Jun 27, 2018
6bd549a
Inline NibbleVec::len and NibbleVec::at
dvdplm Jun 27, 2018
e4cb2ac
Aggressive inlining of NibbleSlice::at and NibbleSlice::encoded
dvdplm Jun 27, 2018
fc3d48c
Remove bench profile
dvdplm Jun 27, 2018
1cf7c28
Fix tests
dvdplm Jun 27, 2018
44bb8b6
Rename NodeCodec::E to Error
dvdplm Jun 27, 2018
29bf049
Make Recorder and Record generic over the Hasher output rather than t…
dvdplm Jun 27, 2018
7be2120
Fix trat descr
dvdplm Jun 28, 2018
211882e
Remove dead code
dvdplm Jun 28, 2018
609ebde
Ditch ethcore-logger for env-logger
dvdplm Jun 28, 2018
308f281
Parametrize the HashMap builder from the Hasher
dvdplm Jun 28, 2018
1aaaa63
Less aggressive inlining
dvdplm Jun 28, 2018
b76fffa
Add convenience types and use them for less line noise
dvdplm Jun 28, 2018
92582c7
Remove unused deps
dvdplm Jun 29, 2018
988bfaf
Clean up tests
dvdplm Jun 29, 2018
f487a69
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jul 2, 2018
d625d8e
Address grumbles
dvdplm Jul 2, 2018
67003be
Remove tempdir as test dep
dvdplm Jul 2, 2018
412ae9c
Boxed errors
dvdplm Jul 2, 2018
07dfdc7
Remove spurious whitespace after use blocks
dvdplm Jul 2, 2018
afbf3a9
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jul 2, 2018
b9e310b
Merge branch 'master' into refactor/hashdb-generic
dvdplm Jul 2, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ node_modules

# Build artifacts
out/
parity-clib-example/build/

.vscode
rls/
Expand Down
90 changes: 63 additions & 27 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ members = [
"transaction-pool",
"whisper",
"whisper/cli",
"util/keccak-hasher",
"util/patricia-trie-ethereum",
]

[patch.crates-io]
Expand Down
5 changes: 4 additions & 1 deletion ethcore/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ fetch = { path = "../util/fetch" }
hashdb = { path = "../util/hashdb" }
memorydb = { path = "../util/memorydb" }
patricia-trie = { path = "../util/patricia_trie" }
patricia-trie-ethereum = { path = "../util/patricia-trie-ethereum" }
ethcore-crypto = { path = "crypto" }
error-chain = { version = "0.12", default-features = false }
ethcore-io = { path = "../util/io" }
Expand Down Expand Up @@ -66,8 +67,9 @@ keccak-hash = { path = "../util/hash" }
triehash = { path = "../util/triehash" }
unexpected = { path = "../util/unexpected" }
journaldb = { path = "../util/journaldb" }
tempdir = { version = "0.3", optional = true }
keccak-hasher = { path = "../util/keccak-hasher" }
kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
tempdir = {version="0.3", optional = true}

[target.'cfg(any(target_os = "linux", target_os = "macos", target_os = "windows", target_os = "android"))'.dependencies]
hardware-wallet = { path = "../hw" }
Expand All @@ -76,6 +78,7 @@ hardware-wallet = { path = "../hw" }
fake-hardware-wallet = { path = "../util/fake-hardware-wallet" }

[dev-dependencies]
tempdir = "0.3"
trie-standardmap = { path = "../util/trie-standardmap" }

[features]
Expand Down
2 changes: 2 additions & 0 deletions ethcore/light/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ ethcore-transaction = { path = "../transaction" }
ethereum-types = "0.3"
memorydb = { path = "../../util/memorydb" }
patricia-trie = { path = "../../util/patricia_trie" }
patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
ethcore-network = { path = "../../util/network" }
ethcore-io = { path = "../../util/io" }
hashdb = { path = "../../util/hashdb" }
Expand All @@ -32,6 +33,7 @@ serde_derive = "1.0"
parking_lot = "0.5"
stats = { path = "../../util/stats" }
keccak-hash = { path = "../../util/hash" }
keccak-hasher = { path = "../../util/keccak-hasher" }
triehash = { path = "../../util/triehash" }
kvdb = { path = "../../util/kvdb" }
memory-cache = { path = "../../util/memory_cache" }
Expand Down
23 changes: 13 additions & 10 deletions ethcore/light/src/cht.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,14 @@
use ethcore::ids::BlockId;
use ethereum_types::{H256, U256};
use hashdb::HashDB;
use keccak_hasher::KeccakHasher;
use memorydb::MemoryDB;
use bytes::Bytes;
use trie::{self, TrieMut, TrieDBMut, Trie, TrieDB, Recorder};
use trie::{TrieMut, TrieDBMut, Trie, TrieDB, Recorder};
use ethtrie::{self, RlpCodec};
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just commenting here because it's the first file I see this in)

It can be overly verbose to rely on 3 crates in order to use patricia-trie-ethereum. It would be easier to use if we re-exported the API of keccak_hasher and patricia-trie from patricia-trie-ethereum, maybe using pub extern crate declarations there.

use rlp::{RlpStream, Rlp};


// encode a key.
macro_rules! key {
($num: expr) => { ::rlp::encode(&$num) }
Expand All @@ -50,13 +53,13 @@ pub const SIZE: u64 = 2048;
/// A canonical hash trie. This is generic over any database it can query.
/// See module docs for more details.
#[derive(Debug, Clone)]
pub struct CHT<DB: HashDB> {
pub struct CHT<DB: HashDB<KeccakHasher>> {
db: DB,
root: H256, // the root of this CHT.
number: u64,
}

impl<DB: HashDB> CHT<DB> {
impl<DB: HashDB<KeccakHasher>> CHT<DB> {
/// Query the root of the CHT.
pub fn root(&self) -> H256 { self.root }

Expand All @@ -66,11 +69,11 @@ impl<DB: HashDB> CHT<DB> {
/// Generate an inclusion proof for the entry at a specific block.
/// Nodes before level `from_level` will be omitted.
/// Returns an error on an incomplete trie, and `Ok(None)` on an unprovable request.
pub fn prove(&self, num: u64, from_level: u32) -> trie::Result<Option<Vec<Bytes>>> {
pub fn prove(&self, num: u64, from_level: u32) -> ethtrie::Result<Option<Vec<Bytes>>> {
if block_to_cht_number(num) != Some(self.number) { return Ok(None) }

let mut recorder = Recorder::with_depth(from_level);
let t = TrieDB::new(&self.db, &self.root)?;
let t = TrieDB::<_, RlpCodec>::new(&self.db, &self.root)?;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there ought to be a type alias for TrieDB and TrieDBMut in patricia-trie so we don't need explicit type annotations

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in patricia-trie

You mean in patricia-trie-ethereum?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah

t.get_with(&key!(num), &mut recorder)?;

Ok(Some(recorder.drain().into_iter().map(|x| x.data).collect()))
Expand All @@ -90,10 +93,10 @@ pub struct BlockInfo {
/// Build an in-memory CHT from a closure which provides necessary information
/// about blocks. If the fetcher ever fails to provide the info, the CHT
/// will not be generated.
pub fn build<F>(cht_num: u64, mut fetcher: F) -> Option<CHT<MemoryDB>>
pub fn build<F>(cht_num: u64, mut fetcher: F) -> Option<CHT<MemoryDB<KeccakHasher>>>
where F: FnMut(BlockId) -> Option<BlockInfo>
{
let mut db = MemoryDB::new();
let mut db = MemoryDB::<KeccakHasher>::new();

// start from the last block by number and work backwards.
let last_num = start_number(cht_num + 1) - 1;
Expand All @@ -102,7 +105,7 @@ pub fn build<F>(cht_num: u64, mut fetcher: F) -> Option<CHT<MemoryDB>>
let mut root = H256::default();

{
let mut t = TrieDBMut::new(&mut db, &mut root);
let mut t = TrieDBMut::<_, RlpCodec>::new(&mut db, &mut root);
for blk_num in (0..SIZE).map(|n| last_num - n) {
let info = match fetcher(id) {
Some(info) => info,
Expand Down Expand Up @@ -147,10 +150,10 @@ pub fn compute_root<I>(cht_num: u64, iterable: I) -> Option<H256>
/// verify the given trie branch and extract the canonical hash and total difficulty.
// TODO: better support for partially-checked queries.
pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256)> {
let mut db = MemoryDB::new();
let mut db = MemoryDB::<KeccakHasher>::new();

for node in proof { db.insert(&node[..]); }
let res = match TrieDB::new(&db, &root) {
let res = match TrieDB::<_, RlpCodec>::new(&db, &root) {
Err(_) => return None,
Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| {
let rlp = Rlp::new(val);
Expand Down
Loading