From 6fc5014b4dbef43435b4531e4b70df36dc478ff5 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 19 Jun 2019 13:54:05 +0200 Subject: [PATCH 01/15] Remove calls to heapsize (#10432) * update memorydb trait * use malloc_size_of instead of heapsize_of * use jemalloc as default allocator for parity client. --- Cargo.lock | 210 +++++++----- Cargo.toml | 5 +- accounts/ethkey/Cargo.toml | 2 +- accounts/ethkey/src/lib.rs | 2 +- accounts/ethkey/src/secret.rs | 2 +- ethcore/Cargo.toml | 10 +- ethcore/blockchain/Cargo.toml | 2 +- ethcore/blockchain/src/blockchain.rs | 24 +- ethcore/blockchain/src/lib.rs | 3 + ethcore/db/Cargo.toml | 2 +- ethcore/db/src/keys.rs | 24 +- ethcore/db/src/lib.rs | 3 + ethcore/evm/Cargo.toml | 2 +- ethcore/evm/benches/basic.rs | 2 +- ethcore/evm/src/interpreter/shared_cache.rs | 9 +- ethcore/evm/src/lib.rs | 2 +- ethcore/light/Cargo.toml | 8 +- ethcore/light/src/cache.rs | 12 +- ethcore/light/src/cht.rs | 5 +- ethcore/light/src/client/header_chain.rs | 19 +- ethcore/light/src/client/mod.rs | 4 +- ethcore/light/src/lib.rs | 4 +- ethcore/light/src/on_demand/request.rs | 2 +- ethcore/private-tx/Cargo.toml | 4 +- ethcore/private-tx/src/lib.rs | 2 +- .../private-tx/src/private_transactions.rs | 4 +- ethcore/src/account_db.rs | 70 ++-- ethcore/src/block.rs | 7 +- ethcore/src/client/client.rs | 4 +- .../src/engines/validator_set/simple_list.rs | 10 +- ethcore/src/engines/validator_set/test.rs | 11 +- ethcore/src/lib.rs | 4 +- ethcore/src/pod_account.rs | 2 +- ethcore/src/snapshot/account.rs | 10 +- ethcore/src/snapshot/mod.rs | 6 +- ethcore/src/snapshot/tests/state.rs | 8 +- ethcore/src/spec/spec.rs | 2 +- ethcore/src/state/account.rs | 6 +- ethcore/src/state/backend.rs | 52 +-- ethcore/src/state/mod.rs | 2 +- ethcore/src/trace/db.rs | 6 +- ethcore/src/trace/types/flat.rs | 28 +- ethcore/src/verification/queue/kind.rs | 21 +- ethcore/src/verification/queue/mod.rs | 21 +- ethcore/src/verification/verification.rs | 11 +- ethcore/sync/Cargo.toml | 4 +- ethcore/sync/src/api.rs | 2 +- ethcore/sync/src/block_sync.rs | 10 +- ethcore/sync/src/blocks.rs | 41 +-- ethcore/sync/src/chain/mod.rs | 16 +- ethcore/sync/src/lib.rs | 4 +- ethcore/sync/src/snapshot.rs | 1 + ethcore/sync/src/transactions_stats.rs | 4 +- ethcore/types/Cargo.toml | 2 +- ethcore/types/src/encoded.rs | 20 +- ethcore/types/src/header.rs | 10 +- ethcore/types/src/lib.rs | 4 +- ethcore/types/src/log_entry.rs | 10 +- ethcore/types/src/receipt.rs | 12 +- ethcore/types/src/transaction/transaction.rs | 29 +- ethcore/vm/Cargo.toml | 2 +- miner/Cargo.toml | 2 +- miner/src/lib.rs | 3 +- miner/src/pool/mod.rs | 4 +- util/journaldb/Cargo.toml | 6 +- util/journaldb/src/archivedb.rs | 180 +++++----- util/journaldb/src/earlymergedb.rs | 296 ++++++++-------- util/journaldb/src/lib.rs | 16 +- util/journaldb/src/overlaydb.rs | 123 ++++--- util/journaldb/src/overlayrecentdb.rs | 324 +++++++++--------- util/journaldb/src/refcounteddb.rs | 126 +++---- util/journaldb/src/traits.rs | 2 +- util/keccak-hasher/Cargo.toml | 2 +- util/memory-cache/Cargo.toml | 2 +- util/memory-cache/src/lib.rs | 12 +- util/memzero/Cargo.toml | 10 - util/memzero/src/lib.rs | 54 --- util/patricia-trie-ethereum/Cargo.toml | 6 +- util/triehash-ethereum/Cargo.toml | 2 +- whisper/Cargo.toml | 2 +- whisper/src/lib.rs | 2 +- whisper/src/rpc/crypto.rs | 2 +- whisper/src/rpc/key_store.rs | 2 +- whisper/src/rpc/mod.rs | 2 +- 84 files changed, 926 insertions(+), 1074 deletions(-) delete mode 100644 util/memzero/Cargo.toml delete mode 100644 util/memzero/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3eb1a0c8465..0fd6289f6b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,7 +121,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -312,7 +312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cfg-if" -version = "0.1.5" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -357,6 +357,14 @@ dependencies = [ "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "clear_on_drop" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cli-signer" version = "1.4.0" @@ -403,9 +411,9 @@ dependencies = [ "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethjson 0.1.0", "ethkey 0.3.0", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp_derive 0.1.0", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -520,7 +528,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -534,7 +542,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -547,7 +555,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -559,7 +567,7 @@ name = "crossbeam-utils" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -572,7 +580,7 @@ name = "crossbeam-utils" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -736,10 +744,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "elastic-array" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -777,7 +785,7 @@ source = "git+https://github.com/paritytech/rust-secp256k1#9791e79f21a5309dcb6e0 dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -873,8 +881,7 @@ dependencies = [ "evm 0.1.0", "fetch 0.1.0", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -888,13 +895,14 @@ dependencies = [ "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "macros 0.1.0", "memory-cache 0.1.0", - "memory-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-crypto 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", "parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -910,8 +918,8 @@ dependencies = [ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "time-utils 0.1.0", "trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-standardmap 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", "unexpected 0.1.0", "using_queue 0.1.0", @@ -945,13 +953,13 @@ dependencies = [ "ethcore-db 0.1.0", "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethkey 0.3.0", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -983,8 +991,8 @@ version = "0.1.0" dependencies = [ "common-types 0.1.0", "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp_derive 0.1.0", @@ -1025,8 +1033,7 @@ dependencies = [ "failsafe 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "fastmap 0.1.0", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1035,8 +1042,9 @@ dependencies = [ "kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "memory-cache 0.1.0", - "memory-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1047,7 +1055,7 @@ dependencies = [ "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "triehash-ethereum 0.2.0", "vm 0.1.0", ] @@ -1084,12 +1092,12 @@ dependencies = [ "ethkey 0.3.0", "fetch 0.1.0", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "hyper 0.12.19 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "price-info 1.12.0", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1176,11 +1184,11 @@ dependencies = [ "ethkey 0.3.0", "fetch 0.1.0", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-crypto 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1193,7 +1201,7 @@ dependencies = [ "time-utils 0.1.0", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "transaction-pool 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1290,8 +1298,7 @@ dependencies = [ "ethstore 0.2.1", "fastmap 0.1.0", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1300,6 +1307,7 @@ dependencies = [ "macros 0.1.0", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1343,8 +1351,8 @@ dependencies = [ "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memzero 0.1.0", "parity-crypto 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-wordlist 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1418,12 +1426,12 @@ dependencies = [ "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "memory-cache 0.1.0", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "vm 0.1.0", @@ -1531,7 +1539,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1680,28 +1688,27 @@ dependencies = [ [[package]] name = "hash-db" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "hash-db" -version = "0.12.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "hash256-std-hasher" -version = "0.12.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crunchy 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hashmap_core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "heapsize" version = "0.4.2" -source = "git+https://github.com/cheme/heapsize.git?branch=ec-macfix#421df390a930cb523a09e5528e6fe57b534b3b26" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "jemallocator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1985,15 +1992,15 @@ dependencies = [ "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "fastmap 0.1.0", - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memory-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2141,18 +2148,18 @@ name = "keccak-hasher" version = "0.1.1" dependencies = [ "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "plain_hasher 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "keccak-hasher" -version = "0.12.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hash256-std-hasher 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", + "hash256-std-hasher 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2170,7 +2177,7 @@ name = "kvdb" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2188,7 +2195,7 @@ name = "kvdb-rocksdb" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "fs-swap 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2272,7 +2279,7 @@ name = "log" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2295,6 +2302,16 @@ dependencies = [ name = "macros" version = "0.1.0" +[[package]] +name = "malloc_size_of_derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.26 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "matches" version = "0.1.8" @@ -2305,7 +2322,7 @@ name = "memchr" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2328,17 +2345,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "memory-cache" version = "0.1.0" dependencies = [ - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "memory-db" -version = "0.11.0" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", + "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2346,10 +2364,6 @@ name = "memory_units" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "memzero" -version = "0.1.0" - [[package]] name = "memzero" version = "0.1.0" @@ -2483,7 +2497,7 @@ name = "net2" version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2739,6 +2753,7 @@ dependencies = [ "parity-rpc 1.12.0", "parity-runtime 0.1.0", "parity-updater 1.12.0", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-version 2.6.0", "parity-whisper 0.1.0", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2989,6 +3004,21 @@ dependencies = [ "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "parity-util-mem" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "jemallocator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "malloc_size_of_derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "parity-version" version = "2.6.0" @@ -3023,8 +3053,8 @@ dependencies = [ "jsonrpc-derive 10.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-pubsub 10.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memzero 0.1.0", "ordered-float 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3115,16 +3145,16 @@ dependencies = [ name = "patricia-trie-ethereum" version = "0.1.0" dependencies = [ - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.2.0", "keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", - "memory-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3627,7 +3657,7 @@ dependencies = [ name = "rlp_compress" version = "0.1.0" dependencies = [ - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3893,7 +3923,7 @@ name = "socket2" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4380,30 +4410,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "trie-db" -version = "0.11.0" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", + "hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "trie-standardmap" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hasher 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "triehash" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4413,7 +4444,7 @@ version = "0.2.0" dependencies = [ "ethereum-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher 0.1.1", - "triehash 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "triehash 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4443,7 +4474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "crunchy 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)", + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4591,7 +4622,7 @@ dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie-ethereum 0.1.0", "rlp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4809,10 +4840,11 @@ dependencies = [ "checksum cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "926013f2860c46252efceabb19f4a6b308197505082c609025aa6706c011d427" "checksum cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4a8b715cb4597106ea87c7c84b2f1d452c7492033765df7f32651e66fcf749" "checksum cesu8 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -"checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" "checksum chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" "checksum cid 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0e37fba0087d9f3f4e269827a55dc511abf3e440cc097a0c154ff4e6584f988" "checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" +"checksum clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a" "checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc" @@ -4847,7 +4879,7 @@ dependencies = [ "checksum docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "db2906c2579b5b7207fc1e328796a9a8835dc44e22dbe8e460b1d636f9a7b225" "checksum edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3bd26878c3d921f89797a4e1a1711919f999a9f6946bb6f5a4ffda126d297b7e" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" -"checksum elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88d4851b005ef16de812ea9acdb7bece2f0a40dd86c07b85631d7dafa54537bb" +"checksum elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "073be79b6538296faf81c631872676600616073817dd9a440c477ad09b408983" "checksum enum_primitive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" "checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" "checksum error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07e791d3be96241c77c43846b665ef1384606da2cd2a48730abe606a12906e02" @@ -4880,10 +4912,10 @@ dependencies = [ "checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c" "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" "checksum handlebars 0.32.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d89ec99d1594f285d4590fc32bac5f75cdab383f1123d504d27862c644a807dd" -"checksum hash-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1b03501f6e1a2a97f1618879aba3156f14ca2847faa530c4e28859638bd11483" -"checksum hash-db 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ba7fb417e5c470acdd61068c79767d0e65962e70836cf6c9dfd2409f06345ce0" -"checksum hash256-std-hasher 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f8b2027c19ec91eb304999abae7307d225cf93be42af53b0039f76e98ed5af86" -"checksum heapsize 0.4.2 (git+https://github.com/cheme/heapsize.git?branch=ec-macfix)" = "" +"checksum hash-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3c95a428c86ed4633d83e07ef9e0a147a906da01e931f07e74a85bedce5a43" +"checksum hash256-std-hasher 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "663ce20dae36902c16d12c6aaae400ca40d922407a8cf2b4caf8cae9b39b4f03" +"checksum hashmap_core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8e04cb7a5051270ef3fa79f8c7604d581ecfa73d520e74f554e45541c4b5881a" +"checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" "checksum hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" "checksum hmac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f127a908633569f208325f86f71255d3363c79721d7f9fe31cd5569908819771" @@ -4924,7 +4956,7 @@ dependencies = [ "checksum jsonrpc-tcp-server 10.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c873dac37a601fb88d40ba49eeac3f1aa60953c06b2e99ddbf0569b6f8028478" "checksum jsonrpc-ws-server 10.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20b8333a5a6e6ccbcf5c90f90919de557cba4929efa164e9bd0e8e497eb20e46" "checksum keccak-hash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "69e8ee697b9aa6dcc34d7657565fa5052763a1627a5b59e4c3c0ae3ed0d70a65" -"checksum keccak-hasher 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "af672553b2abac1c86c29fd62c79880638b6abc91d96db4aa42a5baab2bc1ca9" +"checksum keccak-hasher 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6c936c737d79690593c34275faf583151a0e8c0abf34eaecad10399eed0beb7d" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum kvdb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "72ae89206cea31c32014b39d5a454b96135894221610dbfd19cf4d2d044fa546" "checksum kvdb-memorydb 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "45bcdf5eb083602cff61a6f8438dce2a7900d714e893fc48781c39fb119d37aa" @@ -4941,11 +4973,12 @@ dependencies = [ "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" "checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" "checksum lunarity-lexer 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a1670671f305792567116d4660e6e5bd785d6fa973e817c3445c0a7a54cecb6" +"checksum malloc_size_of_derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "35adee9ed962cf7d07d62cb58bc45029f3227f5b5b86246caa8632f06c187bc3" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4b3629fe9fdbff6daa6c33b90f7c08355c1aca05a3d01fa8063b822fcf185f3b" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" -"checksum memory-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94da53143d45f6bad3753f532e56ad57a6a26c0ca6881794583310c7cb4c885f" +"checksum memory-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1eeeeab44c01c7da4409e68ec5b5db74c92305386efab3615e495b1dacaec196" "checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" "checksum memzero 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "93c0d11ac30a033511ae414355d80f70d9f29a44a49140face477117a1ee90db" "checksum mime 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)" = "0a907b83e7b9e987032439a387e187119cddafc92d5c2aaeb1d92580a793f630" @@ -4985,6 +5018,7 @@ dependencies = [ "checksum parity-snappy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2c5f9d149b13134b8b354d93a92830efcbee6fe5b73a2e6e540fe70d4dd8a63" "checksum parity-snappy-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1a413d51e5e1927320c9de992998e4a279dffb8c8a7363570198bd8383e66f1b" "checksum parity-tokio-ipc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eb002c2d3539ccd3b82bd915ec060028d4ab350ad203dbffa20028c1e483af5b" +"checksum parity-util-mem 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "89e80f22052161e0cb55cb5a8a75890420c525031f95c9d262dbb0434aa85dc1" "checksum parity-wasm 0.31.3 (registry+https://github.com/rust-lang/crates.io-index)" = "511379a8194230c2395d2f5fa627a5a7e108a9f976656ce723ae68fca4097bfc" "checksum parity-wordlist 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf13102febd98f4ad416a526b42deb82daf482626ba6ab10d0ebf8f45327514c" "checksum parity-ws 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2fec5048fba72a2e01baeb0d08089db79aead4b57e2443df172fb1840075a233" @@ -5128,9 +5162,9 @@ dependencies = [ "checksum trace-time 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe82f2f0bf1991e163e757baf044282823155dd326e70f44ce2186c3c320cc9" "checksum transaction-pool 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d8bd3123931aa6e49dd03bc8a2400490e14701d779458d1f1fff1f04c6f666" "checksum transient-hashmap 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aeb4b191d033a35edfce392a38cdcf9790b6cebcb30fa690c312c29da4dc433e" -"checksum trie-db 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c7319e28ca295f27359d944a682f7f65b419158bf1590c92cadc0000258d788" -"checksum trie-standardmap 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ebaa4b340046196efad8872b2dffe585b5ea330230dc44ee14e399f77da29f51" -"checksum triehash 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "92148b4d8d55eff71bc8c9e3c5f714e266c2a05e724dce5405a10deabbf449a8" +"checksum trie-db 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ae063390324bfcf36c7e8e4fb1f85f6f0fb5dd04e1cd282581eb7b8b34b32de7" +"checksum trie-standardmap 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "40787fb1a63a97ed56d12bc303937ea274e09d1afa2e20e4f074eff2074b24d3" +"checksum triehash 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b645ad3fc9871596897fb64a57c9c29adc9f5ece87c2d78766e3fc5a5da56b56" "checksum try-lock 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2aa4715743892880f70885373966c83d73ef1b0838a664ef0c76fffd35e7c2" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" diff --git a/Cargo.toml b/Cargo.toml index 1a5e32d1011..39a9bbd3645 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,8 @@ ethcore-secretstore = { path = "secret-store", optional = true } registrar = { path = "util/registrar" } +parity-util-mem = { version = "0.1", features = ["jemalloc-global"] } + [build-dependencies] rustc_version = "0.2" @@ -139,6 +141,3 @@ members = [ "util/fastmap", "util/time-utils" ] - -[patch.crates-io] -heapsize = { git = "https://github.com/cheme/heapsize.git", branch = "ec-macfix" } diff --git a/accounts/ethkey/Cargo.toml b/accounts/ethkey/Cargo.toml index a6c1a974ba0..9d91d909c91 100644 --- a/accounts/ethkey/Cargo.toml +++ b/accounts/ethkey/Cargo.toml @@ -11,7 +11,7 @@ eth-secp256k1 = { git = "https://github.com/paritytech/rust-secp256k1" } ethereum-types = "0.6.0" lazy_static = "1.0" log = "0.4" -memzero = { path = "../../util/memzero" } +parity-util-mem = "0.1" parity-wordlist = "1.2" quick-error = "1.2.2" rand = "0.6" diff --git a/accounts/ethkey/src/lib.rs b/accounts/ethkey/src/lib.rs index 5c58333c711..f22c77e3c59 100644 --- a/accounts/ethkey/src/lib.rs +++ b/accounts/ethkey/src/lib.rs @@ -20,7 +20,7 @@ extern crate byteorder; extern crate edit_distance; extern crate parity_crypto; extern crate ethereum_types; -extern crate memzero; +extern crate parity_util_mem; extern crate parity_wordlist; #[macro_use] extern crate quick_error; diff --git a/accounts/ethkey/src/secret.rs b/accounts/ethkey/src/secret.rs index c9ab410f81a..25136ee8330 100644 --- a/accounts/ethkey/src/secret.rs +++ b/accounts/ethkey/src/secret.rs @@ -21,7 +21,7 @@ use rustc_hex::ToHex; use secp256k1::constants::{SECRET_KEY_SIZE as SECP256K1_SECRET_KEY_SIZE}; use secp256k1::key; use ethereum_types::H256; -use memzero::Memzero; +use parity_util_mem::Memzero; use {Error, SECP256K1}; #[derive(Clone, PartialEq, Eq)] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 92d3bf3e2c7..57e28f6e15d 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -31,8 +31,8 @@ ethjson = { path = "../json" } ethkey = { path = "../accounts/ethkey" } evm = { path = "evm" } futures = "0.1" -hash-db = "0.11.0" -heapsize = "0.4" +hash-db = "0.12.4" +parity-util-mem = "0.1" itertools = "0.5" journaldb = { path = "../util/journaldb" } keccak-hash = "0.2.0" @@ -46,14 +46,14 @@ log = "0.4" lru-cache = "0.1" macros = { path = "../util/macros" } memory-cache = { path = "../util/memory-cache" } -memory-db = "0.11.0" +memory-db = "0.12.4" num = { version = "0.1", default-features = false, features = ["bigint"] } num_cpus = "1.2" parity-bytes = "0.1" parity-crypto = "0.4.0" parity-snappy = "0.1" parking_lot = "0.7" -trie-db = "0.11.0" +trie-db = "0.12.4" patricia-trie-ethereum = { path = "../util/patricia-trie-ethereum" } rand = "0.6" rayon = "1.0" @@ -83,7 +83,7 @@ kvdb-rocksdb = "0.1.3" parity-runtime = { path = "../util/runtime" } rlp_compress = { path = "../util/rlp-compress" } tempdir = "0.3" -trie-standardmap = "0.12.3" +trie-standardmap = "0.12.4" [features] parity = ["work-notify", "price-info", "stratum"] diff --git a/ethcore/blockchain/Cargo.toml b/ethcore/blockchain/Cargo.toml index 17ef11a997a..ac2955f9db7 100644 --- a/ethcore/blockchain/Cargo.toml +++ b/ethcore/blockchain/Cargo.toml @@ -13,7 +13,7 @@ blooms-db = { path = "../../util/blooms-db" } common-types = { path = "../types" } ethcore-db = { path = "../db" } ethereum-types = "0.6.0" -heapsize = "0.4" +parity-util-mem = "0.1" itertools = "0.5" kvdb = "0.1" log = "0.4" diff --git a/ethcore/blockchain/src/blockchain.rs b/ethcore/blockchain/src/blockchain.rs index afebacad9b4..6656fcc0cb7 100644 --- a/ethcore/blockchain/src/blockchain.rs +++ b/ethcore/blockchain/src/blockchain.rs @@ -39,7 +39,7 @@ use ethcore_db::cache_manager::CacheManager; use ethcore_db::keys::{BlockReceipts, BlockDetails, TransactionAddress, EPOCH_KEY_PREFIX, EpochTransitions}; use ethcore_db::{self as db, Writable, Readable, CacheUpdatePolicy}; use ethereum_types::{H256, Bloom, BloomRef, U256}; -use heapsize::HeapSizeOf; +use util_mem::{MallocSizeOf, allocators::new_malloc_size_ops}; use itertools::Itertools; use kvdb::{DBTransaction, KeyValueDB}; use log::{trace, warn, info}; @@ -1489,11 +1489,12 @@ impl BlockChain { /// Get current cache size. pub fn cache_size(&self) -> CacheSize { + let mut ops = new_malloc_size_ops(); CacheSize { - blocks: self.block_headers.read().heap_size_of_children() + self.block_bodies.read().heap_size_of_children(), - block_details: self.block_details.read().heap_size_of_children(), - transaction_addresses: self.transaction_addresses.read().heap_size_of_children(), - block_receipts: self.block_receipts.read().heap_size_of_children(), + blocks: self.block_headers.size_of(&mut ops) + self.block_bodies.size_of(&mut ops), + block_details: self.block_details.size_of(&mut ops), + transaction_addresses: self.transaction_addresses.size_of(&mut ops), + block_receipts: self.block_receipts.size_of(&mut ops), } } @@ -1528,12 +1529,13 @@ impl BlockChain { transaction_addresses.shrink_to_fit(); block_receipts.shrink_to_fit(); - block_headers.heap_size_of_children() + - block_bodies.heap_size_of_children() + - block_details.heap_size_of_children() + - block_hashes.heap_size_of_children() + - transaction_addresses.heap_size_of_children() + - block_receipts.heap_size_of_children() + let mut ops = new_malloc_size_ops(); + block_headers.size_of(&mut ops) + + block_bodies.size_of(&mut ops) + + block_details.size_of(&mut ops) + + block_hashes.size_of(&mut ops) + + transaction_addresses.size_of(&mut ops) + + block_receipts.size_of(&mut ops) }); } diff --git a/ethcore/blockchain/src/lib.rs b/ethcore/blockchain/src/lib.rs index 3f07a6d8070..004817138de 100644 --- a/ethcore/blockchain/src/lib.rs +++ b/ethcore/blockchain/src/lib.rs @@ -18,6 +18,9 @@ #![warn(missing_docs)] +extern crate parity_util_mem as util_mem; +extern crate parity_util_mem as malloc_size_of; + mod best_block; mod block_info; mod blockchain; diff --git a/ethcore/db/Cargo.toml b/ethcore/db/Cargo.toml index ad02e4680b8..f8caf7040a9 100644 --- a/ethcore/db/Cargo.toml +++ b/ethcore/db/Cargo.toml @@ -10,8 +10,8 @@ edition = "2018" [dependencies] common-types = { path = "../types" } ethereum-types = "0.6.0" -heapsize = "0.4" kvdb = "0.1" +parity-util-mem = "0.1" parking_lot = "0.7" rlp = "0.4.0" rlp_derive = { path = "../../util/rlp-derive" } diff --git a/ethcore/db/src/keys.rs b/ethcore/db/src/keys.rs index 3e6a2db21c4..ceab94211ec 100644 --- a/ethcore/db/src/keys.rs +++ b/ethcore/db/src/keys.rs @@ -23,7 +23,7 @@ use common_types::BlockNumber; use common_types::engines::epoch::Transition as EpochTransition; use common_types::receipt::Receipt; use ethereum_types::{H256, H264, U256}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use kvdb::PREFIX_LEN as DB_PREFIX_LEN; use rlp; use rlp_derive::{RlpEncodableWrapper, RlpDecodableWrapper, RlpEncodable, RlpDecodable}; @@ -140,7 +140,7 @@ impl Key for u64 { } /// Familial details concerning a block -#[derive(Debug, Clone)] +#[derive(Debug, Clone, MallocSizeOf)] pub struct BlockDetails { /// Block number pub number: BlockNumber, @@ -195,14 +195,8 @@ impl rlp::Decodable for BlockDetails { } } -impl HeapSizeOf for BlockDetails { - fn heap_size_of_children(&self) -> usize { - self.children.heap_size_of_children() - } -} - /// Represents address of certain transaction within block -#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable, MallocSizeOf)] pub struct TransactionAddress { /// Block hash pub block_hash: H256, @@ -210,12 +204,8 @@ pub struct TransactionAddress { pub index: usize } -impl HeapSizeOf for TransactionAddress { - fn heap_size_of_children(&self) -> usize { 0 } -} - /// Contains all block receipts. -#[derive(Clone, RlpEncodableWrapper, RlpDecodableWrapper)] +#[derive(Clone, RlpEncodableWrapper, RlpDecodableWrapper, MallocSizeOf)] pub struct BlockReceipts { /// Block receipts pub receipts: Vec, @@ -230,12 +220,6 @@ impl BlockReceipts { } } -impl HeapSizeOf for BlockReceipts { - fn heap_size_of_children(&self) -> usize { - self.receipts.heap_size_of_children() - } -} - /// Candidate transitions to an epoch with specific number. #[derive(Clone, RlpEncodable, RlpDecodable)] pub struct EpochTransitions { diff --git a/ethcore/db/src/lib.rs b/ethcore/db/src/lib.rs index 3fdb368a1aa..492d255c497 100644 --- a/ethcore/db/src/lib.rs +++ b/ethcore/db/src/lib.rs @@ -18,6 +18,9 @@ #![warn(missing_docs)] +extern crate parity_util_mem as mem; +extern crate parity_util_mem as malloc_size_of; + mod db; pub mod keys; diff --git a/ethcore/evm/Cargo.toml b/ethcore/evm/Cargo.toml index 4c6d679a9f6..ca9df4d1bf2 100644 --- a/ethcore/evm/Cargo.toml +++ b/ethcore/evm/Cargo.toml @@ -7,7 +7,7 @@ authors = ["Parity Technologies "] bit-set = "0.4" parity-bytes = "0.1" ethereum-types = "0.6.0" -heapsize = "0.4" +parity-util-mem = "0.1" lazy_static = "1.0" log = "0.4" vm = { path = "../vm" } diff --git a/ethcore/evm/benches/basic.rs b/ethcore/evm/benches/basic.rs index c86afcc5756..9affec15b65 100644 --- a/ethcore/evm/benches/basic.rs +++ b/ethcore/evm/benches/basic.rs @@ -21,7 +21,7 @@ extern crate criterion; extern crate bit_set; extern crate ethereum_types; extern crate parking_lot; -extern crate heapsize; +extern crate parity_util_mem as mem; extern crate vm; extern crate evm; extern crate keccak_hash as hash; diff --git a/ethcore/evm/src/interpreter/shared_cache.rs b/ethcore/evm/src/interpreter/shared_cache.rs index da7c03efa01..2507d274d63 100644 --- a/ethcore/evm/src/interpreter/shared_cache.rs +++ b/ethcore/evm/src/interpreter/shared_cache.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use hash::KECCAK_EMPTY; -use heapsize::HeapSizeOf; +use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; use ethereum_types::H256; use parking_lot::Mutex; use memory_cache::MemoryLruCache; @@ -25,11 +25,12 @@ use super::super::instructions::{self, Instruction}; const DEFAULT_CACHE_SIZE: usize = 4 * 1024 * 1024; -// stub for a HeapSizeOf implementation. +/// Stub for a sharing `BitSet` data in cache (reference counted) +/// and implementing MallocSizeOf on it. struct Bits(Arc); -impl HeapSizeOf for Bits { - fn heap_size_of_children(&self) -> usize { +impl MallocSizeOf for Bits { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { // dealing in bits here self.0.capacity() * 8 } diff --git a/ethcore/evm/src/lib.rs b/ethcore/evm/src/lib.rs index 3548a1fe5e5..7cfa5a69eaa 100644 --- a/ethcore/evm/src/lib.rs +++ b/ethcore/evm/src/lib.rs @@ -19,7 +19,7 @@ extern crate bit_set; extern crate ethereum_types; extern crate parking_lot; -extern crate heapsize; +extern crate parity_util_mem; extern crate vm; extern crate keccak_hash as hash; extern crate memory_cache; diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index fe0901dd3c5..62d70fec4b8 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -15,14 +15,14 @@ ethcore = { path = ".."} ethcore-db = { path = "../db" } ethcore-blockchain = { path = "../blockchain" } ethereum-types = "0.6.0" -memory-db = "0.11.0" -trie-db = "0.11.0" +memory-db = "0.12.4" +trie-db = "0.12.4" patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } ethcore-network = { path = "../../util/network" } ethcore-miner = { path = "../../miner" } ethcore-io = { path = "../../util/io" } -hash-db = "0.11.0" -heapsize = "0.4" +hash-db = "0.12.4" +parity-util-mem = "0.1" vm = { path = "../vm" } fastmap = { path = "../../util/fastmap" } failsafe = { version = "0.3.0", default-features = false, features = ["parking_lot_mutex"] } diff --git a/ethcore/light/src/cache.rs b/ethcore/light/src/cache.rs index d75e0ff7d31..5f56c1ad936 100644 --- a/ethcore/light/src/cache.rs +++ b/ethcore/light/src/cache.rs @@ -21,12 +21,12 @@ //! vector of all gas prices from a recent range of blocks. use std::time::{Instant, Duration}; +use parity_util_mem::{MallocSizeOf, MallocSizeOfOps, MallocSizeOfExt}; use common_types::encoded; use common_types::BlockNumber; use common_types::receipt::Receipt; use ethereum_types::{H256, U256}; -use heapsize::HeapSizeOf; use memory_cache::MemoryLruCache; use stats::Corpus; @@ -157,18 +157,20 @@ impl Cache { /// Get the memory used. pub fn mem_used(&self) -> usize { - self.heap_size_of_children() + self.malloc_size_of() } } -impl HeapSizeOf for Cache { - fn heap_size_of_children(&self) -> usize { + +// This is fast method: it is possible to have a more exhaustive implementation +impl MallocSizeOf for Cache { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { self.headers.current_size() + self.canon_hashes.current_size() + self.bodies.current_size() + self.receipts.current_size() + self.chain_score.current_size() - // TODO: + corpus + // `self.corpus` is skipped } } diff --git a/ethcore/light/src/cht.rs b/ethcore/light/src/cht.rs index 8aadbfc53c1..d6a8dc0bca1 100644 --- a/ethcore/light/src/cht.rs +++ b/ethcore/light/src/cht.rs @@ -95,7 +95,8 @@ pub struct BlockInfo { /// Build an in-memory CHT from a closure which provides necessary information /// about blocks. If the fetcher ever fails to provide the info, the CHT /// will not be generated. -pub fn build(cht_num: u64, mut fetcher: F) -> Option>> +pub fn build(cht_num: u64, mut fetcher: F) + -> Option, DBValue>>> where F: FnMut(BlockId) -> Option { let mut db = new_memory_db(); @@ -154,7 +155,7 @@ pub fn compute_root(cht_num: u64, iterable: I) -> Option pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256)> { let mut db = new_memory_db(); - for node in proof { db.insert(&node[..]); } + for node in proof { db.insert(hash_db::EMPTY_PREFIX, &node[..]); } let res = match TrieDB::new(&db, &root) { Err(_) => return None, Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| { diff --git a/ethcore/light/src/client/header_chain.rs b/ethcore/light/src/client/header_chain.rs index be52185bc53..2a3e8d5d0ad 100644 --- a/ethcore/light/src/client/header_chain.rs +++ b/ethcore/light/src/client/header_chain.rs @@ -38,7 +38,7 @@ use ethcore::engines::epoch::{Transition as EpochTransition, PendingTransition a use ethcore::error::{Error, EthcoreResult, BlockError}; use ethcore::spec::{Spec, SpecHardcodedSync}; use ethereum_types::{H256, H264, U256}; -use heapsize::HeapSizeOf; +use parity_util_mem::{MallocSizeOf, MallocSizeOfOps}; use kvdb::{DBTransaction, KeyValueDB}; use parking_lot::{Mutex, RwLock}; use fastmap::H256FastMap; @@ -95,8 +95,8 @@ struct Entry { canonical_hash: H256, } -impl HeapSizeOf for Entry { - fn heap_size_of_children(&self) -> usize { +impl MallocSizeOf for Entry { + fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { if self.candidates.spilled() { self.candidates.capacity() * ::std::mem::size_of::() } else { @@ -202,14 +202,21 @@ pub enum HardcodedSync { Deny, } +#[derive(MallocSizeOf)] /// Header chain. See module docs for more details. pub struct HeaderChain { + #[ignore_malloc_size_of = "ignored for performance reason"] genesis_header: encoded::Header, // special-case the genesis. candidates: RwLock>, + #[ignore_malloc_size_of = "ignored for performance reason"] best_block: RwLock, + #[ignore_malloc_size_of = "ignored for performance reason"] live_epoch_proofs: RwLock>, + #[ignore_malloc_size_of = "ignored for performance reason"] db: Arc, + #[ignore_malloc_size_of = "ignored for performance reason"] col: Option, + #[ignore_malloc_size_of = "ignored for performance reason"] cache: Arc>, } @@ -838,12 +845,6 @@ impl HeaderChain { } } -impl HeapSizeOf for HeaderChain { - fn heap_size_of_children(&self) -> usize { - self.candidates.read().heap_size_of_children() - } -} - /// Iterator over a block's ancestry. pub struct AncestryIter<'a> { next: Option, diff --git a/ethcore/light/src/client/mod.rs b/ethcore/light/src/client/mod.rs index d20617f4f12..6ed2b222ecd 100644 --- a/ethcore/light/src/client/mod.rs +++ b/ethcore/light/src/client/mod.rs @@ -362,9 +362,9 @@ impl Client { /// Get blockchain mem usage in bytes. pub fn chain_mem_used(&self) -> usize { - use heapsize::HeapSizeOf; + use parity_util_mem::MallocSizeOfExt; - self.chain.heap_size_of_children() + self.chain.malloc_size_of() } /// Set a closure to call when the client wants to be restarted. diff --git a/ethcore/light/src/lib.rs b/ethcore/light/src/lib.rs index 5221830af90..d169c6925e2 100644 --- a/ethcore/light/src/lib.rs +++ b/ethcore/light/src/lib.rs @@ -64,7 +64,9 @@ extern crate ethereum_types; extern crate ethcore_miner as miner; extern crate ethcore; extern crate hash_db; -extern crate heapsize; +extern crate parity_util_mem; +extern crate parity_util_mem as mem; +extern crate parity_util_mem as malloc_size_of; extern crate failsafe; extern crate futures; extern crate itertools; diff --git a/ethcore/light/src/on_demand/request.rs b/ethcore/light/src/on_demand/request.rs index db174219a4c..09433ea236d 100644 --- a/ethcore/light/src/on_demand/request.rs +++ b/ethcore/light/src/on_demand/request.rs @@ -981,7 +981,7 @@ impl Account { let state_root = header.state_root(); let mut db = journaldb::new_memory_db(); - for node in proof { db.insert(&node[..]); } + for node in proof { db.insert(hash_db::EMPTY_PREFIX, &node[..]); } match TrieDB::new(&db, &state_root).and_then(|t| t.get(keccak(&self.address).as_bytes()))? { Some(val) => { diff --git a/ethcore/private-tx/Cargo.toml b/ethcore/private-tx/Cargo.toml index 23cd51cdaa7..ec2f66382b3 100644 --- a/ethcore/private-tx/Cargo.toml +++ b/ethcore/private-tx/Cargo.toml @@ -20,13 +20,13 @@ ethjson = { path = "../../json" } ethkey = { path = "../../accounts/ethkey" } fetch = { path = "../../util/fetch" } futures = "0.1" -heapsize = "0.4" +parity-util-mem = "0.1" keccak-hash = "0.2.0" log = "0.4" parity-bytes = "0.1" parity-crypto = "0.4.0" parking_lot = "0.7" -trie-db = "0.11.0" +trie-db = "0.12.4" patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } rand = "0.3" rlp = "0.4.0" diff --git a/ethcore/private-tx/src/lib.rs b/ethcore/private-tx/src/lib.rs index b9de9e16735..2f23de3fc86 100644 --- a/ethcore/private-tx/src/lib.rs +++ b/ethcore/private-tx/src/lib.rs @@ -34,7 +34,7 @@ extern crate ethjson; extern crate ethkey; extern crate fetch; extern crate futures; -extern crate heapsize; +extern crate parity_util_mem; extern crate keccak_hash as hash; extern crate parity_bytes as bytes; extern crate parity_crypto as crypto; diff --git a/ethcore/private-tx/src/private_transactions.rs b/ethcore/private-tx/src/private_transactions.rs index dd263ee1868..b9108289135 100644 --- a/ethcore/private-tx/src/private_transactions.rs +++ b/ethcore/private-tx/src/private_transactions.rs @@ -21,7 +21,7 @@ use std::collections::{HashMap, HashSet}; use bytes::Bytes; use ethcore_miner::pool; use ethereum_types::{H256, U256, Address}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOfExt; use ethkey::Signature; use messages::PrivateTransaction; use parking_lot::RwLock; @@ -59,7 +59,7 @@ impl txpool::VerifiedTransaction for VerifiedPrivateTransaction { } fn mem_usage(&self) -> usize { - self.transaction.heap_size_of_children() + self.transaction.malloc_size_of() } fn sender(&self) -> &Address { diff --git a/ethcore/src/account_db.rs b/ethcore/src/account_db.rs index bf1380908f8..85eeb7bb691 100644 --- a/ethcore/src/account_db.rs +++ b/ethcore/src/account_db.rs @@ -17,7 +17,7 @@ //! DB backend wrapper for Account trie use ethereum_types::H256; use hash::{KECCAK_NULL_RLP, keccak}; -use hash_db::{HashDB, AsHashDB}; +use hash_db::{HashDB, AsHashDB, Prefix}; use keccak_hasher::KeccakHasher; use kvdb::DBValue; use rlp::NULL_RLP; @@ -103,29 +103,29 @@ impl<'db> AsHashDB for AccountDB<'db> { } impl<'db> HashDB for AccountDB<'db> { - fn get(&self, key: &H256) -> Option { + fn get(&self, key: &H256, prefix: Prefix) -> Option { if key == &KECCAK_NULL_RLP { return Some(DBValue::from_slice(&NULL_RLP)); } - self.db.get(&combine_key(&self.address_hash, key)) + self.db.get(&combine_key(&self.address_hash, key), prefix) } - fn contains(&self, key: &H256) -> bool { + fn contains(&self, key: &H256, prefix: Prefix) -> bool { if key == &KECCAK_NULL_RLP { return true; } - self.db.contains(&combine_key(&self.address_hash, key)) + self.db.contains(&combine_key(&self.address_hash, key), prefix) } - fn insert(&mut self, _value: &[u8]) -> H256 { + fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H256 { unimplemented!() } - fn emplace(&mut self, _key: H256, _value: DBValue) { + fn emplace(&mut self, _key: H256, _prefix: Prefix, _value: DBValue) { unimplemented!() } - fn remove(&mut self, _key: &H256) { + fn remove(&mut self, _key: &H256, _prefix: Prefix) { unimplemented!() } } @@ -158,44 +158,44 @@ impl<'db> AccountDBMut<'db> { } impl<'db> HashDB for AccountDBMut<'db>{ - fn get(&self, key: &H256) -> Option { + fn get(&self, key: &H256, prefix: Prefix) -> Option { if key == &KECCAK_NULL_RLP { return Some(DBValue::from_slice(&NULL_RLP)); } - self.db.get(&combine_key(&self.address_hash, key)) + self.db.get(&combine_key(&self.address_hash, key), prefix) } - fn contains(&self, key: &H256) -> bool { + fn contains(&self, key: &H256, prefix: Prefix) -> bool { if key == &KECCAK_NULL_RLP { return true; } - self.db.contains(&combine_key(&self.address_hash, key)) + self.db.contains(&combine_key(&self.address_hash, key), prefix) } - fn insert(&mut self, value: &[u8]) -> H256 { + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { if value == &NULL_RLP { return KECCAK_NULL_RLP.clone(); } let k = keccak(value); let ak = combine_key(&self.address_hash, &k); - self.db.emplace(ak, DBValue::from_slice(value)); + self.db.emplace(ak, prefix, DBValue::from_slice(value)); k } - fn emplace(&mut self, key: H256, value: DBValue) { + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { if key == KECCAK_NULL_RLP { return; } let key = combine_key(&self.address_hash, &key); - self.db.emplace(key, value) + self.db.emplace(key, prefix, value) } - fn remove(&mut self, key: &H256) { + fn remove(&mut self, key: &H256, prefix: Prefix) { if key == &KECCAK_NULL_RLP { return; } let key = combine_key(&self.address_hash, key); - self.db.remove(&key) + self.db.remove(&key, prefix) } } @@ -212,29 +212,29 @@ impl<'db> AsHashDB for Wrapping<'db> { } impl<'db> HashDB for Wrapping<'db> { - fn get(&self, key: &H256) -> Option { + fn get(&self, key: &H256, prefix: Prefix) -> Option { if key == &KECCAK_NULL_RLP { return Some(DBValue::from_slice(&NULL_RLP)); } - self.0.get(key) + self.0.get(key, prefix) } - fn contains(&self, key: &H256) -> bool { + fn contains(&self, key: &H256, prefix: Prefix) -> bool { if key == &KECCAK_NULL_RLP { return true; } - self.0.contains(key) + self.0.contains(key, prefix) } - fn insert(&mut self, _value: &[u8]) -> H256 { + fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H256 { unimplemented!() } - fn emplace(&mut self, _key: H256, _value: DBValue) { + fn emplace(&mut self, _key: H256, _prefix: Prefix, _value: DBValue) { unimplemented!() } - fn remove(&mut self, _key: &H256) { + fn remove(&mut self, _key: &H256, _prefix: Prefix) { unimplemented!() } } @@ -246,38 +246,38 @@ impl<'db> AsHashDB for WrappingMut<'db> { } impl<'db> HashDB for WrappingMut<'db>{ - fn get(&self, key: &H256) -> Option { + fn get(&self, key: &H256, prefix: Prefix) -> Option { if key == &KECCAK_NULL_RLP { return Some(DBValue::from_slice(&NULL_RLP)); } - self.0.get(key) + self.0.get(key, prefix) } - fn contains(&self, key: &H256) -> bool { + fn contains(&self, key: &H256, prefix: Prefix) -> bool { if key == &KECCAK_NULL_RLP { return true; } - self.0.contains(key) + self.0.contains(key, prefix) } - fn insert(&mut self, value: &[u8]) -> H256 { + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { if value == &NULL_RLP { return KECCAK_NULL_RLP.clone(); } - self.0.insert(value) + self.0.insert(prefix, value) } - fn emplace(&mut self, key: H256, value: DBValue) { + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { if key == KECCAK_NULL_RLP { return; } - self.0.emplace(key, value) + self.0.emplace(key, prefix, value) } - fn remove(&mut self, key: &H256) { + fn remove(&mut self, key: &H256, prefix: Prefix) { if key == &KECCAK_NULL_RLP { return; } - self.0.remove(key) + self.0.remove(key, prefix) } } diff --git a/ethcore/src/block.rs b/ethcore/src/block.rs index 4a75034f025..d672668f733 100644 --- a/ethcore/src/block.rs +++ b/ethcore/src/block.rs @@ -566,6 +566,7 @@ mod tests { use types::header::Header; use types::view; use types::views::BlockView; + use hash_db::EMPTY_PREFIX; /// Enact the block given by `block_bytes` using `engine` on the database `db` with given `parent` block header fn enact_bytes( @@ -668,7 +669,8 @@ mod tests { let db = e.drain().state.drop().1; assert_eq!(orig_db.journal_db().keys(), db.journal_db().keys()); - assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0) != db.journal_db().get(k.0)).next() == None); + assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0, EMPTY_PREFIX) + != db.journal_db().get(k.0, EMPTY_PREFIX)).next() == None); } #[test] @@ -702,6 +704,7 @@ mod tests { let db = e.drain().state.drop().1; assert_eq!(orig_db.journal_db().keys(), db.journal_db().keys()); - assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0) != db.journal_db().get(k.0)).next() == None); + assert!(orig_db.journal_db().keys().iter().filter(|k| orig_db.journal_db().get(k.0, EMPTY_PREFIX) + != db.journal_db().get(k.0, EMPTY_PREFIX)).next() == None); } } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 75227af5aa4..0a755bbe0ee 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -43,7 +43,7 @@ use types::log_entry::LocalizedLogEntry; use types::receipt::{Receipt, LocalizedReceipt}; use types::{BlockNumber, header::{Header, ExtendedHeader}}; use vm::{EnvInfo, LastHashes}; - +use hash_db::EMPTY_PREFIX; use block::{LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock}; use client::ancient_import::AncientVerifier; use client::{ @@ -743,7 +743,7 @@ impl Client { config.history }; - if !chain.block_header_data(&chain.best_block_hash()).map_or(true, |h| state_db.journal_db().contains(&h.state_root())) { + if !chain.block_header_data(&chain.best_block_hash()).map_or(true, |h| state_db.journal_db().contains(&h.state_root(), EMPTY_PREFIX)) { warn!("State root not found for block #{} ({:x})", chain.best_block_number(), chain.best_block_hash()); } diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index ea1a1200ea7..d9a4c6e1c23 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -16,7 +16,7 @@ /// Preconfigured validator list. -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use ethereum_types::{H256, Address}; use machine::{AuxiliaryData, Call, EthereumMachine}; @@ -25,7 +25,7 @@ use types::header::Header; use super::ValidatorSet; /// Validator set containing a known set of addresses. -#[derive(Clone, Debug, PartialEq, Eq, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Default, MallocSizeOf)] pub struct SimpleList { validators: Vec
, } @@ -58,12 +58,6 @@ impl From> for SimpleList { } } -impl HeapSizeOf for SimpleList { - fn heap_size_of_children(&self) -> usize { - self.validators.heap_size_of_children() - } -} - impl ValidatorSet for SimpleList { fn default_caller(&self, _block_id: ::types::ids::BlockId) -> Box { Box::new(|_, _| Err("Simple list doesn't require calls.".into())) diff --git a/ethcore/src/engines/validator_set/test.rs b/ethcore/src/engines/validator_set/test.rs index c66ff14ad4e..c81259f5b90 100644 --- a/ethcore/src/engines/validator_set/test.rs +++ b/ethcore/src/engines/validator_set/test.rs @@ -19,10 +19,10 @@ use std::str::FromStr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; +use parity_util_mem::MallocSizeOf; use bytes::Bytes; use ethereum_types::{H256, Address}; -use heapsize::HeapSizeOf; use types::BlockNumber; use types::header::Header; @@ -30,9 +30,12 @@ use machine::{AuxiliaryData, Call, EthereumMachine}; use super::{ValidatorSet, SimpleList}; /// Set used for testing with a single validator. +#[derive(MallocSizeOf)] pub struct TestSet { validator: SimpleList, + #[ignore_malloc_size_of = "zero sized"] last_malicious: Arc, + #[ignore_malloc_size_of = "zero sized"] last_benign: Arc, } @@ -52,12 +55,6 @@ impl TestSet { } } -impl HeapSizeOf for TestSet { - fn heap_size_of_children(&self) -> usize { - self.validator.heap_size_of_children() - } -} - impl ValidatorSet for TestSet { fn default_caller(&self, _block_id: ::types::ids::BlockId) -> Box { Box::new(|_, _| Err("Test set doesn't require calls.".into())) diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index a5f3adf5891..4c729147a8e 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -72,7 +72,6 @@ extern crate ethjson; extern crate ethkey; extern crate futures; extern crate hash_db; -extern crate heapsize; extern crate itertools; extern crate journaldb; extern crate keccak_hash as hash; @@ -98,6 +97,9 @@ extern crate patricia_trie_ethereum as ethtrie; extern crate rand; extern crate rayon; extern crate rlp; +extern crate parity_util_mem; +extern crate parity_util_mem as mem; +extern crate parity_util_mem as malloc_size_of; extern crate rustc_hex; extern crate serde; extern crate stats; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index 5bdb05f49d0..087577b02c6 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -81,7 +81,7 @@ impl PodAccount { /// Place additional data into given hash DB. pub fn insert_additional(&self, db: &mut dyn HashDB, factory: &TrieFactory) { match self.code { - Some(ref c) if !c.is_empty() => { db.insert(c); } + Some(ref c) if !c.is_empty() => { db.insert(hash_db::EMPTY_PREFIX, c); } _ => {} } let mut r = H256::zero(); diff --git a/ethcore/src/snapshot/account.rs b/ethcore/src/snapshot/account.rs index 2a9ac911f12..84baf543d7b 100644 --- a/ethcore/src/snapshot/account.rs +++ b/ethcore/src/snapshot/account.rs @@ -95,7 +95,7 @@ pub fn to_fat_rlps( } else if used_code.contains(&acc.code_hash) { account_stream.append(&CodeState::Hash.raw()).append(&acc.code_hash); } else { - match acct_db.get(&acc.code_hash) { + match acct_db.get(&acc.code_hash, hash_db::EMPTY_PREFIX) { Some(c) => { used_code.insert(acc.code_hash.clone()); account_stream.append(&CodeState::Inline.raw()).append(&&*c); @@ -182,7 +182,7 @@ pub fn from_fat_rlp( CodeState::Empty => (KECCAK_EMPTY, None), CodeState::Inline => { let code: Bytes = rlp.val_at(3)?; - let code_hash = acct_db.insert(&code); + let code_hash = acct_db.insert(hash_db::EMPTY_PREFIX, &code); (code_hash, Some(code)) } @@ -228,7 +228,7 @@ mod tests { use hash::{KECCAK_EMPTY, KECCAK_NULL_RLP, keccak}; use ethereum_types::{H256, Address}; - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; use kvdb::DBValue; use rlp::Rlp; @@ -324,12 +324,12 @@ mod tests { let code_hash = { let mut acct_db = AccountDBMut::new(db.as_hash_db_mut(), &addr1); - acct_db.insert(b"this is definitely code") + acct_db.insert(EMPTY_PREFIX, b"this is definitely code") }; { let mut acct_db = AccountDBMut::new(db.as_hash_db_mut(), &addr2); - acct_db.emplace(code_hash.clone(), DBValue::from_slice(b"this is definitely code")); + acct_db.emplace(code_hash.clone(), EMPTY_PREFIX, DBValue::from_slice(b"this is definitely code")); } let account1 = BasicAccount { diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index 51f6f6de8bb..96136b042fd 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -448,7 +448,7 @@ impl StateRebuilder { for (code_hash, code, first_with) in status.new_code { for addr_hash in self.missing_code.remove(&code_hash).unwrap_or_else(Vec::new) { let mut db = AccountDBMut::from_hash(self.db.as_hash_db_mut(), addr_hash); - db.emplace(code_hash, DBValue::from_slice(&code)); + db.emplace(code_hash, hash_db::EMPTY_PREFIX, DBValue::from_slice(&code)); } self.known_code.insert(code_hash, first_with); @@ -545,11 +545,11 @@ fn rebuild_accounts( Some(&first_with) => { // if so, load it from the database. let code = AccountDB::from_hash(db, first_with) - .get(&code_hash) + .get(&code_hash, hash_db::EMPTY_PREFIX) .ok_or_else(|| Error::MissingCode(vec![first_with]))?; // and write it again under a different mangled key - AccountDBMut::from_hash(db, hash).emplace(code_hash, code); + AccountDBMut::from_hash(db, hash).emplace(code_hash, hash_db::EMPTY_PREFIX, code); } // if not, queue it up to be filled later None => status.missing_code.push((hash, code_hash)), diff --git a/ethcore/src/snapshot/tests/state.rs b/ethcore/src/snapshot/tests/state.rs index 769534e0298..b62775c547e 100644 --- a/ethcore/src/snapshot/tests/state.rs +++ b/ethcore/src/snapshot/tests/state.rs @@ -40,7 +40,7 @@ const RNG_SEED: [u8; 16] = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]; #[test] fn snap_and_restore() { - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; let mut producer = StateProducer::new(); let mut rng = XorShiftRng::from_seed(RNG_SEED); let mut old_db = journaldb::new_memory_db(); @@ -97,7 +97,7 @@ fn snap_and_restore() { let keys = old_db.keys(); for key in keys.keys() { - assert_eq!(old_db.get(&key).unwrap(), new_db.as_hash_db().get(&key).unwrap()); + assert_eq!(old_db.get(&key, EMPTY_PREFIX).unwrap(), new_db.as_hash_db().get(&key, EMPTY_PREFIX).unwrap()); } } @@ -106,7 +106,7 @@ fn get_code_from_prev_chunk() { use std::collections::HashSet; use rlp::RlpStream; use ethereum_types::{H256, U256}; - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; use account_db::{AccountDBMut, AccountDB}; @@ -128,7 +128,7 @@ fn get_code_from_prev_chunk() { let mut make_chunk = |acc, hash| { let mut db = journaldb::new_memory_db(); - AccountDBMut::from_hash(&mut db, hash).insert(&code[..]); + AccountDBMut::from_hash(&mut db, hash).insert(EMPTY_PREFIX, &code[..]); let p = Progress::default(); let fat_rlp = account::to_fat_rlps(&hash, &acc, &AccountDB::from_hash(&db, hash), &mut used_code, usize::max_value(), usize::max_value(), &p).unwrap(); let mut stream = RlpStream::new_list(1); diff --git a/ethcore/src/spec/spec.rs b/ethcore/src/spec/spec.rs index 87822226127..c09bbc8d153 100644 --- a/ethcore/src/spec/spec.rs +++ b/ethcore/src/spec/spec.rs @@ -813,7 +813,7 @@ impl Spec { /// Ensure that the given state DB has the trie nodes in for the genesis state. pub fn ensure_db_good(&self, db: T, factories: &Factories) -> Result { - if db.as_hash_db().contains(&self.state_root()) { + if db.as_hash_db().contains(&self.state_root(), hash_db::EMPTY_PREFIX) { return Ok(db); } diff --git a/ethcore/src/state/account.rs b/ethcore/src/state/account.rs index 5daaa139839..2c9abc0df67 100644 --- a/ethcore/src/state/account.rs +++ b/ethcore/src/state/account.rs @@ -364,7 +364,7 @@ impl Account { if self.is_cached() { return Some(self.code_cache.clone()); } - match db.get(&self.code_hash) { + match db.get(&self.code_hash, hash_db::EMPTY_PREFIX) { Some(x) => { self.code_size = Some(x.len()); self.code_cache = Arc::new(x.into_vec()); @@ -393,7 +393,7 @@ impl Account { trace!("Account::cache_code_size: ic={}; self.code_hash={:?}, self.code_cache={}", self.is_cached(), self.code_hash, self.code_cache.pretty()); self.code_size.is_some() || if self.code_hash != KECCAK_EMPTY { - match db.get(&self.code_hash) { + match db.get(&self.code_hash, hash_db::EMPTY_PREFIX) { Some(x) => { self.code_size = Some(x.len()); true @@ -507,7 +507,7 @@ impl Account { self.code_filth = Filth::Clean; }, (true, false) => { - db.emplace(self.code_hash.clone(), DBValue::from_slice(&*self.code_cache)); + db.emplace(self.code_hash.clone(), hash_db::EMPTY_PREFIX, DBValue::from_slice(&*self.code_cache)); self.code_size = Some(self.code_cache.len()); self.code_filth = Filth::Clean; }, diff --git a/ethcore/src/state/backend.rs b/ethcore/src/state/backend.rs index 9d9a2a9b388..c9175f26980 100644 --- a/ethcore/src/state/backend.rs +++ b/ethcore/src/state/backend.rs @@ -27,8 +27,8 @@ use std::sync::Arc; use state::Account; use parking_lot::Mutex; use ethereum_types::{Address, H256}; -use memory_db::MemoryDB; -use hash_db::{AsHashDB, HashDB}; +use memory_db::{MemoryDB, HashKey}; +use hash_db::{AsHashDB, HashDB, Prefix, EMPTY_PREFIX}; use kvdb::DBValue; use keccak_hasher::KeccakHasher; use journaldb::AsKeyedHashDB; @@ -78,13 +78,13 @@ pub trait Backend: Send { // TODO: when account lookup moved into backends, this won't rely as tenuously on intended // usage. #[derive(Clone, PartialEq)] -pub struct ProofCheck(MemoryDB); +pub struct ProofCheck(MemoryDB, DBValue>); impl ProofCheck { /// Create a new `ProofCheck` backend from the given state items. pub fn new(proof: &[DBValue]) -> Self { let mut db = journaldb::new_memory_db(); - for item in proof { db.insert(item); } + for item in proof { db.insert(EMPTY_PREFIX, item); } ProofCheck(db) } } @@ -94,23 +94,23 @@ impl journaldb::KeyedHashDB for ProofCheck { } impl HashDB for ProofCheck { - fn get(&self, key: &H256) -> Option { - self.0.get(key) + fn get(&self, key: &H256, prefix: Prefix) -> Option { + self.0.get(key, prefix) } - fn contains(&self, key: &H256) -> bool { - self.0.contains(key) + fn contains(&self, key: &H256, prefix: Prefix) -> bool { + self.0.contains(key, prefix) } - fn insert(&mut self, value: &[u8]) -> H256 { - self.0.insert(value) + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { + self.0.insert(prefix, value) } - fn emplace(&mut self, key: H256, value: DBValue) { - self.0.emplace(key, value) + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { + self.0.emplace(key, prefix, value) } - fn remove(&mut self, _key: &H256) { } + fn remove(&mut self, _key: &H256, _prefix: Prefix) { } } impl AsHashDB for ProofCheck { @@ -141,7 +141,7 @@ impl Backend for ProofCheck { /// This doesn't cache anything or rely on the canonical state caches. pub struct Proving { base: H, // state we're proving values from. - changed: MemoryDB, // changed state via insertions. + changed: MemoryDB, DBValue>, // changed state via insertions. proof: Mutex>, } @@ -163,32 +163,32 @@ impl journaldb::KeyedHashDB for Proving { } impl + Send + Sync> HashDB for Proving { - fn get(&self, key: &H256) -> Option { - match self.base.as_hash_db().get(key) { + fn get(&self, key: &H256, prefix: Prefix) -> Option { + match self.base.as_hash_db().get(key, prefix) { Some(val) => { self.proof.lock().insert(val.clone()); Some(val) } - None => self.changed.get(key) + None => self.changed.get(key, prefix) } } - fn contains(&self, key: &H256) -> bool { - self.get(key).is_some() + fn contains(&self, key: &H256, prefix: Prefix) -> bool { + self.get(key, prefix).is_some() } - fn insert(&mut self, value: &[u8]) -> H256 { - self.changed.insert(value) + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { + self.changed.insert(prefix, value) } - fn emplace(&mut self, key: H256, value: DBValue) { - self.changed.emplace(key, value) + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { + self.changed.emplace(key, prefix, value) } - fn remove(&mut self, key: &H256) { + fn remove(&mut self, key: &H256, prefix: Prefix) { // only remove from `changed` - if self.changed.contains(key) { - self.changed.remove(key) + if self.changed.contains(key, prefix) { + self.changed.remove(key, prefix) } } } diff --git a/ethcore/src/state/mod.rs b/ethcore/src/state/mod.rs index 95e31489c84..a2f07e1417c 100644 --- a/ethcore/src/state/mod.rs +++ b/ethcore/src/state/mod.rs @@ -381,7 +381,7 @@ impl State { /// Creates new state with existing state root pub fn from_existing(db: B, root: H256, account_start_nonce: U256, factories: Factories) -> TrieResult> { - if !db.as_hash_db().contains(&root) { + if !db.as_hash_db().contains(&root, hash_db::EMPTY_PREFIX) { return Err(Box::new(TrieError::InvalidStateRoot(root))); } diff --git a/ethcore/src/trace/db.rs b/ethcore/src/trace/db.rs index 4c8c2cd7f0e..baeca316665 100644 --- a/ethcore/src/trace/db.rs +++ b/ethcore/src/trace/db.rs @@ -17,12 +17,12 @@ //! Trace database. use std::collections::HashMap; use std::sync::Arc; +use parity_util_mem::MallocSizeOfExt; use blockchain::BlockChainDB; use db::cache_manager::CacheManager; use db::{self, Key, Writable, Readable, CacheUpdatePolicy}; use ethereum_types::{H256, H264}; -use heapsize::HeapSizeOf; use kvdb::{DBTransaction}; use parking_lot::RwLock; use types::BlockNumber; @@ -91,7 +91,7 @@ impl TraceDB where T: DatabaseExtras { } fn cache_size(&self) -> usize { - self.traces.read().heap_size_of_children() + self.traces.read().malloc_size_of() } /// Let the cache system know that a cacheable item has been used. @@ -113,7 +113,7 @@ impl TraceDB where T: DatabaseExtras { } traces.shrink_to_fit(); - traces.heap_size_of_children() + traces.malloc_size_of() }); } diff --git a/ethcore/src/trace/types/flat.rs b/ethcore/src/trace/types/flat.rs index cb3e1229b71..17ea1d37887 100644 --- a/ethcore/src/trace/types/flat.rs +++ b/ethcore/src/trace/types/flat.rs @@ -17,17 +17,19 @@ //! Flat trace module use rlp::{Rlp, RlpStream, Decodable, Encodable, DecoderError}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use ethereum_types::Bloom; use super::trace::{Action, Res}; /// Trace localized in vector of traces produced by a single transaction. /// /// Parent and children indexes refer to positions in this vector. -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, MallocSizeOf)] pub struct FlatTrace { + #[ignore_malloc_size_of = "ignored for performance reason"] /// Type of action performed by a transaction. pub action: Action, + #[ignore_malloc_size_of = "ignored for performance reason"] /// Result of this action. pub result: Res, /// Number of subtraces. @@ -45,12 +47,6 @@ impl FlatTrace { } } -impl HeapSizeOf for FlatTrace { - fn heap_size_of_children(&self) -> usize { - self.trace_address.heap_size_of_children() - } -} - impl Encodable for FlatTrace { fn rlp_append(&self, s: &mut RlpStream) { s.begin_list(4); @@ -76,7 +72,7 @@ impl Decodable for FlatTrace { } /// Represents all traces produced by a single transaction. -#[derive(Debug, PartialEq, Clone, RlpEncodableWrapper, RlpDecodableWrapper)] +#[derive(Debug, PartialEq, Clone, RlpEncodableWrapper, RlpDecodableWrapper, MallocSizeOf)] pub struct FlatTransactionTraces(Vec); impl From> for FlatTransactionTraces { @@ -85,12 +81,6 @@ impl From> for FlatTransactionTraces { } } -impl HeapSizeOf for FlatTransactionTraces { - fn heap_size_of_children(&self) -> usize { - self.0.heap_size_of_children() - } -} - impl FlatTransactionTraces { /// Returns bloom of all traces in the collection. pub fn bloom(&self) -> Bloom { @@ -105,15 +95,9 @@ impl Into> for FlatTransactionTraces { } /// Represents all traces produced by transactions in a single block. -#[derive(Debug, PartialEq, Clone, Default, RlpEncodableWrapper, RlpDecodableWrapper)] +#[derive(Debug, PartialEq, Clone, Default, RlpEncodableWrapper, RlpDecodableWrapper, MallocSizeOf)] pub struct FlatBlockTraces(Vec); -impl HeapSizeOf for FlatBlockTraces { - fn heap_size_of_children(&self) -> usize { - self.0.heap_size_of_children() - } -} - impl From> for FlatBlockTraces { fn from(v: Vec) -> Self { FlatBlockTraces(v) diff --git a/ethcore/src/verification/queue/kind.rs b/ethcore/src/verification/queue/kind.rs index ed96ebc9d0e..508503a04b0 100644 --- a/ethcore/src/verification/queue/kind.rs +++ b/ethcore/src/verification/queue/kind.rs @@ -19,7 +19,7 @@ use engines::EthEngine; use error::Error; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use ethereum_types::{H256, U256}; pub use self::blocks::Blocks; @@ -49,13 +49,13 @@ pub trait BlockLike { /// consistent. pub trait Kind: 'static + Sized + Send + Sync { /// The first stage: completely unverified. - type Input: Sized + Send + BlockLike + HeapSizeOf; + type Input: Sized + Send + BlockLike + MallocSizeOf; /// The second stage: partially verified. - type Unverified: Sized + Send + BlockLike + HeapSizeOf; + type Unverified: Sized + Send + BlockLike + MallocSizeOf; /// The third stage: completely verified. - type Verified: Sized + Send + BlockLike + HeapSizeOf; + type Verified: Sized + Send + BlockLike + MallocSizeOf; /// Attempt to create the `Unverified` item from the input. fn create(input: Self::Input, engine: &dyn EthEngine, check_seal: bool) -> Result; @@ -74,7 +74,7 @@ pub mod blocks { use verification::{PreverifiedBlock, verify_block_basic, verify_block_unordered}; use types::transaction::UnverifiedTransaction; - use heapsize::HeapSizeOf; + use parity_util_mem::MallocSizeOf; use ethereum_types::{H256, U256}; use bytes::Bytes; @@ -113,7 +113,7 @@ pub mod blocks { } /// An unverified block. - #[derive(PartialEq, Debug)] + #[derive(PartialEq, Debug, MallocSizeOf)] pub struct Unverified { /// Unverified block header. pub header: Header, @@ -146,15 +146,6 @@ pub mod blocks { } } - impl HeapSizeOf for Unverified { - fn heap_size_of_children(&self) -> usize { - self.header.heap_size_of_children() - + self.transactions.heap_size_of_children() - + self.uncles.heap_size_of_children() - + self.bytes.heap_size_of_children() - } - } - impl BlockLike for Unverified { fn hash(&self) -> H256 { self.header.hash() diff --git a/ethcore/src/verification/queue/mod.rs b/ethcore/src/verification/queue/mod.rs index 3f3484ddc44..7acc496ebb8 100644 --- a/ethcore/src/verification/queue/mod.rs +++ b/ethcore/src/verification/queue/mod.rs @@ -22,7 +22,7 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering}; use std::sync::Arc; use std::cmp; use std::collections::{VecDeque, HashSet, HashMap}; -use heapsize::HeapSizeOf; +use parity_util_mem::{MallocSizeOf, MallocSizeOfExt}; use ethereum_types::{H256, U256}; use parking_lot::{Condvar, Mutex, RwLock}; use io::*; @@ -96,17 +96,12 @@ enum State { } /// An item which is in the process of being verified. +#[derive(MallocSizeOf)] pub struct Verifying { hash: H256, output: Option, } -impl HeapSizeOf for Verifying { - fn heap_size_of_children(&self) -> usize { - self.output.heap_size_of_children() - } -} - /// Status of items in the queue. pub enum Status { /// Currently queued. @@ -353,7 +348,7 @@ impl VerificationQueue { None => continue, }; - verification.sizes.unverified.fetch_sub(item.heap_size_of_children(), AtomicOrdering::SeqCst); + verification.sizes.unverified.fetch_sub(item.malloc_size_of(), AtomicOrdering::SeqCst); verifying.push_back(Verifying { hash: item.hash(), output: None }); item }; @@ -367,7 +362,7 @@ impl VerificationQueue { if e.hash == hash { idx = Some(i); - verification.sizes.verifying.fetch_add(verified.heap_size_of_children(), AtomicOrdering::SeqCst); + verification.sizes.verifying.fetch_add(verified.malloc_size_of(), AtomicOrdering::SeqCst); e.output = Some(verified); break; } @@ -417,7 +412,7 @@ impl VerificationQueue { while let Some(output) = verifying.front_mut().and_then(|x| x.output.take()) { assert!(verifying.pop_front().is_some()); - let size = output.heap_size_of_children(); + let size = output.malloc_size_of(); removed_size += size; if bad.contains(&output.parent_hash()) { @@ -490,7 +485,7 @@ impl VerificationQueue { match K::create(input, &*self.engine, self.verification.check_seal) { Ok(item) => { - self.verification.sizes.unverified.fetch_add(item.heap_size_of_children(), AtomicOrdering::SeqCst); + self.verification.sizes.unverified.fetch_add(item.malloc_size_of(), AtomicOrdering::SeqCst); self.processing.write().insert(hash, item.difficulty()); { @@ -537,7 +532,7 @@ impl VerificationQueue { let mut removed_size = 0; for output in verified.drain(..) { if bad.contains(&output.parent_hash()) { - removed_size += output.heap_size_of_children(); + removed_size += output.malloc_size_of(); bad.insert(output.hash()); if let Some(difficulty) = processing.remove(&output.hash()) { let mut td = self.total_difficulty.write(); @@ -574,7 +569,7 @@ impl VerificationQueue { let count = cmp::min(max, verified.len()); let result = verified.drain(..count).collect::>(); - let drained_size = result.iter().map(HeapSizeOf::heap_size_of_children).fold(0, |a, c| a + c); + let drained_size = result.iter().map(MallocSizeOfExt::malloc_size_of).fold(0, |a, c| a + c); self.verification.sizes.verified.fetch_sub(drained_size, AtomicOrdering::SeqCst); self.ready_signal.reset(); diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index fa458803cd5..7440ad5c645 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -26,7 +26,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use bytes::Bytes; use hash::keccak; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use rlp::Rlp; use triehash::ordered_trie_root; use unexpected::{Mismatch, OutOfBounds}; @@ -44,6 +44,7 @@ use verification::queue::kind::blocks::Unverified; use time_utils::CheckedSystemTime; /// Preprocessed block data gathered in `verify_block_unordered` call +#[derive(MallocSizeOf)] pub struct PreverifiedBlock { /// Populated block header pub header: Header, @@ -55,14 +56,6 @@ pub struct PreverifiedBlock { pub bytes: Bytes, } -impl HeapSizeOf for PreverifiedBlock { - fn heap_size_of_children(&self) -> usize { - self.header.heap_size_of_children() - + self.transactions.heap_size_of_children() - + self.bytes.heap_size_of_children() - } -} - /// Phase 1 quick block verification. Only does checks that are cheap. Operates on a single block pub fn verify_block_basic(block: &Unverified, engine: &dyn EthEngine, check_seal: bool) -> Result<(), Error> { verify_header_params(&block.header, engine, true, check_seal)?; diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 83284063934..1bec74526af 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -19,8 +19,7 @@ ethereum-types = "0.6.0" ethkey = { path = "../../accounts/ethkey" } ethstore = { path = "../../accounts/ethstore" } fastmap = { path = "../../util/fastmap" } -hash-db = "0.11.0" -heapsize = "0.4" +hash-db = "0.12.4" keccak-hash = "0.2.0" keccak-hasher = { path = "../../util/keccak-hasher" } kvdb = "0.1" @@ -28,6 +27,7 @@ log = "0.4" macros = { path = "../../util/macros" } parity-bytes = "0.1" parking_lot = "0.7" +parity-util-mem = "0.1" rand = "0.6" rlp = "0.4.0" trace-time = "0.1" diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index 7aa9e71ae0a..82558235792 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -65,7 +65,7 @@ pub const ETH_PROTOCOL: ProtocolId = *b"eth"; pub const LIGHT_PROTOCOL: ProtocolId = *b"pip"; /// Determine warp sync status. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, MallocSizeOf)] pub enum WarpSync { /// Warp sync is enabled. Enabled, diff --git a/ethcore/sync/src/block_sync.rs b/ethcore/sync/src/block_sync.rs index 490bd39a809..ca521790dc0 100644 --- a/ethcore/sync/src/block_sync.rs +++ b/ethcore/sync/src/block_sync.rs @@ -20,7 +20,7 @@ use std::collections::{HashSet, VecDeque}; use std::cmp; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use ethereum_types::H256; use rlp::{self, Rlp}; use types::BlockNumber; @@ -60,7 +60,7 @@ macro_rules! debug_sync { }; } -#[derive(Copy, Clone, Eq, PartialEq, Debug)] +#[derive(Copy, Clone, Eq, PartialEq, Debug, MallocSizeOf)] /// Downloader state pub enum State { /// No active downloads. @@ -113,6 +113,7 @@ impl From for BlockDownloaderImportError { /// Block downloader strategy. /// Manages state and block data for a block download process. +#[derive(MallocSizeOf)] pub struct BlockDownloader { /// Which set of blocks to download block_set: BlockSet, @@ -223,11 +224,6 @@ impl BlockDownloader { self.state = State::Blocks; } - /// Returns used heap memory size. - pub fn heap_size(&self) -> usize { - self.blocks.heap_size() + self.round_parents.heap_size_of_children() - } - /// Returns best imported block number. pub fn last_imported_block_number(&self) -> BlockNumber { self.last_imported_block diff --git a/ethcore/sync/src/blocks.rs b/ethcore/sync/src/blocks.rs index 2ce2be33348..e9b7000a4a7 100644 --- a/ethcore/sync/src/blocks.rs +++ b/ethcore/sync/src/blocks.rs @@ -16,7 +16,7 @@ use std::collections::{HashSet, HashMap, hash_map}; use hash::{keccak, KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use ethereum_types::H256; use triehash_ethereum::ordered_trie_root; use bytes::Bytes; @@ -26,21 +26,15 @@ use ethcore::verification::queue::kind::blocks::Unverified; use types::transaction::UnverifiedTransaction; use types::header::Header as BlockHeader; -known_heap_size!(0, HeaderId); +malloc_size_of_is_0!(HeaderId); #[derive(PartialEq, Debug, Clone)] +#[derive(MallocSizeOf)] pub struct SyncHeader { pub bytes: Bytes, pub header: BlockHeader, } -impl HeapSizeOf for SyncHeader { - fn heap_size_of_children(&self) -> usize { - self.bytes.heap_size_of_children() - + self.header.heap_size_of_children() - } -} - impl SyncHeader { pub fn from_rlp(bytes: Bytes) -> Result { let result = SyncHeader { @@ -52,6 +46,7 @@ impl SyncHeader { } } +#[derive(MallocSizeOf)] pub struct SyncBody { pub transactions_bytes: Bytes, pub transactions: Vec, @@ -85,16 +80,8 @@ impl SyncBody { } } -impl HeapSizeOf for SyncBody { - fn heap_size_of_children(&self) -> usize { - self.transactions_bytes.heap_size_of_children() - + self.transactions.heap_size_of_children() - + self.uncles_bytes.heap_size_of_children() - + self.uncles.heap_size_of_children() - } -} - /// Block data with optional body. +#[derive(MallocSizeOf)] struct SyncBlock { header: SyncHeader, body: Option, @@ -102,12 +89,6 @@ struct SyncBlock { receipts_root: H256, } -impl HeapSizeOf for SyncBlock { - fn heap_size_of_children(&self) -> usize { - self.header.heap_size_of_children() + self.body.heap_size_of_children() - } -} - fn unverified_from_sync(header: SyncHeader, body: Option) -> Unverified { let mut stream = RlpStream::new_list(3); stream.append_raw(&header.bytes, 1); @@ -141,7 +122,7 @@ struct HeaderId { /// A collection of blocks and subchain pointers being downloaded. This keeps track of /// which headers/bodies need to be downloaded, which are being downloaded and also holds /// the downloaded blocks. -#[derive(Default)] +#[derive(Default, MallocSizeOf)] pub struct BlockCollection { /// Does this collection need block receipts. need_receipts: bool, @@ -399,16 +380,6 @@ impl BlockCollection { self.heads.len() } - /// Return used heap size. - pub fn heap_size(&self) -> usize { - self.heads.heap_size_of_children() - + self.blocks.heap_size_of_children() - + self.parents.heap_size_of_children() - + self.header_ids.heap_size_of_children() - + self.downloading_headers.heap_size_of_children() - + self.downloading_bodies.heap_size_of_children() - } - /// Check if given block hash is marked as being downloaded. pub fn is_downloading(&self, hash: &H256) -> bool { self.downloading_headers.contains(hash) || self.downloading_bodies.contains(hash) diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 20d5b2cd580..bde38cda399 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -98,7 +98,7 @@ use std::collections::{HashSet, HashMap, BTreeMap}; use std::cmp; use std::time::{Duration, Instant}; use hash::keccak; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOfExt; use futures::sync::mpsc as futures_mpsc; use api::Notification; use ethereum_types::{H256, U256}; @@ -132,7 +132,7 @@ use self::propagator::SyncPropagator; use self::requester::SyncRequester; pub(crate) use self::supplier::SyncSupplier; -known_heap_size!(0, PeerInfo); +malloc_size_of_is_0!(PeerInfo); pub type PacketDecodeError = DecoderError; @@ -179,7 +179,7 @@ const SNAPSHOT_DATA_TIMEOUT: Duration = Duration::from_secs(120); /// (so we might sent only to some part of the peers we originally intended to send to) const PRIORITY_TASK_DEADLINE: Duration = Duration::from_millis(100); -#[derive(Copy, Clone, Eq, PartialEq, Debug)] +#[derive(Copy, Clone, Eq, PartialEq, Debug, MallocSizeOf)] /// Sync state pub enum SyncState { /// Collecting enough peers to start syncing. @@ -273,7 +273,7 @@ pub enum PeerAsking { SnapshotData, } -#[derive(PartialEq, Eq, Debug, Clone, Copy)] +#[derive(PartialEq, Eq, Debug, Clone, Copy, MallocSizeOf)] /// Block downloader channel. pub enum BlockSet { /// New blocks better than out best blocks @@ -585,6 +585,7 @@ enum PeerState { /// Blockchain sync handler. /// See module documentation for more details. +#[derive(MallocSizeOf)] pub struct ChainSync { /// Sync state state: SyncState, @@ -618,10 +619,12 @@ pub struct ChainSync { /// Enable ancient block downloading download_old_blocks: bool, /// Shared private tx service. + #[ignore_malloc_size_of = "arc on dyn trait here seems tricky, ignoring"] private_tx_handler: Option>, /// Enable warp sync. warp_sync: WarpSync, + #[ignore_malloc_size_of = "mpsc unmettered, ignoring"] status_sinks: Vec> } @@ -677,10 +680,7 @@ impl ChainSync { num_active_peers: self.peers.values().filter(|p| p.is_allowed() && p.asking != PeerAsking::Nothing).count(), num_snapshot_chunks: self.snapshot.total_chunks(), snapshot_chunks_done: self.snapshot.done_chunks(), - mem_used: - self.new_blocks.heap_size() - + self.old_blocks.as_ref().map_or(0, |d| d.heap_size()) - + self.peers.heap_size_of_children(), + mem_used: self.malloc_size_of(), } } diff --git a/ethcore/sync/src/lib.rs b/ethcore/sync/src/lib.rs index 8ed8befc87a..805e9f0c15f 100644 --- a/ethcore/sync/src/lib.rs +++ b/ethcore/sync/src/lib.rs @@ -53,8 +53,10 @@ extern crate enum_primitive; extern crate macros; #[macro_use] extern crate log; +extern crate parity_util_mem; +extern crate parity_util_mem as mem; #[macro_use] -extern crate heapsize; +extern crate parity_util_mem as malloc_size_of; #[macro_use] extern crate trace_time; diff --git a/ethcore/sync/src/snapshot.rs b/ethcore/sync/src/snapshot.rs index e7c2c3dcef2..3445a9f630d 100644 --- a/ethcore/sync/src/snapshot.rs +++ b/ethcore/sync/src/snapshot.rs @@ -27,6 +27,7 @@ pub enum ChunkType { Block(H256), } +#[derive(MallocSizeOf)] pub struct Snapshot { pending_state_chunks: Vec, pending_block_chunks: Vec, diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index 5f4ab0ab3a2..853cf232d36 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -23,7 +23,7 @@ use types::BlockNumber; type NodeId = H512; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, MallocSizeOf)] pub struct Stats { first_seen: BlockNumber, propagated_to: HashMap, @@ -50,7 +50,7 @@ impl<'a> From<&'a Stats> for TransactionStats { } } -#[derive(Debug, Default)] +#[derive(Debug, Default, MallocSizeOf)] pub struct TransactionsStats { pending_transactions: H256FastMap, } diff --git a/ethcore/types/Cargo.toml b/ethcore/types/Cargo.toml index 98e7c325bf6..e702b6af3b0 100644 --- a/ethcore/types/Cargo.toml +++ b/ethcore/types/Cargo.toml @@ -7,8 +7,8 @@ authors = ["Parity Technologies "] [dependencies] ethereum-types = "0.6.0" ethjson = { path = "../../json" } +parity-util-mem = "0.1" ethkey = { path = "../../accounts/ethkey" } -heapsize = "0.4" keccak-hash = "0.2.0" parity-bytes = "0.1" rlp = "0.4.0" diff --git a/ethcore/types/src/encoded.rs b/ethcore/types/src/encoded.rs index 4680f95af50..c402d7e497d 100644 --- a/ethcore/types/src/encoded.rs +++ b/ethcore/types/src/encoded.rs @@ -27,20 +27,16 @@ use block::Block as FullBlock; use ethereum_types::{H256, Bloom, U256, Address}; use hash::keccak; use header::{Header as FullHeader}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use rlp::{self, Rlp, RlpStream}; use transaction::UnverifiedTransaction; use views::{self, BlockView, HeaderView, BodyView}; use BlockNumber; /// Owning header view. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)] pub struct Header(Vec); -impl HeapSizeOf for Header { - fn heap_size_of_children(&self) -> usize { self.0.heap_size_of_children() } -} - impl Header { /// Create a new owning header view. /// Expects the data to be an RLP-encoded header -- any other case will likely lead to @@ -113,13 +109,9 @@ impl Header { } /// Owning block body view. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)] pub struct Body(Vec); -impl HeapSizeOf for Body { - fn heap_size_of_children(&self) -> usize { self.0.heap_size_of_children() } -} - impl Body { /// Create a new owning block body view. The raw bytes passed in must be an rlp-encoded block /// body. @@ -178,13 +170,9 @@ impl Body { } /// Owning block view. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)] pub struct Block(Vec); -impl HeapSizeOf for Block { - fn heap_size_of_children(&self) -> usize { self.0.heap_size_of_children() } -} - impl Block { /// Create a new owning block view. The raw bytes passed in must be an rlp-encoded block. pub fn new(raw: Vec) -> Self { Block(raw) } diff --git a/ethcore/types/src/header.rs b/ethcore/types/src/header.rs index 408253bc5f8..b5245fcbb28 100644 --- a/ethcore/types/src/header.rs +++ b/ethcore/types/src/header.rs @@ -17,7 +17,7 @@ //! Block header. use hash::{KECCAK_NULL_RLP, KECCAK_EMPTY_LIST_RLP, keccak}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use ethereum_types::{H256, U256, Address, Bloom}; use bytes::Bytes; use rlp::{Rlp, RlpStream, Encodable, DecoderError, Decodable}; @@ -49,7 +49,7 @@ pub struct ExtendedHeader { /// which is non-specific. /// /// Doesn't do all that much on its own. -#[derive(Debug, Clone, Eq)] +#[derive(Debug, Clone, Eq, MallocSizeOf)] pub struct Header { /// Parent hash. parent_hash: H256, @@ -361,12 +361,6 @@ impl Encodable for Header { } } -impl HeapSizeOf for Header { - fn heap_size_of_children(&self) -> usize { - self.extra_data.heap_size_of_children() + self.seal.heap_size_of_children() - } -} - impl ExtendedHeader { /// Returns combined difficulty of all ancestors together with the difficulty of this header. pub fn total_score(&self) -> U256 { diff --git a/ethcore/types/src/lib.rs b/ethcore/types/src/lib.rs index 3223db72206..93145b2e9c5 100644 --- a/ethcore/types/src/lib.rs +++ b/ethcore/types/src/lib.rs @@ -36,7 +36,6 @@ extern crate ethereum_types; extern crate ethjson; extern crate ethkey; -extern crate heapsize; extern crate keccak_hash as hash; extern crate parity_bytes as bytes; extern crate rlp; @@ -44,6 +43,9 @@ extern crate unexpected; #[macro_use] extern crate rlp_derive; +extern crate parity_util_mem; +extern crate parity_util_mem as mem; +extern crate parity_util_mem as malloc_size_of; #[cfg(test)] extern crate rustc_hex; diff --git a/ethcore/types/src/log_entry.rs b/ethcore/types/src/log_entry.rs index cbbfb55bb9c..2d74554a610 100644 --- a/ethcore/types/src/log_entry.rs +++ b/ethcore/types/src/log_entry.rs @@ -17,7 +17,7 @@ //! Log entry type definition. use std::ops::Deref; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use bytes::Bytes; use ethereum_types::{H256, Address, Bloom, BloomInput}; @@ -25,7 +25,7 @@ use {BlockNumber}; use ethjson; /// A record of execution for a `LOG` operation. -#[derive(Default, Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] +#[derive(Default, Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable, MallocSizeOf)] pub struct LogEntry { /// The address of the contract executing at the point of the `LOG` operation. pub address: Address, @@ -35,12 +35,6 @@ pub struct LogEntry { pub data: Bytes, } -impl HeapSizeOf for LogEntry { - fn heap_size_of_children(&self) -> usize { - self.topics.heap_size_of_children() + self.data.heap_size_of_children() - } -} - impl LogEntry { /// Calculates the bloom of this log entry. pub fn bloom(&self) -> Bloom { diff --git a/ethcore/types/src/receipt.rs b/ethcore/types/src/receipt.rs index 797eea81680..95b39e7de4b 100644 --- a/ethcore/types/src/receipt.rs +++ b/ethcore/types/src/receipt.rs @@ -17,14 +17,14 @@ //! Receipt use ethereum_types::{H160, H256, U256, Address, Bloom}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; use rlp::{Rlp, RlpStream, Encodable, Decodable, DecoderError}; use BlockNumber; use log_entry::{LogEntry, LocalizedLogEntry}; /// Transaction outcome store in the receipt. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)] pub enum TransactionOutcome { /// Status and state root are unknown under EIP-98 rules. Unknown, @@ -35,7 +35,7 @@ pub enum TransactionOutcome { } /// Information describing execution of a transaction. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)] pub struct Receipt { /// The total gas used in the block following execution of the transaction. pub gas_used: U256, @@ -110,12 +110,6 @@ impl Decodable for Receipt { } } -impl HeapSizeOf for Receipt { - fn heap_size_of_children(&self) -> usize { - self.logs.heap_size_of_children() - } -} - /// Receipt with additional info. #[derive(Debug, Clone, PartialEq)] pub struct RichReceipt { diff --git a/ethcore/types/src/transaction/transaction.rs b/ethcore/types/src/transaction/transaction.rs index 87241591cd7..12e0d712563 100644 --- a/ethcore/types/src/transaction/transaction.rs +++ b/ethcore/types/src/transaction/transaction.rs @@ -22,7 +22,8 @@ use ethereum_types::{H256, H160, Address, U256, BigEndianHash}; use ethjson; use ethkey::{self, Signature, Secret, Public, recover, public_to_address}; use hash::keccak; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOf; + use rlp::{self, RlpStream, Rlp, DecoderError, Encodable}; use transaction::error; @@ -37,7 +38,7 @@ pub const UNSIGNED_SENDER: Address = H160([0xff; 20]); pub const SYSTEM_ADDRESS: Address = H160([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,0xff, 0xff, 0xff, 0xff,0xff, 0xff, 0xff, 0xff,0xff, 0xff, 0xff, 0xfe]); /// Transaction action type. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)] pub enum Action { /// Create creates new contract. Create, @@ -99,7 +100,7 @@ pub mod signature { /// A set of information describing an externally-originating message call /// or contract creation operation. -#[derive(Default, Debug, Clone, PartialEq, Eq)] +#[derive(Default, Debug, Clone, PartialEq, Eq, MallocSizeOf)] pub struct Transaction { /// Nonce. pub nonce: U256, @@ -133,12 +134,6 @@ impl Transaction { } } -impl HeapSizeOf for Transaction { - fn heap_size_of_children(&self) -> usize { - self.data.heap_size_of_children() - } -} - impl From for SignedTransaction { fn from(t: ethjson::state::Transaction) -> Self { let to: Option = t.to.into(); @@ -255,7 +250,7 @@ impl Transaction { } /// Signed transaction information without verified signature. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, MallocSizeOf)] pub struct UnverifiedTransaction { /// Plain Transaction. unsigned: Transaction, @@ -270,12 +265,6 @@ pub struct UnverifiedTransaction { hash: H256, } -impl HeapSizeOf for UnverifiedTransaction { - fn heap_size_of_children(&self) -> usize { - self.unsigned.heap_size_of_children() - } -} - impl Deref for UnverifiedTransaction { type Target = Transaction; @@ -407,19 +396,13 @@ impl UnverifiedTransaction { } /// A `UnverifiedTransaction` with successfully recovered `sender`. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, MallocSizeOf)] pub struct SignedTransaction { transaction: UnverifiedTransaction, sender: Address, public: Option, } -impl HeapSizeOf for SignedTransaction { - fn heap_size_of_children(&self) -> usize { - self.transaction.heap_size_of_children() - } -} - impl rlp::Encodable for SignedTransaction { fn rlp_append(&self, s: &mut RlpStream) { self.transaction.rlp_append_sealed_transaction(s) } } diff --git a/ethcore/vm/Cargo.toml b/ethcore/vm/Cargo.toml index cde8ed79e6e..02504372bc3 100644 --- a/ethcore/vm/Cargo.toml +++ b/ethcore/vm/Cargo.toml @@ -7,7 +7,7 @@ authors = ["Parity Technologies "] byteorder = "1.0" parity-bytes = "0.1" ethereum-types = "0.6.0" -trie-db = "0.11.0" +trie-db = "0.12.4" patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" } log = "0.4" ethjson = { path = "../../json" } diff --git a/miner/Cargo.toml b/miner/Cargo.toml index 68de893ec81..a9ac13c2e66 100644 --- a/miner/Cargo.toml +++ b/miner/Cargo.toml @@ -23,7 +23,7 @@ ethabi-contract = "8.0" ethcore-call-contract = { path = "../ethcore/call-contract" } ethereum-types = "0.6.0" futures = "0.1" -heapsize = "0.4" +parity-util-mem = "0.1" keccak-hash = "0.2.0" linked-hash-map = "0.5" log = "0.4" diff --git a/miner/src/lib.rs b/miner/src/lib.rs index 99f7a293b55..2880a645c36 100644 --- a/miner/src/lib.rs +++ b/miner/src/lib.rs @@ -25,7 +25,8 @@ extern crate ethabi; extern crate ethcore_call_contract as call_contract; extern crate ethereum_types; extern crate futures; -extern crate heapsize; + +extern crate parity_util_mem; extern crate keccak_hash as hash; extern crate linked_hash_map; extern crate parity_runtime; diff --git a/miner/src/pool/mod.rs b/miner/src/pool/mod.rs index fcd3859e731..6dfb26ffeb5 100644 --- a/miner/src/pool/mod.rs +++ b/miner/src/pool/mod.rs @@ -17,7 +17,7 @@ //! Transaction Pool use ethereum_types::{U256, H256, Address}; -use heapsize::HeapSizeOf; +use parity_util_mem::MallocSizeOfExt; use types::transaction; use txpool; @@ -176,7 +176,7 @@ impl txpool::VerifiedTransaction for VerifiedTransaction { } fn mem_usage(&self) -> usize { - self.transaction.heap_size_of_children() + self.transaction.malloc_size_of() } fn sender(&self) -> &Address { diff --git a/util/journaldb/Cargo.toml b/util/journaldb/Cargo.toml index eaae9404304..64c7d4db181 100644 --- a/util/journaldb/Cargo.toml +++ b/util/journaldb/Cargo.toml @@ -8,12 +8,12 @@ license = "GPL3" [dependencies] parity-bytes = "0.1" ethereum-types = "0.6.0" -hash-db = "0.11.0" -heapsize = "0.4" +hash-db = "0.12.4" +parity-util-mem = "0.1" keccak-hasher = { path = "../keccak-hasher" } kvdb = "0.1" log = "0.4" -memory-db = "0.11.0" +memory-db = "0.12.4" parking_lot = "0.7" fastmap = { path = "../../util/fastmap" } rlp = "0.4.0" diff --git a/util/journaldb/src/archivedb.rs b/util/journaldb/src/archivedb.rs index 6dd1504eb4f..edeac382690 100644 --- a/util/journaldb/src/archivedb.rs +++ b/util/journaldb/src/archivedb.rs @@ -23,12 +23,12 @@ use std::sync::Arc; use bytes::Bytes; use ethereum_types::H256; -use hash_db::{HashDB}; +use parity_util_mem::MallocSizeOfExt; +use hash_db::{HashDB, Prefix}; use keccak_hasher::KeccakHasher; use kvdb::{KeyValueDB, DBTransaction, DBValue}; use rlp::{encode, decode}; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, error_key_already_exists, error_negatively_reference_hash}; -use super::memory_db::*; use traits::JournalDB; /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay @@ -39,7 +39,7 @@ use traits::JournalDB; /// immediately. As this is an "archive" database, nothing is ever removed. This means /// that the states of any block the node has ever processed will be accessible. pub struct ArchiveDB { - overlay: MemoryDB, + overlay: super::MemoryDB, backing: Arc, latest_era: Option, column: Option, @@ -66,8 +66,8 @@ impl ArchiveDB { } impl HashDB for ArchiveDB { - fn get(&self, key: &H256) -> Option { - if let Some((d, rc)) = self.overlay.raw(key) { + fn get(&self, key: &H256, prefix: Prefix) -> Option { + if let Some((d, rc)) = self.overlay.raw(key, prefix) { if rc > 0 { return Some(d.clone()); } @@ -75,20 +75,20 @@ impl HashDB for ArchiveDB { self.payload(key) } - fn contains(&self, key: &H256) -> bool { - self.get(key).is_some() + fn contains(&self, key: &H256, prefix: Prefix) -> bool { + self.get(key, prefix).is_some() } - fn insert(&mut self, value: &[u8]) -> H256 { - self.overlay.insert(value) + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { + self.overlay.insert(prefix, value) } - fn emplace(&mut self, key: H256, value: DBValue) { - self.overlay.emplace(key, value); + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { + self.overlay.emplace(key, prefix, value); } - fn remove(&mut self, key: &H256) { - self.overlay.remove(key); + fn remove(&mut self, key: &H256, prefix: Prefix) { + self.overlay.remove(key, prefix); } } @@ -124,7 +124,7 @@ impl JournalDB for ArchiveDB { } fn mem_used(&self) -> usize { - self.overlay.mem_used() + self.overlay.malloc_size_of() } fn is_empty(&self) -> bool { @@ -197,7 +197,7 @@ impl JournalDB for ArchiveDB { &self.backing } - fn consolidate(&mut self, with: MemoryDB) { + fn consolidate(&mut self, with: super::MemoryDB) { self.overlay.consolidate(with); } } @@ -206,7 +206,7 @@ impl JournalDB for ArchiveDB { mod tests { use keccak::keccak; - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; use super::*; use {kvdb_memorydb, JournalDB}; @@ -215,50 +215,50 @@ mod tests { // history is 1 let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); - let x = jdb.insert(b"X"); + let x = jdb.insert(EMPTY_PREFIX, b"X"); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); jdb.commit_batch(3, &keccak(b"1002a"), Some((1, keccak(b"1")))).unwrap(); jdb.commit_batch(4, &keccak(b"1003a"), Some((2, keccak(b"2")))).unwrap(); - jdb.remove(&x); + jdb.remove(&x, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"1002b"), Some((1, keccak(b"1")))).unwrap(); - let x = jdb.insert(b"X"); + let x = jdb.insert(EMPTY_PREFIX, b"X"); jdb.commit_batch(4, &keccak(b"1003b"), Some((2, keccak(b"2")))).unwrap(); jdb.commit_batch(5, &keccak(b"1004a"), Some((3, keccak(b"1002a")))).unwrap(); jdb.commit_batch(6, &keccak(b"1005a"), Some((4, keccak(b"1003a")))).unwrap(); - assert!(jdb.contains(&x)); + assert!(jdb.contains(&x, EMPTY_PREFIX)); } #[test] fn long_history() { // history is 3 let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); - let h = jdb.insert(b"foo"); + let h = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&h)); - jdb.remove(&h); + assert!(jdb.contains(&h, EMPTY_PREFIX)); + jdb.remove(&h, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap(); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap(); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); } #[test] #[should_panic] fn multiple_owed_removal_not_allowed() { let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); - let h = jdb.insert(b"foo"); + let h = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&h)); - jdb.remove(&h); - jdb.remove(&h); + assert!(jdb.contains(&h, EMPTY_PREFIX)); + jdb.remove(&h, EMPTY_PREFIX); + jdb.remove(&h, EMPTY_PREFIX); // commit_batch would call journal_under(), // and we don't allow multiple owned removals. jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); @@ -269,29 +269,29 @@ mod tests { // history is 1 let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - jdb.remove(&bar); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + jdb.remove(&bar, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - let foo = jdb.insert(b"foo"); - jdb.remove(&baz); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.remove(&baz, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap(); } @@ -301,25 +301,25 @@ mod tests { // history is 1 let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -327,18 +327,18 @@ mod tests { // history is 1 let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); - jdb.insert(b"foo"); - assert!(jdb.contains(&foo)); + jdb.insert(EMPTY_PREFIX, b"foo"); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(3, &keccak(b"2"), Some((0, keccak(b"2")))).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -347,15 +347,15 @@ mod tests { let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2a"), Some((1, keccak(b"1a")))).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -366,22 +366,22 @@ mod tests { let foo = { let mut jdb = ArchiveDB::new(shared_db.clone(), None); // history is 1 - let foo = jdb.insert(b"foo"); - jdb.emplace(bar.clone(), DBValue::from_slice(b"bar")); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.emplace(bar.clone(), EMPTY_PREFIX, DBValue::from_slice(b"bar")); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); foo }; { let mut jdb = ArchiveDB::new(shared_db.clone(), None); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); } { let mut jdb = ArchiveDB::new(shared_db, None); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); } } @@ -393,23 +393,23 @@ mod tests { let foo = { let mut jdb = ArchiveDB::new(shared_db.clone(), None); // history is 1 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); // foo is ancient history. - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); foo }; { let mut jdb = ArchiveDB::new(shared_db, None); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); - assert!(jdb.contains(&foo)); - jdb.remove(&foo); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap(); jdb.commit_batch(5, &keccak(b"5"), Some((4, keccak(b"4")))).unwrap(); } @@ -421,14 +421,14 @@ mod tests { let (foo, _, _) = { let mut jdb = ArchiveDB::new(shared_db.clone(), None); // history is 1 - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - jdb.remove(&foo); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); (foo, bar, baz) }; @@ -436,7 +436,7 @@ mod tests { { let mut jdb = ArchiveDB::new(shared_db, None); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } } @@ -446,7 +446,7 @@ mod tests { let key = { let mut jdb = ArchiveDB::new(shared_db.clone(), None); - let key = jdb.insert(b"foo"); + let key = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); key }; @@ -461,13 +461,13 @@ mod tests { #[test] fn inject() { let mut jdb = ArchiveDB::new(Arc::new(kvdb_memorydb::create(0)), None); - let key = jdb.insert(b"dog"); + let key = jdb.insert(EMPTY_PREFIX, b"dog"); jdb.inject_batch().unwrap(); - assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog")); - jdb.remove(&key); + assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog")); + jdb.remove(&key, EMPTY_PREFIX); jdb.inject_batch().unwrap(); - assert!(jdb.get(&key).is_none()); + assert!(jdb.get(&key, EMPTY_PREFIX).is_none()); } } diff --git a/util/journaldb/src/earlymergedb.rs b/util/journaldb/src/earlymergedb.rs index 563df751b8b..5d12cd480c0 100644 --- a/util/journaldb/src/earlymergedb.rs +++ b/util/journaldb/src/earlymergedb.rs @@ -23,27 +23,22 @@ use std::sync::Arc; use bytes::Bytes; use ethereum_types::H256; -use hash_db::{HashDB}; -use heapsize::HeapSizeOf; +use hash_db::{HashDB, Prefix}; +use parity_util_mem::{MallocSizeOf, allocators::new_malloc_size_ops}; use keccak_hasher::KeccakHasher; use kvdb::{KeyValueDB, DBTransaction, DBValue}; -use memory_db::*; use parking_lot::RwLock; use rlp::{encode, decode}; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY, error_negatively_reference_hash, error_key_already_exists}; use super::traits::JournalDB; use util::{DatabaseKey, DatabaseValueView, DatabaseValueRef}; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, MallocSizeOf)] struct RefInfo { queue_refs: usize, in_archive: bool, } -impl HeapSizeOf for RefInfo { - fn heap_size_of_children(&self) -> usize { 0 } -} - #[derive(Clone, PartialEq, Eq)] enum RemoveFrom { Queue, @@ -107,7 +102,7 @@ enum RemoveFrom { /// /// TODO: `store_reclaim_period` pub struct EarlyMergeDB { - overlay: MemoryDB, + overlay: super::MemoryDB, backing: Arc, refs: Option>>>, latest_era: Option, @@ -292,8 +287,8 @@ impl EarlyMergeDB { } impl HashDB for EarlyMergeDB { - fn get(&self, key: &H256) -> Option { - if let Some((d, rc)) = self.overlay.raw(key) { + fn get(&self, key: &H256, prefix: Prefix) -> Option { + if let Some((d, rc)) = self.overlay.raw(key, prefix) { if rc > 0 { return Some(d.clone()) } @@ -301,18 +296,18 @@ impl HashDB for EarlyMergeDB { self.payload(key) } - fn contains(&self, key: &H256) -> bool { - self.get(key).is_some() + fn contains(&self, key: &H256, prefix: Prefix) -> bool { + self.get(key, prefix).is_some() } - fn insert(&mut self, value: &[u8]) -> H256 { - self.overlay.insert(value) + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { + self.overlay.insert(prefix, value) } - fn emplace(&mut self, key: H256, value: DBValue) { - self.overlay.emplace(key, value); + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { + self.overlay.emplace(key, prefix, value); } - fn remove(&mut self, key: &H256) { - self.overlay.remove(key); + fn remove(&mut self, key: &H256, prefix: Prefix) { + self.overlay.remove(key, prefix); } } @@ -358,8 +353,9 @@ impl JournalDB for EarlyMergeDB { fn latest_era(&self) -> Option { self.latest_era } fn mem_used(&self) -> usize { - self.overlay.mem_used() + match self.refs { - Some(ref c) => c.read().heap_size_of_children(), + let mut ops = new_malloc_size_ops(); + self.overlay.size_of(&mut ops) + match self.refs { + Some(ref c) => c.read().size_of(&mut ops), None => 0 } } @@ -520,7 +516,7 @@ impl JournalDB for EarlyMergeDB { Ok(ops) } - fn consolidate(&mut self, with: MemoryDB) { + fn consolidate(&mut self, with: super::MemoryDB) { self.overlay.consolidate(with); } } @@ -529,7 +525,7 @@ impl JournalDB for EarlyMergeDB { mod tests { use keccak::keccak; - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; use super::*; use super::super::traits::JournalDB; use kvdb_memorydb; @@ -539,7 +535,7 @@ mod tests { // history is 1 let mut jdb = new_db(); - let x = jdb.insert(b"X"); + let x = jdb.insert(EMPTY_PREFIX, b"X"); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); @@ -549,10 +545,10 @@ mod tests { jdb.commit_batch(4, &keccak(b"1003a"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&x); + jdb.remove(&x, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"1002b"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - let x = jdb.insert(b"X"); + let x = jdb.insert(EMPTY_PREFIX, b"X"); jdb.commit_batch(4, &keccak(b"1003b"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -561,50 +557,50 @@ mod tests { jdb.commit_batch(6, &keccak(b"1005a"), Some((4, keccak(b"1003a")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&x)); + assert!(jdb.contains(&x, EMPTY_PREFIX)); } #[test] fn insert_older_era() { let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let bar = jdb.insert(b"bar"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0a")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(0, &keccak(b"0b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); } #[test] fn long_history() { // history is 3 let mut jdb = new_db(); - let h = jdb.insert(b"foo"); + let h = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); - jdb.remove(&h); + assert!(jdb.contains(&h, EMPTY_PREFIX)); + jdb.remove(&h, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&h)); + assert!(!jdb.contains(&h, EMPTY_PREFIX)); } #[test] @@ -612,42 +608,42 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - jdb.remove(&bar); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + jdb.remove(&bar, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - let foo = jdb.insert(b"foo"); - jdb.remove(&baz); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.remove(&baz, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(!jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(!jdb.contains(&baz)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); } #[test] @@ -655,31 +651,31 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&baz)); - assert!(!jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); } #[test] @@ -687,22 +683,22 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); - assert!(jdb.contains(&foo)); + jdb.insert(EMPTY_PREFIX, b"foo"); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(3, &keccak(b"2"), Some((0, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -712,23 +708,23 @@ mod tests { jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2a"), Some((1, keccak(b"1a")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -737,23 +733,23 @@ mod tests { jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -762,23 +758,23 @@ mod tests { jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3a"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3b"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -802,8 +798,8 @@ mod tests { let foo = { let mut jdb = EarlyMergeDB::new(shared_db.clone(), None); // history is 1 - let foo = jdb.insert(b"foo"); - jdb.emplace(bar.clone(), DBValue::from_slice(b"bar")); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.emplace(bar.clone(), EMPTY_PREFIX, DBValue::from_slice(b"bar")); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); foo @@ -811,18 +807,18 @@ mod tests { { let mut jdb = EarlyMergeDB::new(shared_db.clone(), None); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); } { let mut jdb = EarlyMergeDB::new(shared_db, None); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); } } @@ -833,19 +829,19 @@ mod tests { let mut jdb = new_db(); // history is 4 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(4, &keccak(b"4"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); // expunge foo @@ -859,39 +855,39 @@ mod tests { let mut jdb = new_db(); // history is 4 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(2, &keccak(b"2a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(2, &keccak(b"2b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(4, &keccak(b"4a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(4, &keccak(b"4b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -905,28 +901,28 @@ mod tests { let mut jdb = new_db(); // history is 1 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); // BROKEN assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(5, &keccak(b"5"), Some((4, keccak(b"4")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -934,7 +930,7 @@ mod tests { let mut jdb = new_db(); // history is 4 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); @@ -948,16 +944,16 @@ mod tests { // foo is ancient history. - jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(5, &keccak(b"5"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); - jdb.remove(&bar); + jdb.remove(&foo, EMPTY_PREFIX); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(6, &keccak(b"6"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); - jdb.insert(b"bar"); + jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(7, &keccak(b"7"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); } @@ -972,7 +968,7 @@ mod tests { { let mut jdb = EarlyMergeDB::new(shared_db.clone(), None); // history is 1 - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); @@ -980,24 +976,24 @@ mod tests { // foo is ancient history. - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); // incantation to reopen the db }; { let mut jdb = EarlyMergeDB::new(shared_db.clone(), None); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(4, &keccak(b"4"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); // incantation to reopen the db }; { @@ -1005,7 +1001,7 @@ mod tests { jdb.commit_batch(5, &keccak(b"5"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); // incantation to reopen the db }; { @@ -1013,7 +1009,7 @@ mod tests { jdb.commit_batch(6, &keccak(b"6"), Some((4, keccak(b"4")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); } } @@ -1024,16 +1020,16 @@ mod tests { let (foo, bar, baz) = { let mut jdb = EarlyMergeDB::new(shared_db.clone(), None); // history is 1 - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); (foo, bar, baz) @@ -1043,22 +1039,22 @@ mod tests { let mut jdb = EarlyMergeDB::new(shared_db, None); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&baz)); - assert!(!jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); } } #[test] fn inject() { let mut jdb = new_db(); - let key = jdb.insert(b"dog"); + let key = jdb.insert(EMPTY_PREFIX, b"dog"); jdb.inject_batch().unwrap(); - assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog")); - jdb.remove(&key); + assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog")); + jdb.remove(&key, EMPTY_PREFIX); jdb.inject_batch().unwrap(); - assert!(jdb.get(&key).is_none()); + assert!(jdb.get(&key, EMPTY_PREFIX).is_none()); } } diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index 3eb6ee38c84..b141a90ab02 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -16,7 +16,9 @@ //! `JournalDB` interface and implementation. -extern crate heapsize; +extern crate parity_util_mem; +extern crate parity_util_mem as mem; +extern crate parity_util_mem as malloc_size_of; #[macro_use] extern crate log; @@ -59,6 +61,14 @@ pub use self::traits::KeyedHashDB; /// Export as keyed hash trait pub use self::traits::AsKeyedHashDB; + +/// Alias to ethereum MemoryDB +type MemoryDB = memory_db::MemoryDB< + keccak_hasher::KeccakHasher, + memory_db::HashKey, + kvdb::DBValue, +>; + /// Journal database operating strategy. #[derive(Debug, PartialEq, Clone, Copy)] pub enum Algorithm { @@ -163,8 +173,8 @@ fn error_negatively_reference_hash(hash: ðereum_types::H256) -> io::Error { io::Error::new(io::ErrorKind::Other, format!("Entry {} removed from database more times than it was added.", hash)) } -pub fn new_memory_db() -> memory_db::MemoryDB { - memory_db::MemoryDB::from_null_node(&rlp::NULL_RLP, rlp::NULL_RLP.as_ref().into()) +pub fn new_memory_db() -> MemoryDB { + MemoryDB::from_null_node(&rlp::NULL_RLP, rlp::NULL_RLP.as_ref().into()) } #[cfg(test)] diff --git a/util/journaldb/src/overlaydb.rs b/util/journaldb/src/overlaydb.rs index 2e7dc34ca3a..a542f787b27 100644 --- a/util/journaldb/src/overlaydb.rs +++ b/util/journaldb/src/overlaydb.rs @@ -23,9 +23,8 @@ use std::sync::Arc; use ethereum_types::H256; use rlp::{Rlp, RlpStream, Encodable, DecoderError, Decodable, encode, decode}; -use hash_db::{HashDB}; +use hash_db::{HashDB, Prefix, EMPTY_PREFIX}; use keccak_hasher::KeccakHasher; -use memory_db::*; use kvdb::{KeyValueDB, DBTransaction, DBValue}; use super::{error_negatively_reference_hash}; @@ -39,7 +38,7 @@ use super::{error_negatively_reference_hash}; /// queries have an immediate effect in terms of these functions. #[derive(Clone)] pub struct OverlayDB { - overlay: MemoryDB, + overlay: super::MemoryDB, backing: Arc, column: Option, } @@ -134,7 +133,7 @@ impl OverlayDB { pub fn revert(&mut self) { self.overlay.clear(); } /// Get the number of references that would be committed. - pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(key).map_or(0, |(_, refs)| refs) } + pub fn commit_refs(&self, key: &H256) -> i32 { self.overlay.raw(key, EMPTY_PREFIX).map_or(0, |(_, refs)| refs) } /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option { @@ -182,10 +181,10 @@ impl crate::KeyedHashDB for OverlayDB { } impl HashDB for OverlayDB { - fn get(&self, key: &H256) -> Option { + fn get(&self, key: &H256, prefix: Prefix) -> Option { // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. - let k = self.overlay.raw(key); + let k = self.overlay.raw(key, prefix); let memrc = { if let Some((d, rc)) = k { if rc > 0 { return Some(d.clone()); } @@ -209,10 +208,10 @@ impl HashDB for OverlayDB { } } - fn contains(&self, key: &H256) -> bool { + fn contains(&self, key: &H256, prefix: Prefix) -> bool { // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. - let k = self.overlay.raw(key); + let k = self.overlay.raw(key, prefix); match k { Some((_, rc)) if rc > 0 => true, _ => { @@ -229,111 +228,111 @@ impl HashDB for OverlayDB { } } - fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) } - fn emplace(&mut self, key: H256, value: DBValue) { self.overlay.emplace(key, value); } - fn remove(&mut self, key: &H256) { self.overlay.remove(key); } + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { self.overlay.insert(prefix, value) } + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { self.overlay.emplace(key, prefix, value); } + fn remove(&mut self, key: &H256, prefix: Prefix) { self.overlay.remove(key, prefix); } } #[test] fn overlaydb_revert() { let mut m = OverlayDB::new_temp(); - let foo = m.insert(b"foo"); // insert foo. + let foo = m.insert(EMPTY_PREFIX, b"foo"); // insert foo. let mut batch = m.backing.transaction(); m.commit_to_batch(&mut batch).unwrap(); // commit - new operations begin here... m.backing.write(batch).unwrap(); - let bar = m.insert(b"bar"); // insert bar. - m.remove(&foo); // remove foo. - assert!(!m.contains(&foo)); // foo is gone. - assert!(m.contains(&bar)); // bar is here. + let bar = m.insert(EMPTY_PREFIX, b"bar"); // insert bar. + m.remove(&foo, EMPTY_PREFIX); // remove foo. + assert!(!m.contains(&foo, EMPTY_PREFIX)); // foo is gone. + assert!(m.contains(&bar, EMPTY_PREFIX)); // bar is here. m.revert(); // revert the last two operations. - assert!(m.contains(&foo)); // foo is here. - assert!(!m.contains(&bar)); // bar is gone. + assert!(m.contains(&foo, EMPTY_PREFIX)); // foo is here. + assert!(!m.contains(&bar, EMPTY_PREFIX)); // bar is gone. } #[test] fn overlaydb_overlay_insert_and_remove() { let mut trie = OverlayDB::new_temp(); - let h = trie.insert(b"hello world"); - assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world")); - trie.remove(&h); - assert_eq!(trie.get(&h), None); + let h = trie.insert(EMPTY_PREFIX, b"hello world"); + assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world")); + trie.remove(&h, EMPTY_PREFIX); + assert_eq!(trie.get(&h, EMPTY_PREFIX), None); } #[test] fn overlaydb_backing_insert_revert() { let mut trie = OverlayDB::new_temp(); - let h = trie.insert(b"hello world"); - assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world")); + let h = trie.insert(EMPTY_PREFIX, b"hello world"); + assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world")); trie.commit().unwrap(); - assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world")); + assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world")); trie.revert(); - assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world")); + assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world")); } #[test] fn overlaydb_backing_remove() { let mut trie = OverlayDB::new_temp(); - let h = trie.insert(b"hello world"); + let h = trie.insert(EMPTY_PREFIX, b"hello world"); trie.commit().unwrap(); - trie.remove(&h); - assert_eq!(trie.get(&h), None); + trie.remove(&h, EMPTY_PREFIX); + assert_eq!(trie.get(&h, EMPTY_PREFIX), None); trie.commit().unwrap(); - assert_eq!(trie.get(&h), None); + assert_eq!(trie.get(&h, EMPTY_PREFIX), None); trie.revert(); - assert_eq!(trie.get(&h), None); + assert_eq!(trie.get(&h, EMPTY_PREFIX), None); } #[test] fn overlaydb_backing_remove_revert() { let mut trie = OverlayDB::new_temp(); - let h = trie.insert(b"hello world"); + let h = trie.insert(EMPTY_PREFIX, b"hello world"); trie.commit().unwrap(); - trie.remove(&h); - assert_eq!(trie.get(&h), None); + trie.remove(&h, EMPTY_PREFIX); + assert_eq!(trie.get(&h, EMPTY_PREFIX), None); trie.revert(); - assert_eq!(trie.get(&h).unwrap(), DBValue::from_slice(b"hello world")); + assert_eq!(trie.get(&h, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"hello world")); } #[test] fn overlaydb_negative() { let mut trie = OverlayDB::new_temp(); - let h = trie.insert(b"hello world"); + let h = trie.insert(EMPTY_PREFIX, b"hello world"); trie.commit().unwrap(); - trie.remove(&h); - trie.remove(&h); //bad - sends us into negative refs. - assert_eq!(trie.get(&h), None); + trie.remove(&h, EMPTY_PREFIX); + trie.remove(&h, EMPTY_PREFIX); //bad - sends us into negative refs. + assert_eq!(trie.get(&h, EMPTY_PREFIX), None); assert!(trie.commit().is_err()); } #[test] fn overlaydb_complex() { let mut trie = OverlayDB::new_temp(); - let hfoo = trie.insert(b"foo"); - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); - let hbar = trie.insert(b"bar"); - assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar")); + let hfoo = trie.insert(EMPTY_PREFIX, b"foo"); + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); + let hbar = trie.insert(EMPTY_PREFIX, b"bar"); + assert_eq!(trie.get(&hbar, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"bar")); trie.commit().unwrap(); - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); - assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar")); - trie.insert(b"foo"); // two refs - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); + assert_eq!(trie.get(&hbar, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"bar")); + trie.insert(EMPTY_PREFIX, b"foo"); // two refs + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); trie.commit().unwrap(); - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); - assert_eq!(trie.get(&hbar).unwrap(), DBValue::from_slice(b"bar")); - trie.remove(&hbar); // zero refs - delete - assert_eq!(trie.get(&hbar), None); - trie.remove(&hfoo); // one ref - keep - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); + assert_eq!(trie.get(&hbar, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"bar")); + trie.remove(&hbar, EMPTY_PREFIX); // zero refs - delete + assert_eq!(trie.get(&hbar, EMPTY_PREFIX), None); + trie.remove(&hfoo, EMPTY_PREFIX); // one ref - keep + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); trie.commit().unwrap(); - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); - trie.remove(&hfoo); // zero ref - would delete, but... - assert_eq!(trie.get(&hfoo), None); - trie.insert(b"foo"); // one ref - keep after all. - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); + trie.remove(&hfoo, EMPTY_PREFIX); // zero ref - would delete, but... + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX), None); + trie.insert(EMPTY_PREFIX, b"foo"); // one ref - keep after all. + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); trie.commit().unwrap(); - assert_eq!(trie.get(&hfoo).unwrap(), DBValue::from_slice(b"foo")); - trie.remove(&hfoo); // zero ref - delete - assert_eq!(trie.get(&hfoo), None); + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"foo")); + trie.remove(&hfoo, EMPTY_PREFIX); // zero ref - delete + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX), None); trie.commit().unwrap(); // - assert_eq!(trie.get(&hfoo), None); + assert_eq!(trie.get(&hfoo, EMPTY_PREFIX), None); } diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index 6487dffda6b..ea389e0ce19 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -23,11 +23,10 @@ use std::sync::Arc; use bytes::Bytes; use ethereum_types::H256; -use hash_db::{HashDB}; -use heapsize::HeapSizeOf; +use hash_db::{HashDB, Prefix, EMPTY_PREFIX}; +use parity_util_mem::{MallocSizeOf, allocators::new_malloc_size_ops}; use keccak_hasher::KeccakHasher; use kvdb::{KeyValueDB, DBTransaction, DBValue}; -use memory_db::*; use parking_lot::RwLock; use fastmap::H256FastMap; use rlp::{Rlp, RlpStream, encode, decode, DecoderError, Decodable, Encodable}; @@ -66,7 +65,7 @@ use util::DatabaseKey; /// 7. Delete ancient record from memory and disk. pub struct OverlayRecentDB { - transaction_overlay: MemoryDB, + transaction_overlay: super::MemoryDB, backing: Arc, journal_overlay: Arc>, column: Option, @@ -120,7 +119,7 @@ impl<'a> Encodable for DatabaseValueRef<'a> { #[derive(PartialEq)] struct JournalOverlay { - backing_overlay: MemoryDB, // Nodes added in the history period + backing_overlay: super::MemoryDB, // Nodes added in the history period pending_overlay: H256FastMap, // Nodes being transfered from backing_overlay to backing db journal: HashMap>, latest_era: Option, @@ -128,19 +127,13 @@ struct JournalOverlay { cumulative_size: usize, // cumulative size of all entries. } -#[derive(PartialEq)] +#[derive(PartialEq, MallocSizeOf)] struct JournalEntry { id: H256, insertions: Vec, deletions: Vec, } -impl HeapSizeOf for JournalEntry { - fn heap_size_of_children(&self) -> usize { - self.insertions.heap_size_of_children() + self.deletions.heap_size_of_children() - } -} - impl Clone for OverlayRecentDB { fn clone(&self) -> OverlayRecentDB { OverlayRecentDB { @@ -204,11 +197,11 @@ impl OverlayRecentDB { for (k, v) in value.inserts { let short_key = to_short_key(&k); - if !overlay.contains(&short_key) { + if !overlay.contains(&short_key, EMPTY_PREFIX) { cumulative_size += v.len(); } - overlay.emplace(short_key, v); + overlay.emplace(short_key, EMPTY_PREFIX, v); inserted_keys.push(k); } journal.entry(era).or_insert_with(Vec::new).push(JournalEntry { @@ -272,12 +265,13 @@ impl JournalDB for OverlayRecentDB { } fn mem_used(&self) -> usize { - let mut mem = self.transaction_overlay.mem_used(); + let mut ops = new_malloc_size_ops(); + let mut mem = self.transaction_overlay.size_of(&mut ops); let overlay = self.journal_overlay.read(); - mem += overlay.backing_overlay.mem_used(); - mem += overlay.pending_overlay.heap_size_of_children(); - mem += overlay.journal.heap_size_of_children(); + mem += overlay.backing_overlay.size_of(&mut ops); + mem += overlay.pending_overlay.size_of(&mut ops); + mem += overlay.journal.size_of(&mut ops); mem } @@ -302,7 +296,7 @@ impl JournalDB for OverlayRecentDB { fn state(&self, key: &H256) -> Option { let journal_overlay = self.journal_overlay.read(); let key = to_short_key(key); - journal_overlay.backing_overlay.get(&key).map(|v| v.into_vec()) + journal_overlay.backing_overlay.get(&key, EMPTY_PREFIX).map(|v| v.into_vec()) .or_else(|| journal_overlay.pending_overlay.get(&key).map(|d| d.clone().into_vec())) .or_else(|| self.backing.get_by_prefix(self.column, &key[0..DB_PREFIX_LEN]).map(|b| b.into_vec())) } @@ -334,11 +328,11 @@ impl JournalDB for OverlayRecentDB { for (k, v) in insertions { let short_key = to_short_key(&k); - if !journal_overlay.backing_overlay.contains(&short_key) { + if !journal_overlay.backing_overlay.contains(&short_key, EMPTY_PREFIX) { journal_overlay.cumulative_size += v.len(); } - journal_overlay.backing_overlay.emplace(short_key, v); + journal_overlay.backing_overlay.emplace(short_key, EMPTY_PREFIX, v); } let index = journal_overlay.journal.get(&now).map_or(0, |j| j.len()); @@ -387,7 +381,7 @@ impl JournalDB for OverlayRecentDB { { if *canon_id == journal.id { for h in &journal.insertions { - if let Some((d, rc)) = journal_overlay.backing_overlay.raw(&to_short_key(h)) { + if let Some((d, rc)) = journal_overlay.backing_overlay.raw(&to_short_key(h), EMPTY_PREFIX) { if rc > 0 { canon_insertions.push((h.clone(), d.clone())); //TODO: optimize this to avoid data copy } @@ -410,13 +404,13 @@ impl JournalDB for OverlayRecentDB { } // update the overlay for k in overlay_deletions { - if let Some(val) = journal_overlay.backing_overlay.remove_and_purge(&to_short_key(&k)) { + if let Some(val) = journal_overlay.backing_overlay.remove_and_purge(&to_short_key(&k), EMPTY_PREFIX) { journal_overlay.cumulative_size -= val.len(); } } // apply canon deletions for k in canon_deletions { - if !journal_overlay.backing_overlay.contains(&to_short_key(&k)) { + if !journal_overlay.backing_overlay.contains(&to_short_key(&k), EMPTY_PREFIX) { batch.delete(self.column, k.as_bytes()); } } @@ -458,14 +452,14 @@ impl JournalDB for OverlayRecentDB { Ok(ops) } - fn consolidate(&mut self, with: MemoryDB) { + fn consolidate(&mut self, with: super::MemoryDB) { self.transaction_overlay.consolidate(with); } } impl HashDB for OverlayRecentDB { - fn get(&self, key: &H256) -> Option { - if let Some((d, rc)) = self.transaction_overlay.raw(key) { + fn get(&self, key: &H256, prefix: Prefix) -> Option { + if let Some((d, rc)) = self.transaction_overlay.raw(key, prefix) { if rc > 0 { return Some(d.clone()) } @@ -473,24 +467,24 @@ impl HashDB for OverlayRecentDB { let v = { let journal_overlay = self.journal_overlay.read(); let key = to_short_key(key); - journal_overlay.backing_overlay.get(&key) + journal_overlay.backing_overlay.get(&key, prefix) .or_else(|| journal_overlay.pending_overlay.get(&key).cloned()) }; v.or_else(|| self.payload(key)) } - fn contains(&self, key: &H256) -> bool { - self.get(key).is_some() + fn contains(&self, key: &H256, prefix: Prefix) -> bool { + self.get(key, prefix).is_some() } - fn insert(&mut self, value: &[u8]) -> H256 { - self.transaction_overlay.insert(value) + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { + self.transaction_overlay.insert(prefix, value) } - fn emplace(&mut self, key: H256, value: DBValue) { - self.transaction_overlay.emplace(key, value); + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { + self.transaction_overlay.emplace(key, prefix, value); } - fn remove(&mut self, key: &H256) { - self.transaction_overlay.remove(key); + fn remove(&mut self, key: &H256, prefix: Prefix) { + self.transaction_overlay.remove(key, prefix); } } @@ -499,7 +493,7 @@ mod tests { use keccak::keccak; use super::*; - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; use {kvdb_memorydb, JournalDB}; fn new_db() -> OverlayRecentDB { @@ -512,7 +506,7 @@ mod tests { // history is 1 let mut jdb = new_db(); - let x = jdb.insert(b"X"); + let x = jdb.insert(EMPTY_PREFIX, b"X"); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); @@ -522,10 +516,10 @@ mod tests { jdb.commit_batch(4, &keccak(b"1003a"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&x); + jdb.remove(&x, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"1002b"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - let x = jdb.insert(b"X"); + let x = jdb.insert(EMPTY_PREFIX, b"X"); jdb.commit_batch(4, &keccak(b"1003b"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -534,30 +528,30 @@ mod tests { jdb.commit_batch(6, &keccak(b"1005a"), Some((4, keccak(b"1003a")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&x)); + assert!(jdb.contains(&x, EMPTY_PREFIX)); } #[test] fn long_history() { // history is 3 let mut jdb = new_db(); - let h = jdb.insert(b"foo"); + let h = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); - jdb.remove(&h); + assert!(jdb.contains(&h, EMPTY_PREFIX)); + jdb.remove(&h, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&h)); + assert!(!jdb.contains(&h, EMPTY_PREFIX)); } #[test] @@ -565,42 +559,42 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - jdb.remove(&bar); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + jdb.remove(&bar, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - let foo = jdb.insert(b"foo"); - jdb.remove(&baz); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.remove(&baz, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(!jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(!jdb.contains(&baz)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); } #[test] @@ -608,31 +602,31 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&baz)); - assert!(!jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); } #[test] @@ -640,22 +634,22 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); - assert!(jdb.contains(&foo)); + jdb.insert(EMPTY_PREFIX, b"foo"); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(3, &keccak(b"2"), Some((0, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -664,23 +658,23 @@ mod tests { jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2a"), Some((1, keccak(b"1a")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -690,23 +684,23 @@ mod tests { jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1c"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); } #[test] @@ -716,23 +710,23 @@ mod tests { jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3a"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3b"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -751,8 +745,8 @@ mod tests { let foo = { let mut jdb = OverlayRecentDB::new(shared_db.clone(), None); // history is 1 - let foo = jdb.insert(b"foo"); - jdb.emplace(bar.clone(), DBValue::from_slice(b"bar")); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.emplace(bar.clone(), EMPTY_PREFIX, DBValue::from_slice(b"bar")); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); foo @@ -760,18 +754,18 @@ mod tests { { let mut jdb = OverlayRecentDB::new(shared_db.clone(), None); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); } { let mut jdb = OverlayRecentDB::new(shared_db.clone(), None); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); } } @@ -781,19 +775,19 @@ mod tests { let mut jdb = new_db(); // history is 4 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(4, &keccak(b"4"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); // expunge foo @@ -807,39 +801,39 @@ mod tests { let mut jdb = new_db(); // history is 4 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(2, &keccak(b"2a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(2, &keccak(b"2b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(4, &keccak(b"4a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(4, &keccak(b"4b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); @@ -852,35 +846,35 @@ mod tests { fn broken_assert() { let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); // foo is ancient history. - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); // BROKEN assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(5, &keccak(b"5"), Some((4, keccak(b"4")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); } #[test] fn reopen_test() { let mut jdb = new_db(); // history is 4 - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); @@ -894,16 +888,16 @@ mod tests { // foo is ancient history. - jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(5, &keccak(b"5"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); - jdb.remove(&bar); + jdb.remove(&foo, EMPTY_PREFIX); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(6, &keccak(b"6"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.insert(b"foo"); - jdb.insert(b"bar"); + jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(7, &keccak(b"7"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); } @@ -918,7 +912,7 @@ mod tests { { let mut jdb = OverlayRecentDB::new(shared_db.clone(), None); // history is 1 - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); @@ -926,24 +920,24 @@ mod tests { // foo is ancient history. - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); - jdb.insert(b"foo"); + jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(3, &keccak(b"3"), Some((1, keccak(b"1")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); // incantation to reopen the db }; { let mut jdb = OverlayRecentDB::new(shared_db.clone(), None); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(4, &keccak(b"4"), Some((2, keccak(b"2")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); // incantation to reopen the db }; { @@ -951,7 +945,7 @@ mod tests { jdb.commit_batch(5, &keccak(b"5"), Some((3, keccak(b"3")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); // incantation to reopen the db }; { @@ -959,7 +953,7 @@ mod tests { jdb.commit_batch(6, &keccak(b"6"), Some((4, keccak(b"4")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(!jdb.contains(&foo)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); } } @@ -970,16 +964,16 @@ mod tests { let (foo, bar, baz) = { let mut jdb = OverlayRecentDB::new(shared_db.clone(), None); // history is 1 - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&foo); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); assert!(jdb.can_reconstruct_refs()); (foo, bar, baz) @@ -989,43 +983,43 @@ mod tests { let mut jdb = OverlayRecentDB::new(shared_db, None); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&baz)); - assert!(!jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); } } #[test] fn insert_older_era() { let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0a"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); - let bar = jdb.insert(b"bar"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0a")))).unwrap(); assert!(jdb.can_reconstruct_refs()); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(0, &keccak(b"0b"), None).unwrap(); assert!(jdb.can_reconstruct_refs()); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); } #[test] fn inject() { let mut jdb = new_db(); - let key = jdb.insert(b"dog"); + let key = jdb.insert(EMPTY_PREFIX, b"dog"); jdb.inject_batch().unwrap(); - assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog")); - jdb.remove(&key); + assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog")); + jdb.remove(&key, EMPTY_PREFIX); jdb.inject_batch().unwrap(); - assert!(jdb.get(&key).is_none()); + assert!(jdb.get(&key, EMPTY_PREFIX).is_none()); } #[test] @@ -1037,7 +1031,7 @@ mod tests { assert!(jdb.earliest_era().is_none()); // single journalled era. - let _key = jdb.insert(b"hello!"); + let _key = jdb.insert(EMPTY_PREFIX, b"hello!"); let mut batch = jdb.backing().transaction(); jdb.journal_under(&mut batch, 0, &keccak(b"0")).unwrap(); jdb.backing().write_buffered(batch); diff --git a/util/journaldb/src/refcounteddb.rs b/util/journaldb/src/refcounteddb.rs index bdd396a481e..b0eb7809d1b 100644 --- a/util/journaldb/src/refcounteddb.rs +++ b/util/journaldb/src/refcounteddb.rs @@ -22,11 +22,10 @@ use std::sync::Arc; use bytes::Bytes; use ethereum_types::H256; -use hash_db::{HashDB}; -use heapsize::HeapSizeOf; +use hash_db::{HashDB, Prefix, EMPTY_PREFIX}; +use parity_util_mem::{MallocSizeOf, allocators::new_malloc_size_ops}; use keccak_hasher::KeccakHasher; use kvdb::{KeyValueDB, DBTransaction, DBValue}; -use memory_db::MemoryDB; use overlaydb::OverlayDB; use rlp::{encode, decode}; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; @@ -81,11 +80,11 @@ impl RefCountedDB { } impl HashDB for RefCountedDB { - fn get(&self, key: &H256) -> Option { self.forward.get(key) } - fn contains(&self, key: &H256) -> bool { self.forward.contains(key) } - fn insert(&mut self, value: &[u8]) -> H256 { let r = self.forward.insert(value); self.inserts.push(r.clone()); r } - fn emplace(&mut self, key: H256, value: DBValue) { self.inserts.push(key.clone()); self.forward.emplace(key, value); } - fn remove(&mut self, key: &H256) { self.removes.push(key.clone()); } + fn get(&self, key: &H256, prefix: Prefix) -> Option { self.forward.get(key, prefix) } + fn contains(&self, key: &H256, prefix: Prefix) -> bool { self.forward.contains(key, prefix) } + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H256 { let r = self.forward.insert(prefix, value); self.inserts.push(r.clone()); r } + fn emplace(&mut self, key: H256, prefix: Prefix, value: DBValue) { self.inserts.push(key.clone()); self.forward.emplace(key, prefix, value); } + fn remove(&mut self, key: &H256, _prefix: Prefix) { self.removes.push(key.clone()); } } impl ::traits::KeyedHashDB for RefCountedDB { @@ -105,7 +104,8 @@ impl JournalDB for RefCountedDB { } fn mem_used(&self) -> usize { - self.inserts.heap_size_of_children() + self.removes.heap_size_of_children() + let mut ops = new_malloc_size_ops(); + self.inserts.size_of(&mut ops) + self.removes.size_of(&mut ops) } fn is_empty(&self) -> bool { @@ -184,7 +184,7 @@ impl JournalDB for RefCountedDB { }.expect("rlp read from db; qed"); trace!(target: "rcdb", "delete journal for time #{}.{}=>{}, (canon was {}): deleting {:?}", end_era, db_key.index, our_id, canon_id, to_remove); for i in &to_remove { - self.forward.remove(i); + self.forward.remove(i, EMPTY_PREFIX); } batch.delete(self.column, &last); db_key.index += 1; @@ -197,19 +197,19 @@ impl JournalDB for RefCountedDB { fn inject(&mut self, batch: &mut DBTransaction) -> io::Result { self.inserts.clear(); for remove in self.removes.drain(..) { - self.forward.remove(&remove); + self.forward.remove(&remove, EMPTY_PREFIX); } self.forward.commit_to_batch(batch) } - fn consolidate(&mut self, mut with: MemoryDB) { + fn consolidate(&mut self, mut with: super::MemoryDB) { for (key, (value, rc)) in with.drain() { for _ in 0..rc { - self.emplace(key, value.clone()); + self.emplace(key, EMPTY_PREFIX, value.clone()); } for _ in rc..0 { - self.remove(&key); + self.remove(&key, EMPTY_PREFIX); } } } @@ -219,7 +219,7 @@ impl JournalDB for RefCountedDB { mod tests { use keccak::keccak; - use hash_db::HashDB; + use hash_db::{HashDB, EMPTY_PREFIX}; use super::*; use {JournalDB, kvdb_memorydb}; @@ -232,18 +232,18 @@ mod tests { fn long_history() { // history is 3 let mut jdb = new_db(); - let h = jdb.insert(b"foo"); + let h = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&h)); - jdb.remove(&h); + assert!(jdb.contains(&h, EMPTY_PREFIX)); + jdb.remove(&h, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(3, &keccak(b"3"), Some((0, keccak(b"0")))).unwrap(); - assert!(jdb.contains(&h)); + assert!(jdb.contains(&h, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((1, keccak(b"1")))).unwrap(); - assert!(!jdb.contains(&h)); + assert!(!jdb.contains(&h, EMPTY_PREFIX)); } #[test] @@ -251,10 +251,10 @@ mod tests { // history is 3 let mut jdb = new_db(); assert_eq!(jdb.latest_era(), None); - let h = jdb.insert(b"foo"); + let h = jdb.insert(EMPTY_PREFIX, b"foo"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); assert_eq!(jdb.latest_era(), Some(0)); - jdb.remove(&h); + jdb.remove(&h, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1"), None).unwrap(); assert_eq!(jdb.latest_era(), Some(1)); jdb.commit_batch(2, &keccak(b"2"), None).unwrap(); @@ -270,37 +270,37 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - jdb.remove(&bar); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + jdb.remove(&bar, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1"), Some((0, keccak(b"0")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - let foo = jdb.insert(b"foo"); - jdb.remove(&baz); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + jdb.remove(&baz, EMPTY_PREFIX); jdb.commit_batch(2, &keccak(b"2"), Some((1, keccak(b"1")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); - jdb.remove(&foo); + jdb.remove(&foo, EMPTY_PREFIX); jdb.commit_batch(3, &keccak(b"3"), Some((2, keccak(b"2")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(!jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); jdb.commit_batch(4, &keccak(b"4"), Some((3, keccak(b"3")))).unwrap(); - assert!(!jdb.contains(&foo)); - assert!(!jdb.contains(&bar)); - assert!(!jdb.contains(&baz)); + assert!(!jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); } #[test] @@ -308,39 +308,39 @@ mod tests { // history is 1 let mut jdb = new_db(); - let foo = jdb.insert(b"foo"); - let bar = jdb.insert(b"bar"); + let foo = jdb.insert(EMPTY_PREFIX, b"foo"); + let bar = jdb.insert(EMPTY_PREFIX, b"bar"); jdb.commit_batch(0, &keccak(b"0"), None).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); - jdb.remove(&foo); - let baz = jdb.insert(b"baz"); + jdb.remove(&foo, EMPTY_PREFIX); + let baz = jdb.insert(EMPTY_PREFIX, b"baz"); jdb.commit_batch(1, &keccak(b"1a"), Some((0, keccak(b"0")))).unwrap(); - jdb.remove(&bar); + jdb.remove(&bar, EMPTY_PREFIX); jdb.commit_batch(1, &keccak(b"1b"), Some((0, keccak(b"0")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(jdb.contains(&bar)); - assert!(jdb.contains(&baz)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(jdb.contains(&bar, EMPTY_PREFIX)); + assert!(jdb.contains(&baz, EMPTY_PREFIX)); jdb.commit_batch(2, &keccak(b"2b"), Some((1, keccak(b"1b")))).unwrap(); - assert!(jdb.contains(&foo)); - assert!(!jdb.contains(&baz)); - assert!(!jdb.contains(&bar)); + assert!(jdb.contains(&foo, EMPTY_PREFIX)); + assert!(!jdb.contains(&baz, EMPTY_PREFIX)); + assert!(!jdb.contains(&bar, EMPTY_PREFIX)); } #[test] fn inject() { let mut jdb = new_db(); - let key = jdb.insert(b"dog"); + let key = jdb.insert(EMPTY_PREFIX, b"dog"); jdb.inject_batch().unwrap(); - assert_eq!(jdb.get(&key).unwrap(), DBValue::from_slice(b"dog")); - jdb.remove(&key); + assert_eq!(jdb.get(&key, EMPTY_PREFIX).unwrap(), DBValue::from_slice(b"dog")); + jdb.remove(&key, EMPTY_PREFIX); jdb.inject_batch().unwrap(); - assert!(jdb.get(&key).is_none()); + assert!(jdb.get(&key, EMPTY_PREFIX).is_none()); } } diff --git a/util/journaldb/src/traits.rs b/util/journaldb/src/traits.rs index e6ce8acb040..470598a02db 100644 --- a/util/journaldb/src/traits.rs +++ b/util/journaldb/src/traits.rs @@ -93,7 +93,7 @@ pub trait JournalDB: KeyedHashDB { fn flush(&self) {} /// Consolidate all the insertions and deletions in the given memory overlay. - fn consolidate(&mut self, overlay: ::memory_db::MemoryDB); + fn consolidate(&mut self, overlay: super::MemoryDB); /// Commit all changes in a single batch #[cfg(test)] diff --git a/util/keccak-hasher/Cargo.toml b/util/keccak-hasher/Cargo.toml index c33dab9aa49..b15a2872482 100644 --- a/util/keccak-hasher/Cargo.toml +++ b/util/keccak-hasher/Cargo.toml @@ -8,5 +8,5 @@ license = "GPL-3.0" [dependencies] ethereum-types = "0.6.0" tiny-keccak = "1.4.2" -hash-db = "0.11.0" +hash-db = "0.12.4" plain_hasher = "0.2" diff --git a/util/memory-cache/Cargo.toml b/util/memory-cache/Cargo.toml index f2b612463ac..ceb49c4dc44 100644 --- a/util/memory-cache/Cargo.toml +++ b/util/memory-cache/Cargo.toml @@ -6,5 +6,5 @@ description = "An LRU-cache which operates on memory used" license = "GPL3" [dependencies] -heapsize = "0.4" +parity-util-mem = "0.1" lru-cache = "0.1" diff --git a/util/memory-cache/src/lib.rs b/util/memory-cache/src/lib.rs index 64d10336fe2..960ebc20a49 100644 --- a/util/memory-cache/src/lib.rs +++ b/util/memory-cache/src/lib.rs @@ -18,10 +18,10 @@ //! crate. // TODO: push changes upstream in a clean way. -extern crate heapsize; +extern crate parity_util_mem; extern crate lru_cache; -use heapsize::HeapSizeOf; +use parity_util_mem::{MallocSizeOf, MallocSizeOfExt}; use lru_cache::LruCache; use std::hash::Hash; @@ -29,18 +29,18 @@ use std::hash::Hash; const INITIAL_CAPACITY: usize = 4; /// An LRU-cache which operates on memory used. -pub struct MemoryLruCache { +pub struct MemoryLruCache { inner: LruCache, cur_size: usize, max_size: usize, } // amount of memory used when the item will be put on the heap. -fn heap_size_of(val: &T) -> usize { - ::std::mem::size_of::() + val.heap_size_of_children() +fn heap_size_of(val: &T) -> usize { + ::std::mem::size_of::() + val.malloc_size_of() } -impl MemoryLruCache { +impl MemoryLruCache { /// Create a new cache with a maximum size in bytes. pub fn new(max_size: usize) -> Self { MemoryLruCache { diff --git a/util/memzero/Cargo.toml b/util/memzero/Cargo.toml deleted file mode 100644 index 67d17d26d07..00000000000 --- a/util/memzero/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "memzero" -version = "0.1.0" -description = "A wrapper for zero-ing out memory when dropped" -license = "GPL-3.0" -homepage = "https://parity.io" -repository = "https://github.com/paritytech/parity-ethereum" -documentation = "https://docs.rs/crate/memzero" -authors = ["Parity Technologies "] -edition = "2018" diff --git a/util/memzero/src/lib.rs b/util/memzero/src/lib.rs deleted file mode 100644 index 827407a9133..00000000000 --- a/util/memzero/src/lib.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . - -use std::ops::{Deref, DerefMut}; -use std::ptr; - -/// Wrapper to zero out memory when dropped. -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Memzero> { - mem: T, -} - -impl> From for Memzero { - fn from(mem: T) -> Memzero { - Memzero { mem } - } -} - -impl> Drop for Memzero { - fn drop(&mut self) { - unsafe { - for byte_ref in self.mem.as_mut() { - ptr::write_volatile(byte_ref, 0) - } - } - } -} - -impl> Deref for Memzero { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.mem - } -} - -impl> DerefMut for Memzero { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.mem - } -} diff --git a/util/patricia-trie-ethereum/Cargo.toml b/util/patricia-trie-ethereum/Cargo.toml index f6a25b876fd..52f6d7dc716 100644 --- a/util/patricia-trie-ethereum/Cargo.toml +++ b/util/patricia-trie-ethereum/Cargo.toml @@ -6,15 +6,15 @@ description = "Merkle-Patricia Trie (Ethereum Style)" license = "GPL-3.0" [dependencies] -trie-db = "0.11.0" +trie-db = "0.12.4" keccak-hasher = { version = "0.1.1", path = "../keccak-hasher" } -hash-db = "0.11.0" +hash-db = "0.12.4" rlp = "0.4.0" parity-bytes = "0.1" ethereum-types = "0.6.0" elastic-array = "0.10" [dev-dependencies] -memory-db = "0.11.0" +memory-db = "0.12.4" keccak-hash = "0.2.0" journaldb = { path = "../journaldb" } diff --git a/util/triehash-ethereum/Cargo.toml b/util/triehash-ethereum/Cargo.toml index b1f7b12168a..07f720be189 100644 --- a/util/triehash-ethereum/Cargo.toml +++ b/util/triehash-ethereum/Cargo.toml @@ -6,6 +6,6 @@ description = "Trie-root helpers, ethereum style" license = "GPL-3.0" [dependencies] -triehash = "0.5.0" +triehash = "0.6.0" ethereum-types = "0.6.0" keccak-hasher = { path = "../keccak-hasher" } diff --git a/whisper/Cargo.toml b/whisper/Cargo.toml index 1998de55a8f..64bd33594ac 100644 --- a/whisper/Cargo.toml +++ b/whisper/Cargo.toml @@ -13,7 +13,7 @@ ring = "0.14.6" ethkey = { path = "../accounts/ethkey" } hex = "0.2" log = "0.4" -memzero = { path = "../util/memzero" } +parity-util-mem = "0.1" ordered-float = "0.5" parking_lot = "0.7" rand = "0.6" diff --git a/whisper/src/lib.rs b/whisper/src/lib.rs index 62a44ac9340..7ff7794b77a 100644 --- a/whisper/src/lib.rs +++ b/whisper/src/lib.rs @@ -24,7 +24,7 @@ extern crate ethcore_network as network; extern crate ethereum_types; extern crate ethkey; extern crate hex; -extern crate memzero; +extern crate parity_util_mem; extern crate ordered_float; extern crate parking_lot; extern crate rand; diff --git a/whisper/src/rpc/crypto.rs b/whisper/src/rpc/crypto.rs index 418f4037413..3f26e0350a0 100644 --- a/whisper/src/rpc/crypto.rs +++ b/whisper/src/rpc/crypto.rs @@ -20,7 +20,7 @@ use aes_gcm::{Encryptor, Decryptor}; use ethkey::crypto::ecies; use ethereum_types::H256; use ethkey::{self, Public, Secret}; -use memzero::Memzero; +use parity_util_mem::Memzero; /// Length of AES key pub const AES_KEY_LEN: usize = 32; diff --git a/whisper/src/rpc/key_store.rs b/whisper/src/rpc/key_store.rs index 687eab48ad7..3f84e59ad3a 100644 --- a/whisper/src/rpc/key_store.rs +++ b/whisper/src/rpc/key_store.rs @@ -23,7 +23,7 @@ use std::collections::HashMap; use ethereum_types::H256; use ethkey::{KeyPair, Public, Secret}; -use memzero::Memzero; +use parity_util_mem::Memzero; use rand::{Rng, rngs::OsRng}; use rpc::crypto::{AES_KEY_LEN, EncryptionInstance, DecryptionInstance}; diff --git a/whisper/src/rpc/mod.rs b/whisper/src/rpc/mod.rs index a5ed05da73b..61cd14d4b9b 100644 --- a/whisper/src/rpc/mod.rs +++ b/whisper/src/rpc/mod.rs @@ -28,7 +28,7 @@ use jsonrpc_derive::rpc; use jsonrpc_pubsub::{Session, PubSubMetadata, SubscriptionId, typed::Subscriber}; use ethereum_types::H256; -use memzero::Memzero; +use parity_util_mem::Memzero; use parking_lot::RwLock; use self::filter::Filter; From 4ee217ba7c8744938af4eb053370de51e54f96c0 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 19 Jun 2019 16:08:21 +0200 Subject: [PATCH 02/15] Remove unused code (#10762) --- ethcore/service/src/lib.rs | 1 - ethcore/service/src/service.rs | 5 ---- ethcore/service/src/stop_guard.rs | 40 ------------------------------- 3 files changed, 46 deletions(-) delete mode 100644 ethcore/service/src/stop_guard.rs diff --git a/ethcore/service/src/lib.rs b/ethcore/service/src/lib.rs index 67292e1d8da..b9a8c15b38c 100644 --- a/ethcore/service/src/lib.rs +++ b/ethcore/service/src/lib.rs @@ -36,7 +36,6 @@ extern crate tempdir; mod error; mod service; -mod stop_guard; #[cfg(test)] extern crate kvdb_rocksdb; diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index bdc743581ad..17690805642 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -23,7 +23,6 @@ use std::time::Duration; use ansi_term::Colour; use ethereum_types::H256; use io::{IoContext, TimerToken, IoHandler, IoService, IoError}; -use stop_guard::StopGuard; use sync::PrivateTxHandler; use blockchain::{BlockChainDB, BlockChainDBHandler}; @@ -84,7 +83,6 @@ pub struct ClientService { snapshot: Arc, private_tx: Arc, database: Arc, - _stop_guard: StopGuard, } impl ClientService { @@ -152,15 +150,12 @@ impl ClientService { spec.engine.register_client(Arc::downgrade(&client) as _); - let stop_guard = StopGuard::new(); - Ok(ClientService { io_service: Arc::new(io_service), client: client, snapshot: snapshot, private_tx, database: blockchain_db, - _stop_guard: stop_guard, }) } diff --git a/ethcore/service/src/stop_guard.rs b/ethcore/service/src/stop_guard.rs deleted file mode 100644 index 168219520a1..00000000000 --- a/ethcore/service/src/stop_guard.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2015-2019 Parity Technologies (UK) Ltd. -// This file is part of Parity Ethereum. - -// Parity Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Parity Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Parity Ethereum. If not, see . - -//! Stop guard mod - -use std::sync::Arc; -use std::sync::atomic::*; - -/// Stop guard that will set a stop flag on drop -pub struct StopGuard { - flag: Arc, -} - -impl StopGuard { - /// Create a stop guard - pub fn new() -> StopGuard { - StopGuard { - flag: Arc::new(AtomicBool::new(false)) - } - } -} - -impl Drop for StopGuard { - fn drop(&mut self) { - self.flag.store(true, Ordering::Relaxed) - } -} From e1333ea1e99f1ce6780bedb2280e7d9fb1b71be1 Mon Sep 17 00:00:00 2001 From: Anton Gavrilov Date: Wed, 19 Jun 2019 16:28:31 +0200 Subject: [PATCH 03/15] Initialize private tx logger only if private tx functionality is enabled (#10758) * Initialize private tx logger only if private tx functionality is enabled * Fix configuration test * Use default contructor for config --- parity/configuration.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/parity/configuration.rs b/parity/configuration.rs index 0d6c2d86fe0..2a807e99b07 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -917,7 +917,10 @@ impl Configuration { let provider_conf = ProviderConfig { validator_accounts: to_addresses(&self.args.arg_private_validators)?, signer_account: self.args.arg_private_signer.clone().and_then(|account| to_address(Some(account)).ok()), - logs_path: Some(dirs.base), + logs_path: match self.args.flag_private_enabled { + true => Some(dirs.base), + false => None, + } }; let encryptor_conf = EncryptorConfig { @@ -1455,11 +1458,7 @@ mod tests { net_settings: Default::default(), ipfs_conf: Default::default(), secretstore_conf: Default::default(), - private_provider_conf: ProviderConfig { - validator_accounts: Default::default(), - signer_account: Default::default(), - logs_path: Some(Directories::default().base), - }, + private_provider_conf: Default::default(), private_encryptor_conf: Default::default(), private_tx_enabled: false, name: "".into(), From 78d0a8696ffec4eb6b0e2214b476fed388e14bed Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Thu, 20 Jun 2019 09:00:50 +0200 Subject: [PATCH 04/15] fix: aura don't add `SystemTime::now()` (#10720) This commit does the following: - Prevent overflow in `verify_timestamp()` by not adding `now` to found faulty timestamp - Use explicit `CheckedSystemTime::checked_add` to prevent potential consensus issues because SystemTime is platform depedent - remove `#[cfg(not(time_checked_add))]` conditional compilation --- ethcore/src/engines/authority_round/mod.rs | 14 ++++++-------- ethcore/src/engines/clique/block_state.rs | 6 ++---- ethcore/src/engines/clique/mod.rs | 12 +++++------- ethcore/src/lib.rs | 5 +---- ethcore/src/verification/verification.rs | 7 +++---- 5 files changed, 17 insertions(+), 27 deletions(-) diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index 6f2723bba71..11fab75ab9b 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -22,7 +22,7 @@ use std::iter::FromIterator; use std::ops::Deref; use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; use std::sync::{Weak, Arc}; -use std::time::{UNIX_EPOCH, SystemTime, Duration}; +use std::time::{UNIX_EPOCH, Duration}; use block::*; use client::EngineClient; @@ -42,14 +42,12 @@ use itertools::{self, Itertools}; use rlp::{encode, Decodable, DecoderError, Encodable, RlpStream, Rlp}; use ethereum_types::{H256, H520, Address, U128, U256}; use parking_lot::{Mutex, RwLock}; +use time_utils::CheckedSystemTime; use types::BlockNumber; use types::header::{Header, ExtendedHeader}; use types::ancestry_action::AncestryAction; use unexpected::{Mismatch, OutOfBounds}; -#[cfg(not(time_checked_add))] -use time_utils::CheckedSystemTime; - mod finality; /// `AuthorityRound` params. @@ -584,10 +582,10 @@ fn verify_timestamp(step: &Step, header_step: u64) -> Result<(), BlockError> { // Returning it further won't recover the sync process. trace!(target: "engine", "verify_timestamp: block too early"); - let now = SystemTime::now(); - let found = now.checked_add(Duration::from_secs(oob.found)).ok_or(BlockError::TimestampOverflow)?; - let max = oob.max.and_then(|m| now.checked_add(Duration::from_secs(m))); - let min = oob.min.and_then(|m| now.checked_add(Duration::from_secs(m))); + let found = CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::from_secs(oob.found)) + .ok_or(BlockError::TimestampOverflow)?; + let max = oob.max.and_then(|m| CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::from_secs(m))); + let min = oob.min.and_then(|m| CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::from_secs(m))); let new_oob = OutOfBounds { min, max, found }; diff --git a/ethcore/src/engines/clique/block_state.rs b/ethcore/src/engines/clique/block_state.rs index 5e1c5f2fb46..913053eaa57 100644 --- a/ethcore/src/engines/clique/block_state.rs +++ b/ethcore/src/engines/clique/block_state.rs @@ -24,13 +24,11 @@ use engines::clique::{VoteType, DIFF_INTURN, DIFF_NOTURN, NULL_AUTHOR, SIGNING_D use error::{Error, BlockError}; use ethereum_types::{Address, H64}; use rand::Rng; +use time_utils::CheckedSystemTime; use types::BlockNumber; use types::header::Header; use unexpected::Mismatch; -#[cfg(not(feature = "time_checked_add"))] -use time_utils::CheckedSystemTime; - /// Type that keeps track of the state for a given vote // Votes that go against the proposal aren't counted since it's equivalent to not voting #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] @@ -268,7 +266,7 @@ impl CliqueBlockState { // This is a quite bad API because we must mutate both variables even when already `inturn` fails // That's why we can't return early and must have the `if-else` in the end pub fn calc_next_timestamp(&mut self, timestamp: u64, period: u64) -> Result<(), Error> { - let inturn = UNIX_EPOCH.checked_add(Duration::from_secs(timestamp.saturating_add(period))); + let inturn = CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::from_secs(timestamp.saturating_add(period))); self.next_timestamp_inturn = inturn; diff --git a/ethcore/src/engines/clique/mod.rs b/ethcore/src/engines/clique/mod.rs index 052f0a578ce..15827477863 100644 --- a/ethcore/src/engines/clique/mod.rs +++ b/ethcore/src/engines/clique/mod.rs @@ -81,12 +81,10 @@ use parking_lot::RwLock; use rand::Rng; use super::signer::EngineSigner; use unexpected::{Mismatch, OutOfBounds}; +use time_utils::CheckedSystemTime; use types::BlockNumber; use types::header::{ExtendedHeader, Header}; -#[cfg(not(feature = "time_checked_add"))] -use time_utils::CheckedSystemTime; - use self::block_state::CliqueBlockState; use self::params::CliqueParams; @@ -545,7 +543,7 @@ impl Engine for Clique { // Don't waste time checking blocks from the future { - let limit = SystemTime::now().checked_add(Duration::from_secs(self.period)) + let limit = CheckedSystemTime::checked_add(SystemTime::now(), Duration::from_secs(self.period)) .ok_or(BlockError::TimestampOverflow)?; // This should succeed under the contraints that the system clock works @@ -555,7 +553,7 @@ impl Engine for Clique { let hdr = Duration::from_secs(header.timestamp()); if hdr > limit_as_dur { - let found = UNIX_EPOCH.checked_add(hdr).ok_or(BlockError::TimestampOverflow)?; + let found = CheckedSystemTime::checked_add(UNIX_EPOCH, hdr).ok_or(BlockError::TimestampOverflow)?; Err(BlockError::TemporarilyInvalid(OutOfBounds { min: None, @@ -666,8 +664,8 @@ impl Engine for Clique { // Ensure that the block's timestamp isn't too close to it's parent let limit = parent.timestamp().saturating_add(self.period); if limit > header.timestamp() { - let max = UNIX_EPOCH.checked_add(Duration::from_secs(header.timestamp())); - let found = UNIX_EPOCH.checked_add(Duration::from_secs(limit)) + let max = CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::from_secs(header.timestamp())); + let found = CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::from_secs(limit)) .ok_or(BlockError::TimestampOverflow)?; Err(BlockError::InvalidTimestamp(OutOfBounds { diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 4c729147a8e..e7b2fe0faf3 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -15,7 +15,6 @@ // along with Parity Ethereum. If not, see . #![warn(missing_docs, unused_extern_crates)] -#![cfg_attr(feature = "time_checked_add", feature(time_checked_add))] //! Ethcore library //! @@ -103,6 +102,7 @@ extern crate parity_util_mem as malloc_size_of; extern crate rustc_hex; extern crate serde; extern crate stats; +extern crate time_utils; extern crate triehash_ethereum as triehash; extern crate unexpected; extern crate using_queue; @@ -153,9 +153,6 @@ extern crate fetch; #[cfg(all(test, feature = "price-info"))] extern crate parity_runtime; -#[cfg(not(time_checked_add))] -extern crate time_utils; - pub mod block; pub mod builtin; pub mod client; diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 7440ad5c645..beebb155a38 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -40,7 +40,6 @@ use types::{BlockNumber, header::Header}; use types::transaction::SignedTransaction; use verification::queue::kind::blocks::Unverified; -#[cfg(not(time_checked_add))] use time_utils::CheckedSystemTime; /// Preprocessed block data gathered in `verify_block_unordered` call @@ -303,7 +302,7 @@ pub fn verify_header_params(header: &Header, engine: &dyn EthEngine, is_full: bo // this will resist overflow until `year 2037` let max_time = SystemTime::now() + ACCEPTABLE_DRIFT; let invalid_threshold = max_time + ACCEPTABLE_DRIFT * 9; - let timestamp = UNIX_EPOCH.checked_add(Duration::from_secs(header.timestamp())) + let timestamp = CheckedSystemTime::checked_add(UNIX_EPOCH, Duration::from_secs(header.timestamp())) .ok_or(BlockError::TimestampOverflow)?; if timestamp > invalid_threshold { @@ -327,9 +326,9 @@ fn verify_parent(header: &Header, parent: &Header, engine: &dyn EthEngine) -> Re if !engine.is_timestamp_valid(header.timestamp(), parent.timestamp()) { let now = SystemTime::now(); - let min = now.checked_add(Duration::from_secs(parent.timestamp().saturating_add(1))) + let min = CheckedSystemTime::checked_add(now, Duration::from_secs(parent.timestamp().saturating_add(1))) .ok_or(BlockError::TimestampOverflow)?; - let found = now.checked_add(Duration::from_secs(header.timestamp())) + let found = CheckedSystemTime::checked_add(now, Duration::from_secs(header.timestamp())) .ok_or(BlockError::TimestampOverflow)?; return Err(From::from(BlockError::InvalidTimestamp(OutOfBounds { max: None, min: Some(min), found }.into()))) } From 213e007a4b6f181921da4a6bb8757c5b05ab4147 Mon Sep 17 00:00:00 2001 From: Seun LanLege Date: Thu, 20 Jun 2019 22:16:11 +0100 Subject: [PATCH 05/15] Updated Bn128PairingImpl to use optimized batch pairing (#10765) * Bn128PairingImpl uses pairing_batch method in bn * updated Cargo.lock --- Cargo.lock | 4 +++- ethcore/src/builtin.rs | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0fd6289f6b6..f5fc14cada3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,11 +264,13 @@ dependencies = [ [[package]] name = "bn" version = "0.4.4" -source = "git+https://github.com/paritytech/bn#2a71dbde5ca93451c8da2135767896a64483759e" +source = "git+https://github.com/paritytech/bn#6beba2ed6c9351622f9e948ccee4063846b2b39a" dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "crunchy 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/ethcore/src/builtin.rs b/ethcore/src/builtin.rs index 76d00ba5269..bae1c75da36 100644 --- a/ethcore/src/builtin.rs +++ b/ethcore/src/builtin.rs @@ -498,7 +498,7 @@ impl Impl for Bn128PairingImpl { impl Bn128PairingImpl { fn execute_with_error(&self, input: &[u8], output: &mut BytesRef) -> Result<(), Error> { - use bn::{AffineG1, AffineG2, Fq, Fq2, pairing, G1, G2, Gt, Group}; + use bn::{AffineG1, AffineG2, Fq, Fq2, pairing_batch, G1, G2, Gt, Group}; let elements = input.len() / 192; // (a, b_a, b_b - each 64-byte affine coordinates) let ret_val = if input.len() == 0 { @@ -539,7 +539,7 @@ impl Bn128PairingImpl { vals.push((a, b)); }; - let mul = vals.into_iter().fold(Gt::one(), |s, (a, b)| s * pairing(a, b)); + let mul = pairing_batch(&vals); if mul == Gt::one() { U256::one() From 7701f73cdfac34b2171c99b75184062271853991 Mon Sep 17 00:00:00 2001 From: Luke Schoen Date: Fri, 21 Jun 2019 18:03:38 +1000 Subject: [PATCH 06/15] refactor: Fix indentation (#10740) --- json/src/state/test.rs | 108 ++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/json/src/state/test.rs b/json/src/state/test.rs index 3521c797785..b8e5f35f694 100644 --- a/json/src/state/test.rs +++ b/json/src/state/test.rs @@ -125,13 +125,13 @@ mod tests { #[test] fn multi_transaction_deserialization() { let s = r#"{ - "data" : [ "" ], - "gasLimit" : [ "0x2dc6c0", "0x222222" ], - "gasPrice" : "0x01", - "nonce" : "0x00", - "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", - "to" : "1000000000000000000000000000000000000000", - "value" : [ "0x00", "0x01", "0x02" ] + "data": [ "" ], + "gasLimit": [ "0x2dc6c0", "0x222222" ], + "gasPrice": "0x01", + "nonce": "0x00", + "secretKey": "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "to": "1000000000000000000000000000000000000000", + "value": [ "0x00", "0x01", "0x02" ] }"#; let _deserialized: MultiTransaction = serde_json::from_str(s).unwrap(); } @@ -139,74 +139,74 @@ mod tests { #[test] fn state_deserialization() { let s = r#"{ - "env" : { - "currentCoinbase" : "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty" : "0x0100", - "currentGasLimit" : "0x01c9c380", - "currentNumber" : "0x00", - "currentTimestamp" : "0x01", - "previousHash" : "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" + "env": { + "currentCoinbase": "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x0100", + "currentGasLimit": "0x01c9c380", + "currentNumber": "0x00", + "currentTimestamp": "0x01", + "previousHash": "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" }, - "post" : { - "EIP150" : [ + "post": { + "EIP150": [ { - "hash" : "3e6dacc1575c6a8c76422255eca03529bbf4c0dda75dfc110b22d6dc4152396f", - "indexes" : { "data" : 0, "gas" : 0, "value" : 0 } + "hash": "3e6dacc1575c6a8c76422255eca03529bbf4c0dda75dfc110b22d6dc4152396f", + "indexes": { "data": 0, "gas": 0, "value": 0 } }, { - "hash" : "99a450d8ce5b987a71346d8a0a1203711f770745c7ef326912e46761f14cd764", - "indexes" : { "data" : 0, "gas" : 0, "value" : 1 } + "hash": "99a450d8ce5b987a71346d8a0a1203711f770745c7ef326912e46761f14cd764", + "indexes": { "data": 0, "gas": 0, "value": 1 } } ], - "EIP158" : [ + "EIP158": [ { - "hash" : "3e6dacc1575c6a8c76422255eca03529bbf4c0dda75dfc110b22d6dc4152396f", - "indexes" : { "data" : 0, "gas" : 0, "value" : 0 } + "hash": "3e6dacc1575c6a8c76422255eca03529bbf4c0dda75dfc110b22d6dc4152396f", + "indexes": { "data": 0, "gas": 0, "value": 0 } }, { - "hash" : "99a450d8ce5b987a71346d8a0a1203711f770745c7ef326912e46761f14cd764", - "indexes" : { "data" : 0, "gas" : 0, "value" : 1 } + "hash": "99a450d8ce5b987a71346d8a0a1203711f770745c7ef326912e46761f14cd764", + "indexes": { "data": 0, "gas": 0, "value": 1 } } ] }, - "pre" : { - "1000000000000000000000000000000000000000" : { - "balance" : "0x0de0b6b3a7640000", - "code" : "0x6040600060406000600173100000000000000000000000000000000000000162055730f1600055", - "nonce" : "0x00", - "storage" : { + "pre": { + "1000000000000000000000000000000000000000": { + "balance": "0x0de0b6b3a7640000", + "code": "0x6040600060406000600173100000000000000000000000000000000000000162055730f1600055", + "nonce": "0x00", + "storage": { } }, - "1000000000000000000000000000000000000001" : { - "balance" : "0x0de0b6b3a7640000", - "code" : "0x604060006040600060027310000000000000000000000000000000000000026203d090f1600155", - "nonce" : "0x00", - "storage" : { + "1000000000000000000000000000000000000001": { + "balance": "0x0de0b6b3a7640000", + "code": "0x604060006040600060027310000000000000000000000000000000000000026203d090f1600155", + "nonce": "0x00", + "storage": { } }, - "1000000000000000000000000000000000000002" : { - "balance" : "0x00", - "code" : "0x600160025533600455346007553060e6553260e8553660ec553860ee553a60f055", - "nonce" : "0x00", - "storage" : { + "1000000000000000000000000000000000000002": { + "balance": "0x00", + "code": "0x600160025533600455346007553060e6553260e8553660ec553860ee553a60f055", + "nonce": "0x00", + "storage": { } }, - "a94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { - "balance" : "0x0de0b6b3a7640000", - "code" : "0x", - "nonce" : "0x00", - "storage" : { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": { } } }, - "transaction" : { - "data" : [ "" ], - "gasLimit" : [ "285000", "100000", "6000" ], - "gasPrice" : "0x01", - "nonce" : "0x00", - "secretKey" : "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", - "to" : "095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value" : [ "10", "0" ] + "transaction": { + "data": [ "" ], + "gasLimit": [ "285000", "100000", "6000" ], + "gasPrice": "0x01", + "nonce": "0x00", + "secretKey": "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "to": "095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ "10", "0" ] } }"#; let _deserialized: State = serde_json::from_str(s).unwrap(); From 55c046cb882393a6f22d0dc51f8b1e0a58eb5b29 Mon Sep 17 00:00:00 2001 From: Talha Cross <47772477+soc1c@users.noreply.github.com> Date: Fri, 21 Jun 2019 19:38:53 +0200 Subject: [PATCH 07/15] ethcore/res: activate atlantis classic hf on block 8772000 (#10766) --- ethcore/res/ethereum/classic.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ethcore/res/ethereum/classic.json b/ethcore/res/ethereum/classic.json index 682b752991e..2750f6f6391 100644 --- a/ethcore/res/ethereum/classic.json +++ b/ethcore/res/ethereum/classic.json @@ -12,7 +12,7 @@ "ecip1010PauseTransition": "0x2dc6c0", "ecip1010ContinueTransition": "0x4c4b40", "ecip1017EraRounds": "0x4c4b40", - "eip100bTransition": "0x7fffffffffffffff", + "eip100bTransition": "0x85d9a0", "bombDefuseTransition": "0x5a06e0" } } @@ -29,15 +29,15 @@ "forkCanonHash": "0x94365e3a8c0b35089c1d1195081fe7489b528a84b22199c916180db8b28ade7f", "eip150Transition": "0x2625a0", "eip160Transition": "0x2dc6c0", - "eip161abcTransition": "0x7fffffffffffffff", - "eip161dTransition": "0x7fffffffffffffff", + "eip161abcTransition": "0x85d9a0", + "eip161dTransition": "0x85d9a0", "eip155Transition": "0x2dc6c0", "maxCodeSize": "0x6000", - "maxCodeSizeTransition": "0x7fffffffffffffff", - "eip140Transition": "0x7fffffffffffffff", - "eip211Transition": "0x7fffffffffffffff", - "eip214Transition": "0x7fffffffffffffff", - "eip658Transition": "0x7fffffffffffffff" + "maxCodeSizeTransition": "0x85d9a0", + "eip140Transition": "0x85d9a0", + "eip211Transition": "0x85d9a0", + "eip214Transition": "0x85d9a0", + "eip658Transition": "0x85d9a0" }, "genesis": { "seal": { @@ -3905,7 +3905,7 @@ "0x0000000000000000000000000000000000000005": { "builtin": { "name": "modexp", - "activate_at": "0x7fffffffffffffff", + "activate_at": "0x85d9a0", "pricing": { "modexp": { "divisor": 20 @@ -3916,7 +3916,7 @@ "0x0000000000000000000000000000000000000006": { "builtin": { "name": "alt_bn128_add", - "activate_at": "0x7fffffffffffffff", + "activate_at": "0x85d9a0", "pricing": { "linear": { "base": 500, @@ -3928,7 +3928,7 @@ "0x0000000000000000000000000000000000000007": { "builtin": { "name": "alt_bn128_mul", - "activate_at": "0x7fffffffffffffff", + "activate_at": "0x85d9a0", "pricing": { "linear": { "base": 40000, @@ -3940,7 +3940,7 @@ "0x0000000000000000000000000000000000000008": { "builtin": { "name": "alt_bn128_pairing", - "activate_at": "0x7fffffffffffffff", + "activate_at": "0x85d9a0", "pricing": { "alt_bn128_pairing": { "base": 100000, From f2dd032884107f30c4b6e6573febbf84b2215fd2 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 24 Jun 2019 14:50:11 +0200 Subject: [PATCH 08/15] Print warnings when using dangerous settings for ValidatorSet (#10733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Stop breaking out of loop if a non-canonical hash is found * include expected hash in log msg * More logging * Scope * Syntax * Log in blank RollingFinality Escalate bad proposer to warning * Check validator set size: warn if 1 or even number * More readable code * Use SimpleList::new * Extensive logging on unexpected non-canonical hash * Wording * Update ethcore/blockchain/src/blockchain.rs Co-Authored-By: Tomasz Drwięga * Improved logging, address grumbles * Update ethcore/src/engines/validator_set/simple_list.rs Co-Authored-By: Luke Schoen --- ethcore/blockchain/src/blockchain.rs | 10 +++++----- ethcore/src/engines/authority_round/finality.rs | 2 -- ethcore/src/engines/validator_set/safe_contract.rs | 14 +++++++++----- ethcore/src/engines/validator_set/simple_list.rs | 13 ++++++++----- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/ethcore/blockchain/src/blockchain.rs b/ethcore/blockchain/src/blockchain.rs index 6656fcc0cb7..dbe18f25310 100644 --- a/ethcore/blockchain/src/blockchain.rs +++ b/ethcore/blockchain/src/blockchain.rs @@ -42,7 +42,7 @@ use ethereum_types::{H256, Bloom, BloomRef, U256}; use util_mem::{MallocSizeOf, allocators::new_malloc_size_ops}; use itertools::Itertools; use kvdb::{DBTransaction, KeyValueDB}; -use log::{trace, warn, info}; +use log::{trace, debug, warn, info}; use parity_bytes::Bytes; use parking_lot::{Mutex, RwLock}; use rayon::prelude::*; @@ -963,7 +963,7 @@ impl BlockChain { /// Iterate over all epoch transitions. /// This will only return transitions within the canonical chain. pub fn epoch_transitions(&self) -> EpochTransitionIter { - trace!(target: "blockchain", "Iterating over all epoch transitions"); + debug!(target: "blockchain", "Iterating over all epoch transitions"); let iter = self.db.key_value().iter_from_prefix(db::COL_EXTRA, &EPOCH_KEY_PREFIX[..]); EpochTransitionIter { chain: self, @@ -991,7 +991,7 @@ impl BlockChain { for hash in self.ancestry_iter(parent_hash)? { trace!(target: "blockchain", "Got parent hash {} from ancestry_iter", hash); let details = self.block_details(&hash)?; - trace!(target: "blockchain", "Block #{}: Got block details", details.number); + trace!(target: "blockchain", "Block #{}: Got block details for parent hash {}", details.number, hash); // look for transition in database. if let Some(transition) = self.epoch_transition(details.number, hash) { @@ -1013,8 +1013,8 @@ impl BlockChain { Some(h) => { warn!(target: "blockchain", "Block #{}: Found non-canonical block hash {} (expected {})", details.number, h, hash); - trace!(target: "blockchain", "Block #{} Mismatched hashes. Ancestor {} != Own {} – Own block #{}", details.number, hash, h, self.block_number(&h).unwrap_or_default() ); - trace!(target: "blockchain", " Ancestor {}: #{:#?}", hash, self.block_details(&hash)); + trace!(target: "blockchain", "Block #{} Mismatched hashes. Ancestor {} != Own {}", details.number, hash, h); + trace!(target: "blockchain", " Ancestor {}: #{:#?}", hash, details); trace!(target: "blockchain", " Own {}: #{:#?}", h, self.block_details(&h)); }, diff --git a/ethcore/src/engines/authority_round/finality.rs b/ethcore/src/engines/authority_round/finality.rs index e88458d81aa..e1f050393ea 100644 --- a/ethcore/src/engines/authority_round/finality.rs +++ b/ethcore/src/engines/authority_round/finality.rs @@ -111,8 +111,6 @@ impl RollingFinality { /// Returns a list of all newly finalized headers. // TODO: optimize with smallvec. pub fn push_hash(&mut self, head: H256, signers: Vec
) -> Result, UnknownValidator> { - // TODO: seems bad to iterate over signers twice like this. - // Can do the work in a single loop and call `clear()` if an unknown validator was found? for their_signer in signers.iter() { if !self.signers.contains(their_signer) { warn!(target: "finality", "Unknown validator: {}", their_signer); diff --git a/ethcore/src/engines/validator_set/safe_contract.rs b/ethcore/src/engines/validator_set/safe_contract.rs index e8db31423bb..18477d64de0 100644 --- a/ethcore/src/engines/validator_set/safe_contract.rs +++ b/ethcore/src/engines/validator_set/safe_contract.rs @@ -208,11 +208,11 @@ impl ValidatorSafeContract { match value { Ok(new) => { - debug!(target: "engine", "Set of validators obtained: {:?}", new); + debug!(target: "engine", "Got validator set from contract: {:?}", new); Some(SimpleList::new(new)) }, Err(s) => { - debug!(target: "engine", "Set of validators could not be updated: {}", s); + debug!(target: "engine", "Could not get validator set from contract: {}", s); None }, } @@ -335,7 +335,7 @@ impl ValidatorSet for ValidatorSafeContract { Some(receipts) => match self.extract_from_event(bloom, header, receipts) { None => ::engines::EpochChange::No, Some(list) => { - info!(target: "engine", "Signal for transition within contract. New list: {:?}", + info!(target: "engine", "Signal for transition within contract. New validator list: {:?}", &*list); let proof = encode_proof(&header, receipts); @@ -359,7 +359,7 @@ impl ValidatorSet for ValidatorSafeContract { let addresses = check_first_proof(machine, self.contract_address, old_header, &state_items) .map_err(::engines::EngineError::InsufficientProof)?; - trace!(target: "engine", "extracted epoch set at #{}: {} addresses", + trace!(target: "engine", "Extracted epoch validator set at block #{}: {} addresses", number, addresses.len()); Ok((SimpleList::new(addresses), Some(old_hash))) @@ -380,7 +380,11 @@ impl ValidatorSet for ValidatorSafeContract { let bloom = self.expected_bloom(&old_header); match self.extract_from_event(bloom, &old_header, &receipts) { - Some(list) => Ok((list, Some(old_header.hash()))), + Some(list) => { + trace!(target: "engine", "Extracted epoch validator set at block #{}: {} addresses", old_header.number(), list.len()); + + Ok((list, Some(old_header.hash()))) + }, None => Err(::engines::EngineError::InsufficientProof("No log event in proof.".into()).into()), } } diff --git a/ethcore/src/engines/validator_set/simple_list.rs b/ethcore/src/engines/validator_set/simple_list.rs index d9a4c6e1c23..958bda20246 100644 --- a/ethcore/src/engines/validator_set/simple_list.rs +++ b/ethcore/src/engines/validator_set/simple_list.rs @@ -33,9 +33,14 @@ pub struct SimpleList { impl SimpleList { /// Create a new `SimpleList`. pub fn new(validators: Vec
) -> Self { - SimpleList { - validators: validators, + let validator_count = validators.len(); + if validator_count == 1 { + warn!(target: "engine", "Running AuRa with a single validator implies instant finality. Use a database?"); } + if validator_count % 2 == 0 { + warn!(target: "engine", "Running AuRa with an even number of validators is not recommended (risk of network split)."); + } + SimpleList { validators } } /// Convert into inner representation. @@ -52,9 +57,7 @@ impl ::std::ops::Deref for SimpleList { impl From> for SimpleList { fn from(validators: Vec
) -> Self { - SimpleList { - validators: validators, - } + SimpleList::new(validators) } } From 1a304d38c0b9c3e6baec44e6e81ae8363bdad426 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 24 Jun 2019 15:04:28 +0200 Subject: [PATCH 09/15] cleanup --- ethcore/src/engines/authority_round/mod.rs | 44 ++++++++++------------ ethcore/src/engines/validator_set/test.rs | 1 + 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index e3a17daef6c..4fb4f551613 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -1878,13 +1878,11 @@ mod tests { assert!(aura.verify_block_family(&header, &parent_header).is_ok()); assert_eq!(last_benign.load(AtomicOrdering::SeqCst), 0); - aura.set_signer(Box::new( - ( - Arc::new(AccountProvider::transient_provider()), - validator2, - "".into() - ) - )); + aura.set_signer(Box::new(( + Arc::new(AccountProvider::transient_provider()), + validator2, + "".into(), + ))); // Do not report on steps skipped between genesis and first block. header.set_number(1); @@ -1906,13 +1904,11 @@ mod tests { let validator_set = TestSet::default().with_last_benign(last_benign.clone()); p.validators = Box::new(validator_set); }); - aura.set_signer(Box::new( - ( - Arc::new(AccountProvider::transient_provider()), - Address::from_low_u64_be(123), - "".into() - ) - )); + aura.set_signer(Box::new(( + Arc::new(AccountProvider::transient_provider()), + Address::from_low_u64_be(123), + "".into() + ))); let mut parent_header: Header = Header::default(); parent_header.set_seal(vec![encode(&1usize)]); parent_header.set_gas_limit("222222".parse::().unwrap()); @@ -1943,16 +1939,16 @@ mod tests { assert_eq!(aura.maximum_uncle_count(100), 0); } - #[test] - #[should_panic(expected="counter is too high")] - fn test_counter_increment_too_high() { - use super::Step; - let step = Step { - calibrate: false, - inner: AtomicUsize::new(::std::usize::MAX), - duration: 1, - }; - step.increment(); + #[test] + #[should_panic(expected="counter is too high")] + fn test_counter_increment_too_high() { + use super::Step; + let step = Step { + calibrate: false, + inner: AtomicUsize::new(::std::usize::MAX), + duration: 1, + }; + step.increment(); } #[test] diff --git a/ethcore/src/engines/validator_set/test.rs b/ethcore/src/engines/validator_set/test.rs index 5c13c512294..3ddf1776d16 100644 --- a/ethcore/src/engines/validator_set/test.rs +++ b/ethcore/src/engines/validator_set/test.rs @@ -58,6 +58,7 @@ impl TestSet { } } + /// Build a `TestSet` with the given number of benign misbehaviour reports pub fn with_last_benign(mut self, last_benign: Arc) -> Self { self.last_benign = last_benign; self From 02eb0467749673c3d752290c93de297a75104206 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 24 Jun 2019 15:10:42 +0200 Subject: [PATCH 10/15] Move Engine::register_client to be before other I/O handler registration (#10767) --- ethcore/light/src/client/service.rs | 3 +-- ethcore/service/src/service.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ethcore/light/src/client/service.rs b/ethcore/light/src/client/service.rs index 9672974fc25..cb89659536b 100644 --- a/ethcore/light/src/client/service.rs +++ b/ethcore/light/src/client/service.rs @@ -75,9 +75,8 @@ impl Service { io_service.channel(), cache, )?); - - io_service.register_handler(Arc::new(ImportBlocks(client.clone()))).map_err(Error::Io)?; spec.engine.register_client(Arc::downgrade(&client) as _); + io_service.register_handler(Arc::new(ImportBlocks(client.clone()))).map_err(Error::Io)?; Ok(Service { client, diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index 17690805642..60caefe9a74 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -113,6 +113,7 @@ impl ClientService { miner.clone(), io_service.channel(), )?; + spec.engine.register_client(Arc::downgrade(&client) as _); miner.set_io_channel(io_service.channel()); miner.set_in_chain_checker(&client.clone()); @@ -148,8 +149,6 @@ impl ClientService { }); io_service.register_handler(client_io)?; - spec.engine.register_client(Arc::downgrade(&client) as _); - Ok(ClientService { io_service: Arc::new(io_service), client: client, From 1a0f82b74ea06feca3ab3d45084fba135a0f96e2 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 24 Jun 2019 20:47:03 +0200 Subject: [PATCH 11/15] On second thought non-validators are allowed to report --- ethcore/src/engines/authority_round/mod.rs | 99 ++++++---------------- 1 file changed, 28 insertions(+), 71 deletions(-) diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index 4fb4f551613..3d43a2e5480 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -813,24 +813,22 @@ impl AuthorityRound { return; } - if let (true, me) = (current_step > parent_step + 1, self.address()) { + if let (true, Some(me)) = (current_step > parent_step + 1, self.address()) { debug!(target: "engine", "Author {} built block with step gap. current step: {}, parent step: {}", header.author(), current_step, parent_step); - if validators.contains(header.parent_hash(), &me) { - let mut reported = HashSet::new(); - for step in parent_step + 1..current_step { - let skipped_primary = step_proposer(validators, header.parent_hash(), step); - // Do not report this signer. - if skipped_primary != me { - // Stop reporting once validators start repeating. - if !reported.insert(skipped_primary) { break; } - self.validators.report_benign(&skipped_primary, set_number, header.number()); - } else { - trace!(target: "engine", "Primary that skipped is self, not self-reporting. Own address: {}", me); - } + let mut reported = HashSet::new(); + for step in parent_step + 1..current_step { + let skipped_primary = step_proposer(validators, header.parent_hash(), step); + // Do not report this signer. + if skipped_primary != me { + // Stop reporting once validators start repeating. + if !reported.insert(skipped_primary) { break; } + trace!(target: "engine", "Reporting benign misbehaviour (cause: skipped step) at block #{}, epoch set number {}. Own address: {}", + header.number(), set_number, me); + self.validators.report_benign(&skipped_primary, set_number, header.number()); + } else { + trace!(target: "engine", "Primary that skipped is self, not self-reporting. Own address: {}", me); } - } else { - warn!(target: "engine", "We are not part of the validator set so not reporting (skipped steps). Own address: {}", self.address()); } } } @@ -898,8 +896,8 @@ impl AuthorityRound { finalized.unwrap_or_default() } - fn address(&self) -> Address { - self.signer.read().as_ref().map_or_else(|| Address::zero(), |s| s.address()) + fn address(&self) -> Option
{ + self.signer.read().as_ref().map(|s| s.address() ) } } @@ -1300,19 +1298,17 @@ impl Engine for AuthorityRound { // This check runs in Phase 1 where there is no guarantee that the parent block is // already imported, therefore the call to `epoch_set` may fail. In that case we // won't report the misbehavior but this is not a concern because: - // - Only authorities can report and it's expected that they'll be up-to-date and - // importing, therefore the parent header will most likely be available + // - Authorities will have a signing key available to report and it's expected that + // they'll be up-to-date and importing, therefore the parent header will most likely + // be available // - Even if you are an authority that is syncing the chain, the contract will most // likely ignore old reports // - This specific check is only relevant if you're importing (since it checks // against wall clock) - if let Ok((set, set_number)) = self.epoch_set(header) { - let me = self.address(); - if set.contains(header.parent_hash(), &me) { - self.validators.report_benign(header.author(), set_number, header.number()); - } else { - debug!(target: "engine", "Not reporting benign misbehaviour (cause: InvalidSeal) because we're not part of the validator set, Own address: {}", me); - } + if let Ok((_, set_number)) = self.epoch_set(header) { + trace!(target: "engine", "Reporting benign misbehaviour (cause: InvalidSeal) at block #{}, epoch set number {}. Own address: {}", + header.number(), set_number, self.address().unwrap_or_default()); + self.validators.report_benign(header.author(), set_number, header.number()); } Err(BlockError::InvalidSeal.into()) @@ -1382,12 +1378,8 @@ impl Engine for AuthorityRound { match validate_empty_steps() { Ok(len) => len, Err(err) => { - if self.validators.contains(header.parent_hash(), &self.address()) { - self.validators.report_benign(header.author(), set_number, header.number()); - } else { - warn!(target: "engine", "Not reporting benign misbehaviour (cause: invalid empty steps) because we're not a validator. Own address: {}", - self.address()); - } + trace!(target: "engine", "Reporting benign misbehaviour (cause: invalid empty steps) at block #{}, epoch set number {}. Own address: {}", header.number(), set_number, self.address().unwrap_or_default()); + self.validators.report_benign(header.author(), set_number, header.number()); return Err(err); }, } @@ -1416,11 +1408,9 @@ impl Engine for AuthorityRound { let res = verify_external(header, &*validators, self.empty_steps_transition); match res { Err(Error::Engine(EngineError::NotProposer(_))) => { - if self.validators.contains(header.parent_hash(), &self.address()) { - self.validators.report_benign(header.author(), set_number, header.number()); - } else { - warn!(target: "engine", "Not reporting benign misbehaviour (cause:block from incorrect proposer) because we're not a validator. Own address: {}", self.address()); - } + trace!(target: "engine", "Reporting benign misbehaviour (cause: block from incorrect proposer) at block #{}, epoch set number {}. Own address: {}", + header.number(), set_number, self.address().unwrap_or_default()); + self.validators.report_benign(header.author(), set_number, header.number()); }, Ok(_) => { // we can drop all accumulated empty step messages that are older than this header's step @@ -1893,40 +1883,7 @@ mod tests { header.set_number(2); assert!(aura.verify_block_family(&header, &parent_header).is_ok()); assert_eq!(last_benign.load(AtomicOrdering::SeqCst), 2); - } - - #[test] - fn skips_benign_reporting_unless_validator() { - let _ = ::env_logger::try_init(); - - let last_benign = Arc::new(AtomicUsize::new(0)); - let aura = build_aura(|p| { - let validator_set = TestSet::default().with_last_benign(last_benign.clone()); - p.validators = Box::new(validator_set); - }); - aura.set_signer(Box::new(( - Arc::new(AccountProvider::transient_provider()), - Address::from_low_u64_be(123), - "".into() - ))); - let mut parent_header: Header = Header::default(); - parent_header.set_seal(vec![encode(&1usize)]); - parent_header.set_gas_limit("222222".parse::().unwrap()); - let mut header: Header = Header::default(); - header.set_difficulty(calculate_score(1, 3, 0)); - header.set_gas_limit("222222".parse::().unwrap()); - header.set_seal(vec![encode(&3usize)]); - - header.set_number(1); - aura.verify_block_family(&header, &parent_header).unwrap(); - header.set_number(2); - aura.verify_block_family(&header, &parent_header).unwrap(); - header.set_number(3); - aura.verify_block_family(&header, &parent_header).unwrap(); - - // No reporting happened: we're not a validator - assert_eq!(last_benign.load(AtomicOrdering::SeqCst), 0); - } + }if self.validators.contains(header.parent_hash() #[test] fn test_uncles_transition() { From 05df05300f32a5c94b7469dc74e2472c251cb054 Mon Sep 17 00:00:00 2001 From: David Palm Date: Mon, 24 Jun 2019 21:05:17 +0200 Subject: [PATCH 12/15] cleanup --- ethcore/src/engines/authority_round/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index 3d43a2e5480..44c10b4c2a0 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -1378,7 +1378,8 @@ impl Engine for AuthorityRound { match validate_empty_steps() { Ok(len) => len, Err(err) => { - trace!(target: "engine", "Reporting benign misbehaviour (cause: invalid empty steps) at block #{}, epoch set number {}. Own address: {}", header.number(), set_number, self.address().unwrap_or_default()); + trace!(target: "engine", "Reporting benign misbehaviour (cause: invalid empty steps) at block #{}, epoch set number {}. Own address: {}", + header.number(), set_number, self.address().unwrap_or_default()); self.validators.report_benign(header.author(), set_number, header.number()); return Err(err); }, @@ -1883,7 +1884,7 @@ mod tests { header.set_number(2); assert!(aura.verify_block_family(&header, &parent_header).is_ok()); assert_eq!(last_benign.load(AtomicOrdering::SeqCst), 2); - }if self.validators.contains(header.parent_hash() + } #[test] fn test_uncles_transition() { From 6022c47b53427852286e9c5e73ccc9ee64f00bf2 Mon Sep 17 00:00:00 2001 From: Luke Schoen Date: Tue, 25 Jun 2019 16:15:13 +1000 Subject: [PATCH 13/15] docs: Update Readme with TOC, Contributor Guideline. Update Cargo package descriptions (#10652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: Update Readme with TOC, Contributor Guide and License sections * docs: Simplify package descriptions in Readme * docs: Fix typos * docs: Update Cargo package configs adding missing descriptions * fix: Remove machine since was accidental from diff branch. Fix contribting indentation. * fix: Fix formatting * remove details snippet since not code and formats badly * review-fix: Remove duplicate description * fix dot point formatting * fix: Add missing description to vm package * fix: Remove duplicate Code of Conduct and Contributor guidelines * docs: Update Contributing for consistency with other repos * docs: Update Readme to link to existing Code of Conduct and Contributor guidelines in .github folder * refactor: Add missing space * fix: Fix links to be markdown format * review-fix: Update CONTRIBUTING.md to link to Parity Ethereum Style Guide * docs: Fix type in .github/CONTRIBUTING.md Co-Authored-By: David * docs: Fix typo in .github/CONTRIBUTING.md Co-Authored-By: David * docs: Rephrase parity-clib/Cargo.toml Co-Authored-By: David * review-fix: Fix whole paragraph * review-fix: Add comment to prevent direct pushes to master branch * review-fix: Change some rules to recommendations * Update .github/CONTRIBUTING.md Co-Authored-By: Tomasz Drwięga * review-fix: Replace feature reduction with breaking changes dot point * review-fix: Rephrase what to do when reviewing PR * review-fix: Update parity-rpc package description and module rustdocs * docs: Add missing fullstop * review-fix: Update rustdoc section to show all packages first as default * review-fix: Rename Parity Util to Parity Core Libraries * review-fix: Rename readme too for the C bindings name change * review-fix: Remove some docs since we do not teach the reader Rust * review-fix: Wrap Parity Ethereum specific packages in a details section * review-fix: Separate tools that are in this repo vs those that are not * review-fix: Add link to ethabi on crates.io * review-fix: Remove unnecessary extra line talking about pushing to master * review-fix: Remove useless summary of parts of the rust book documentation section * docs: Update .github/CONTRIBUTING.md to specify two reviewers required Co-Authored-By: David * docs: Update README.md with simplification Co-Authored-By: David * docs: Update README.md renaming to Test Runner instead of just Run Co-Authored-By: David * review-fix: Remove hardware wallet from docs * review-fix: Remove veto PR section of contributor doc * review-fix: Change to test generator title --- .github/CONTRIBUTING.md | 37 +++- README.md | 246 ++++++++++++++++++++++-- accounts/Cargo.toml | 2 +- accounts/ethkey/Cargo.toml | 1 + accounts/ethkey/README.md | 2 +- accounts/ethkey/cli/Cargo.toml | 1 + accounts/ethstore/Cargo.toml | 1 + accounts/ethstore/cli/Cargo.toml | 1 + chainspec/Cargo.toml | 1 + cli-signer/Cargo.toml | 4 +- cli-signer/rpc-client/Cargo.toml | 4 +- ethash/Cargo.toml | 1 + ethcore/Cargo.toml | 2 +- ethcore/blockchain/Cargo.toml | 2 +- ethcore/call-contract/Cargo.toml | 1 + ethcore/evm/Cargo.toml | 1 + ethcore/light/Cargo.toml | 2 +- ethcore/node-filter/Cargo.toml | 2 +- ethcore/private-tx/src/encryptor.rs | 2 +- ethcore/service/Cargo.toml | 1 + ethcore/sync/Cargo.toml | 2 +- ethcore/types/Cargo.toml | 2 +- ethcore/types/src/restoration_status.rs | 2 +- ethcore/vm/Cargo.toml | 1 + ethcore/wasm/Cargo.toml | 1 + ethcore/wasm/run/Cargo.toml | 1 + evmbin/Cargo.toml | 2 +- ipfs/Cargo.toml | 2 +- json/Cargo.toml | 1 + miner/Cargo.toml | 2 +- parity/logger/Cargo.toml | 2 +- rpc/Cargo.toml | 2 +- rpc/src/lib.rs | 2 +- secret-store/Cargo.toml | 2 +- whisper/Cargo.toml | 2 +- 35 files changed, 300 insertions(+), 40 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b51cca8821e..6d58aa9412e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -24,7 +24,42 @@ Also, try to include **steps to reproduce** the issue and expand on the **actual If you would like to contribute to Parity Ethereum, please **fork it**, fix bugs or implement features, and [propose a pull request](https://github.com/paritytech/parity-ethereum/compare). -Please, refer to the [Coding Guide](https://wiki.parity.io/Coding-guide) in our wiki for more details about hacking on Parity. +### Labels & Milestones + +We use [labels](https://github.com/paritytech/parity-ethereum/labels) to manage PRs and issues and communicate the state of a PR. Please familiarize yourself with them. Furthermore we are organizing issues in [milestones](https://github.com/paritytech/parity-ethereum/milestones). Best way to get started is to a pick a ticket from the current milestone tagged [`easy`](https://github.com/paritytech/parity-ethereum/labels/Q2-easy%20%F0%9F%92%83) and get going, or [`mentor`](https://github.com/paritytech/parity-ethereum/labels/Q1-mentor%20%F0%9F%95%BA) and get in contact with the mentor offering their support on that larger task. + +### Rules + +There are a few basic ground-rules for contributors (including the maintainer(s) of the project): + +* **No pushing directly to the master branch**. +* **All modifications** must be made in a **pull-request** to solicit feedback from other contributors. +* Pull-requests cannot be merged before CI runs green and two reviewers have given their approval. +* Contributors should adhere to the [Parity Ethereum Style Guide](https://wiki.parity.io/Parity-Ethereum-Style-Guide). + +### Recommendations + +* **Non-master branch names** *should* be prefixed with a short name moniker, followed by the associated Github Issue ID (if any), and a brief description of the task using the format `--` (e.g. `gavin-123-readme`). The name moniker helps people to inquiry about their unfinished work, and the GitHub Issue ID helps your future self and other developers (particularly those who are onboarding) find out about and understand the original scope of the task, and where it fits into Parity Ethereum [Projects](https://github.com/paritytech/parity-ethereum/projects). +* **Remove stale branches periodically** + +### Preparing Pull Requests + +* If your PR does not alter any logic (e.g. comments, dependencies, docs), then it may be tagged [`insubstantial`](https://github.com/paritytech/parity-ethereum/pulls?q=is%3Aopen+is%3Apr+label%3A%22A2-insubstantial+%F0%9F%91%B6%22). + +* Once a PR is ready for review please add the [`pleasereview`](https://github.com/paritytech/parity-ethereum/pulls?utf8=%E2%9C%93&q=is%3Aopen+is%3Apr+label%3A%22A0-pleasereview+%F0%9F%A4%93%22+) label. + +### Reviewing Pull Requests*: + +* At least two reviewers are required to review PRs (even for PRs tagged [`insubstantial`](https://github.com/paritytech/parity-ethereum/pulls?q=is%3Aopen+is%3Apr+label%3A%22A2-insubstantial+%F0%9F%91%B6%22)). + +When doing a review, make sure to look for any: + +* Buggy behavior. +* Undue maintenance burden. +* Breaking with house coding style. +* Pessimization (i.e. reduction of speed as measured in the projects benchmarks). +* Breaking changes should be carefuly reviewed and tagged as such so they end up in the [changelog](../CHANGELOG.md). +* Uselessness (i.e. it does not strictly add a feature or fix a known issue). ## License. diff --git a/README.md b/README.md index 42e56735063..a8489013a3c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,24 @@

+## Table of Contents + +1. [Description](#chapter-001) +2. [Technical Overview](#chapter-002) +3. [Building](#chapter-003)
+ 3.1 [Building Dependencies](#chapter-0031)
+ 3.2 [Building from Source Code](#chapter-0032)
+ 3.3 [Simple One-Line Installer for Mac and Linux](#chapter-0033)
+ 3.4 [Starting Parity Ethereum](#chapter-0034) +4. [Documentation](#chapter-004) +5. [Toolchain](#chapter-005) +6. [Community](#chapter-006) +7. [Contributing](#chapter-007) +8. [License](#chapter-008) + + +## 1. Description + **Built for mission-critical use**: Miners, service providers, and exchanges need fast synchronisation and maximum uptime. Parity Ethereum provides the core infrastructure essential for speedy and reliable services. - Clean, modular codebase for easy customisation @@ -15,7 +33,7 @@ - Synchronise in hours, not days with Warp Sync - Modular for light integration into your service or product -## Technical Overview +## 2. Technical Overview Parity Ethereum's goal is to be the fastest, lightest, and most secure Ethereum client. We are developing Parity Ethereum using the sophisticated and cutting-edge **Rust programming language**. Parity Ethereum is licensed under the GPLv3 and can be used for all your Ethereum needs. @@ -25,7 +43,9 @@ If you run into problems while using Parity Ethereum, check out the [wiki for do Parity Ethereum's current beta-release is 2.1. You can download it at [the releases page](https://github.com/paritytech/parity-ethereum/releases) or follow the instructions below to build from source. Please, mind the [CHANGELOG.md](CHANGELOG.md) for a list of all changes between different versions. -## Build Dependencies +## 3. Building + +### 3.1 Build Dependencies Parity Ethereum requires **latest stable Rust version** to build. @@ -58,7 +78,7 @@ Once you have `rustup` installed, then you need to install: Make sure that these binaries are in your `PATH`. After that, you should be able to build Parity Ethereum from source. -## Build from Source Code +### 3.2 Build from Source Code ```bash # download Parity Ethereum code @@ -95,7 +115,7 @@ or $ git checkout beta ``` -## Simple One-Line Installer for Mac and Linux +### 3.3 Simple One-Line Installer for Mac and Linux ```bash bash <(curl https://get.parity.io -L) @@ -107,9 +127,9 @@ The one-line installer always defaults to the latest beta release. To install a bash <(curl https://get.parity.io -L) -r stable ``` -## Start Parity Ethereum +### 3.4 Starting Parity Ethereum -### Manually +#### Manually To start Parity Ethereum manually, just run @@ -119,7 +139,7 @@ $ ./target/release/parity so Parity Ethereum begins syncing the Ethereum blockchain. -### Using `systemd` service file +#### Using `systemd` service file To start Parity Ethereum as a regular user using `systemd` init: @@ -128,17 +148,203 @@ To start Parity Ethereum as a regular user using `systemd` init: 2. Copy release to bin folder, write `sudo install ./target/release/parity /usr/bin/parity` 3. To configure Parity Ethereum, write a `/etc/parity/config.toml` config file, see [Configuring Parity Ethereum](https://paritytech.github.io/wiki/Configuring-Parity) for details. -## Parity Ethereum toolchain +## 4. Documentation + +Official website: https://parity.io + +Be sure to [check out our wiki](https://wiki.parity.io) for more information. + +### Viewing documentation for Parity Ethereum packages + +You can generate documentation for Parity Ethereum Rust packages that automatically opens in your web browser using [rustdoc with Cargo](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html#using-rustdoc-with-cargo) (of the The Rustdoc Book), by running the the following commands: + +* **All** packages + ``` + cargo doc --open + ``` + +* Specific package + ``` + cargo doc --package --open + ``` + +Replacing `` with one of the following from the details section below (i.e. `cargo doc --package parity-ethereum --open`): + +

+ +* Parity Ethereum (EthCore) Client Application + ```bash + parity-ethereum + ``` +* Parity Ethereum Account Management, Key Management Tool, and Keys Generator + ```bash + ethcore-accounts, ethkey-cli, ethstore, ethstore-cli + ``` +* Parity Chain Specification + ```bash + chainspec + ``` +* Parity CLI Signer Tool & RPC Client + ```bash + cli-signer parity-rpc-client + ``` +* Parity Ethereum Ethash & ProgPoW Implementations + ```bash + ethash + ``` +* Parity (EthCore) Library + ```bash + ethcore + ``` + * Parity Ethereum Blockchain Database, Test Generator, Configuration, +Caching, Importing Blocks, and Block Information + ```bash + ethcore-blockchain + ``` + * Parity Ethereum (EthCore) Contract Calls and Blockchain Service & Registry Information + ```bash + ethcore-call-contract + ``` + * Parity Ethereum (EthCore) Database Access & Utilities, Database Cache Manager + ```bash + ethcore-db + ``` + * Parity Ethereum Virtual Machine (EVM) Rust Implementation + ```bash + evm + ``` + * Parity Ethereum (EthCore) Light Client Implementation + ```bash + ethcore-light + ``` + * Parity Smart Contract based Node Filter, Manage Permissions of Network Connections + ```bash + node-filter + ``` + * Parity Private Transactions + ```bash + ethcore-private-tx + ``` + * Parity Ethereum (EthCore) Client & Network Service Creation & Registration with the I/O Subsystem + ```bash + ethcore-service + ``` + * Parity Ethereum (EthCore) Blockchain Synchronization + ```bash + ethcore-sync + ``` + * Parity Ethereum Common Types + ```bash + common-types + ``` + * Parity Ethereum Virtual Machines (VM) Support Library + ```bash + vm + ``` + * Parity Ethereum WASM Interpreter + ```bash + wasm + ``` + * Parity Ethereum WASM Test Runner + ```bash + pwasm-run-test + ``` + * Parity EVM Implementation + ```bash + evmbin + ``` + * Parity Ethereum IPFS-compatible API + ```bash + parity-ipfs-api + ``` + * Parity Ethereum JSON Deserialization + ```bash + ethjson + ``` + * Parity Ethereum State Machine Generalization for Consensus Engines + ```bash + parity-machine + ``` +* Parity Ethereum (EthCore) Miner Interface + ```bash + ethcore-miner parity-local-store price-info ethcore-stratum using_queue + ``` +* Parity Ethereum (EthCore) Logger Implementation + ```bash + ethcore-logger + ``` +* C bindings library for the Parity Ethereum client + ```bash + parity-clib + ``` +* Parity Ethereum JSON-RPC Servers + ```bash + parity-rpc + ``` +* Parity Ethereum (EthCore) Secret Store + ```bash + ethcore-secretstore + ``` +* Parity Updater Service + ```bash + parity-updater parity-hash-fetch + ``` +* Parity Core Libraries (Parity Util) + ```bash + ethcore-bloom-journal blooms-db dir eip-712 fake-fetch fastmap fetch ethcore-io + journaldb keccak-hasher len-caching-lock macros memory-cache memzero + migration-rocksdb ethcore-network ethcore-network-devp2p panic_hook + patricia-trie-ethereum registrar rlp_compress rlp_derive parity-runtime stats + time-utils triehash-ethereum unexpected parity-version + ``` +* Parity Whisper Protocol Implementation + ```bash + parity-whisper whisper-cli + ``` + +

+ +### Contributing to documentation for Parity Ethereum packages + +[Document source code](https://doc.rust-lang.org/1.9.0/book/documentation.html) for Parity Ethereum packages by annotating the source code with documentation comments. + +Example (generic documentation comment): +```markdown +/// Summary +/// +/// Description +/// +/// # Panics +/// +/// # Errors +/// +/// # Safety +/// +/// # Examples +/// +/// Summary of Example 1 +/// +/// ```rust +/// // insert example 1 code here for use with documentation as tests +/// ``` +/// +``` + +## 5. Toolchain In addition to the Parity Ethereum client, there are additional tools in this repository available: -- [evmbin](https://github.com/paritytech/parity-ethereum/blob/master/evmbin/) - EVM implementation for Parity Ethereum. -- [ethabi](https://github.com/paritytech/ethabi) - Parity Ethereum function calls encoding. -- [ethstore](https://github.com/paritytech/parity-ethereum/blob/master/accounts/ethstore) - Parity Ethereum key management. -- [ethkey](https://github.com/paritytech/parity-ethereum/blob/master/accounts/ethkey) - Parity Ethereum keys generator. -- [whisper](https://github.com/paritytech/parity-ethereum/blob/master/whisper/) - Implementation of Whisper-v2 PoC. +- [evmbin](./evmbin) - Parity Ethereum EVM Implementation. +- [ethstore](./accounts/ethstore) - Parity Ethereum Key Management. +- [ethkey](./accounts/ethkey) - Parity Ethereum Keys Generator. +- [whisper](./whisper) - Parity Ethereum Whisper-v2 PoC Implementation. -## Join the chat! +The following tool is available in a separate repository: +- [ethabi](https://github.com/paritytech/ethabi) - Parity Ethereum Encoding of Function Calls. [Docs here](https://crates.io/crates/ethabi) + +## 6. Community + +### Join the chat! Questions? Get in touch with us on Gitter: [![Gitter: Parity](https://img.shields.io/badge/gitter-parity-4AB495.svg)](https://gitter.im/paritytech/parity) @@ -149,8 +355,14 @@ Questions? Get in touch with us on Gitter: Alternatively, join our community on Matrix: [![Riot: +Parity](https://img.shields.io/badge/riot-%2Bparity%3Amatrix.parity.io-orange.svg)](https://riot.im/app/#/group/+parity:matrix.parity.io) -## Documentation +## 7. Contributing -Official website: https://parity.io +An introduction has been provided in the ["So You Want to be a Core Developer" presentation slides by Hernando Castano](http://tiny.cc/contrib-to-parity-eth). Additional guidelines are provided in [CONTRIBUTING](./.github/CONTRIBUTING.md). -Be sure to [check out our wiki](https://wiki.parity.io) for more information. +### Contributor Code of Conduct + +[CODE_OF_CONDUCT](./.github/CODE_OF_CONDUCT.md) + +## 8. License + +[LICENSE](./LICENSE) diff --git a/accounts/Cargo.toml b/accounts/Cargo.toml index 61f82815f9d..bf4300ab255 100644 --- a/accounts/Cargo.toml +++ b/accounts/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Account management for Parity Ethereum" +description = "Parity Ethereum Account Management" homepage = "http://parity.io" license = "GPL-3.0" name = "ethcore-accounts" diff --git a/accounts/ethkey/Cargo.toml b/accounts/ethkey/Cargo.toml index 9d91d909c91..ebb17d21d00 100644 --- a/accounts/ethkey/Cargo.toml +++ b/accounts/ethkey/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum Keys Generator" name = "ethkey" version = "0.3.0" authors = ["Parity Technologies "] diff --git a/accounts/ethkey/README.md b/accounts/ethkey/README.md index 6fc98b020d6..23c57fa4cad 100644 --- a/accounts/ethkey/README.md +++ b/accounts/ethkey/README.md @@ -5,7 +5,7 @@ Parity Ethereum keys generator. ### Usage ``` -Parity Ethereum keys generator. +Parity Ethereum Keys Generator. Copyright 2015-2019 Parity Technologies (UK) Ltd. Usage: diff --git a/accounts/ethkey/cli/Cargo.toml b/accounts/ethkey/cli/Cargo.toml index d43aa2c5c0a..cb57f05053f 100644 --- a/accounts/ethkey/cli/Cargo.toml +++ b/accounts/ethkey/cli/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum Keys Generator CLI" name = "ethkey-cli" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/accounts/ethstore/Cargo.toml b/accounts/ethstore/Cargo.toml index 71ff4768408..7459d3dc9d0 100644 --- a/accounts/ethstore/Cargo.toml +++ b/accounts/ethstore/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum Key Management" name = "ethstore" version = "0.2.1" authors = ["Parity Technologies "] diff --git a/accounts/ethstore/cli/Cargo.toml b/accounts/ethstore/cli/Cargo.toml index 80fcb65397f..3d2349e8546 100644 --- a/accounts/ethstore/cli/Cargo.toml +++ b/accounts/ethstore/cli/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum Key Management CLI" name = "ethstore-cli" version = "0.1.1" authors = ["Parity Technologies "] diff --git a/chainspec/Cargo.toml b/chainspec/Cargo.toml index 8739ca7dcba..c0308edd320 100644 --- a/chainspec/Cargo.toml +++ b/chainspec/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum Chain Specification" name = "chainspec" version = "0.1.0" authors = ["Marek Kotewicz "] diff --git a/cli-signer/Cargo.toml b/cli-signer/Cargo.toml index 522eb796f8f..1f3b40f23c4 100644 --- a/cli-signer/Cargo.toml +++ b/cli-signer/Cargo.toml @@ -1,10 +1,10 @@ [package] -authors = ["Parity "] -description = "Parity Cli Tool" +description = "Parity Ethereum CLI Signer Tool" homepage = "http://parity.io" license = "GPL-3.0" name = "cli-signer" version = "1.4.0" +authors = ["Parity "] [dependencies] ethereum-types = "0.6.0" diff --git a/cli-signer/rpc-client/Cargo.toml b/cli-signer/rpc-client/Cargo.toml index 6a974aa5ab1..0d8b0e2d8c5 100644 --- a/cli-signer/rpc-client/Cargo.toml +++ b/cli-signer/rpc-client/Cargo.toml @@ -1,10 +1,10 @@ [package] -authors = ["Parity "] -description = "Parity Rpc Client" +description = "Parity Ethereum RPC Client" homepage = "http://parity.io" license = "GPL-3.0" name = "parity-rpc-client" version = "1.4.0" +authors = ["Parity "] [dependencies] ethereum-types = "0.6.0" diff --git a/ethash/Cargo.toml b/ethash/Cargo.toml index 5994e338a3d..ec4c1838a6f 100644 --- a/ethash/Cargo.toml +++ b/ethash/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum Ethash & ProgPoW Implementations" name = "ethash" version = "1.12.0" authors = ["Parity Technologies "] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 57e28f6e15d..f7355da85ca 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Ethcore library" +description = "Parity Ethereum (EthCore) Library" homepage = "http://parity.io" license = "GPL-3.0" name = "ethcore" diff --git a/ethcore/blockchain/Cargo.toml b/ethcore/blockchain/Cargo.toml index ac2955f9db7..7b9f72778a8 100644 --- a/ethcore/blockchain/Cargo.toml +++ b/ethcore/blockchain/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Ethcore blockchain database" +description = "Parity Ethereum Blockchain Database, Test Generator, Configuration, Caching, Importing Blocks, and Block Information" homepage = "http://parity.io" license = "GPL-3.0" name = "ethcore-blockchain" diff --git a/ethcore/call-contract/Cargo.toml b/ethcore/call-contract/Cargo.toml index 8836d7b30ff..c67f22aad2a 100644 --- a/ethcore/call-contract/Cargo.toml +++ b/ethcore/call-contract/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum (EthCore) Contract Calls and Blockchain Service & Registry Information" name = "ethcore-call-contract" version = "0.1.0" license = "GPL-3.0" diff --git a/ethcore/evm/Cargo.toml b/ethcore/evm/Cargo.toml index ca9df4d1bf2..43621cb37e6 100644 --- a/ethcore/evm/Cargo.toml +++ b/ethcore/evm/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum Virtual Machine (EVM) Rust Implementation" name = "evm" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/ethcore/light/Cargo.toml b/ethcore/light/Cargo.toml index 62d70fec4b8..e2788ce8315 100644 --- a/ethcore/light/Cargo.toml +++ b/ethcore/light/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Parity Light Client Implementation" +description = "Parity Ethereum (EthCore) Light Client Implementation (Block Import IO Service, Blockchain Data Fetching, Light Client Header Chain Storage, Parity Light Protocol (PLP) Provider, Light Transaction Queue, CHT Definitions, Light Client Data Cache), Parity Light Protocol (PLP) Implementation, P2P Network I/O and Event Context Generalization, Peer Error Handling & Punishment, Request Load Timer & Distribution Manager, Pending Request Set Storage, Request Credit Management, Light Client Request Types, Request Chain Builder Utility, On-demand Chain Request Service over LES (for RPCs), ResponseGuard Implementation)" homepage = "http://parity.io" license = "GPL-3.0" name = "ethcore-light" diff --git a/ethcore/node-filter/Cargo.toml b/ethcore/node-filter/Cargo.toml index 4ad60d0dd5b..fc59fbdf83e 100644 --- a/ethcore/node-filter/Cargo.toml +++ b/ethcore/node-filter/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Parity smart network connections" +description = "Parity Smart Contract based Node Filter, Manage Permissions of Network Connections" homepage = "http://parity.io" license = "GPL-3.0" name = "node-filter" diff --git a/ethcore/private-tx/src/encryptor.rs b/ethcore/private-tx/src/encryptor.rs index 8d5504d34d0..323f44a692e 100644 --- a/ethcore/private-tx/src/encryptor.rs +++ b/ethcore/private-tx/src/encryptor.rs @@ -60,7 +60,7 @@ pub trait Encryptor: Send + Sync + 'static { ) -> Result; } -/// Configurtion for key server encryptor +/// Configuration for key server encryptor #[derive(Default, PartialEq, Debug, Clone)] pub struct EncryptorConfig { /// URL to key server diff --git a/ethcore/service/Cargo.toml b/ethcore/service/Cargo.toml index be9c3d45377..fe6f7c46c42 100644 --- a/ethcore/service/Cargo.toml +++ b/ethcore/service/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum (EthCore) Client & Network Service Creation & Registration with the I/O Subsystem" name = "ethcore-service" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/ethcore/sync/Cargo.toml b/ethcore/sync/Cargo.toml index 1bec74526af..38dbb3ff15c 100644 --- a/ethcore/sync/Cargo.toml +++ b/ethcore/sync/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Ethcore blockchain sync" +description = "Parity Ethereum (EthCore) Blockchain Synchronization (Strategy, Blockchain Downloader, Blockchain Synchronization Implementation of Ethereum Protocol, Propagating Data to Peers, Requesting Data from Peers, Supplying Data in Response to Peer Requests, Handling Peer Responses, Matching Packet IDs and Protocol, Light Client Synchronization of Header Chain, Header Download State Machine, Light Decoding & Verifying Header Responses, Private Transaction Handling, Synchronization Snapshot Service to Download & Verify Block Chunks, Peer Connection Management and Blockchain Client I/O Interface for Synchronization Handler, Transaction Statistics)" name = "ethcore-sync" version = "1.12.0" license = "GPL-3.0" diff --git a/ethcore/types/Cargo.toml b/ethcore/types/Cargo.toml index e702b6af3b0..083205d717a 100644 --- a/ethcore/types/Cargo.toml +++ b/ethcore/types/Cargo.toml @@ -1,6 +1,6 @@ [package] +description = "Parity Ethereum Common Types" name = "common-types" -description = "Common types used throughout the codebase" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/ethcore/types/src/restoration_status.rs b/ethcore/types/src/restoration_status.rs index dea342a3104..b36ec7ef4a9 100644 --- a/ethcore/types/src/restoration_status.rs +++ b/ethcore/types/src/restoration_status.rs @@ -21,7 +21,7 @@ pub enum RestorationStatus { /// No restoration. Inactive, - /// Restoration is initalizing + /// Restoration is initializing Initializing { /// Number of chunks done/imported chunks_done: u32, diff --git a/ethcore/vm/Cargo.toml b/ethcore/vm/Cargo.toml index 02504372bc3..7a3768e1e06 100644 --- a/ethcore/vm/Cargo.toml +++ b/ethcore/vm/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Virtual Machines (VM) Support Library" name = "vm" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/ethcore/wasm/Cargo.toml b/ethcore/wasm/Cargo.toml index 9447642c66d..33dc62a1e8e 100644 --- a/ethcore/wasm/Cargo.toml +++ b/ethcore/wasm/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "WASM Interpreter" name = "wasm" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/ethcore/wasm/run/Cargo.toml b/ethcore/wasm/run/Cargo.toml index 029183d671a..52402e3b8ee 100644 --- a/ethcore/wasm/run/Cargo.toml +++ b/ethcore/wasm/run/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity WASM Test Run" name = "pwasm-run-test" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/evmbin/Cargo.toml b/evmbin/Cargo.toml index a483a4b191a..57ed25289cb 100644 --- a/evmbin/Cargo.toml +++ b/evmbin/Cargo.toml @@ -1,6 +1,6 @@ [package] +description = "Parity EVM Implementation" name = "evmbin" -description = "Parity's EVM implementation" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/ipfs/Cargo.toml b/ipfs/Cargo.toml index 2fc02a7f67c..13e72c67ae2 100644 --- a/ipfs/Cargo.toml +++ b/ipfs/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Parity IPFS-compatible API" +description = "Parity Ethereum IPFS-compatible API" name = "parity-ipfs-api" version = "1.12.0" license = "GPL-3.0" diff --git a/json/Cargo.toml b/json/Cargo.toml index 747042e8543..175155e9d41 100644 --- a/json/Cargo.toml +++ b/json/Cargo.toml @@ -1,4 +1,5 @@ [package] +description = "Parity Ethereum JSON Deserialization" name = "ethjson" version = "0.1.0" authors = ["Parity Technologies "] diff --git a/miner/Cargo.toml b/miner/Cargo.toml index a9ac13c2e66..13a819930ed 100644 --- a/miner/Cargo.toml +++ b/miner/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Parity Miner interface." +description = "Parity Ethereum Miner Interface." name = "ethcore-miner" homepage = "http://parity.io" license = "GPL-3.0" diff --git a/parity/logger/Cargo.toml b/parity/logger/Cargo.toml index 17c8c4396de..217bcf48a61 100644 --- a/parity/logger/Cargo.toml +++ b/parity/logger/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Log implementation for Parity" +description = "Parity Ethereum Logger Implementation" name = "ethcore-logger" version = "1.12.0" license = "GPL-3.0" diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index bd71c81e6a1..7044128d625 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Parity JSON-RPC servers." +description = "Parity Ethereum JSON-RPC Servers (WS, HTTP, IPC)" name = "parity-rpc" version = "1.12.0" license = "GPL-3.0" diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index 324aea1d270..6b856eb5291 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity Ethereum. If not, see . -//! Parity RPC. +//! Parity Ethereum JSON-RPC Servers (WS, HTTP, IPC). #![warn(missing_docs, unused_extern_crates)] #![cfg_attr(feature = "cargo-clippy", warn(clippy::all, clippy::pedantic))] diff --git a/secret-store/Cargo.toml b/secret-store/Cargo.toml index 5139d1b7fae..8f730ec2764 100644 --- a/secret-store/Cargo.toml +++ b/secret-store/Cargo.toml @@ -1,5 +1,5 @@ [package] -description = "Ethcore Secret Store" +description = "Parity Ethereum (EthCore) Secret Store" name = "ethcore-secretstore" version = "1.0.0" license = "GPL-3.0" diff --git a/whisper/Cargo.toml b/whisper/Cargo.toml index 64bd33594ac..46351acc58a 100644 --- a/whisper/Cargo.toml +++ b/whisper/Cargo.toml @@ -1,8 +1,8 @@ [package] +description = "Parity Ethereum Whisper Protocol Implementation" name = "parity-whisper" version = "0.1.0" authors = ["Parity Technologies "] -description = "Whisper Protocol implementation for Parity" [dependencies] bitflags = "0.9" From d743df549bfbcceed223b25097ee117a9005b71c Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 25 Jun 2019 09:14:16 +0200 Subject: [PATCH 14/15] Treat empty account the same as non-exist accounts in EIP-1052 (#10775) --- ethcore/src/externalities.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ethcore/src/externalities.rs b/ethcore/src/externalities.rs index 54c913c73d8..2d831290a4b 100644 --- a/ethcore/src/externalities.rs +++ b/ethcore/src/externalities.rs @@ -315,7 +315,11 @@ impl<'a, T: 'a, V: 'a, B: 'a> Ext for Externalities<'a, T, V, B> } fn extcodehash(&self, address: &Address) -> vm::Result> { - Ok(self.state.code_hash(address)?) + if self.state.exists_and_not_null(address)? { + Ok(self.state.code_hash(address)?) + } else { + Ok(None) + } } fn extcodesize(&self, address: &Address) -> vm::Result> { From 9080630f49f80553996f1eb2412f21514cc1c8f3 Mon Sep 17 00:00:00 2001 From: David Palm Date: Tue, 25 Jun 2019 10:03:11 +0200 Subject: [PATCH 15/15] remove dead code --- ethcore/src/engines/validator_set/test.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ethcore/src/engines/validator_set/test.rs b/ethcore/src/engines/validator_set/test.rs index 3ddf1776d16..b82bcfe1240 100644 --- a/ethcore/src/engines/validator_set/test.rs +++ b/ethcore/src/engines/validator_set/test.rs @@ -57,12 +57,6 @@ impl TestSet { last_benign, } } - - /// Build a `TestSet` with the given number of benign misbehaviour reports - pub fn with_last_benign(mut self, last_benign: Arc) -> Self { - self.last_benign = last_benign; - self - } } impl ValidatorSet for TestSet {