Remove calls to heapsize (#10432)

* update memorydb trait
* use malloc_size_of instead of heapsize_of
* use jemalloc as default allocator for parity client.

Cargo.toml

@@ -73,6 +73,8 @@ ethcore-secretstore = { path = "secret-store", optional = true }
 
 registrar = { path = "util/registrar" }
 
+parity-util-mem = { version = "0.1", features = ["jemalloc-global"] }
+
 [build-dependencies]
 rustc_version = "0.2"
 
@@ -139,6 +141,3 @@ members = [
 	"util/fastmap",
 	"util/time-utils"
 ]
-
-[patch.crates-io]
-heapsize = { git = "https://github.com/cheme/heapsize.git", branch = "ec-macfix" }

accounts/ethkey/Cargo.toml

@@ -11,7 +11,7 @@ eth-secp256k1 = { git = "https://github.com/paritytech/rust-secp256k1" }
 ethereum-types = "0.6.0"
 lazy_static = "1.0"
 log = "0.4"
-memzero = { path = "../../util/memzero" }
+parity-util-mem = "0.1"
 parity-wordlist = "1.2"
 quick-error = "1.2.2"
 rand = "0.6"

accounts/ethkey/src/lib.rs

@@ -20,7 +20,7 @@ extern crate byteorder;
 extern crate edit_distance;
 extern crate parity_crypto;
 extern crate ethereum_types;
-extern crate memzero;
+extern crate parity_util_mem;
 extern crate parity_wordlist;
 #[macro_use]
 extern crate quick_error;

accounts/ethkey/src/secret.rs

@@ -21,7 +21,7 @@ use rustc_hex::ToHex;
 use secp256k1::constants::{SECRET_KEY_SIZE as SECP256K1_SECRET_KEY_SIZE};
 use secp256k1::key;
 use ethereum_types::H256;
-use memzero::Memzero;
+use parity_util_mem::Memzero;
 use {Error, SECP256K1};
 
 #[derive(Clone, PartialEq, Eq)]

ethcore/Cargo.toml

@@ -31,8 +31,8 @@ ethjson = { path = "../json" }
 ethkey = { path = "../accounts/ethkey" }
 evm = { path = "evm" }
 futures = "0.1"
-hash-db = "0.11.0"
-heapsize = "0.4"
+hash-db = "0.12.4"
+parity-util-mem = "0.1"
 itertools = "0.5"
 journaldb = { path = "../util/journaldb" }
 keccak-hash = "0.2.0"
@@ -46,14 +46,14 @@ log = "0.4"
 lru-cache = "0.1"
 macros = { path = "../util/macros" }
 memory-cache = { path = "../util/memory-cache" }
-memory-db = "0.11.0"
+memory-db = "0.12.4"
 num = { version = "0.1", default-features = false, features = ["bigint"] }
 num_cpus = "1.2"
 parity-bytes = "0.1"
 parity-crypto = "0.4.0"
 parity-snappy = "0.1"
 parking_lot = "0.7"
-trie-db = "0.11.0"
+trie-db = "0.12.4"
 patricia-trie-ethereum = { path = "../util/patricia-trie-ethereum" }
 rand = "0.6"
 rayon = "1.0"
@@ -83,7 +83,7 @@ kvdb-rocksdb = "0.1.3"
 parity-runtime = { path = "../util/runtime" }
 rlp_compress = { path = "../util/rlp-compress" }
 tempdir = "0.3"
-trie-standardmap = "0.12.3"
+trie-standardmap = "0.12.4"
 
 [features]
 parity = ["work-notify", "price-info", "stratum"]

ethcore/blockchain/Cargo.toml

@@ -13,7 +13,7 @@ blooms-db = { path = "../../util/blooms-db" }
 common-types = { path = "../types" }
 ethcore-db = { path = "../db" }
 ethereum-types = "0.6.0"
-heapsize = "0.4"
+parity-util-mem = "0.1"
 itertools = "0.5"
 kvdb = "0.1"
 log = "0.4"

ethcore/blockchain/src/blockchain.rs

@@ -39,7 +39,7 @@ use ethcore_db::cache_manager::CacheManager;
 use ethcore_db::keys::{BlockReceipts, BlockDetails, TransactionAddress, EPOCH_KEY_PREFIX, EpochTransitions};
 use ethcore_db::{self as db, Writable, Readable, CacheUpdatePolicy};
 use ethereum_types::{H256, Bloom, BloomRef, U256};
-use heapsize::HeapSizeOf;
+use util_mem::{MallocSizeOf, allocators::new_malloc_size_ops};
 use itertools::Itertools;
 use kvdb::{DBTransaction, KeyValueDB};
 use log::{trace, warn, info};
@@ -1489,11 +1489,12 @@ impl BlockChain {
 
 	/// Get current cache size.
 	pub fn cache_size(&self) -> CacheSize {
+		let mut ops = new_malloc_size_ops();
 		CacheSize {
-			blocks: self.block_headers.read().heap_size_of_children() + self.block_bodies.read().heap_size_of_children(),
-			block_details: self.block_details.read().heap_size_of_children(),
-			transaction_addresses: self.transaction_addresses.read().heap_size_of_children(),
-			block_receipts: self.block_receipts.read().heap_size_of_children(),
+			blocks: self.block_headers.size_of(&mut ops) + self.block_bodies.size_of(&mut ops),
+			block_details: self.block_details.size_of(&mut ops),
+			transaction_addresses: self.transaction_addresses.size_of(&mut ops),
+			block_receipts: self.block_receipts.size_of(&mut ops),
 		}
 	}
 
@@ -1528,12 +1529,13 @@ impl BlockChain {
 			transaction_addresses.shrink_to_fit();
 			block_receipts.shrink_to_fit();
 
-			block_headers.heap_size_of_children() +
-			block_bodies.heap_size_of_children() +
-			block_details.heap_size_of_children() +
-			block_hashes.heap_size_of_children() +
-			transaction_addresses.heap_size_of_children() +
-			block_receipts.heap_size_of_children()
+			let mut ops = new_malloc_size_ops();
+			block_headers.size_of(&mut ops) +
+			block_bodies.size_of(&mut ops) +
+			block_details.size_of(&mut ops) +
+			block_hashes.size_of(&mut ops) +
+			transaction_addresses.size_of(&mut ops) +
+			block_receipts.size_of(&mut ops)
 		});
 	}
 

ethcore/blockchain/src/lib.rs

@@ -18,6 +18,9 @@
 
 #![warn(missing_docs)]
 
+extern crate parity_util_mem as util_mem;
+extern crate parity_util_mem as malloc_size_of;
+
 mod best_block;
 mod block_info;
 mod blockchain;

ethcore/db/Cargo.toml

@@ -10,8 +10,8 @@ edition = "2018"
 [dependencies]
 common-types = { path = "../types" }
 ethereum-types = "0.6.0"
-heapsize = "0.4"
 kvdb = "0.1"
+parity-util-mem = "0.1"
 parking_lot = "0.7"
 rlp = "0.4.0"
 rlp_derive = { path = "../../util/rlp-derive" }

ethcore/db/src/keys.rs

@@ -23,7 +23,7 @@ use common_types::BlockNumber;
 use common_types::engines::epoch::Transition as EpochTransition;
 use common_types::receipt::Receipt;
 use ethereum_types::{H256, H264, U256};
-use heapsize::HeapSizeOf;
+use parity_util_mem::MallocSizeOf;
 use kvdb::PREFIX_LEN as DB_PREFIX_LEN;
 use rlp;
 use rlp_derive::{RlpEncodableWrapper, RlpDecodableWrapper, RlpEncodable, RlpDecodable};
@@ -140,7 +140,7 @@ impl Key<EpochTransitions> for u64 {
 }
 
 /// Familial details concerning a block
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, MallocSizeOf)]
 pub struct BlockDetails {
 	/// Block number
 	pub number: BlockNumber,
@@ -195,27 +195,17 @@ impl rlp::Decodable for BlockDetails {
 	}
 }
 
-impl HeapSizeOf for BlockDetails {
-	fn heap_size_of_children(&self) -> usize {
-		self.children.heap_size_of_children()
-	}
-}
-
 /// Represents address of certain transaction within block
-#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)]
+#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable, MallocSizeOf)]
 pub struct TransactionAddress {
 	/// Block hash
 	pub block_hash: H256,
 	/// Transaction index within the block
 	pub index: usize
 }
 
-impl HeapSizeOf for TransactionAddress {
-	fn heap_size_of_children(&self) -> usize { 0 }
-}
-
 /// Contains all block receipts.
-#[derive(Clone, RlpEncodableWrapper, RlpDecodableWrapper)]
+#[derive(Clone, RlpEncodableWrapper, RlpDecodableWrapper, MallocSizeOf)]
 pub struct BlockReceipts {
 	/// Block receipts
 	pub receipts: Vec<Receipt>,
@@ -230,12 +220,6 @@ impl BlockReceipts {
 	}
 }
 
-impl HeapSizeOf for BlockReceipts {
-	fn heap_size_of_children(&self) -> usize {
-		self.receipts.heap_size_of_children()
-	}
-}
-
 /// Candidate transitions to an epoch with specific number.
 #[derive(Clone, RlpEncodable, RlpDecodable)]
 pub struct EpochTransitions {

ethcore/db/src/lib.rs

@@ -18,6 +18,9 @@
 
 #![warn(missing_docs)]
 
+extern crate parity_util_mem as mem;
+extern crate parity_util_mem as malloc_size_of;
+
 mod db;
 
 pub mod keys;

ethcore/evm/Cargo.toml

@@ -7,7 +7,7 @@ authors = ["Parity Technologies <[email protected]>"]
 bit-set = "0.4"
 parity-bytes = "0.1"
 ethereum-types = "0.6.0"
-heapsize = "0.4"
+parity-util-mem = "0.1"
 lazy_static = "1.0"
 log = "0.4"
 vm = { path = "../vm" }

ethcore/evm/benches/basic.rs

@@ -21,7 +21,7 @@ extern crate criterion;
 extern crate bit_set;
 extern crate ethereum_types;
 extern crate parking_lot;
-extern crate heapsize;
+extern crate parity_util_mem as mem;
 extern crate vm;
 extern crate evm;
 extern crate keccak_hash as hash;

ethcore/evm/src/interpreter/shared_cache.rs

@@ -16,7 +16,7 @@
 
 use std::sync::Arc;
 use hash::KECCAK_EMPTY;
-use heapsize::HeapSizeOf;
+use parity_util_mem::{MallocSizeOf, MallocSizeOfOps};
 use ethereum_types::H256;
 use parking_lot::Mutex;
 use memory_cache::MemoryLruCache;
@@ -25,11 +25,12 @@ use super::super::instructions::{self, Instruction};
 
 const DEFAULT_CACHE_SIZE: usize = 4 * 1024 * 1024;
 
-// stub for a HeapSizeOf implementation.
+/// Stub for a sharing `BitSet` data in cache (reference counted)
+/// and implementing MallocSizeOf on it.
 struct Bits(Arc<BitSet>);
 
-impl HeapSizeOf for Bits {
-	fn heap_size_of_children(&self) -> usize {
+impl MallocSizeOf for Bits {
+	fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize {
 		// dealing in bits here
 		self.0.capacity() * 8
 	}

ethcore/evm/src/lib.rs

@@ -19,7 +19,7 @@
 extern crate bit_set;
 extern crate ethereum_types;
 extern crate parking_lot;
-extern crate heapsize;
+extern crate parity_util_mem;
 extern crate vm;
 extern crate keccak_hash as hash;
 extern crate memory_cache;

ethcore/light/Cargo.toml

@@ -15,14 +15,14 @@ ethcore = { path = ".."}
 ethcore-db = { path = "../db" }
 ethcore-blockchain = { path = "../blockchain" }
 ethereum-types = "0.6.0"
-memory-db = "0.11.0"
-trie-db = "0.11.0"
+memory-db = "0.12.4"
+trie-db = "0.12.4"
 patricia-trie-ethereum = { path = "../../util/patricia-trie-ethereum" }
 ethcore-network = { path = "../../util/network" }
 ethcore-miner = { path = "../../miner" }
 ethcore-io = { path = "../../util/io" }
-hash-db = "0.11.0"
-heapsize = "0.4"
+hash-db = "0.12.4"
+parity-util-mem = "0.1"
 vm = { path = "../vm" }
 fastmap = { path = "../../util/fastmap" }
 failsafe = { version = "0.3.0", default-features = false, features = ["parking_lot_mutex"] }

ethcore/light/src/cache.rs

@@ -21,12 +21,12 @@
 //! vector of all gas prices from a recent range of blocks.
 
 use std::time::{Instant, Duration};
+use parity_util_mem::{MallocSizeOf, MallocSizeOfOps, MallocSizeOfExt};
 
 use common_types::encoded;
 use common_types::BlockNumber;
 use common_types::receipt::Receipt;
 use ethereum_types::{H256, U256};
-use heapsize::HeapSizeOf;
 use memory_cache::MemoryLruCache;
 use stats::Corpus;
 
@@ -157,18 +157,20 @@ impl Cache {
 
 	/// Get the memory used.
 	pub fn mem_used(&self) -> usize {
-		self.heap_size_of_children()
+		self.malloc_size_of()
 	}
 }
 
-impl HeapSizeOf for Cache {
-	fn heap_size_of_children(&self) -> usize {
+
+// This is fast method: it is possible to have a more exhaustive implementation
+impl MallocSizeOf for Cache {
+	fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize {
 		self.headers.current_size()
 			+ self.canon_hashes.current_size()
 			+ self.bodies.current_size()
 			+ self.receipts.current_size()
 			+ self.chain_score.current_size()
-			// TODO: + corpus
+			// `self.corpus` is skipped
 	}
 }
 

ethcore/light/src/cht.rs

@@ -95,7 +95,8 @@ pub struct BlockInfo {
 /// Build an in-memory CHT from a closure which provides necessary information
 /// about blocks. If the fetcher ever fails to provide the info, the CHT
 /// will not be generated.
-pub fn build<F>(cht_num: u64, mut fetcher: F) -> Option<CHT<MemoryDB<KeccakHasher, DBValue>>>
+pub fn build<F>(cht_num: u64, mut fetcher: F)
+	-> Option<CHT<MemoryDB<KeccakHasher, memory_db::HashKey<KeccakHasher>, DBValue>>>
 	where F: FnMut(BlockId) -> Option<BlockInfo>
 {
 	let mut db = new_memory_db();
@@ -154,7 +155,7 @@ pub fn compute_root<I>(cht_num: u64, iterable: I) -> Option<H256>
 pub fn check_proof(proof: &[Bytes], num: u64, root: H256) -> Option<(H256, U256)> {
 	let mut db = new_memory_db();
 
-	for node in proof { db.insert(&node[..]); }
+	for node in proof { db.insert(hash_db::EMPTY_PREFIX, &node[..]); }
 	let res = match TrieDB::new(&db, &root) {
 		Err(_) => return None,
 		Ok(trie) => trie.get_with(&key!(num), |val: &[u8]| {