new blooms database (#8712)

* new blooms database

* fixed conflict in Cargo.lock

* removed bloomchain

* cleanup in progress

* all tests passing in trace db with new blooms-db

* added trace_blooms to BlockChainDB interface, fixed db flushing

* BlockChainDB no longer exposes RwLock in the interface

* automatically flush blooms-db after every insert

* blooms-db uses io::BufReader to read files, wrap blooms-db into Mutex, cause fs::File is just a shared file handle

* fix json_tests

* blooms-db can filter multiple possibilities at the same time

* removed enum trace/db.rs CacheId

* lint fixes

* fixed tests

* kvdb-rocksdb uses fs-swap crate

* update Cargo.lock

* use fs::rename

* fixed failing test on linux

* fix tests

* use fs_swap

* fixed failing test on linux

* cleanup after swap

* fix tests

* fixed osx permissions

* simplify parity database opening functions

* added migration to blooms-db

* address @niklasad1 grumbles

* fix license and authors field of blooms-db Cargo.toml

* restore blooms-db after snapshot

Cargo.toml

@@ -7,6 +7,7 @@ license = "GPL-3.0"
 authors = ["Parity Technologies <[email protected]>"]
 
 [dependencies]
+blooms-db = { path = "util/blooms-db" }
 log = "0.3"
 env_logger = "0.4"
 rustc-hex = "1.0"

ethcore/Cargo.toml

@@ -8,7 +8,7 @@ authors = ["Parity Technologies <[email protected]>"]
 
 [dependencies]
 ansi_term = "0.10"
-bloomchain = { path = "../util/bloomchain" }
+blooms-db = { path = "../util/blooms-db" }
 bn = { git = "https://github.com/paritytech/bn", default-features = false }
 byteorder = "1.0"
 common-types = { path = "types" }
@@ -67,11 +67,11 @@ keccak-hash = { path = "../util/hash" }
 triehash = { path = "../util/triehash" }
 unexpected = { path = "../util/unexpected" }
 journaldb = { path = "../util/journaldb" }
+tempdir = "0.3"
+kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
 
 [dev-dependencies]
-tempdir = "0.3"
 trie-standardmap = { path = "../util/trie-standardmap" }
-kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
 
 [features]
 # Display EVM debug traces.

ethcore/light/src/client/service.rs

@@ -21,11 +21,10 @@ use std::fmt;
 use std::sync::Arc;
 
 use ethcore::client::ClientIoMessage;
-use ethcore::db;
+use ethcore::{db, BlockChainDB};
 use ethcore::error::Error as CoreError;
 use ethcore::spec::Spec;
 use io::{IoContext, IoError, IoHandler, IoService};
-use kvdb::KeyValueDB;
 
 use cache::Cache;
 use parking_lot::Mutex;
@@ -65,11 +64,10 @@ pub struct Service<T> {
 
 impl<T: ChainDataFetcher> Service<T> {
 	/// Start the service: initialize I/O workers and client itself.
-	pub fn start(config: ClientConfig, spec: &Spec, fetcher: T, db: Arc<KeyValueDB>, cache: Arc<Mutex<Cache>>) -> Result<Self, Error> {
-
+	pub fn start(config: ClientConfig, spec: &Spec, fetcher: T, db: Arc<BlockChainDB>, cache: Arc<Mutex<Cache>>) -> Result<Self, Error> {
 		let io_service = IoService::<ClientIoMessage>::start().map_err(Error::Io)?;
 		let client = Arc::new(Client::new(config,
-			db,
+			db.key_value().clone(),
 			db::COL_LIGHT_CHAIN,
 			spec,
 			fetcher,
@@ -122,12 +120,11 @@ mod tests {
 	use client::fetch;
 	use std::time::Duration;
 	use parking_lot::Mutex;
-	use kvdb_memorydb;
-	use ethcore::db::NUM_COLUMNS;
+	use ethcore::test_helpers;
 
 	#[test]
 	fn it_works() {
-		let db = Arc::new(kvdb_memorydb::create(NUM_COLUMNS.unwrap_or(0)));
+		let db = test_helpers::new_db();
 		let spec = Spec::new_test();
 		let cache = Arc::new(Mutex::new(Cache::new(Default::default(), Duration::from_secs(6 * 3600))));
 

ethcore/node_filter/src/lib.rs

@@ -115,6 +115,7 @@ mod test {
 	use ethcore::spec::Spec;
 	use ethcore::client::{BlockChainClient, Client, ClientConfig};
 	use ethcore::miner::Miner;
+	use ethcore::test_helpers;
 	use network::{ConnectionDirection, ConnectionFilter, NodeId};
 	use io::IoChannel;
 	use super::NodeFilter;
@@ -127,7 +128,7 @@ mod test {
 		let data = include_bytes!("../res/node_filter.json");
 		let tempdir = TempDir::new("").unwrap();
 		let spec = Spec::load(&tempdir.path(), &data[..]).unwrap();
-		let client_db = Arc::new(::kvdb_memorydb::create(::ethcore::db::NUM_COLUMNS.unwrap_or(0)));
+		let client_db = test_helpers::new_db();
 
 		let client = Client::new(
 			ClientConfig::default(),

ethcore/service/src/service.rs

@@ -22,10 +22,10 @@ use std::time::Duration;
 
 use ansi_term::Colour;
 use io::{IoContext, TimerToken, IoHandler, IoService, IoError};
-use kvdb::{KeyValueDB, KeyValueDBHandler};
 use stop_guard::StopGuard;
 
 use sync::PrivateTxHandler;
+use ethcore::{BlockChainDB, BlockChainDBHandler};
 use ethcore::client::{Client, ClientConfig, ChainNotify, ClientIoMessage};
 use ethcore::miner::Miner;
 use ethcore::snapshot::service::{Service as SnapshotService, ServiceParams as SnapServiceParams};
@@ -69,7 +69,7 @@ pub struct ClientService {
 	client: Arc<Client>,
 	snapshot: Arc<SnapshotService>,
 	private_tx: Arc<PrivateTxService>,
-	database: Arc<KeyValueDB>,
+	database: Arc<BlockChainDB>,
 	_stop_guard: StopGuard,
 }
 
@@ -78,9 +78,9 @@ impl ClientService {
 	pub fn start(
 		config: ClientConfig,
 		spec: &Spec,
-		client_db: Arc<KeyValueDB>,
+		blockchain_db: Arc<BlockChainDB>,
 		snapshot_path: &Path,
-		restoration_db_handler: Box<KeyValueDBHandler>,
+		restoration_db_handler: Box<BlockChainDBHandler>,
 		_ipc_path: &Path,
 		miner: Arc<Miner>,
 		account_provider: Arc<AccountProvider>,
@@ -93,7 +93,7 @@ impl ClientService {
 		info!("Configured for {} using {} engine", Colour::White.bold().paint(spec.name.clone()), Colour::Yellow.bold().paint(spec.engine.name()));
 
 		let pruning = config.pruning;
-		let client = Client::new(config, &spec, client_db.clone(), miner.clone(), io_service.channel())?;
+		let client = Client::new(config, &spec, blockchain_db.clone(), miner.clone(), io_service.channel())?;
 
 		let snapshot_params = SnapServiceParams {
 			engine: spec.engine.clone(),
@@ -131,7 +131,7 @@ impl ClientService {
 			client: client,
 			snapshot: snapshot,
 			private_tx,
-			database: client_db,
+			database: blockchain_db,
 			_stop_guard: stop_guard,
 		})
 	}
@@ -167,7 +167,7 @@ impl ClientService {
 	}
 
 	/// Get a handle to the database.
-	pub fn db(&self) -> Arc<KeyValueDB> { self.database.clone() }
+	pub fn db(&self) -> Arc<BlockChainDB> { self.database.clone() }
 
 	/// Shutdown the Client Service
 	pub fn shutdown(&self) {
@@ -259,8 +259,8 @@ mod tests {
 	use ethcore::miner::Miner;
 	use ethcore::spec::Spec;
 	use ethcore::db::NUM_COLUMNS;
-	use kvdb::Error;
-	use kvdb_rocksdb::{Database, DatabaseConfig, CompactionProfile};
+	use ethcore::test_helpers;
+	use kvdb_rocksdb::{DatabaseConfig, CompactionProfile};
 	use super::*;
 
 	use ethcore_private_tx;
@@ -278,24 +278,9 @@ mod tests {
 		client_db_config.compaction = CompactionProfile::auto(&client_path);
 		client_db_config.wal = client_config.db_wal;
 
-		let client_db = Arc::new(Database::open(
-			&client_db_config,
-			&client_path.to_str().expect("DB path could not be converted to string.")
-		).unwrap());
-
-		struct RestorationDBHandler {
-			config: DatabaseConfig,
-		}
-
-		impl KeyValueDBHandler for RestorationDBHandler {
-			fn open(&self, db_path: &Path) -> Result<Arc<KeyValueDB>, Error> {
-				Ok(Arc::new(Database::open(&self.config, &db_path.to_string_lossy())?))
-			}
-		}
-
-		let restoration_db_handler = Box::new(RestorationDBHandler {
-			config: client_db_config,
-		});
+		let client_db_handler = test_helpers::restoration_db_handler(client_db_config.clone());
+		let client_db = client_db_handler.open(&client_path).unwrap();
+		let restoration_db_handler = test_helpers::restoration_db_handler(client_db_config);
 
 		let spec = Spec::new_test();
 		let service = ClientService::start(