Transaction Pool re-implementation (#6994)

* Initial design and some tests.

* Insertion & limits.

* Constructing pending block.

* Change to PendingIterator.

* Removing/cancelling transactions.

* Full status.

* Culling transactions.

* Use bigint.

* Add listener tests.

* Clean up listener types.

* Split into multiple files.

* Add copyright notice.

* Documentation.

* Don't require ownership.

* Fix cull to remove from by_hash.

* Make the queue generic over transactions.

* Address code review.

* Update wasm submodules.

* Fix review grumbles.

* Add some docs.

Cargo.toml

@@ -116,4 +116,13 @@ lto = false
 panic = "abort"
 
 [workspace]
-members = ["ethstore/cli", "ethkey/cli", "evmbin", "whisper", "chainspec", "dapps/js-glue", "ethcore/wasm/run"]
+members = [
+	"chainspec",
+	"dapps/js-glue",
+	"ethcore/wasm/run",
+	"ethkey/cli",
+	"ethstore/cli",
+	"evmbin",
+	"transaction-pool",
+	"whisper",
+]

transaction-pool/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+description = "Generic transaction pool."
+name = "transaction-pool"
+version = "1.9.0"
+license = "GPL-3.0"
+authors = ["Parity Technologies <[email protected]>"]
+
+[dependencies]
+error-chain = "0.11"
+log="0.3"
+smallvec = "0.4"
+ethcore-bigint = { features = ["heapsizeof"], version="0.2" }

transaction-pool/src/error.rs

@@ -0,0 +1,48 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use bigint::hash::H256;
+
+error_chain! {
+	errors {
+		AlreadyImported(hash: H256) {
+			description("transaction is already in the queue"),
+			display("[{:?}] transaction already imported", hash)
+		}
+		TooCheapToEnter(hash: H256) {
+			description("the pool is full and transaction is too cheap to replace any transaction"),
+			display("[{:?}] transaction too cheap to enter the pool", hash)
+		}
+		TooCheapToReplace(old_hash: H256, hash: H256) {
+			description("transaction is too cheap to replace existing transaction in the queue"),
+			display("[{:?}] transaction too cheap to replace: {:?}", hash, old_hash)
+		}
+	}
+}
+
+#[cfg(test)]
+impl PartialEq for ErrorKind {
+	fn eq(&self, other: &Self) -> bool {
+		use self::ErrorKind::*;
+
+		match (self, other) {
+			(&AlreadyImported(ref h1), &AlreadyImported(ref h2)) => h1 == h2,
+			(&TooCheapToEnter(ref h1), &TooCheapToEnter(ref h2)) => h1 == h2,
+			(&TooCheapToReplace(ref old1, ref new1), &TooCheapToReplace(ref old2, ref new2)) => old1 == old2 && new1 == new2,
+			_ => false,
+		}
+	}
+}

transaction-pool/src/lib.rs

@@ -0,0 +1,118 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Generic Transaction Pool
+//!
+//! An extensible and performant implementation of Ethereum Transaction Pool.
+//! The pool stores ordered, verified transactions according to some pluggable
+//! `Scoring` implementation.
+//! The pool also allows you to construct a set of `pending` transactions according
+//! to some notion of `Readiness` (pluggable).
+//!
+//! The pool is generic over transactions and should make no assumptions about them.
+//! The only thing we can rely on is the `Scoring` that defines:
+//!  - the ordering of transactions from a single sender
+//!  - the priority of the transaction compared to other transactions from different senders
+//!
+//! NOTE: the transactions from a single sender are not ordered by priority,
+//! but still when constructing pending set we always need to maintain the ordering
+//! (i.e. `txs[1]` always needs to be included after `txs[0]` even if it has higher priority)
+//!
+//! ### Design Details
+//!
+//! Performance assumptions:
+//! - Possibility to handle tens of thousands of transactions
+//! - Fast insertions and replacements `O(per-sender + log(senders))`
+//! - Reasonably fast removal of stalled transactions `O(per-sender)`
+//! - Reasonably fast construction of pending set `O(txs * (log(senders) + log(per-sender))`
+//!
+//! The removal performance could be improved by trading some memory. Currently `SmallVec` is used
+//! to store senders transactions, instead we could use `VecDeque` and efficiently `pop_front`
+//! the best transactions.
+//!
+//! The pending set construction and insertion complexity could be reduced by introducing
+//! a notion of `nonce` - an absolute, numeric ordering of transactions.
+//! We don't do that because of possible implications of EIP208 where nonce might not be
+//! explicitly available.
+//!
+//! 1. The pool groups transactions from particular sender together
+//!    and stores them ordered by `Scoring` within that group
+//!    i.e. `HashMap<Sender, Vec<Transaction>>`.
+//! 2. Additionaly we maintain the best and the worst transaction from each sender
+//!    (by `Scoring` not `priority`) ordered by `priority`.
+//!    It means that we can easily identify the best transaction inside the entire pool
+//!    and the worst transaction.
+//! 3. Whenever new transaction is inserted to the queue:
+//!    - first check all the limits (overall, memory, per-sender)
+//!    - retrieve all transactions from a sender
+//!    - binary search for position to insert the transaction
+//!    - decide if we are replacing existing transaction (3 outcomes: drop, replace, insert)
+//!    - update best and worst transaction from that sender if affected
+//! 4. Pending List construction:
+//!    - Take the best transaction (by priority) from all senders to the List
+//!    - Replace the transaction with next transaction (by ordering) from that sender (if any)
+//!    - Repeat
+
+#![warn(missing_docs)]
+
+extern crate smallvec;
+extern crate ethcore_bigint as bigint;
+
+#[macro_use]
+extern crate error_chain;
+#[macro_use]
+extern crate log;
+
+#[cfg(test)]
+mod tests;
+
+mod error;
+mod listener;
+mod options;
+mod pool;
+mod ready;
+mod status;
+mod transactions;
+mod verifier;
+
+pub mod scoring;
+
+pub use self::listener::{Listener, NoopListener};
+pub use self::options::Options;
+pub use self::pool::Pool;
+pub use self::ready::{Ready, Readiness};
+pub use self::scoring::Scoring;
+pub use self::status::{LightStatus, Status};
+pub use self::verifier::Verifier;
+
+use std::fmt;
+
+use self::bigint::prelude::{H256, H160 as Address};
+
+/// Already verified transaction that can be safely queued.
+pub trait VerifiedTransaction: fmt::Debug {
+	/// Transaction hash
+	fn hash(&self) -> &H256;
+
+	/// Memory usage
+	fn mem_usage(&self) -> usize;
+
+	/// Transaction sender
+	fn sender(&self) -> &Address;
+
+	/// Unique index of insertion (lower = older).
+	fn insertion_id(&self) -> u64;
+}

transaction-pool/src/listener.rs

@@ -0,0 +1,48 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+/// Transaction pool listener.
+///
+/// Listener is being notified about status of every transaction in the pool.
+pub trait Listener<T> {
+	/// The transaction has been successfuly added to the pool.
+	/// If second argument is `Some` the transaction has took place of some other transaction
+	/// which was already in pool.
+	/// NOTE: You won't be notified about drop of `old` transaction separately.
+	fn added(&mut self, _tx: &Arc<T>, _old: Option<&Arc<T>>) {}
+
+	/// The transaction was rejected from the pool.
+	/// It means that it was too cheap to replace any transaction already in the pool.
+	fn rejected(&mut self, _tx: T) {}
+
+	/// The transaction was dropped from the pool because of a limit.
+	fn dropped(&mut self, _tx: &Arc<T>) {}
+
+	/// The transaction was marked as invalid by executor.
+	fn invalid(&mut self, _tx: &Arc<T>) {}
+
+	/// The transaction has been cancelled.
+	fn cancelled(&mut self, _tx: &Arc<T>) {}
+
+	/// The transaction has been mined.
+	fn mined(&mut self, _tx: &Arc<T>) {}
+}
+
+/// A no-op implementation of `Listener`.
+pub struct NoopListener;
+impl<T> Listener<T> for NoopListener {}

transaction-pool/src/options.rs

@@ -0,0 +1,36 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+/// Transaction Pool options.
+#[derive(Debug)]
+pub struct Options {
+	/// Maximal number of transactions in the pool.
+	pub max_count: usize,
+	/// Maximal number of transactions from single sender.
+	pub max_per_sender: usize,
+	/// Maximal memory usage.
+	pub max_mem_usage: usize,
+}
+
+impl Default for Options {
+	fn default() -> Self {
+		Options {
+			max_count: 1024,
+			max_per_sender: 16,
+			max_mem_usage: 8 * 1024 * 1024,
+		}
+	}
+}