From 4c5753e903060ceaf1121633951a82f9739e2027 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 21 Jun 2018 18:08:04 +0800 Subject: [PATCH 01/12] Make it possible to expose jsontests using feature flag --- ethcore/Cargo.toml | 4 ++-- ethcore/src/client/evm_test_client.rs | 23 +++++++---------------- ethcore/src/json_tests/chain.rs | 14 +++++++++++++- ethcore/src/json_tests/executive.rs | 11 +++++++++++ ethcore/src/json_tests/mod.rs | 17 +++++++++++++++++ ethcore/src/json_tests/state.rs | 14 +++++++++++++- ethcore/src/json_tests/test_common.rs | 5 +++++ ethcore/src/json_tests/transaction.rs | 11 +++++++++++ ethcore/src/json_tests/trie.rs | 27 +++++++++++++++++++++++++++ ethcore/src/lib.rs | 5 ++--- 10 files changed, 108 insertions(+), 23 deletions(-) diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 2b6ab9ebc3a..f862325e8d7 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -67,9 +67,9 @@ keccak-hash = { path = "../util/hash" } triehash = { path = "../util/triehash" } unexpected = { path = "../util/unexpected" } journaldb = { path = "../util/journaldb" } +tempdir = { version = "0.3", optional = true } [dev-dependencies] -tempdir = "0.3" trie-standardmap = { path = "../util/trie-standardmap" } kvdb-rocksdb = { path = "../util/kvdb-rocksdb" } @@ -84,7 +84,7 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"] # EVM debug traces are printed. slow-blocks = [] # Run JSON consensus tests. -json-tests = ["ethcore-transaction/json-tests"] +json-tests = ["ethcore-transaction/json-tests", "test-helpers", "tempdir"] # Run memory/cpu heavy tests. test-heavy = [] # Compile benches diff --git a/ethcore/src/client/evm_test_client.rs b/ethcore/src/client/evm_test_client.rs index fbf57cbdc72..2e5e912cc03 100644 --- a/ethcore/src/client/evm_test_client.rs +++ b/ethcore/src/client/evm_test_client.rs @@ -64,15 +64,6 @@ impl fmt::Display for EvmTestError { use ethereum; use ethjson::state::test::ForkSpec; -lazy_static! { - pub static ref FRONTIER: spec::Spec = ethereum::new_frontier_test(); - pub static ref HOMESTEAD: spec::Spec = ethereum::new_homestead_test(); - pub static ref EIP150: spec::Spec = ethereum::new_eip150_test(); - pub static ref EIP161: spec::Spec = ethereum::new_eip161_test(); - pub static ref BYZANTIUM: spec::Spec = ethereum::new_byzantium_test(); - pub static ref BYZANTIUM_TRANSITION: spec::Spec = ethereum::new_transition_test(); -} - /// Simplified, single-block EVM test client. pub struct EvmTestClient<'a> { state: state::State, @@ -90,14 +81,14 @@ impl<'a> fmt::Debug for EvmTestClient<'a> { impl<'a> EvmTestClient<'a> { /// Converts a json spec definition into spec. - pub fn spec_from_json(spec: &ForkSpec) -> Option<&'static spec::Spec> { + pub fn spec_from_json(spec: &ForkSpec) -> Option { match *spec { - ForkSpec::Frontier => Some(&*FRONTIER), - ForkSpec::Homestead => Some(&*HOMESTEAD), - ForkSpec::EIP150 => Some(&*EIP150), - ForkSpec::EIP158 => Some(&*EIP161), - ForkSpec::Byzantium => Some(&*BYZANTIUM), - ForkSpec::EIP158ToByzantiumAt5 => Some(&BYZANTIUM_TRANSITION), + ForkSpec::Frontier => Some(ethereum::new_frontier_test()), + ForkSpec::Homestead => Some(ethereum::new_homestead_test()), + ForkSpec::EIP150 => Some(ethereum::new_eip150_test()), + ForkSpec::EIP158 => Some(ethereum::new_eip161_test()), + ForkSpec::Byzantium => Some(ethereum::new_byzantium_test()), + ForkSpec::EIP158ToByzantiumAt5 => Some(ethereum::new_transition_test()), ForkSpec::FrontierToHomesteadAt5 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => None, _ => None, } diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 814538cdbd3..64d149f0728 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use std::sync::Arc; use client::{EvmTestClient, Client, ClientConfig, ChainInfo, ImportBlock}; use block::Block; @@ -22,6 +23,16 @@ use ethjson; use miner::Miner; use io::IoChannel; +/// Run chain jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str]) { + ::json_tests::test_common::run_test_path(p, skip, json_chain_test) +} + +/// Run chain jsontests on a given file. +pub fn run_test_file(p: &Path) { + ::json_tests::test_common::run_test_file(p, json_chain_test) +} + pub fn json_chain_test(json_data: &[u8]) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::blockchain::Test::load(json_data).unwrap(); @@ -41,7 +52,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { let spec = { let mut spec = match EvmTestClient::spec_from_json(&blockchain.network) { - Some(spec) => (*spec).clone(), + Some(spec) => spec, None => { println!(" - {} | {:?} Ignoring tests because of missing spec", name, blockchain.network); continue; @@ -87,6 +98,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { failed } +#[cfg(test)] mod block_tests { use super::json_chain_test; diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 5bda6c55a39..e5eed1e8e8b 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use std::sync::Arc; use super::test_common::*; use state::{Backend as StateBackend, State, Substate}; @@ -35,6 +36,16 @@ use rlp::RlpStream; use hash::keccak; use machine::EthereumMachine as Machine; +/// Run executive jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str]) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test) +} + +/// Run executive jsontests on a given file. +pub fn run_test_file(p: &Path) { + ::json_tests::test_common::run_test_file(p, do_json_test) +} + #[derive(Debug, PartialEq, Clone)] struct CallCreate { data: Bytes, diff --git a/ethcore/src/json_tests/mod.rs b/ethcore/src/json_tests/mod.rs index 65cc6d21340..b12189d0a22 100644 --- a/ethcore/src/json_tests/mod.rs +++ b/ethcore/src/json_tests/mod.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +//! Helpers and tests for operating on jsontests. + #[macro_use] mod test_common; @@ -22,4 +24,19 @@ mod executive; mod state; mod chain; mod trie; + +#[cfg(test)] mod difficulty; + +pub use self::transaction::run_test_path as run_transaction_test_path; +pub use self::transaction::run_test_file as run_transaction_test_file; +pub use self::executive::run_test_path as run_executive_test_path; +pub use self::executive::run_test_file as run_executive_test_file; +pub use self::state::run_test_path as run_state_test_path; +pub use self::state::run_test_file as run_state_test_file; +pub use self::chain::run_test_path as run_chain_test_path; +pub use self::chain::run_test_file as run_chain_test_file; +pub use self::trie::run_generic_test_path as run_generic_trie_test_path; +pub use self::trie::run_generic_test_file as run_generic_trie_test_file; +pub use self::trie::run_secure_test_path as run_secure_trie_test_path; +pub use self::trie::run_secure_test_file as run_secure_trie_test_file; diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 45ec6f3fb0c..d101e7c88d3 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use super::test_common::*; use pod_state::PodState; use trace; @@ -22,6 +23,16 @@ use ethjson; use transaction::SignedTransaction; use vm::EnvInfo; +/// Run state jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str]) { + ::json_tests::test_common::run_test_path(p, skip, json_chain_test) +} + +/// Run state jsontests on a given file. +pub fn run_test_file(p: &Path) { + ::json_tests::test_common::run_test_file(p, json_chain_test) +} + pub fn json_chain_test(json_data: &[u8]) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::state::test::Test::load(json_data).unwrap(); @@ -50,7 +61,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { let transaction: SignedTransaction = multitransaction.select(&state.indexes).into(); let result = || -> Result<_, EvmTestError> { - Ok(EvmTestClient::from_pod_state(spec, pre.clone())? + Ok(EvmTestClient::from_pod_state(&spec, pre.clone())? .transact(&env, transaction, trace::NoopTracer, trace::NoopVMTracer)) }; match result() { @@ -89,6 +100,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { failed } +#[cfg(test)] mod state_tests { use super::json_chain_test; diff --git a/ethcore/src/json_tests/test_common.rs b/ethcore/src/json_tests/test_common.rs index 6ce38b27a0c..08bd0e04c6a 100644 --- a/ethcore/src/json_tests/test_common.rs +++ b/ethcore/src/json_tests/test_common.rs @@ -54,6 +54,7 @@ pub fn run_test_file(path: &Path, runner: fn (json_data: &[u8]) -> Vec) assert_eq!(results, empty); } +#[cfg(test)] macro_rules! test { ($name: expr, $skip: expr) => { ::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test); @@ -63,6 +64,7 @@ macro_rules! test { #[macro_export] macro_rules! declare_test { (skip => $arr: expr, $id: ident, $name: expr) => { + #[cfg(test)] #[test] #[allow(non_snake_case)] fn $id() { @@ -70,6 +72,7 @@ macro_rules! declare_test { } }; (ignore => $id: ident, $name: expr) => { + #[cfg(test)] #[ignore] #[test] #[allow(non_snake_case)] @@ -78,6 +81,7 @@ macro_rules! declare_test { } }; (heavy => $id: ident, $name: expr) => { + #[cfg(test)] #[cfg(feature = "test-heavy")] #[test] #[allow(non_snake_case)] @@ -86,6 +90,7 @@ macro_rules! declare_test { } }; ($id: ident, $name: expr) => { + #[cfg(test)] #[test] #[allow(non_snake_case)] fn $id() { diff --git a/ethcore/src/json_tests/transaction.rs b/ethcore/src/json_tests/transaction.rs index 295093305d7..ccac13457e5 100644 --- a/ethcore/src/json_tests/transaction.rs +++ b/ethcore/src/json_tests/transaction.rs @@ -14,12 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . +use std::path::Path; use super::test_common::*; use evm; use ethjson; use rlp::Rlp; use transaction::{Action, UnverifiedTransaction, SignedTransaction}; +/// Run transaction jsontests on a given folder. +pub fn run_test_path(p: &Path, skip: &[&'static str]) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test) +} + +/// Run transaction jsontests on a given file. +pub fn run_test_file(p: &Path) { + ::json_tests::test_common::run_test_file(p, do_json_test) +} + fn do_json_test(json_data: &[u8]) -> Vec { let tests = ethjson::transaction::Test::load(json_data).unwrap(); let mut failed = Vec::new(); diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index fae7cc7380a..91eb46cbd59 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -19,6 +19,11 @@ use trie::{TrieFactory, TrieSpec}; use ethereum_types::H256; use memorydb::MemoryDB; +pub use self::generic::run_test_path as run_generic_test_path; +pub use self::generic::run_test_file as run_generic_test_file; +pub use self::secure::run_test_path as run_secure_test_path; +pub use self::secure::run_test_file as run_secure_test_file; + fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { let tests = ethjson::trie::Test::load(json).unwrap(); let factory = TrieFactory::new(trie); @@ -49,8 +54,19 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { } mod generic { + use std::path::Path; use trie::TrieSpec; + /// Run generic trie jsontests on a given folder. + pub fn run_test_path(p: &Path, skip: &[&'static str]) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test) + } + + /// Run generic trie jsontests on a given file. + pub fn run_test_file(p: &Path) { + ::json_tests::test_common::run_test_file(p, do_json_test) + } + fn do_json_test(json: &[u8]) -> Vec { super::test_trie(json, TrieSpec::Generic) } @@ -60,8 +76,19 @@ mod generic { } mod secure { + use std::path::Path; use trie::TrieSpec; + /// Run secure trie jsontests on a given folder. + pub fn run_test_path(p: &Path, skip: &[&'static str]) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test) + } + + /// Run secure trie jsontests on a given file. + pub fn run_test_file(p: &Path) { + ::json_tests::test_common::run_test_file(p, do_json_test) + } + fn do_json_test(json: &[u8]) -> Vec { super::test_trie(json, TrieSpec::Secure) } diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index 51e75d4b40a..ddf40f316d8 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -108,7 +108,7 @@ extern crate vm; extern crate wasm; extern crate memory_cache; extern crate journaldb; -#[cfg(test)] +#[cfg(any(test, feature = "json-tests"))] extern crate tempdir; #[macro_use] @@ -172,9 +172,8 @@ mod tx_filter; #[cfg(test)] mod tests; -#[cfg(test)] #[cfg(feature = "json-tests")] -mod json_tests; +pub mod json_tests; #[cfg(any(test, feature = "test-helpers"))] pub mod test_helpers; #[cfg(test)] From cd0c8570deac79c2a61eb16c963dda3eaed15636 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 21 Jun 2018 18:46:04 +0800 Subject: [PATCH 02/12] Add start_stop_hook for jsontests --- ethcore/src/json_tests/chain.rs | 21 ++++++++++------ ethcore/src/json_tests/difficulty.rs | 18 ++++++++++---- ethcore/src/json_tests/executive.rs | 20 +++++++++------ ethcore/src/json_tests/mod.rs | 2 ++ ethcore/src/json_tests/state.rs | 20 +++++++++------ ethcore/src/json_tests/test_common.rs | 29 ++++++++++++++++----- ethcore/src/json_tests/transaction.rs | 14 +++++++---- ethcore/src/json_tests/trie.rs | 36 +++++++++++++++++---------- 8 files changed, 110 insertions(+), 50 deletions(-) diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 64d149f0728..bad64e5f12e 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -23,22 +23,26 @@ use ethjson; use miner::Miner; use io::IoChannel; +use super::HookType; + /// Run chain jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str]) { - ::json_tests::test_common::run_test_path(p, skip, json_chain_test) +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, json_chain_test, h) } /// Run chain jsontests on a given file. -pub fn run_test_file(p: &Path) { - ::json_tests::test_common::run_test_file(p, json_chain_test) +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, json_chain_test, h) } -pub fn json_chain_test(json_data: &[u8]) -> Vec { +pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::blockchain::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, blockchain) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + let mut fail = false; { let mut fail_unless = |cond: bool| if !cond && !fail { @@ -92,6 +96,8 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { if !fail { flushln!("ok"); } + + start_stop_hook(&name, HookType::OnStop); } println!("!!! {:?} tests from failed.", failed.len()); @@ -101,9 +107,10 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { #[cfg(test)] mod block_tests { use super::json_chain_test; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { - json_chain_test(json_data) + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { + json_chain_test(json_data, h) } declare_test!{BlockchainTests_bcBlockGasLimitTest, "BlockchainTests/bcBlockGasLimitTest"} diff --git a/ethcore/src/json_tests/difficulty.rs b/ethcore/src/json_tests/difficulty.rs index d111f0890db..2647e16595f 100644 --- a/ethcore/src/json_tests/difficulty.rs +++ b/ethcore/src/json_tests/difficulty.rs @@ -19,12 +19,16 @@ use header::Header; use ethereum_types::U256; use spec::Spec; -pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec { +use super::HookType; + +pub fn json_difficulty_test(json_data: &[u8], spec: Spec, start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::test::DifficultyTest::load(json_data).unwrap(); let engine = &spec.engine; for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + flush!(" - {}...", name); println!(" - {}...", name); @@ -42,15 +46,18 @@ pub fn json_difficulty_test(json_data: &[u8], spec: Spec) -> Vec { let expected_difficulty: U256 = test.current_difficulty.into(); assert_eq!(header.difficulty(), &expected_difficulty); flushln!("ok"); + + start_stop_hook(&name, HookType::OnStop); } vec![] } mod difficulty_test_byzantium { use super::json_difficulty_test; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { - json_difficulty_test(json_data, ::ethereum::new_byzantium_test()) + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { + json_difficulty_test(json_data, ::ethereum::new_byzantium_test(), h) } declare_test!{DifficultyTests_difficultyByzantium, "BasicTests/difficultyByzantium.json"} @@ -59,10 +66,11 @@ mod difficulty_test_byzantium { mod difficulty_test_foundation { use super::json_difficulty_test; use tempdir::TempDir; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { let tempdir = TempDir::new("").unwrap(); - json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path())) + json_difficulty_test(json_data, ::ethereum::new_foundation(&tempdir.path()), h) } declare_test!{DifficultyTests_difficultyMainNetwork, "BasicTests/difficultyMainNetwork.json"} diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index e5eed1e8e8b..44c1e8950b4 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -36,14 +36,16 @@ use rlp::RlpStream; use hash::keccak; use machine::EthereumMachine as Machine; +use super::HookType; + /// Run executive jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str]) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test) +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) } /// Run executive jsontests on a given file. -pub fn run_test_file(p: &Path) { - ::json_tests::test_common::run_test_file(p, do_json_test) +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) } #[derive(Debug, PartialEq, Clone)] @@ -204,19 +206,21 @@ impl<'a, T: 'a, V: 'a, B: 'a> Ext for TestExt<'a, T, V, B> } } -fn do_json_test(json_data: &[u8]) -> Vec { +fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { let vms = VMType::all(); vms .iter() - .flat_map(|vm| do_json_test_for(vm, json_data)) + .flat_map(|vm| do_json_test_for(vm, json_data, h)) .collect() } -fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec { +fn do_json_test_for(vm_type: &VMType, json_data: &[u8], start_stop_hook: &mut H) -> Vec { let tests = ethjson::vm::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, vm) in tests.into_iter() { + start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStart); + println!("name: {:?}", name); let mut fail = false; @@ -316,6 +320,8 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8]) -> Vec { fail_unless(Some(callcreates) == calls, "callcreates does not match"); } }; + + start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStop); } for f in &failed { diff --git a/ethcore/src/json_tests/mod.rs b/ethcore/src/json_tests/mod.rs index b12189d0a22..fa1c822ce2a 100644 --- a/ethcore/src/json_tests/mod.rs +++ b/ethcore/src/json_tests/mod.rs @@ -28,6 +28,8 @@ mod trie; #[cfg(test)] mod difficulty; +pub use self::test_common::HookType; + pub use self::transaction::run_test_path as run_transaction_test_path; pub use self::transaction::run_test_file as run_transaction_test_file; pub use self::executive::run_test_path as run_executive_test_path; diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index d101e7c88d3..da171697216 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -23,22 +23,26 @@ use ethjson; use transaction::SignedTransaction; use vm::EnvInfo; +use super::HookType; + /// Run state jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str]) { - ::json_tests::test_common::run_test_path(p, skip, json_chain_test) +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, json_chain_test, h) } /// Run state jsontests on a given file. -pub fn run_test_file(p: &Path) { - ::json_tests::test_common::run_test_file(p, json_chain_test) +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, json_chain_test, h) } -pub fn json_chain_test(json_data: &[u8]) -> Vec { +pub fn json_chain_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { ::ethcore_logger::init_log(); let tests = ethjson::state::test::Test::load(json_data).unwrap(); let mut failed = Vec::new(); for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + { let multitransaction = test.transaction; let env: EnvInfo = test.env.into(); @@ -92,6 +96,7 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { } } + start_stop_hook(&name, HookType::OnStop); } if !failed.is_empty() { @@ -103,9 +108,10 @@ pub fn json_chain_test(json_data: &[u8]) -> Vec { #[cfg(test)] mod state_tests { use super::json_chain_test; + use json_tests::HookType; - fn do_json_test(json_data: &[u8]) -> Vec { - json_chain_test(json_data) + fn do_json_test(json_data: &[u8], h: &mut H) -> Vec { + json_chain_test(json_data, h) } declare_test!{GeneralStateTest_stAttackTest, "GeneralStateTests/stAttackTest/"} diff --git a/ethcore/src/json_tests/test_common.rs b/ethcore/src/json_tests/test_common.rs index 08bd0e04c6a..bdb76a48522 100644 --- a/ethcore/src/json_tests/test_common.rs +++ b/ethcore/src/json_tests/test_common.rs @@ -21,7 +21,20 @@ use std::path::Path; use std::ffi::OsString; pub use ethereum_types::{H256, U256, Address}; -pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u8]) -> Vec) { +/// Indicate the type of the hook passed to test functions. +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum HookType { + /// Hook passed on starting of a test. + OnStart, + /// Hook passed on a test is finished. + OnStop +} + +pub fn run_test_path( + p: &Path, skip: &[&'static str], + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H +) { let path = Path::new(p); let s: HashSet = skip.iter().map(|s| { let mut os: OsString = s.into(); @@ -36,20 +49,24 @@ pub fn run_test_path(p: &Path, skip: &[&'static str], runner: fn (json_data: &[u } else { Some(e.path()) }}) { - run_test_path(&p, skip, runner) + run_test_path(&p, skip, runner, start_stop_hook) } } else { let mut path = p.to_path_buf(); path.set_extension("json"); - run_test_file(&path, runner) + run_test_file(&path, runner, start_stop_hook) } } -pub fn run_test_file(path: &Path, runner: fn (json_data: &[u8]) -> Vec) { +pub fn run_test_file( + path: &Path, + runner: fn(json_data: &[u8], start_stop_hook: &mut H) -> Vec, + start_stop_hook: &mut H +) { let mut data = Vec::new(); let mut file = File::open(&path).expect("Error opening test file"); file.read_to_end(&mut data).expect("Error reading test file"); - let results = runner(&data); + let results = runner(&data, start_stop_hook); let empty: [String; 0] = []; assert_eq!(results, empty); } @@ -57,7 +74,7 @@ pub fn run_test_file(path: &Path, runner: fn (json_data: &[u8]) -> Vec) #[cfg(test)] macro_rules! test { ($name: expr, $skip: expr) => { - ::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test); + ::json_tests::test_common::run_test_path(::std::path::Path::new(concat!("res/ethereum/tests/", $name)), &$skip, do_json_test, &mut |_, _| ()); } } diff --git a/ethcore/src/json_tests/transaction.rs b/ethcore/src/json_tests/transaction.rs index ccac13457e5..af16481f41f 100644 --- a/ethcore/src/json_tests/transaction.rs +++ b/ethcore/src/json_tests/transaction.rs @@ -22,22 +22,24 @@ use rlp::Rlp; use transaction::{Action, UnverifiedTransaction, SignedTransaction}; /// Run transaction jsontests on a given folder. -pub fn run_test_path(p: &Path, skip: &[&'static str]) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test) +pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) } /// Run transaction jsontests on a given file. -pub fn run_test_file(p: &Path) { - ::json_tests::test_common::run_test_file(p, do_json_test) +pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) } -fn do_json_test(json_data: &[u8]) -> Vec { +fn do_json_test(json_data: &[u8], start_stop_hook: &mut H) -> Vec { let tests = ethjson::transaction::Test::load(json_data).unwrap(); let mut failed = Vec::new(); let frontier_schedule = evm::Schedule::new_frontier(); let homestead_schedule = evm::Schedule::new_homestead(); let byzantium_schedule = evm::Schedule::new_byzantium(); for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + let mut fail_unless = |cond: bool, title: &str| if !cond { failed.push(name.clone()); println!("Transaction failed: {:?}: {:?}", name, title); }; let number: Option = test.block_number.map(Into::into); @@ -80,6 +82,8 @@ fn do_json_test(json_data: &[u8]) -> Vec { Action::Create => fail_unless(None == to, "create mismatch"), } } + + start_stop_hook(&name, HookType::OnStop); } for f in &failed { diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index 91eb46cbd59..81bc5bcc012 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -19,17 +19,21 @@ use trie::{TrieFactory, TrieSpec}; use ethereum_types::H256; use memorydb::MemoryDB; +use super::HookType; + pub use self::generic::run_test_path as run_generic_test_path; pub use self::generic::run_test_file as run_generic_test_file; pub use self::secure::run_test_path as run_secure_test_path; pub use self::secure::run_test_file as run_secure_test_file; -fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { +fn test_trie(json: &[u8], trie: TrieSpec, start_stop_hook: &mut H) -> Vec { let tests = ethjson::trie::Test::load(json).unwrap(); let factory = TrieFactory::new(trie); let mut result = vec![]; for (name, test) in tests.into_iter() { + start_stop_hook(&name, HookType::OnStart); + let mut memdb = MemoryDB::new(); let mut root = H256::default(); let mut t = factory.create(&mut memdb, &mut root); @@ -44,6 +48,8 @@ fn test_trie(json: &[u8], trie: TrieSpec) -> Vec { if *t.root() != test.root.into() { result.push(format!("Trie test '{:?}' failed.", name)); } + + start_stop_hook(&name, HookType::OnStop); } for i in &result { @@ -57,18 +63,20 @@ mod generic { use std::path::Path; use trie::TrieSpec; + use super::HookType; + /// Run generic trie jsontests on a given folder. - pub fn run_test_path(p: &Path, skip: &[&'static str]) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test) + pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) } /// Run generic trie jsontests on a given file. - pub fn run_test_file(p: &Path) { - ::json_tests::test_common::run_test_file(p, do_json_test) + pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) } - fn do_json_test(json: &[u8]) -> Vec { - super::test_trie(json, TrieSpec::Generic) + fn do_json_test(json: &[u8], h: &mut H) -> Vec { + super::test_trie(json, TrieSpec::Generic, h) } declare_test!{TrieTests_trietest, "TrieTests/trietest"} @@ -79,18 +87,20 @@ mod secure { use std::path::Path; use trie::TrieSpec; + use super::HookType; + /// Run secure trie jsontests on a given folder. - pub fn run_test_path(p: &Path, skip: &[&'static str]) { - ::json_tests::test_common::run_test_path(p, skip, do_json_test) + pub fn run_test_path(p: &Path, skip: &[&'static str], h: &mut H) { + ::json_tests::test_common::run_test_path(p, skip, do_json_test, h) } /// Run secure trie jsontests on a given file. - pub fn run_test_file(p: &Path) { - ::json_tests::test_common::run_test_file(p, do_json_test) + pub fn run_test_file(p: &Path, h: &mut H) { + ::json_tests::test_common::run_test_file(p, do_json_test, h) } - fn do_json_test(json: &[u8]) -> Vec { - super::test_trie(json, TrieSpec::Secure) + fn do_json_test(json: &[u8], h: &mut H) -> Vec { + super::test_trie(json, TrieSpec::Secure, h) } declare_test!{TrieTests_hex_encoded_secure, "TrieTests/hex_encoded_securetrie_test"} From 230708dab04acc6502cb4df2288ed7e98c51b148 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 21 Jun 2018 18:52:46 +0800 Subject: [PATCH 03/12] Fix evmbin compile --- evmbin/src/info.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evmbin/src/info.rs b/evmbin/src/info.rs index d1cd3cf6fc9..ce824fe17d3 100644 --- a/evmbin/src/info.rs +++ b/evmbin/src/info.rs @@ -105,7 +105,7 @@ pub fn run_transaction( informant.set_gas(env_info.gas_limit); - let result = run(spec, env_info.gas_limit, pre_state, |mut client| { + let result = run(&spec, env_info.gas_limit, pre_state, |mut client| { let result = client.transact(env_info, transaction, trace::NoopTracer, informant); match result { TransactResult::Ok { state_root, .. } if state_root != post_root => { From f2f8c00fccab1e4121aaf0aecc548e8301fe5bbb Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 21 Jun 2018 19:29:07 +0800 Subject: [PATCH 04/12] Implement vm jsontests times to tsv result --- ethcore/src/json_tests/executive.rs | 4 +-- evmbin/Cargo.toml | 2 +- evmbin/src/main.rs | 44 ++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 44c1e8950b4..86c6d6efcd2 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -221,7 +221,7 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8] for (name, vm) in tests.into_iter() { start_stop_hook(&format!("{}-{}", name, vm_type), HookType::OnStart); - println!("name: {:?}", name); + info!(target: "jsontests", "name: {:?}", name); let mut fail = false; let mut fail_unless = |cond: bool, s: &str | if !cond && !fail { @@ -325,7 +325,7 @@ fn do_json_test_for(vm_type: &VMType, json_data: &[u8] } for f in &failed { - println!("FAILED: {:?}", f); + error!("FAILED: {:?}", f); } failed diff --git a/evmbin/Cargo.toml b/evmbin/Cargo.toml index 51865091a90..9ff27a2e6b6 100644 --- a/evmbin/Cargo.toml +++ b/evmbin/Cargo.toml @@ -10,7 +10,7 @@ path = "./src/main.rs" [dependencies] docopt = "0.8" -ethcore = { path = "../ethcore", features = ["test-helpers"] } +ethcore = { path = "../ethcore", features = ["test-helpers", "json-tests"] } ethjson = { path = "../json" } ethcore-bytes = { path = "../util/bytes" } ethcore-transaction = { path = "../ethcore/transaction" } diff --git a/evmbin/src/main.rs b/evmbin/src/main.rs index 4620143f767..64531fe9175 100644 --- a/evmbin/src/main.rs +++ b/evmbin/src/main.rs @@ -46,7 +46,7 @@ use docopt::Docopt; use rustc_hex::FromHex; use ethereum_types::{U256, Address}; use bytes::Bytes; -use ethcore::spec; +use ethcore::{spec, json_tests}; use vm::{ActionParams, CallType}; mod info; @@ -61,6 +61,7 @@ EVM implementation for Parity. Usage: parity-evm state-test [--json --std-json --only NAME --chain CHAIN] parity-evm stats [options] + parity-evm stats-jsontests-vm [--folder] parity-evm [options] parity-evm [-h | --help] @@ -76,6 +77,9 @@ State test options: --only NAME Runs only a single test matching the name. --chain CHAIN Run only tests from specific chain. +Stats jsontests VM options: + --folder Run jsontest for a folder instead of a single file. + General options: --json Display verbose results in JSON. --std-json Display results in standardized JSON format. @@ -90,6 +94,8 @@ fn main() { if args.cmd_state_test { run_state_test(args) + } else if args.cmd_stats_jsontests_vm { + run_stats_jsontests_vm(args) } else if args.flag_json { run_call(args, display::json::Informant::default()) } else if args.flag_std_json { @@ -99,6 +105,40 @@ fn main() { } } +fn run_stats_jsontests_vm(args: Args) { + use json_tests::HookType; + use std::collections::HashMap; + use std::time::{Instant, Duration}; + + let file = args.arg_file.expect("FILE is required"); + + let mut times: HashMap)> = HashMap::new(); + + { + let mut record_time = |name: &str, typ: HookType| { + match typ { + HookType::OnStart => { + times.insert(name.to_string(), (Instant::now(), None)); + }, + HookType::OnStop => { + times.entry(name.to_string()).and_modify(|v| { + v.1 = Some(v.0.elapsed()); + }); + }, + } + }; + if args.flag_folder { + json_tests::run_executive_test_path(&file, &[], &mut record_time); + } else { + json_tests::run_executive_test_file(&file, &mut record_time); + } + } + + for (name, v) in times { + println!("{}\t{}", name, display::as_micros(&v.1.expect("All hooks are called with OnStop; qed"))); + } +} + fn run_state_test(args: Args) { use ethjson::state::test::Test; @@ -179,6 +219,7 @@ fn run_call(args: Args, informant: T) { struct Args { cmd_stats: bool, cmd_state_test: bool, + cmd_stats_jsontests_vm: bool, arg_file: Option, flag_only: Option, flag_from: Option, @@ -190,6 +231,7 @@ struct Args { flag_chain: Option, flag_json: bool, flag_std_json: bool, + flag_folder: bool, } impl Args { From 07e180722ae9ffae2260be018afc153c6d81150e Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 21 Jun 2018 19:31:54 +0800 Subject: [PATCH 05/12] Use /usr/bin/env to avoid errors on non-Debian systems --- scripts/aura-test.sh | 2 +- scripts/cov.sh | 2 +- scripts/doc.sh | 2 +- scripts/docker-build.sh | 2 +- scripts/gitlab-build.sh | 2 +- scripts/gitlab-push-release.sh | 2 +- scripts/gitlab-test.sh | 2 +- scripts/hook.sh | 2 +- scripts/safe_curl.sh | 3 +-- scripts/validate_chainspecs.sh | 3 +-- 10 files changed, 10 insertions(+), 12 deletions(-) diff --git a/scripts/aura-test.sh b/scripts/aura-test.sh index 1cd6bf53687..0c39e12f4d1 100755 --- a/scripts/aura-test.sh +++ b/scripts/aura-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/cov.sh b/scripts/cov.sh index b199da31aa3..578fc715ce0 100755 --- a/scripts/cov.sh +++ b/scripts/cov.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Installing KCOV under ubuntu # https://users.rust-lang.org/t/tutorial-how-to-collect-test-coverages-for-rust-project/650# ### Install deps diff --git a/scripts/doc.sh b/scripts/doc.sh index f0022610a54..44e544c993e 100755 --- a/scripts/doc.sh +++ b/scripts/doc.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh # generate documentation only for partiy and ethcore libraries cargo doc --no-deps --verbose --all --exclude parity-ipfs-api && diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index ceb085f511b..0a09fee5814 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash cd docker/hub DOCKER_BUILD_TAG=$1 echo "Docker build tag: " $DOCKER_BUILD_TAG diff --git a/scripts/gitlab-build.sh b/scripts/gitlab-build.sh index 5e272f6216c..14a9fd76953 100755 --- a/scripts/gitlab-build.sh +++ b/scripts/gitlab-build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/gitlab-push-release.sh b/scripts/gitlab-push-release.sh index 275576b91ec..207d66c11cb 100755 --- a/scripts/gitlab-push-release.sh +++ b/scripts/gitlab-push-release.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/gitlab-test.sh b/scripts/gitlab-test.sh index 57cf7c6b6c5..fbd93167a33 100755 --- a/scripts/gitlab-test.sh +++ b/scripts/gitlab-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #ARGUMENT test for RUST and COVERAGE set -e # fail on any error set -u # treat unset variables as error diff --git a/scripts/hook.sh b/scripts/hook.sh index 2d64d5782f2..ed7e173c505 100755 --- a/scripts/hook.sh +++ b/scripts/hook.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh FILE=./.git/hooks/pre-push echo "#!/bin/sh\n" > $FILE diff --git a/scripts/safe_curl.sh b/scripts/safe_curl.sh index 5990d1e24b3..f5bb2ee4ef5 100755 --- a/scripts/safe_curl.sh +++ b/scripts/safe_curl.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu @@ -17,4 +17,3 @@ else echo 'Unable to push info to updater service.'; exit 2 fi - diff --git a/scripts/validate_chainspecs.sh b/scripts/validate_chainspecs.sh index a0e2d9b1739..c350445dd9b 100755 --- a/scripts/validate_chainspecs.sh +++ b/scripts/validate_chainspecs.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh ERR=0 cargo build --release -p chainspec @@ -12,4 +12,3 @@ for spec in ethcore/res/ethereum/*.json; do done exit $ERR - From 937b0d6e8fb4d3e502a173468159ecf01bd0e9ce Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 21 Jun 2018 19:39:22 +0800 Subject: [PATCH 06/12] Move evmbin/bench.sh to scripts and add vm_jsontests script for convenience --- scripts/evm_jsontests_bench.sh | 16 ++++++++++++ evmbin/bench.sh => scripts/evm_uint_bench.sh | 26 +++++++++++--------- 2 files changed, 30 insertions(+), 12 deletions(-) create mode 100755 scripts/evm_jsontests_bench.sh rename evmbin/bench.sh => scripts/evm_uint_bench.sh (54%) diff --git a/scripts/evm_jsontests_bench.sh b/scripts/evm_jsontests_bench.sh new file mode 100755 index 00000000000..6f6372b72fe --- /dev/null +++ b/scripts/evm_jsontests_bench.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +cargo build --release -p evmbin + +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmArithmeticTest --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBitwiseLogicOperation --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBlockInfoTest --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmEnvironmentalInfo --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmIOandFlowOperations --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmLogTest --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPerformance --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPushDupSwapTest --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmRandomTest --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSha3Test --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSystemOperations --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmTests --folder diff --git a/evmbin/bench.sh b/scripts/evm_uint_bench.sh similarity index 54% rename from evmbin/bench.sh rename to scripts/evm_uint_bench.sh index 9d19729a681..b0ca1c4f7bc 100755 --- a/evmbin/bench.sh +++ b/scripts/evm_uint_bench.sh @@ -1,23 +1,25 @@ -#!/bin/bash +#!/usr/bin/env bash -set -e - -cargo build --release +cargo build --release -p evmbin # LOOP TEST CODE1=606060405260005b620f42408112156019575b6001016007565b600081905550600680602b6000396000f3606060405200 -ethvm --code $CODE1 -echo "^^^^ ethvm" -./target/release/evm stats --code $CODE1 --gas 4402000 +if [ -x "$(command -v ethvm)" ]; then + ethvm --code $CODE1 + echo "^^^^ ethvm" +fi +./target/release/parity-evm stats --code $CODE1 --gas 4402000 echo "^^^^ usize" -./target/release/evm stats --code $CODE1 +./target/release/parity-evm stats --code $CODE1 echo "^^^^ U256" # RNG TEST CODE2=6060604052600360056007600b60005b620f4240811215607f5767ffe7649d5eca84179490940267f47ed85c4b9a6379019367f8e5dd9a5c994bba9390930267f91d87e4b8b74e55019267ff97f6f3b29cda529290920267f393ada8dd75c938019167fe8d437c45bb3735830267f47d9a7b5428ffec019150600101600f565b838518831882186000555050505050600680609a6000396000f3606060405200 -ethvm --code $CODE2 -echo "^^^^ ethvm" -./target/release/evm stats --code $CODE2 --gas 143020115 +if [ -x "$(command -v ethvm)" ]; then + ethvm --code $CODE2 + echo "^^^^ ethvm" +fi +./target/release/parity-evm stats --code $CODE2 --gas 143020115 echo "^^^^ usize" -./target/release/evm stats --code $CODE2 +./target/release/parity-evm stats --code $CODE2 echo "^^^^ U256" From 5a8861274572d13cfbd4ac0c882567d46db456cf Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Thu, 21 Jun 2018 20:46:45 +0800 Subject: [PATCH 07/12] Add tempdir as required deps for test-helpers --- ethcore/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index e651039970a..eb185091545 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -90,4 +90,4 @@ test-heavy = [] # Compile benches benches = [] # Compile test helpers -test-helpers = [] +test-helpers = ["tempdir"] From 17abcf534c15cf8cf69d1e5fed9102d9851c2bb1 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 25 Jun 2018 15:22:51 +0800 Subject: [PATCH 08/12] Address grumbles on comments --- ethcore/src/json_tests/test_common.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ethcore/src/json_tests/test_common.rs b/ethcore/src/json_tests/test_common.rs index bdb76a48522..39b7447c7e8 100644 --- a/ethcore/src/json_tests/test_common.rs +++ b/ethcore/src/json_tests/test_common.rs @@ -21,12 +21,12 @@ use std::path::Path; use std::ffi::OsString; pub use ethereum_types::{H256, U256, Address}; -/// Indicate the type of the hook passed to test functions. +/// Indicate when to run the hook passed to test functions. #[derive(Copy, Clone, Eq, PartialEq)] pub enum HookType { - /// Hook passed on starting of a test. + /// Hook to code to run on test start. OnStart, - /// Hook passed on a test is finished. + /// Hook to code to run on test end. OnStop } From 1b30fcf7e1202a2c5633af8e094f4bdaf3d93afc Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 25 Jun 2018 15:33:02 +0800 Subject: [PATCH 09/12] Detect file/folder automatically and add command docs --- evmbin/src/main.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/evmbin/src/main.rs b/evmbin/src/main.rs index 64531fe9175..69481e910dc 100644 --- a/evmbin/src/main.rs +++ b/evmbin/src/main.rs @@ -61,10 +61,16 @@ EVM implementation for Parity. Usage: parity-evm state-test [--json --std-json --only NAME --chain CHAIN] parity-evm stats [options] - parity-evm stats-jsontests-vm [--folder] + parity-evm stats-jsontests-vm parity-evm [options] parity-evm [-h | --help] +Commands: + state-test Run a state test from a json file. + stats Execute EVM runtime code and return the statistics. + stats-jsontests-vm Execute standard jsontests format VMTests and return + timing statistcis in tsv format. + Transaction options: --code CODE Contract code as hex (without 0x). --to ADDRESS Recipient address (without 0x). @@ -77,9 +83,6 @@ State test options: --only NAME Runs only a single test matching the name. --chain CHAIN Run only tests from specific chain. -Stats jsontests VM options: - --folder Run jsontest for a folder instead of a single file. - General options: --json Display verbose results in JSON. --std-json Display results in standardized JSON format. @@ -110,7 +113,7 @@ fn run_stats_jsontests_vm(args: Args) { use std::collections::HashMap; use std::time::{Instant, Duration}; - let file = args.arg_file.expect("FILE is required"); + let file = args.arg_file.expect("FILE (or PATH) is required"); let mut times: HashMap)> = HashMap::new(); @@ -127,7 +130,7 @@ fn run_stats_jsontests_vm(args: Args) { }, } }; - if args.flag_folder { + if file.is_file() { json_tests::run_executive_test_path(&file, &[], &mut record_time); } else { json_tests::run_executive_test_file(&file, &mut record_time); @@ -231,7 +234,6 @@ struct Args { flag_chain: Option, flag_json: bool, flag_std_json: bool, - flag_folder: bool, } impl Args { From 477574f38f6f1f654505b1e265969ebaf1b05956 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 25 Jun 2018 15:33:55 +0800 Subject: [PATCH 10/12] Fix bench script --- scripts/evm_jsontests_bench.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/evm_jsontests_bench.sh b/scripts/evm_jsontests_bench.sh index 6f6372b72fe..acec902196d 100755 --- a/scripts/evm_jsontests_bench.sh +++ b/scripts/evm_jsontests_bench.sh @@ -2,15 +2,15 @@ cargo build --release -p evmbin -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmArithmeticTest --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBitwiseLogicOperation --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBlockInfoTest --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmEnvironmentalInfo --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmIOandFlowOperations --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmLogTest --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPerformance --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPushDupSwapTest --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmRandomTest --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSha3Test --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSystemOperations --folder -./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmTests --folder +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmArithmeticTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBitwiseLogicOperation +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmBlockInfoTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmEnvironmentalInfo +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmIOandFlowOperations +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmLogTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPerformance +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmPushDupSwapTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmRandomTest +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSha3Test +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmSystemOperations +./target/release/parity-evm stats-jsontests-vm ./ethcore/res/ethereum/tests/VMTests/vmTests From 9c8897e2b7aa7c8acdf217b294de2a3ea27f04ab Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 25 Jun 2018 15:34:50 +0800 Subject: [PATCH 11/12] times -> timings --- evmbin/src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evmbin/src/main.rs b/evmbin/src/main.rs index 69481e910dc..882994d99fe 100644 --- a/evmbin/src/main.rs +++ b/evmbin/src/main.rs @@ -115,16 +115,16 @@ fn run_stats_jsontests_vm(args: Args) { let file = args.arg_file.expect("FILE (or PATH) is required"); - let mut times: HashMap)> = HashMap::new(); + let mut timings: HashMap)> = HashMap::new(); { let mut record_time = |name: &str, typ: HookType| { match typ { HookType::OnStart => { - times.insert(name.to_string(), (Instant::now(), None)); + timings.insert(name.to_string(), (Instant::now(), None)); }, HookType::OnStop => { - times.entry(name.to_string()).and_modify(|v| { + timings.entry(name.to_string()).and_modify(|v| { v.1 = Some(v.0.elapsed()); }); }, @@ -137,7 +137,7 @@ fn run_stats_jsontests_vm(args: Args) { } } - for (name, v) in times { + for (name, v) in timings { println!("{}\t{}", name, display::as_micros(&v.1.expect("All hooks are called with OnStop; qed"))); } } From a7d05b57c9930f729f83382a44dbcecac473d8e9 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 25 Jun 2018 15:39:26 +0800 Subject: [PATCH 12/12] typo: wrong if condition --- evmbin/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evmbin/src/main.rs b/evmbin/src/main.rs index 882994d99fe..fb0c35b07be 100644 --- a/evmbin/src/main.rs +++ b/evmbin/src/main.rs @@ -130,7 +130,7 @@ fn run_stats_jsontests_vm(args: Args) { }, } }; - if file.is_file() { + if !file.is_file() { json_tests::run_executive_test_path(&file, &[], &mut record_time); } else { json_tests::run_executive_test_file(&file, &mut record_time);