Various improvements to tracing & diagnostics. (#1707)

* Various improvements to tracing & diagnostics.

- Manage possibility of `Account` not having code for `PodAccount`
- New RPC: `trace_sendRawTransaction`
- See raw transaction dump when inspecting over RPC

* Fix test

* Remove one of the dupe error messages

* Remove unneeded `&`s

* Reformat and extremely minor optimisation

* Minor optimisation

* Remove unneeded let

* Fix tests.

* Additional fix.

* Minor rename.

[ci:skip]

* Bowing to the pressure.

ethcore/src/account.rs

@@ -58,8 +58,8 @@ impl Account {
 			nonce: pod.nonce,
 			storage_root: SHA3_NULL_RLP,
 			storage_overlay: RefCell::new(pod.storage.into_iter().map(|(k, v)| (k, (Filth::Dirty, v))).collect()),
-			code_hash: Some(pod.code.sha3()),
-			code_cache: pod.code
+			code_hash: pod.code.as_ref().map(|c| c.sha3()),
+			code_cache: pod.code.as_ref().map_or_else(|| { warn!("POD account with unknown code is being created! Assuming no code."); vec![] }, |c| c.clone()),
 		}
 	}
 

ethcore/src/pod_account.rs

@@ -28,8 +28,8 @@ pub struct PodAccount {
 	pub balance: U256,
 	/// The nonce of the account.
 	pub nonce: U256,
-	/// The code of the account.
-	pub code: Bytes,
+	/// The code of the account or `None` in the special case that it is unknown.
+	pub code: Option<Bytes>,
 	/// The storage of the account.
 	pub storage: BTreeMap<H256, H256>,
 }
@@ -38,7 +38,7 @@ impl PodAccount {
 	/// Construct new object.
 	#[cfg(test)]
 	pub fn new(balance: U256, nonce: U256, code: Bytes, storage: BTreeMap<H256, H256>) -> PodAccount {
-		PodAccount { balance: balance, nonce: nonce, code: code, storage: storage }
+		PodAccount { balance: balance, nonce: nonce, code: Some(code), storage: storage }
 	}
 
 	/// Convert Account to a PodAccount.
@@ -48,7 +48,7 @@ impl PodAccount {
 			balance: *acc.balance(),
 			nonce: *acc.nonce(),
 			storage: acc.storage_overlay().iter().fold(BTreeMap::new(), |mut m, (k, &(_, ref v))| {m.insert(k.clone(), v.clone()); m}),
-			code: acc.code().unwrap().to_vec(),
+			code: acc.code().map(|x| x.to_vec()),
 		}
 	}
 
@@ -58,14 +58,15 @@ impl PodAccount {
 		stream.append(&self.nonce);
 		stream.append(&self.balance);
 		stream.append(&sec_trie_root(self.storage.iter().map(|(k, v)| (k.to_vec(), encode(&U256::from(v.as_slice())).to_vec())).collect()));
-		stream.append(&self.code.sha3());
+		stream.append(&self.code.as_ref().unwrap_or(&vec![]).sha3());
 		stream.out()
 	}
 
 	/// Place additional data into given hash DB.
 	pub fn insert_additional(&self, db: &mut AccountDBMut) {
-		if !self.code.is_empty() {
-			db.insert(&self.code);
+		match self.code {
+			Some(ref c) if !c.is_empty() => { db.insert(c); }
+			_ => {}
 		}
 		let mut r = H256::new();
 		let mut t = SecTrieDBMut::new(db, &mut r);
@@ -80,7 +81,7 @@ impl From<ethjson::blockchain::Account> for PodAccount {
 		PodAccount {
 			balance: a.balance.into(),
 			nonce: a.nonce.into(),
-			code: a.code.into(),
+			code: Some(a.code.into()),
 			storage: a.storage.into_iter().map(|(key, value)| {
 				let key: U256 = key.into();
 				let value: U256 = value.into();
@@ -95,15 +96,21 @@ impl From<ethjson::spec::Account> for PodAccount {
 		PodAccount {
 			balance: a.balance.map_or_else(U256::zero, Into::into),
 			nonce: a.nonce.map_or_else(U256::zero, Into::into),
-			code: vec![],
+			code: Some(vec![]),
 			storage: BTreeMap::new()
 		}
 	}
 }
 
 impl fmt::Display for PodAccount {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-		write!(f, "(bal={}; nonce={}; code={} bytes, #{}; storage={} items)", self.balance, self.nonce, self.code.len(), self.code.sha3(), self.storage.len())
+		write!(f, "(bal={}; nonce={}; code={} bytes, #{}; storage={} items)",
+			self.balance,
+			self.nonce,
+			self.code.as_ref().map_or(0, |c| c.len()),
+			self.code.as_ref().map_or_else(H256::new, |c| c.sha3()),
+			self.storage.len()
+		)
 	}
 }
 
@@ -114,13 +121,13 @@ pub fn diff_pod(pre: Option<&PodAccount>, post: Option<&PodAccount>) -> Option<A
 		(None, Some(x)) => Some(AccountDiff {
 			balance: Diff::Born(x.balance),
 			nonce: Diff::Born(x.nonce),
-			code: Diff::Born(x.code.clone()),
+			code: Diff::Born(x.code.as_ref().expect("account is newly created; newly created accounts must be given code; all caches should remain in place; qed").clone()),
 			storage: x.storage.iter().map(|(k, v)| (k.clone(), Diff::Born(v.clone()))).collect(),
 		}),
 		(Some(x), None) => Some(AccountDiff {
 			balance: Diff::Died(x.balance),
 			nonce: Diff::Died(x.nonce),
-			code: Diff::Died(x.code.clone()),
+			code: Diff::Died(x.code.as_ref().expect("account is deleted; only way to delete account is running SUICIDE; account must have had own code cached to make operation; all caches should remain in place; qed").clone()),
 			storage: x.storage.iter().map(|(k, v)| (k.clone(), Diff::Died(v.clone()))).collect(),
 		}),
 		(Some(pre), Some(post)) => {
@@ -130,7 +137,10 @@ pub fn diff_pod(pre: Option<&PodAccount>, post: Option<&PodAccount>) -> Option<A
 			let r = AccountDiff {
 				balance: Diff::new(pre.balance, post.balance),
 				nonce: Diff::new(pre.nonce, post.nonce),
-				code: Diff::new(pre.code.clone(), post.code.clone()),
+				code: match (pre.code.clone(), post.code.clone()) {
+					(Some(pre_code), Some(post_code)) => Diff::new(pre_code, post_code),
+					_ => Diff::Same,
+				},
 				storage: storage.into_iter().map(|k|
 					(k.clone(), Diff::new(
 						pre.storage.get(&k).cloned().unwrap_or_else(H256::new),
@@ -156,7 +166,7 @@ mod test {
 
 	#[test]
 	fn existence() {
-		let a = PodAccount{balance: 69.into(), nonce: 0.into(), code: vec![], storage: map![]};
+		let a = PodAccount{balance: 69.into(), nonce: 0.into(), code: Some(vec![]), storage: map![]};
 		assert_eq!(diff_pod(Some(&a), Some(&a)), None);
 		assert_eq!(diff_pod(None, Some(&a)), Some(AccountDiff{
 			balance: Diff::Born(69.into()),
@@ -168,8 +178,8 @@ mod test {
 
 	#[test]
 	fn basic() {
-		let a = PodAccount{balance: 69.into(), nonce: 0.into(), code: vec![], storage: map![]};
-		let b = PodAccount{balance: 42.into(), nonce: 1.into(), code: vec![], storage: map![]};
+		let a = PodAccount{balance: 69.into(), nonce: 0.into(), code: Some(vec![]), storage: map![]};
+		let b = PodAccount{balance: 42.into(), nonce: 1.into(), code: Some(vec![]), storage: map![]};
 		assert_eq!(diff_pod(Some(&a), Some(&b)), Some(AccountDiff {
 			balance: Diff::Changed(69.into(), 42.into()),
 			nonce: Diff::Changed(0.into(), 1.into()),
@@ -180,8 +190,8 @@ mod test {
 
 	#[test]
 	fn code() {
-		let a = PodAccount{balance: 0.into(), nonce: 0.into(), code: vec![], storage: map![]};
-		let b = PodAccount{balance: 0.into(), nonce: 1.into(), code: vec![0], storage: map![]};
+		let a = PodAccount{balance: 0.into(), nonce: 0.into(), code: Some(vec![]), storage: map![]};
+		let b = PodAccount{balance: 0.into(), nonce: 1.into(), code: Some(vec![0]), storage: map![]};
 		assert_eq!(diff_pod(Some(&a), Some(&b)), Some(AccountDiff {
 			balance: Diff::Same,
 			nonce: Diff::Changed(0.into(), 1.into()),
@@ -195,13 +205,13 @@ mod test {
 		let a = PodAccount {
 			balance: 0.into(),
 			nonce: 0.into(),
-			code: vec![],
+			code: Some(vec![]),
 			storage: map_into![1 => 1, 2 => 2, 3 => 3, 4 => 4, 5 => 0, 6 => 0, 7 => 0]
 		};
 		let b = PodAccount {
 			balance: 0.into(),
 			nonce: 0.into(),
-			code: vec![],
+			code: Some(vec![]),
 			storage: map_into![1 => 1, 2 => 3, 3 => 0, 5 => 0, 7 => 7, 8 => 0, 9 => 9]
 		};
 		assert_eq!(diff_pod(Some(&a), Some(&b)), Some(AccountDiff {

ethcore/src/trace/db.rs

@@ -23,7 +23,7 @@ use bloomchain::{Number, Config as BloomConfig};
 use bloomchain::group::{BloomGroupDatabase, BloomGroupChain, GroupPosition, BloomGroup};
 use util::{H256, H264, Database, DatabaseConfig, DBTransaction};
 use header::BlockNumber;
-use trace::{BlockTraces, LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest,
+use trace::{LocalizedTrace, Config, Switch, Filter, Database as TraceDatabase, ImportRequest,
 DatabaseExtras, Error};
 use db::{Key, Writable, Readable, CacheUpdatePolicy};
 use blooms;

ethcore/src/trace/flat.rs

@@ -74,6 +74,7 @@ impl Decodable for FlatTrace {
 pub struct FlatTransactionTraces(Vec<FlatTrace>);
 
 impl FlatTransactionTraces {
+	/// Returns bloom of the trace.
 	pub fn bloom(&self) -> LogBloom {
 		self.0.iter().fold(Default::default(), | bloom, trace | bloom | trace.bloom())
 	}
@@ -102,6 +103,7 @@ impl Into<Vec<FlatTrace>> for FlatTransactionTraces {
 pub struct FlatBlockTraces(Vec<FlatTransactionTraces>);
 
 impl FlatBlockTraces {
+	/// Returns bloom of the trace.
 	pub fn bloom(&self) -> LogBloom {
 		self.0.iter().fold(Default::default(), | bloom, tx_traces | bloom | tx_traces.bloom())
 	}

rpc/src/v1/impls/traces.rs

@@ -18,13 +18,13 @@
 
 use std::sync::{Weak, Arc};
 use jsonrpc_core::*;
-use std::collections::BTreeMap;
 use util::H256;
+use util::rlp::{UntrustedRlp, View};
 use ethcore::client::{BlockChainClient, CallAnalytics, TransactionID, TraceId};
 use ethcore::miner::MinerService;
 use ethcore::transaction::{Transaction as EthTransaction, SignedTransaction, Action};
 use v1::traits::Traces;
-use v1::types::{TraceFilter, LocalizedTrace, Trace, BlockNumber, Index, CallRequest, Bytes, StateDiff, VMTrace};
+use v1::types::{TraceFilter, LocalizedTrace, BlockNumber, Index, CallRequest, Bytes, TraceResults};
 
 /// Traces api implementation.
 pub struct TracesClient<C, M> where C: BlockChainClient, M: MinerService {
@@ -113,22 +113,30 @@ impl<C, M> Traces for TracesClient<C, M> where C: BlockChainClient + 'static, M:
 					state_diffing: flags.contains(&("stateDiff".to_owned())),
 				};
 				let signed = try!(self.sign_call(request));
-				let r = take_weak!(self.client).call(&signed, analytics);
-				if let Ok(executed) = r {
-					// TODO maybe add other stuff to this?
-					let mut ret = map!["output".to_owned() => to_value(&Bytes(executed.output)).unwrap()];
-					if let Some(trace) = executed.trace {
-						ret.insert("trace".to_owned(), to_value(&Trace::from(trace)).unwrap());
-					}
-					if let Some(vm_trace) = executed.vm_trace {
-						ret.insert("vmTrace".to_owned(), to_value(&VMTrace::from(vm_trace)).unwrap());
-					}
-					if let Some(state_diff) = executed.state_diff {
-						ret.insert("stateDiff".to_owned(), to_value(&StateDiff::from(state_diff)).unwrap());
-					}
-					return Ok(Value::Object(ret))
+				match take_weak!(self.client).call(&signed, analytics) {
+					Ok(e) => to_value(&TraceResults::from(e)),
+					_ => Ok(Value::Null),
+				}
+			})
+	}
+
+	fn raw_transaction(&self, params: Params) -> Result<Value, Error> {
+		trace!(target: "jsonrpc", "call: {:?}", params);
+		from_params::<(Bytes, Vec<String>)>(params)
+			.and_then(|(raw_transaction, flags)| {
+				let raw_transaction = raw_transaction.to_vec();
+				let analytics = CallAnalytics {
+					transaction_tracing: flags.contains(&("trace".to_owned())),
+					vm_tracing: flags.contains(&("vmTrace".to_owned())),
+					state_diffing: flags.contains(&("stateDiff".to_owned())),
+				};
+				match UntrustedRlp::new(&raw_transaction).as_val() {
+					Ok(signed) => match take_weak!(self.client).call(&signed, analytics) {
+						Ok(e) => to_value(&TraceResults::from(e)),
+						_ => Ok(Value::Null),
+					},
+					Err(_) => Err(Error::invalid_params()),
 				}
-				Ok(Value::Null)
 			})
 	}
 }

rpc/src/v1/tests/mocked/eth.rs

@@ -366,7 +366,7 @@ fn rpc_eth_pending_transaction_by_hash() {
 		tester.miner.pending_transactions.lock().unwrap().insert(H256::zero(), tx);
 	}
 
-	let response = r#"{"jsonrpc":"2.0","result":{"blockHash":null,"blockNumber":null,"creates":null,"from":"0x0f65fe9276bc9a24ae7083ae28e2660ef72df99e","gas":"0x5208","gasPrice":"0x01","hash":"0x41df922fd0d4766fcc02e161f8295ec28522f329ae487f14d811e4b64c8d6e31","input":"0x","nonce":"0x00","to":"0x095e7baea6a6c7c4c2dfeb977efac326af552d87","transactionIndex":null,"value":"0x0a"},"id":1}"#;
+	let response = r#"{"jsonrpc":"2.0","result":{"blockHash":null,"blockNumber":null,"creates":null,"from":"0x0f65fe9276bc9a24ae7083ae28e2660ef72df99e","gas":"0x5208","gasPrice":"0x01","hash":"0x41df922fd0d4766fcc02e161f8295ec28522f329ae487f14d811e4b64c8d6e31","input":"0x","nonce":"0x00","raw":"0xf85f800182520894095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba048b55bfa915ac795c431978d8a6a992b628d557da5ff759b307d495a36649353a0efffd310ac743f371de3b9f7f9cb56c0b28ad43601b4ab949f53faa07bd2c804","to":"0x095e7baea6a6c7c4c2dfeb977efac326af552d87","transactionIndex":null,"value":"0x0a"},"id":1}"#;
 	let request = r#"{
 		"jsonrpc": "2.0",
 		"method": "eth_getTransactionByHash",

rpc/src/v1/traits/traces.rs

@@ -35,6 +35,9 @@ pub trait Traces: Sized + Send + Sync + 'static {
 	/// Executes the given call and returns a number of possible traces for it.
 	fn call(&self, _: Params) -> Result<Value, Error>;
 
+	/// Executes the given raw transaction and returns a number of possible traces for it.
+	fn raw_transaction(&self, _: Params) -> Result<Value, Error>;
+
 	/// Should be used to convert object to io delegate.
 	fn to_delegate(self) -> IoDelegate<Self> {
 		let mut delegate = IoDelegate::new(Arc::new(self));
@@ -43,6 +46,7 @@ pub trait Traces: Sized + Send + Sync + 'static {
 		delegate.add_method("trace_transaction", Traces::transaction_traces);
 		delegate.add_method("trace_block", Traces::block_traces);
 		delegate.add_method("trace_call", Traces::call);
+		delegate.add_method("trace_rawTransaction", Traces::raw_transaction);
 
 		delegate
 	}

rpc/src/v1/types/block.rs

@@ -103,7 +103,7 @@ mod tests {
 	fn test_serialize_block_transactions() {
 		let t = BlockTransactions::Full(vec![Transaction::default()]);
 		let serialized = serde_json::to_string(&t).unwrap();
-		assert_eq!(serialized, r#"[{"hash":"0x0000000000000000000000000000000000000000000000000000000000000000","nonce":"0x00","blockHash":null,"blockNumber":null,"transactionIndex":null,"from":"0x0000000000000000000000000000000000000000","to":null,"value":"0x00","gasPrice":"0x00","gas":"0x00","input":"0x","creates":null}]"#);
+		assert_eq!(serialized, r#"[{"hash":"0x0000000000000000000000000000000000000000000000000000000000000000","nonce":"0x00","blockHash":null,"blockNumber":null,"transactionIndex":null,"from":"0x0000000000000000000000000000000000000000","to":null,"value":"0x00","gasPrice":"0x00","gas":"0x00","input":"0x","creates":null,"raw":"0x"}]"#);
 
 		let t = BlockTransactions::Hashes(vec![H256::default()]);
 		let serialized = serde_json::to_string(&t).unwrap();

rpc/src/v1/types/mod.rs.in

@@ -41,5 +41,5 @@ pub use self::transaction::Transaction;
 pub use self::transaction_request::{TransactionRequest, TransactionConfirmation, TransactionModification};
 pub use self::call_request::CallRequest;
 pub use self::receipt::Receipt;
-pub use self::trace::{Trace, LocalizedTrace, StateDiff, VMTrace};
+pub use self::trace::{LocalizedTrace, TraceResults};
 pub use self::trace_filter::TraceFilter;

rpc/src/v1/types/trace.rs

@@ -22,8 +22,9 @@ use ethcore::trace::{Trace as EthTrace, LocalizedTrace as EthLocalizedTrace};
 use ethcore::trace as et;
 use ethcore::state_diff;
 use ethcore::account_diff;
-use ethcore::executed;
 use v1::types::{Bytes};
+use ethcore::client::Executed;
+use ethcore::executed;
 
 #[derive(Debug, Serialize)]
 /// A diff of some chunk of memory.
@@ -194,6 +195,7 @@ impl From<account_diff::AccountDiff> for AccountDiff {
 	}
 }
 
+#[derive(Debug)]
 /// Serde-friendly `StateDiff` shadow.
 pub struct StateDiff(BTreeMap<Address, AccountDiff>);
 
@@ -477,6 +479,32 @@ impl From<EthTrace> for Trace {
 	}
 }
 
+#[derive(Debug, Serialize)]
+/// A diff of some chunk of memory.
+pub struct TraceResults {
+	/// The output of the call/create
+	pub output: Vec<u8>,
+	/// The transaction trace.
+	pub trace: Option<Trace>,
+	/// The transaction trace.
+	#[serde(rename="vmTrace")]
+	pub vm_trace: Option<VMTrace>,
+	/// The transaction trace.
+	#[serde(rename="stateDiff")]
+	pub state_diff: Option<StateDiff>,
+}
+
+impl From<Executed> for TraceResults {
+	fn from(t: Executed) -> Self {
+		TraceResults {
+			output: t.output.into(),
+			trace: t.trace.map(Into::into),
+			vm_trace: t.vm_trace.map(Into::into),
+			state_diff: t.state_diff.map(Into::into),
+		}
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use serde_json;
@@ -485,6 +513,18 @@ mod tests {
 	use v1::types::Bytes;
 	use super::*;
 
+	#[test]
+	fn should_serialize_trace_results() {
+		let r = TraceResults {
+			output: vec![0x60],
+			trace: None,
+			vm_trace: None,
+			state_diff: None,
+		};
+		let serialized = serde_json::to_string(&r).unwrap();
+		assert_eq!(serialized, r#"{"output":[96],"trace":null,"vmTrace":null,"stateDiff":null}"#);
+	}
+
 	#[test]
 	fn test_trace_serialize() {
 		let t = LocalizedTrace {