[WIP] Integrate v14 to txdecoder (#46)

* add v14 branch as dependency

* integrate v14 to txdecoder

* update to latest v14

* change log level to change with --verbose flag

* remove exclude and patch

* clippy lints

* fix v14 dependency

Cargo.toml

@@ -11,14 +11,3 @@ members = [
 	"substrate-metadata-versions/metadatav10",
 	"substrate-metadata-versions/metadatav11",
 ]
-
-#exclude = [
-#	"bin/tx-decoder"
-#]
-
-[patch.crates-io]
-# codec = { git = "https://github.com/insipx/parity-scale-codec", version = "1.3", branch = "insipx/debugging", package = "parity-scale-codec" }
-# parity-scale-codec-derive = { path = "derive", version = "^1", git = "https://github.com/insipx/parity-scale-codec", branch="insipx/debugging" }
-
-
-

bin/tx-decoder/Cargo.toml

@@ -11,6 +11,7 @@ async-stream = "0.3.2"
 sqlx = { version = "0.5", features = [ "runtime-async-std-rustls", "postgres", "offline" ]}
 desub = { path = "../../core/", package = "desub-core" }
 desub-extras = { path = "../../extras", features = [ "polkadot" ]}
+desub-v14 = { path = "../../core_v14", package = "core_v14" }
 anyhow = "1.0.43"
 futures = "0.3.17"
 serde = { version = "1.0", features = ["derive"] }

bin/tx-decoder/src/app.rs

@@ -15,8 +15,9 @@
 // along with substrate-desub.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::queries::*;
+use crate::decoder::Decoder;
+use desub::decoder::Chain;
 
-use desub::decoder::{Chain, Decoder};
 use desub_extras::runtimes;
 
 use anyhow::Error;
@@ -37,7 +38,7 @@ use std::{
 	},
 };
 
-type SpecVersion = i32;
+pub type SpecVersion = i32;
 
 #[derive(FromArgs, PartialEq, Debug)]
 /// Decode Extrinsics And Storage from Substrate Archive
@@ -119,22 +120,21 @@ impl<'a> AppState<'a> {
 				error_count += 1;
 			}
 			len += 1;
-			self.pb.map(|p| p.inc(1));
+			if let Some(p) = self.pb { p.inc(1) };
 		}
 		Ok((error_count, len))
 	}
 
 	fn decode(decoder: &Decoder, block: BlockModel, spec: SpecVersion, errors: &mut Vec<String>) -> Result<(), Error> {
-		log::debug!("-<<-<<-<<-<<-<<-<<-<<-<<-<< Decoding block {}, ext length {}", block.block_num, block.ext.len());
-		match decoder.decode_extrinsics(spec.try_into()?, block.ext.as_slice()) {
+		log::debug!("Decoding block {}, spec_version {}, ext length {}", block.block_num, spec, block.ext.len());
+		match decoder.decode_extrinsics(spec, &block.ext) {
 			Err(e) => {
-				let e: Error = e.into();
 				let e = e.context(format!("Failed to decode block {}", block.block_num));
 				errors.push(format!("{}", e));
 				Err(e)
 			}
 			Ok(d) => {
-				log::info!("Block {} Decoded Succesfully. {}", block.block_num, serde_json::to_string_pretty(&d)?);
+				log::info!("Block {} Decoded Succesfully. {}", block.block_num, &d);
 				Ok(())
 			}
 		}
@@ -145,30 +145,30 @@ impl<'a> AppState<'a> {
 	async fn register_metadata(&self, conn: &mut PgConnection, version: SpecVersion) -> Result<Option<u32>, Error> {
 		let (past, present) = past_and_present_version(conn, version).await?;
 		let mut decoder = self.decoder.write();
-		if !decoder.has_version(present) {
+		if !decoder.has_version(present.try_into().unwrap()) {
 			let meta = metadata(conn, present.try_into()?).await?;
-			decoder.register_version(present, meta);
+			decoder.register_version(present.try_into()?, &meta)?;
 		}
 
 		if let Some(p) = past {
-			if !decoder.has_version(p) {
+			if !decoder.has_version(p.try_into()?) {
 				let meta = metadata(conn, p.try_into()?).await?;
-				decoder.register_version(p, meta);
+				decoder.register_version(p.try_into()?, &meta)?;
 			}
 		}
-		Ok(past.try_into()?)
+		Ok(past)
 	}
 
 	fn set_message(&self, msg: impl Into<Cow<'static, str>>) {
-		self.pb.map(|p| p.set_message(msg));
+		if let Some(p) = self.pb { p.set_message(msg) }
 	}
 
 	fn set_length(&self, len: u64) {
-		self.pb.map(|p| p.set_length(len));
+		if let Some(p) = self.pb { p.set_length(len) }
 	}
 
 	fn finish_and_clear(&self) {
-		self.pb.map(|p| p.finish_and_clear());
+		if let Some(p) = self.pb { p.finish_and_clear() }
 	}
 }
 
@@ -222,8 +222,8 @@ pub async fn app(app: App) -> Result<(), Error> {
 		let spec_versions = spec_versions(&mut conn).await?;
 		let now = std::time::Instant::now();
 		let count = total_block_count(&mut conn).await?;
-		pb.as_ref().map(|p| p.set_message("decoding all blocks"));
-		pb.as_ref().map(|p| p.set_length(count as u64));
+		if let Some(p) = &pb { p.set_message("decoding all blocks") };
+		if let Some(p) = &pb { p.set_length(count as u64) };
 		let (error_count, length) = state.print_blocks(spec_versions, &mut errors)?;
 		state.finish_and_clear();
 		println!("Took {:?} to decode {} blocks with {} errors.", now.elapsed(), length, error_count);

bin/tx-decoder/src/decoder.rs

@@ -0,0 +1,71 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of substrate-desub.
+//
+// substrate-desub is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// substrate-desub is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with substrate-desub.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use desub::{decoder::Decoder as DecoderOld, TypeDetective};
+use desub_v14::{Decoder as DecoderNew, Metadata as MetadataNew};
+use desub::decoder::Chain;
+use std::{collections::HashMap, convert::TryInto};
+use anyhow::{Error, anyhow};
+
+use crate::app::SpecVersion;
+
+pub struct Decoder {
+	old: DecoderOld,
+	new: HashMap<u32, DecoderNew>,
+}
+
+impl Decoder {
+	pub fn new(types: impl TypeDetective + 'static, chain: Chain) -> Self {
+		Self { old: DecoderOld::new(types, chain), new: HashMap::new() }
+	}
+
+	pub fn register_version(&mut self, version: SpecVersion, meta: &[u8]) -> Result<(), Error> {
+		log::debug!("Registering version {}", version);
+		let new = MetadataNew::from_bytes(meta);
+		if let Err(e) = new {
+			log::error!("{}", e);
+			self.old.register_version(version.try_into()?, meta);
+		} else {
+			self.new.insert(version.try_into()?, DecoderNew::with_metadata(new?));
+		};
+		Ok(())
+	}
+
+	// Decodes extrinsics and serializes to String
+	pub fn decode_extrinsics(&self, version: SpecVersion, data: &[u8]) -> Result<String, Error> {
+		if self.is_version_new(version) {
+			log::debug!("DECODING NEW");
+			let decoder = self.new.get(&version.try_into()?).ok_or_else(|| anyhow!("version {} not found for new decoder", version))?;
+			match decoder.decode_extrinsics(data) {
+				Ok(v) => Ok(format!("{:#?}", v)),
+				Err(e) => Err(e.1.into())
+			}
+		} else {
+			log::debug!("DECODING OLD");
+			let ext = self.old.decode_extrinsics(version.try_into()?, data)?;
+			Ok(serde_json::to_string_pretty(&ext)?)
+		}
+	}
+
+	fn is_version_new(&self, version: SpecVersion) -> bool {
+		self.new.contains_key(&(version as u32))
+	}
+
+	pub fn has_version(&self, version: SpecVersion) -> bool {
+		self.new.contains_key(&(version as u32)) || self.old.has_version(version as u32)
+	}
+}

bin/tx-decoder/src/main.rs

@@ -16,6 +16,7 @@
 
 mod app;
 mod queries;
+mod decoder;
 
 use anyhow::Error;
 use colored::Colorize;
@@ -34,6 +35,7 @@ async fn main() -> Result<(), Error> {
 		.level_for("desub_core", level)
 		.level_for("desub_extras", level)
 		.level_for("tx_decoder", level)
+		.level_for("core_v14", level)
 		.format(move |out, message, record| {
 			out.finish(format_args!(
 				" {} {}::{}		>{} ",

core/src/decoder/metadata.rs

@@ -179,12 +179,6 @@ impl<'a> Metadata {
 					Decode::decode(&mut &*bytes).expect("decode failed");
 				meta.try_into().expect("Conversion failed")
 			}
-			0xE => {
-				log::debug!("Metadata V14");
-				let meta: frame_metadata::RuntimeMetadataPrefixed =
-					Decode::decode(&mut &*bytes).expect("decode failed");
-				meta.try_into().expect("Conversion failed")
-			}
 			/* TODO remove panics */
 			e => panic!("substrate metadata version {} is unknown, invalid or unsupported", e),
 		}

core_v14/src/decoder/extrinsic_bytes.rs

@@ -136,7 +136,7 @@ mod test {
 	use super::*;
 	use codec::{Compact, Encode};
 
-	fn iter_result_to_bytes<'a, E>(res: Option<Result<ExtrinsicBytes<'a>, E>>) -> Option<Result<&'a [u8], E>> {
+	fn iter_result_to_bytes<E>(res: Option<Result<ExtrinsicBytes<'_>, E>>) -> Option<Result<&[u8], E>> {
 		res.map(|r| r.map(|e| e.bytes()))
 	}
 

core_v14/tests/decode_extrinsics.rs

@@ -16,7 +16,7 @@
 
 use core_v14::{value, Decoder, Metadata, Value};
 
-static V14_METADATA_POLKADOT_SCALE: &'static [u8] = include_bytes!("data/v14_metadata_polkadot.scale");
+static V14_METADATA_POLKADOT_SCALE: &[u8] = include_bytes!("data/v14_metadata_polkadot.scale");
 
 fn decoder() -> Decoder {
 	let m = Metadata::from_bytes(V14_METADATA_POLKADOT_SCALE).expect("valid metadata");