Metadata V15: Expose API to fetch metadata for version (paritytech#13287

)

* impl_runtime_apis: Generate getters for `metadata_at` functions

Signed-off-by: Alexandru Vasile <[email protected]>

* runtime: Implement new `Metadata` runtime trait

Signed-off-by: Alexandru Vasile <[email protected]>

* runtime: Move `metadata_at` functions to construct_runtime macro

Signed-off-by: Alexandru Vasile <[email protected]>

* contruct_runtime: Use `OpaqueMetadata` from hidden imports

Signed-off-by: Alexandru Vasile <[email protected]>

* Adjust testing

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/tests: Add tests for the new API

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/tests: Adjust metdata naming

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Expose `metadata-v14` feature flag

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Expose metadata only under feature flags

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Expose v14 metadata by default

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Expose metadata feature for testing

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Test metadata under different feature flags

Signed-off-by: Alexandru Vasile <[email protected]>

* Update primitives/api/src/lib.rs

Co-authored-by: Bastian Köcher <[email protected]>

* Update primitives/api/src/lib.rs

Co-authored-by: Bastian Köcher <[email protected]>

* client/tests: Adjust testing to reflect trait Metadata change

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/metadata-ir: Add intermediate representation types for metadata

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/metadata-ir: Convert metadata to V14

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/metadata-ir: Add API to convert metadata to multiple versions

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/metadata-ir: Expose V14 under feature flag

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Adjust to metadata IR

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: More adjustments

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Guard v14 details under feature flag

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Adjust testing

Signed-off-by: Alexandru Vasile <[email protected]>

* CI: Ensure `quick-benchmarks` uses `metadata-v14`

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Use `metadata-v14` for benchmarks

Signed-off-by: Alexandru Vasile <[email protected]>

* Adjust cargo fmt

Signed-off-by: Alexandru Vasile <[email protected]>

* kitchensink-runtime: Add feature flag for `metadata-v14`

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support/test: Adjust testing

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support/test: Check crates locally

Signed-off-by: Alexandru Vasile <[email protected]>

* Activate metadata-v14 for pallets

Signed-off-by: Alexandru Vasile <[email protected]>

* Remove metadata-v14 feature flag

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/metadata_ir: Move `api.rs` to `mod.rs`

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Handle latest metadata conversion via IR

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/tests: Add constant for metadata version 14

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support/test: Fix merge conflict

Signed-off-by: Alexandru Vasile <[email protected]>

* Update frame/support/Cargo.toml

Co-authored-by: Bastian Köcher <[email protected]>

* Update frame/support/src/metadata_ir/mod.rs

Co-authored-by: Bastian Köcher <[email protected]>

* Update frame/support/test/Cargo.toml

Co-authored-by: Bastian Köcher <[email protected]>

* Update primitives/api/src/lib.rs

Co-authored-by: Bastian Köcher <[email protected]>

* frame/metadata: Collect pallet documentation for MetadataIR

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/tests: Check pallet documentation is propagated to MetadataIR

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Improve documentation

Signed-off-by: Alexandru Vasile <[email protected]>

---------

Signed-off-by: Alexandru Vasile <[email protected]>
Co-authored-by: parity-processbot <>
Co-authored-by: Bastian Köcher <[email protected]>

bin/node-template/runtime/src/lib.rs

@@ -354,6 +354,14 @@ impl_runtime_apis! {
 		fn metadata() -> OpaqueMetadata {
 			OpaqueMetadata::new(Runtime::metadata().into())
 		}
+
+		fn metadata_at_version(version: u32) -> Option<OpaqueMetadata> {
+			Runtime::metadata_at_version(version)
+		}
+
+		fn metadata_versions() -> sp_std::vec::Vec<u32> {
+			Runtime::metadata_versions()
+		}
 	}
 
 	impl sp_block_builder::BlockBuilder<Block> for Runtime {

bin/node/runtime/src/lib.rs

@@ -1942,6 +1942,14 @@ impl_runtime_apis! {
 		fn metadata() -> OpaqueMetadata {
 			OpaqueMetadata::new(Runtime::metadata().into())
 		}
+
+		fn metadata_at_version(version: u32) -> Option<OpaqueMetadata> {
+			Runtime::metadata_at_version(version)
+		}
+
+		fn metadata_versions() -> sp_std::vec::Vec<u32> {
+			Runtime::metadata_versions()
+		}
 	}
 
 	impl sp_block_builder::BlockBuilder<Block> for Runtime {

client/rpc-spec-v2/src/chain_head/tests.rs

@@ -170,7 +170,7 @@ async fn follow_with_runtime() {
 
 	let runtime_str = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\
 		\"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\
-		[\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\
+		[\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\
 		[\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\
 		[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\
 		\"transactionVersion\":1,\"stateVersion\":1}";

client/rpc/src/state/tests.rs

@@ -484,7 +484,7 @@ async fn should_return_runtime_version() {
 
 	let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\
 		\"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\
-		[\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\
+		[\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\
 		[\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\
 		[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\
 		\"transactionVersion\":1,\"stateVersion\":1}";

frame/support/procedural/src/construct_runtime/expand/metadata.rs

@@ -48,6 +48,7 @@ pub fn expand_runtime_metadata(
 			let event = expand_pallet_metadata_events(&filtered_names, runtime, scrate, decl);
 			let constants = expand_pallet_metadata_constants(runtime, decl);
 			let errors = expand_pallet_metadata_errors(runtime, decl);
+			let docs = expand_pallet_metadata_docs(runtime, decl);
 			let attr = decl.cfg_pattern.iter().fold(TokenStream::new(), |acc, pattern| {
 				let attr = TokenStream::from_str(&format!("#[cfg({})]", pattern.original()))
 					.expect("was successfully parsed before; qed");
@@ -59,25 +60,26 @@ pub fn expand_runtime_metadata(
 
 			quote! {
 				#attr
-				#scrate::metadata::PalletMetadata {
+				#scrate::metadata_ir::PalletMetadataIR {
 					name: stringify!(#name),
 					index: #index,
 					storage: #storage,
 					calls: #calls,
 					event: #event,
 					constants: #constants,
 					error: #errors,
+					docs: #docs,
 				}
 			}
 		})
 		.collect::<Vec<_>>();
 
 	quote! {
 		impl #runtime {
-			pub fn metadata() -> #scrate::metadata::RuntimeMetadataPrefixed {
-				#scrate::metadata::RuntimeMetadataLastVersion::new(
-					#scrate::sp_std::vec![ #(#pallets),* ],
-					#scrate::metadata::ExtrinsicMetadata {
+			fn metadata_ir() -> #scrate::metadata_ir::MetadataIR {
+				#scrate::metadata_ir::MetadataIR {
+					pallets: #scrate::sp_std::vec![ #(#pallets),* ],
+					extrinsic: #scrate::metadata_ir::ExtrinsicMetadataIR {
 						ty: #scrate::scale_info::meta_type::<#extrinsic>(),
 						version: <#extrinsic as #scrate::sp_runtime::traits::ExtrinsicMetadata>::VERSION,
 						signed_extensions: <
@@ -86,15 +88,29 @@ pub fn expand_runtime_metadata(
 								>::SignedExtensions as #scrate::sp_runtime::traits::SignedExtension
 							>::metadata()
 								.into_iter()
-								.map(|meta| #scrate::metadata::SignedExtensionMetadata {
+								.map(|meta| #scrate::metadata_ir::SignedExtensionMetadataIR {
 									identifier: meta.identifier,
 									ty: meta.ty,
 									additional_signed: meta.additional_signed,
 								})
 								.collect(),
 					},
-					#scrate::scale_info::meta_type::<#runtime>()
-				).into()
+					ty: #scrate::scale_info::meta_type::<#runtime>()
+				}
+			}
+
+			pub fn metadata() -> #scrate::metadata::RuntimeMetadataPrefixed {
+				#scrate::metadata_ir::into_latest(#runtime::metadata_ir())
+			}
+
+			pub fn metadata_at_version(version: u32) -> Option<#scrate::OpaqueMetadata> {
+				#scrate::metadata_ir::into_version(#runtime::metadata_ir(), version).map(|prefixed| {
+					#scrate::OpaqueMetadata::new(prefixed.into())
+				})
+			}
+
+			pub fn metadata_versions() -> #scrate::sp_std::vec::Vec<u32> {
+				#scrate::metadata_ir::supported_versions()
 			}
 		}
 	}
@@ -157,7 +173,7 @@ fn expand_pallet_metadata_events(
 
 		quote! {
 			Some(
-				#scrate::metadata::PalletEventMetadata {
+				#scrate::metadata_ir::PalletEventMetadataIR {
 					ty: #scrate::scale_info::meta_type::<#pallet_event>()
 				}
 			)
@@ -184,3 +200,12 @@ fn expand_pallet_metadata_errors(runtime: &Ident, decl: &Pallet) -> TokenStream
 		#path::Pallet::<#runtime #(, #path::#instance)*>::error_metadata()
 	}
 }
+
+fn expand_pallet_metadata_docs(runtime: &Ident, decl: &Pallet) -> TokenStream {
+	let path = &decl.path;
+	let instance = decl.instance.as_ref().into_iter();
+
+	quote! {
+		#path::Pallet::<#runtime #(, #path::#instance)*>::pallet_documentation_metadata()
+	}
+}

frame/support/procedural/src/lib.rs

@@ -495,7 +495,7 @@ pub fn construct_runtime(input: TokenStream) -> TokenStream {
 /// #[doc = include_str!("../README.md")]
 /// #[pallet_doc("../doc1.md")]
 /// #[pallet_doc("../doc2.md")]
-/// pub struct Pallet<T>(_);
+/// pub mod pallet {}
 /// ```
 ///
 /// The runtime metadata for this pallet contains the following
@@ -514,7 +514,7 @@ pub fn construct_runtime(input: TokenStream) -> TokenStream {
 /// /// Documentation for pallet 1
 /// /// Documentation for pallet 2
 /// /// Content of README.md
-/// pub struct Pallet<T>(_);
+/// pub mod pallet {}
 /// ```
 ///
 /// If you want to specify the file from which the documentation is loaded, you can use the
@@ -531,7 +531,7 @@ pub fn construct_runtime(input: TokenStream) -> TokenStream {
 ///
 /// This approach is beneficial when you use the `include_str` macro at the beginning of the file
 /// and want that documentation to extend to the runtime metadata, without reiterating the
-/// documentation on the module itself.
+/// documentation on the pallet module itself.
 #[proc_macro_attribute]
 pub fn pallet(attr: TokenStream, item: TokenStream) -> TokenStream {
 	pallet::pallet(attr, item)

frame/support/procedural/src/pallet/expand/call.rs

@@ -362,7 +362,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream {
 
 		impl<#type_impl_gen> #pallet_ident<#type_use_gen> #where_clause {
 			#[doc(hidden)]
-			pub fn call_functions() -> #frame_support::metadata::PalletCallMetadata {
+			pub fn call_functions() -> #frame_support::metadata_ir::PalletCallMetadataIR {
 				#frame_support::scale_info::meta_type::<#call_ident<#type_use_gen>>().into()
 			}
 		}

frame/support/procedural/src/pallet/expand/constants.rs

@@ -85,7 +85,7 @@ pub fn expand_constants(def: &mut Def) -> proc_macro2::TokenStream {
 		let default_byte_impl = &const_.default_byte_impl;
 
 		quote::quote!({
-			#frame_support::metadata::PalletConstantMetadata {
+			#frame_support::metadata_ir::PalletConstantMetadataIR {
 				name: #ident_str,
 				ty: #frame_support::scale_info::meta_type::<#const_type>(),
 				value: { #default_byte_impl },
@@ -99,7 +99,7 @@ pub fn expand_constants(def: &mut Def) -> proc_macro2::TokenStream {
 
 			#[doc(hidden)]
 			pub fn pallet_constants_metadata()
-				-> #frame_support::sp_std::vec::Vec<#frame_support::metadata::PalletConstantMetadata>
+				-> #frame_support::sp_std::vec::Vec<#frame_support::metadata_ir::PalletConstantMetadataIR>
 			{
 				#frame_support::sp_std::vec![ #( #consts ),* ]
 			}

frame/support/procedural/src/pallet/expand/documentation.rs

@@ -145,7 +145,7 @@ impl ToTokens for DocMetaValue {
 /// Implement a `pallet_documentation_metadata` function to fetch the
 /// documentation that is included in the metadata.
 ///
-/// The documentation is placed at the top of the module similar to:
+/// The documentation is placed on the pallet similar to:
 ///
 /// ```ignore
 /// #[pallet]
@@ -163,7 +163,7 @@ impl ToTokens for DocMetaValue {
 /// which is the file path that holds the documentation to be added to the metadata.
 ///
 /// Unlike the `doc` attribute, the documentation provided to the `proc_macro` attribute is
-/// not inserted at the beginning of the module.
+/// not added to the pallet.
 pub fn expand_documentation(def: &mut Def) -> proc_macro2::TokenStream {
 	let frame_support = &def.frame_support;
 	let type_impl_gen = &def.type_impl_generics(proc_macro2::Span::call_site());

frame/support/procedural/src/pallet/expand/pallet_struct.rs

@@ -82,8 +82,8 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream {
 		quote::quote_spanned!(def.pallet_struct.attr_span =>
 			impl<#type_impl_gen> #pallet_ident<#type_use_gen> #config_where_clause {
 				#[doc(hidden)]
-				pub fn error_metadata() -> Option<#frame_support::metadata::PalletErrorMetadata> {
-					Some(#frame_support::metadata::PalletErrorMetadata {
+				pub fn error_metadata() -> Option<#frame_support::metadata_ir::PalletErrorMetadataIR> {
+					Some(#frame_support::metadata_ir::PalletErrorMetadataIR {
 						ty: #frame_support::scale_info::meta_type::<#error_ident<#type_use_gen>>()
 					})
 				}
@@ -93,7 +93,7 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream {
 		quote::quote_spanned!(def.pallet_struct.attr_span =>
 			impl<#type_impl_gen> #pallet_ident<#type_use_gen> #config_where_clause {
 				#[doc(hidden)]
-				pub fn error_metadata() -> Option<#frame_support::metadata::PalletErrorMetadata> {
+				pub fn error_metadata() -> Option<#frame_support::metadata_ir::PalletErrorMetadataIR> {
 					None
 				}
 			}

frame/support/procedural/src/pallet/expand/storage.rs

@@ -639,8 +639,8 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream {
 			#completed_where_clause
 		{
 			#[doc(hidden)]
-			pub fn storage_metadata() -> #frame_support::metadata::PalletStorageMetadata {
-				#frame_support::metadata::PalletStorageMetadata {
+			pub fn storage_metadata() -> #frame_support::metadata_ir::PalletStorageMetadataIR {
+				#frame_support::metadata_ir::PalletStorageMetadataIR {
 					prefix: <
 						<T as #frame_system::Config>::PalletInfo as
 						#frame_support::traits::PalletInfo

frame/support/procedural/src/storage/metadata.rs

@@ -27,7 +27,7 @@ fn storage_line_metadata_type(scrate: &TokenStream, line: &StorageLineDefExt) ->
 	match &line.storage_type {
 		StorageLineTypeDef::Simple(_) => {
 			quote! {
-				#scrate::metadata::StorageEntryType::Plain(
+				#scrate::metadata_ir::StorageEntryTypeIR::Plain(
 					#scrate::scale_info::meta_type::<#value_type>()
 				)
 			}
@@ -36,8 +36,8 @@ fn storage_line_metadata_type(scrate: &TokenStream, line: &StorageLineDefExt) ->
 			let hasher = map.hasher.into_metadata();
 			let key = &map.key;
 			quote! {
-				#scrate::metadata::StorageEntryType::Map {
-					hashers: #scrate::sp_std::vec! [ #scrate::metadata::#hasher ],
+				#scrate::metadata_ir::StorageEntryTypeIR::Map {
+					hashers: #scrate::sp_std::vec! [ #scrate::metadata_ir::#hasher ],
 					key: #scrate::scale_info::meta_type::<#key>(),
 					value: #scrate::scale_info::meta_type::<#value_type>(),
 				}
@@ -49,10 +49,10 @@ fn storage_line_metadata_type(scrate: &TokenStream, line: &StorageLineDefExt) ->
 			let key1 = &map.key1;
 			let key2 = &map.key2;
 			quote! {
-				#scrate::metadata::StorageEntryType::Map {
+				#scrate::metadata_ir::StorageEntryTypeIR::Map {
 					hashers: #scrate::sp_std::vec! [
-						#scrate::metadata::#hasher1,
-						#scrate::metadata::#hasher2,
+						#scrate::metadata_ir::#hasher1,
+						#scrate::metadata_ir::#hasher2,
 					],
 					key: #scrate::scale_info::meta_type::<(#key1, #key2)>(),
 					value: #scrate::scale_info::meta_type::<#value_type>(),
@@ -67,9 +67,9 @@ fn storage_line_metadata_type(scrate: &TokenStream, line: &StorageLineDefExt) ->
 				.map(|hasher| hasher.to_storage_hasher_struct())
 				.collect::<Vec<_>>();
 			quote! {
-				#scrate::metadata::StorageEntryType::Map {
+				#scrate::metadata_ir::StorageEntryTypeIR::Map {
 					hashers: #scrate::sp_std::vec! [
-						#( #scrate::metadata::StorageHasher::#hashers, )*
+						#( #scrate::metadata_ir::StorageHasherIR::#hashers, )*
 					],
 					key: #scrate::scale_info::meta_type::<#key_tuple>(),
 					value: #scrate::scale_info::meta_type::<#value_type>(),
@@ -159,9 +159,9 @@ pub fn impl_metadata(def: &DeclStorageDefExt) -> TokenStream {
 		let str_name = line.name.to_string();
 
 		let modifier = if line.is_option {
-			quote!(#scrate::metadata::StorageEntryModifier::Optional)
+			quote!(#scrate::metadata_ir::StorageEntryModifierIR::Optional)
 		} else {
-			quote!(#scrate::metadata::StorageEntryModifier::Default)
+			quote!(#scrate::metadata_ir::StorageEntryModifierIR::Default)
 		};
 
 		let ty = storage_line_metadata_type(scrate, line);
@@ -172,7 +172,7 @@ pub fn impl_metadata(def: &DeclStorageDefExt) -> TokenStream {
 		let docs = get_doc_literals(&line.attrs);
 
 		let entry = quote! {
-			#scrate::metadata::StorageEntryMetadata {
+			#scrate::metadata_ir::StorageEntryMetadataIR {
 				name: #str_name,
 				modifier: #modifier,
 				ty: #ty,
@@ -194,7 +194,7 @@ pub fn impl_metadata(def: &DeclStorageDefExt) -> TokenStream {
 	};
 
 	let store_metadata = quote!(
-		#scrate::metadata::PalletStorageMetadata {
+		#scrate::metadata_ir::PalletStorageMetadataIR {
 			prefix: #prefix,
 			entries: #scrate::sp_std::vec![ #entries ],
 		}
@@ -209,7 +209,7 @@ pub fn impl_metadata(def: &DeclStorageDefExt) -> TokenStream {
 
 		impl #module_impl #module_struct #where_clause {
 			#[doc(hidden)]
-			pub fn storage_metadata() -> #scrate::metadata::PalletStorageMetadata {
+			pub fn storage_metadata() -> #scrate::metadata_ir::PalletStorageMetadataIR {
 				#store_metadata
 			}
 		}

frame/support/procedural/src/storage/mod.rs

@@ -454,13 +454,13 @@ impl HasherKind {
 
 	fn into_metadata(&self) -> proc_macro2::TokenStream {
 		match self {
-			HasherKind::Blake2_256 => quote!(StorageHasher::Blake2_256),
-			HasherKind::Blake2_128 => quote!(StorageHasher::Blake2_128),
-			HasherKind::Blake2_128Concat => quote!(StorageHasher::Blake2_128Concat),
-			HasherKind::Twox256 => quote!(StorageHasher::Twox256),
-			HasherKind::Twox128 => quote!(StorageHasher::Twox128),
-			HasherKind::Twox64Concat => quote!(StorageHasher::Twox64Concat),
-			HasherKind::Identity => quote!(StorageHasher::Identity),
+			HasherKind::Blake2_256 => quote!(StorageHasherIR::Blake2_256),
+			HasherKind::Blake2_128 => quote!(StorageHasherIR::Blake2_128),
+			HasherKind::Blake2_128Concat => quote!(StorageHasherIR::Blake2_128Concat),
+			HasherKind::Twox256 => quote!(StorageHasherIR::Twox256),
+			HasherKind::Twox128 => quote!(StorageHasherIR::Twox128),
+			HasherKind::Twox64Concat => quote!(StorageHasherIR::Twox64Concat),
+			HasherKind::Identity => quote!(StorageHasherIR::Identity),
 		}
 	}
 }