diff --git a/Cargo.lock b/Cargo.lock index 5bc51b7840ec..166344f0e5f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4857,6 +4857,7 @@ dependencies = [ "pallet-transaction-payment", "parity-scale-codec", "scale-info", + "serde_json", "sp-api 26.0.0", "sp-block-builder", "sp-consensus-aura", @@ -4864,6 +4865,7 @@ dependencies = [ "sp-genesis-builder", "sp-inherents", "sp-io 30.0.0", + "sp-keyring", "sp-offchain", "sp-runtime 31.0.1", "sp-session", @@ -4940,6 +4942,7 @@ dependencies = [ "sp-consensus", "sp-consensus-aura", "sp-core 28.0.0", + "sp-genesis-builder", "sp-io 30.0.0", "sp-keyring", "sp-runtime 31.0.1", @@ -6286,15 +6289,21 @@ dependencies = [ "chrono", "clap 4.5.13", "comfy-table", + "cumulus-client-parachain-inherent", + "cumulus-primitives-proof-size-hostfunction", + "cumulus-test-runtime", "frame-benchmarking", "frame-support", "frame-system", "gethostname", "handlebars", + "hex", "itertools 0.11.0", "linked-hash-map", "log", "parity-scale-codec", + "polkadot-parachain-primitives", + "polkadot-primitives", "rand", "rand_pcg", "sc-block-builder", @@ -6303,13 +6312,16 @@ dependencies = [ "sc-client-api", "sc-client-db", "sc-executor 0.32.0", + "sc-executor-common 0.29.0", "sc-service", "sc-sysinfo", "serde", "serde_json", "sp-api 26.0.0", + "sp-block-builder", "sp-blockchain", "sp-core 28.0.0", + "sp-crypto-hashing 0.1.0", "sp-database", "sp-externalities 0.25.0", "sp-genesis-builder", @@ -6319,10 +6331,17 @@ dependencies = [ "sp-runtime 31.0.1", "sp-state-machine 0.35.0", "sp-storage 19.0.0", + "sp-timestamp", + "sp-transaction-pool", "sp-trie 29.0.0", + "sp-version 29.0.0", "sp-wasm-interface 20.0.0", + "substrate-test-runtime", + "subxt", + "subxt-signer", "thiserror", "thousands", + "westend-runtime", ] [[package]] @@ -6459,13 +6478,19 @@ dependencies = [ name = "frame-omni-bencher" version = "0.1.0" dependencies = [ + "assert_cmd", "clap 4.5.13", "cumulus-primitives-proof-size-hostfunction", + "cumulus-test-runtime", "frame-benchmarking-cli", "log", + "sc-chain-spec", "sc-cli", + "sp-genesis-builder", "sp-runtime 31.0.1", "sp-statement-store", + "sp-tracing 16.0.0", + "tempfile", "tracing-subscriber 0.3.18", ] diff --git a/cumulus/client/parachain-inherent/src/mock.rs b/cumulus/client/parachain-inherent/src/mock.rs index a3f881e6ef9d..950cba2aaa7d 100644 --- a/cumulus/client/parachain-inherent/src/mock.rs +++ b/cumulus/client/parachain-inherent/src/mock.rs @@ -45,6 +45,7 @@ pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; /// in addition to the messages themselves, you must provide some information about /// your parachain's configuration in order to mock the MQC heads properly. /// See [`MockXcmConfig`] for more information +#[derive(Default)] pub struct MockValidationDataInherentDataProvider { /// The current block number of the local block chain (the parachain). pub current_para_block: u32, diff --git a/cumulus/polkadot-omni-node/lib/src/command.rs b/cumulus/polkadot-omni-node/lib/src/command.rs index fe935a03cc95..cf283819966f 100644 --- a/cumulus/polkadot-omni-node/lib/src/command.rs +++ b/cumulus/polkadot-omni-node/lib/src/command.rs @@ -48,6 +48,16 @@ pub struct RunConfig { pub runtime_resolver: Box, } +impl RunConfig { + /// Create a new `RunConfig` + pub fn new( + runtime_resolver: Box, + chain_spec_loader: Box, + ) -> Self { + RunConfig { chain_spec_loader, runtime_resolver } + } +} + pub fn new_aura_node_spec( aura_id: AuraConsensusId, extra_args: &NodeExtraArgs, diff --git a/cumulus/polkadot-omni-node/src/main.rs b/cumulus/polkadot-omni-node/src/main.rs index 5bca81e2e78b..a6c1dd3cadbb 100644 --- a/cumulus/polkadot-omni-node/src/main.rs +++ b/cumulus/polkadot-omni-node/src/main.rs @@ -49,9 +49,6 @@ impl CliConfigT for CliConfig { fn main() -> color_eyre::eyre::Result<()> { color_eyre::install()?; - let config = RunConfig { - chain_spec_loader: Box::new(DiskChainSpecLoader), - runtime_resolver: Box::new(DefaultRuntimeResolver), - }; + let config = RunConfig::new(Box::new(DefaultRuntimeResolver), Box::new(DiskChainSpecLoader)); Ok(run::(config)?) } diff --git a/cumulus/polkadot-parachain/src/main.rs b/cumulus/polkadot-parachain/src/main.rs index c8464be937cc..61764636a060 100644 --- a/cumulus/polkadot-parachain/src/main.rs +++ b/cumulus/polkadot-parachain/src/main.rs @@ -46,9 +46,9 @@ impl CliConfigT for CliConfig { fn main() -> color_eyre::eyre::Result<()> { color_eyre::install()?; - let config = RunConfig { - chain_spec_loader: Box::new(chain_spec::ChainSpecLoader), - runtime_resolver: Box::new(chain_spec::RuntimeResolver), - }; + let config = RunConfig::new( + Box::new(chain_spec::RuntimeResolver), + Box::new(chain_spec::ChainSpecLoader), + ); Ok(run::(config)?) } diff --git a/cumulus/test/client/src/lib.rs b/cumulus/test/client/src/lib.rs index eaf81699f6d7..863a8fa93f6f 100644 --- a/cumulus/test/client/src/lib.rs +++ b/cumulus/test/client/src/lib.rs @@ -39,7 +39,7 @@ use sp_consensus_aura::{AuraApi, Slot}; use sp_core::Pair; use sp_io::TestExternalities; use sp_keystore::testing::MemoryKeystore; -use sp_runtime::{generic::Era, traits::Header, BuildStorage, SaturatedConversion}; +use sp_runtime::{generic::Era, traits::Header, BuildStorage, MultiAddress, SaturatedConversion}; use std::sync::Arc; pub use substrate_test_client::*; @@ -158,7 +158,7 @@ pub fn generate_extrinsic_with_pair( UncheckedExtrinsic::new_signed( function, - origin.public().into(), + MultiAddress::Id(origin.public().into()), Signature::Sr25519(signature), tx_ext, ) @@ -181,7 +181,7 @@ pub fn transfer( value: Balance, ) -> UncheckedExtrinsic { let function = RuntimeCall::Balances(pallet_balances::Call::transfer_allow_death { - dest: dest.public().into(), + dest: MultiAddress::Id(dest.public().into()), value, }); diff --git a/cumulus/test/runtime/Cargo.toml b/cumulus/test/runtime/Cargo.toml index 54b83e2dfeda..8117e6e69709 100644 --- a/cumulus/test/runtime/Cargo.toml +++ b/cumulus/test/runtime/Cargo.toml @@ -11,6 +11,7 @@ workspace = true [dependencies] codec = { features = ["derive"], workspace = true } scale-info = { features = ["derive"], workspace = true } +serde_json = { workspace = true } # Substrate frame-executive = { workspace = true } @@ -38,6 +39,7 @@ sp-session = { workspace = true } sp-consensus-aura = { workspace = true } sp-transaction-pool = { workspace = true } sp-version = { workspace = true } +sp-keyring = { workspace = true } # Cumulus cumulus-pallet-parachain-system = { workspace = true } @@ -76,6 +78,7 @@ std = [ "pallet-transaction-payment/std", "parachain-info/std", "scale-info/std", + "serde_json/std", "sp-api/std", "sp-block-builder/std", "sp-consensus-aura/std", @@ -83,6 +86,7 @@ std = [ "sp-genesis-builder/std", "sp-inherents/std", "sp-io/std", + "sp-keyring/std", "sp-offchain/std", "sp-runtime/std", "sp-session/std", diff --git a/cumulus/test/runtime/src/genesis_config_presets.rs b/cumulus/test/runtime/src/genesis_config_presets.rs new file mode 100644 index 000000000000..6cf56ef5363f --- /dev/null +++ b/cumulus/test/runtime/src/genesis_config_presets.rs @@ -0,0 +1,72 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +use super::{ + AccountId, AuraConfig, AuraId, BalancesConfig, ParachainInfoConfig, RuntimeGenesisConfig, + SudoConfig, +}; +use alloc::{vec, vec::Vec}; + +use cumulus_primitives_core::ParaId; +use frame_support::build_struct_json_patch; +use sp_genesis_builder::PresetId; +use sp_keyring::Sr25519Keyring; + +fn cumulus_test_runtime( + invulnerables: Vec, + endowed_accounts: Vec, + id: ParaId, +) -> serde_json::Value { + build_struct_json_patch!(RuntimeGenesisConfig { + balances: BalancesConfig { + balances: endowed_accounts.iter().cloned().map(|k| (k, 1 << 60)).collect(), + }, + sudo: SudoConfig { key: Some(Sr25519Keyring::Alice.public().into()) }, + parachain_info: ParachainInfoConfig { parachain_id: id }, + aura: AuraConfig { authorities: invulnerables }, + }) +} + +fn testnet_genesis_with_default_endowed(self_para_id: ParaId) -> serde_json::Value { + let endowed = Sr25519Keyring::well_known().map(|x| x.to_account_id()).collect::>(); + + let invulnerables = + Sr25519Keyring::invulnerable().map(|x| x.public().into()).collect::>(); + cumulus_test_runtime(invulnerables, endowed, self_para_id) +} + +/// List of supported presets. +pub fn preset_names() -> Vec { + vec![ + PresetId::from(sp_genesis_builder::DEV_RUNTIME_PRESET), + PresetId::from(sp_genesis_builder::LOCAL_TESTNET_RUNTIME_PRESET), + ] +} + +/// Provides the JSON representation of predefined genesis config for given `id`. +pub fn get_preset(id: &PresetId) -> Option> { + let patch = match id.try_into() { + Ok(sp_genesis_builder::DEV_RUNTIME_PRESET) | + Ok(sp_genesis_builder::LOCAL_TESTNET_RUNTIME_PRESET) => + testnet_genesis_with_default_endowed(100.into()), + _ => return None, + }; + Some( + serde_json::to_string(&patch) + .expect("serialization to json is expected to work. qed.") + .into_bytes(), + ) +} diff --git a/cumulus/test/runtime/src/lib.rs b/cumulus/test/runtime/src/lib.rs index 5443bb5f526b..01c1571f343f 100644 --- a/cumulus/test/runtime/src/lib.rs +++ b/cumulus/test/runtime/src/lib.rs @@ -32,6 +32,7 @@ pub mod elastic_scaling { include!(concat!(env!("OUT_DIR"), "/wasm_binary_elastic_scaling.rs")); } +mod genesis_config_presets; mod test_pallet; extern crate alloc; @@ -45,9 +46,9 @@ use sp_core::{ConstBool, ConstU32, ConstU64, OpaqueMetadata}; use cumulus_primitives_core::{ClaimQueueOffset, CoreSelector}; use sp_runtime::{ create_runtime_str, generic, impl_opaque_keys, - traits::{BlakeTwo256, Block as BlockT, IdentifyAccount, IdentityLookup, Verify}, + traits::{BlakeTwo256, Block as BlockT, IdentifyAccount, Verify}, transaction_validity::{TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, MultiSignature, + ApplyExtrinsicResult, MultiAddress, MultiSignature, }; #[cfg(feature = "std")] use sp_version::NativeVersion; @@ -208,8 +209,6 @@ parameter_types! { impl frame_system::Config for Runtime { /// The identifier used to distinguish between accounts. type AccountId = AccountId; - /// The lookup mechanism to get account ID from whatever is passed in dispatchers. - type Lookup = IdentityLookup; /// The index type for storing how many extrinsics an account has signed. type Nonce = Nonce; /// The type for hashing blocks and tries. @@ -363,7 +362,7 @@ pub type AccountId = <::Signer as IdentifyAccount>::Account pub type NodeBlock = generic::Block; /// The address format for describing accounts. -pub type Address = AccountId; +pub type Address = MultiAddress; /// Block header type as expected by this runtime. pub type Header = generic::Header; /// Block type as expected by this runtime. @@ -544,11 +543,11 @@ impl_runtime_apis! { } fn get_preset(id: &Option) -> Option> { - get_preset::(id, |_| None) + get_preset::(id, genesis_config_presets::get_preset) } fn preset_names() -> Vec { - vec![] + genesis_config_presets::preset_names() } } } diff --git a/cumulus/test/service/Cargo.toml b/cumulus/test/service/Cargo.toml index 3ef9424b9ed6..86a8c48bb54f 100644 --- a/cumulus/test/service/Cargo.toml +++ b/cumulus/test/service/Cargo.toml @@ -49,6 +49,7 @@ sp-core = { workspace = true, default-features = true } sp-io = { workspace = true, default-features = true } sp-api = { workspace = true, default-features = true } sp-keyring = { workspace = true, default-features = true } +sp-genesis-builder = { workspace = true, default-features = true } sp-runtime = { workspace = true } sp-state-machine = { workspace = true, default-features = true } sp-tracing = { workspace = true, default-features = true } diff --git a/cumulus/test/service/benches/transaction_throughput.rs b/cumulus/test/service/benches/transaction_throughput.rs index 011eb4c7d50e..bba624e36ad1 100644 --- a/cumulus/test/service/benches/transaction_throughput.rs +++ b/cumulus/test/service/benches/transaction_throughput.rs @@ -54,7 +54,7 @@ fn create_account_extrinsics(client: &Client, accounts: &[sr25519::Pair]) -> Vec SudoCall::sudo { call: Box::new( BalancesCall::force_set_balance { - who: AccountId::from(a.public()), + who: AccountId::from(a.public()).into(), new_free: 0, } .into(), @@ -69,7 +69,7 @@ fn create_account_extrinsics(client: &Client, accounts: &[sr25519::Pair]) -> Vec SudoCall::sudo { call: Box::new( BalancesCall::force_set_balance { - who: AccountId::from(a.public()), + who: AccountId::from(a.public()).into(), new_free: 1_000_000_000_000 * ExistentialDeposit::get(), } .into(), @@ -96,7 +96,7 @@ fn create_benchmark_extrinsics( construct_extrinsic( client, BalancesCall::transfer_allow_death { - dest: Bob.to_account_id(), + dest: Bob.to_account_id().into(), value: ExistentialDeposit::get(), }, account.clone(), diff --git a/cumulus/test/service/benches/validate_block.rs b/cumulus/test/service/benches/validate_block.rs index 34b09d99ce98..ca20de338f3c 100644 --- a/cumulus/test/service/benches/validate_block.rs +++ b/cumulus/test/service/benches/validate_block.rs @@ -60,7 +60,10 @@ fn create_extrinsics( let extrinsic: UncheckedExtrinsic = generate_extrinsic_with_pair( client, src.clone(), - BalancesCall::transfer_keep_alive { dest: AccountId::from(dst.public()), value: 10000 }, + BalancesCall::transfer_keep_alive { + dest: AccountId::from(dst.public()).into(), + value: 10000, + }, None, ); diff --git a/cumulus/test/service/src/bench_utils.rs b/cumulus/test/service/src/bench_utils.rs index 76717b4136fa..49ba1b230cc3 100644 --- a/cumulus/test/service/src/bench_utils.rs +++ b/cumulus/test/service/src/bench_utils.rs @@ -41,7 +41,7 @@ use sp_core::{sr25519, Pair}; use sp_keyring::Sr25519Keyring::Alice; use sp_runtime::{ transaction_validity::{InvalidTransaction, TransactionValidityError}, - AccountId32, FixedU64, OpaqueExtrinsic, + AccountId32, FixedU64, MultiAddress, OpaqueExtrinsic, }; /// Accounts to use for transfer transactions. Enough for 5000 transactions. @@ -151,7 +151,10 @@ pub fn create_benchmarking_transfer_extrinsics( for (src, dst) in src_accounts.iter().zip(dst_accounts.iter()) { let extrinsic: UncheckedExtrinsic = construct_extrinsic( client, - BalancesCall::transfer_keep_alive { dest: AccountId::from(dst.public()), value: 10000 }, + BalancesCall::transfer_keep_alive { + dest: MultiAddress::Id(AccountId::from(dst.public())), + value: 10000, + }, src.clone(), Some(0), ); diff --git a/cumulus/test/service/src/chain_spec.rs b/cumulus/test/service/src/chain_spec.rs index 3abffcff794f..3d4e4dca5f8d 100644 --- a/cumulus/test/service/src/chain_spec.rs +++ b/cumulus/test/service/src/chain_spec.rs @@ -16,13 +16,13 @@ #![allow(missing_docs)] +use cumulus_client_service::ParachainHostFunctions; use cumulus_primitives_core::ParaId; use cumulus_test_runtime::AccountId; -use parachains_common::AuraId; -use sc_chain_spec::{ChainSpecExtension, ChainSpecGroup}; +use sc_chain_spec::{ChainSpecExtension, ChainSpecGroup, GenesisConfigBuilderRuntimeCaller}; use sc_service::ChainType; use serde::{Deserialize, Serialize}; -use sp_keyring::Sr25519Keyring; +use serde_json::json; /// Specialized `ChainSpec` for the normal parachain runtime. pub type ChainSpec = sc_service::GenericChainSpec; @@ -50,17 +50,51 @@ pub fn get_chain_spec_with_extra_endowed( extra_endowed_accounts: Vec, code: &[u8], ) -> ChainSpec { + let runtime_caller = GenesisConfigBuilderRuntimeCaller::::new(code); + let mut development_preset = runtime_caller + .get_named_preset(Some(&sp_genesis_builder::LOCAL_TESTNET_RUNTIME_PRESET.to_string())) + .expect("development preset is available on test runtime; qed"); + + // Extract existing balances + let existing_balances = development_preset + .get("balances") + .and_then(|b| b.get("balances")) + .and_then(|b| b.as_array()) + .cloned() + .unwrap_or_default(); + + // Create new balances by combining existing and extra accounts + let mut all_balances = existing_balances; + all_balances.extend(extra_endowed_accounts.into_iter().map(|a| json!([a, 1u64 << 60]))); + + let mut patch_json = json!({ + "balances": { + "balances": all_balances, + } + }); + + if let Some(id) = id { + // Merge parachain ID if given, otherwise use the one from the preset. + sc_chain_spec::json_merge( + &mut patch_json, + json!({ + "parachainInfo": { + "parachainId": id, + }, + }), + ); + }; + + sc_chain_spec::json_merge(&mut development_preset, patch_json.into()); + ChainSpec::builder( code, Extensions { para_id: id.unwrap_or(cumulus_test_runtime::PARACHAIN_ID.into()).into() }, ) .with_name("Local Testnet") - .with_id("local_testnet") + .with_id(sp_genesis_builder::LOCAL_TESTNET_RUNTIME_PRESET) .with_chain_type(ChainType::Local) - .with_genesis_config_patch(testnet_genesis_with_default_endowed( - extra_endowed_accounts.clone(), - id, - )) + .with_genesis_config_patch(development_preset) .build() } @@ -82,35 +116,3 @@ pub fn get_elastic_scaling_chain_spec(id: Option) -> ChainSpec { .expect("WASM binary was not built, please build it!"), ) } - -/// Local testnet genesis for testing. -pub fn testnet_genesis_with_default_endowed( - mut extra_endowed_accounts: Vec, - self_para_id: Option, -) -> serde_json::Value { - let mut endowed = Sr25519Keyring::well_known().map(|k| k.to_account_id()).collect::>(); - endowed.append(&mut extra_endowed_accounts); - let invulnerables = - Sr25519Keyring::invulnerable().map(|k| k.public().into()).collect::>(); - testnet_genesis(Sr25519Keyring::Alice.to_account_id(), invulnerables, endowed, self_para_id) -} - -/// Creates a local testnet genesis with endowed accounts. -pub fn testnet_genesis( - root_key: AccountId, - invulnerables: Vec, - endowed_accounts: Vec, - self_para_id: Option, -) -> serde_json::Value { - let self_para_id = self_para_id.unwrap_or(cumulus_test_runtime::PARACHAIN_ID.into()); - serde_json::json!({ - "balances": cumulus_test_runtime::BalancesConfig { - balances: endowed_accounts.iter().cloned().map(|k| (k, 1 << 60)).collect(), - }, - "sudo": cumulus_test_runtime::SudoConfig { key: Some(root_key) }, - "parachainInfo": { - "parachainId": self_para_id, - }, - "aura": cumulus_test_runtime::AuraConfig { authorities: invulnerables } - }) -} diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index a3e519a68b91..fe3cbfbbb498 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -90,7 +90,7 @@ use sp_arithmetic::traits::SaturatedConversion; use sp_blockchain::HeaderBackend; use sp_core::Pair; use sp_keyring::Sr25519Keyring; -use sp_runtime::{codec::Encode, generic}; +use sp_runtime::{codec::Encode, generic, MultiAddress}; use sp_state_machine::BasicExternalities; use std::sync::Arc; use substrate_test_client::{ @@ -991,7 +991,7 @@ pub fn construct_extrinsic( let signature = raw_payload.using_encoded(|e| caller.sign(e)); runtime::UncheckedExtrinsic::new_signed( function, - caller.public().into(), + MultiAddress::Id(caller.public().into()), runtime::Signature::Sr25519(signature), tx_ext, ) diff --git a/cumulus/zombienet/examples/small_network.toml b/cumulus/zombienet/examples/small_network.toml index ab7265712308..64765566471a 100644 --- a/cumulus/zombienet/examples/small_network.toml +++ b/cumulus/zombienet/examples/small_network.toml @@ -3,23 +3,23 @@ default_image = "parity/polkadot:latest" default_command = "polkadot" chain = "rococo-local" - [[relaychain.nodes]] - name = "alice" - validator = true +[[relaychain.nodes]] +name = "alice" +validator = true - [[relaychain.nodes]] - name = "bob" - validator = true +[[relaychain.nodes]] +name = "bob" +validator = true [[parachains]] id = 2000 cumulus_based = true chain = "asset-hub-rococo-local" - # run charlie as parachain collator - [[parachains.collators]] - name = "charlie" - validator = true - image = "parity/polkadot-parachain:latest" - command = "polkadot-parachain" - args = ["--force-authoring"] +# run charlie as parachain collator +[[parachains.collators]] +name = "charlie" +validator = true +image = "parity/polkadot-parachain:latest" +command = "polkadot-parachain" +args = ["--force-authoring"] diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs index 7c904e6658e7..02c9b97150c2 100644 --- a/polkadot/cli/src/command.rs +++ b/polkadot/cli/src/command.rs @@ -15,12 +15,14 @@ // along with Polkadot. If not, see . use crate::cli::{Cli, Subcommand, NODE_VERSION}; -use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; +use frame_benchmarking_cli::{ + BenchmarkCmd, ExtrinsicFactory, SubstrateRemarkBuilder, SUBSTRATE_REFERENCE_HARDWARE, +}; use futures::future::TryFutureExt; use log::info; use polkadot_service::{ self, - benchmarking::{benchmark_inherent_data, RemarkBuilder, TransferKeepAliveBuilder}, + benchmarking::{benchmark_inherent_data, TransferKeepAliveBuilder}, HeaderBackend, IdentifyVariant, }; #[cfg(feature = "pyroscope")] @@ -391,44 +393,40 @@ pub fn run() -> Result<()> { cmd.run(client.clone()).map_err(Error::SubstrateCli) }), - // These commands are very similar and can be handled in nearly the same way. - BenchmarkCmd::Extrinsic(_) | BenchmarkCmd::Overhead(_) => - runner.sync_run(|mut config| { - let (client, _, _, _) = polkadot_service::new_chain_ops(&mut config)?; - let header = client.header(client.info().genesis_hash).unwrap().unwrap(); - let inherent_data = benchmark_inherent_data(header) - .map_err(|e| format!("generating inherent data: {:?}", e))?; - let remark_builder = - RemarkBuilder::new(client.clone(), config.chain_spec.identify_chain()); - - match cmd { - BenchmarkCmd::Extrinsic(cmd) => { - let tka_builder = TransferKeepAliveBuilder::new( - client.clone(), - Sr25519Keyring::Alice.to_account_id(), - config.chain_spec.identify_chain(), - ); - - let ext_factory = ExtrinsicFactory(vec![ - Box::new(remark_builder), - Box::new(tka_builder), - ]); - - cmd.run(client.clone(), inherent_data, Vec::new(), &ext_factory) - .map_err(Error::SubstrateCli) - }, - BenchmarkCmd::Overhead(cmd) => cmd - .run( - config, - client.clone(), - inherent_data, - Vec::new(), - &remark_builder, - ) - .map_err(Error::SubstrateCli), - _ => unreachable!("Ensured by the outside match; qed"), - } - }), + BenchmarkCmd::Overhead(cmd) => runner.sync_run(|config| { + if cmd.params.runtime.is_some() { + return Err(sc_cli::Error::Input( + "Polkadot binary does not support `--runtime` flag for `benchmark overhead`. Please provide a chain spec or use the `frame-omni-bencher`." + .into(), + ) + .into()) + } + + cmd.run_with_default_builder_and_spec::( + Some(config.chain_spec), + ) + .map_err(Error::SubstrateCli) + }), + BenchmarkCmd::Extrinsic(cmd) => runner.sync_run(|mut config| { + let (client, _, _, _) = polkadot_service::new_chain_ops(&mut config)?; + let header = client.header(client.info().genesis_hash).unwrap().unwrap(); + let inherent_data = benchmark_inherent_data(header) + .map_err(|e| format!("generating inherent data: {:?}", e))?; + + let remark_builder = SubstrateRemarkBuilder::new_from_client(client.clone())?; + + let tka_builder = TransferKeepAliveBuilder::new( + client.clone(), + Sr25519Keyring::Alice.to_account_id(), + config.chain_spec.identify_chain(), + ); + + let ext_factory = + ExtrinsicFactory(vec![Box::new(remark_builder), Box::new(tka_builder)]); + + cmd.run(client.clone(), inherent_data, Vec::new(), &ext_factory) + .map_err(Error::SubstrateCli) + }), BenchmarkCmd::Pallet(cmd) => { set_default_ss58_version(chain_spec); diff --git a/polkadot/node/service/src/benchmarking.rs b/polkadot/node/service/src/benchmarking.rs index 186bea3960e8..0cf16edc03cc 100644 --- a/polkadot/node/service/src/benchmarking.rs +++ b/polkadot/node/service/src/benchmarking.rs @@ -79,53 +79,6 @@ macro_rules! identify_chain { }; } -/// Generates `System::Remark` extrinsics for the benchmarks. -/// -/// Note: Should only be used for benchmarking. -pub struct RemarkBuilder { - client: Arc, - chain: Chain, -} - -impl RemarkBuilder { - /// Creates a new [`Self`] from the given client. - pub fn new(client: Arc, chain: Chain) -> Self { - Self { client, chain } - } -} - -impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder { - fn pallet(&self) -> &str { - "system" - } - - fn extrinsic(&self) -> &str { - "remark" - } - - fn build(&self, nonce: u32) -> std::result::Result { - // We apply the extrinsic directly, so let's take some random period. - let period = 128; - let genesis = self.client.usage_info().chain.best_hash; - let signer = Sr25519Keyring::Bob.pair(); - let current_block = 0; - - identify_chain! { - self.chain, - nonce, - current_block, - period, - genesis, - signer, - { - runtime::RuntimeCall::System( - runtime::SystemCall::remark { remark: vec![] } - ) - }, - } - } -} - /// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks. /// /// Note: Should only be used for benchmarking. diff --git a/polkadot/tests/benchmark_overhead.rs b/polkadot/tests/benchmark_overhead.rs index b0912225347d..51f507450f38 100644 --- a/polkadot/tests/benchmark_overhead.rs +++ b/polkadot/tests/benchmark_overhead.rs @@ -29,14 +29,6 @@ fn benchmark_overhead_works() { } } -/// `benchmark overhead` rejects all non-dev runtimes. -#[test] -fn benchmark_overhead_rejects_non_dev_runtimes() { - for runtime in RUNTIMES.into_iter() { - assert!(benchmark_overhead(runtime).is_err()); - } -} - fn benchmark_overhead(runtime: &str) -> Result<(), String> { let tmp_dir = tempdir().expect("could not create a temp dir"); let base_path = tmp_dir.path(); diff --git a/prdoc/pr_5891.prdoc b/prdoc/pr_5891.prdoc new file mode 100644 index 000000000000..4f8252628eb4 --- /dev/null +++ b/prdoc/pr_5891.prdoc @@ -0,0 +1,33 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Add benchmark overhead command to frame-omni-bencher + +doc: + - audience: Runtime Dev + description: | + This adds the benchmark overhead command to the `frame-omni-bencher` library. This allows + para- and relay chain teams to generate extrinsic and block base weights. + +crates: + - name: sc-chain-spec + bump: minor + - name: polkadot-service + bump: major + - name: frame-benchmarking-cli + bump: major + - name: cumulus-client-parachain-inherent + bump: patch + - name: polkadot-cli + bump: patch + - name: polkadot-omni-node-lib + bump: patch + - name: polkadot-omni-node + bump: patch + - name: polkadot-parachain-bin + bump: patch + - name: polkadot + bump: patch + - name: frame-omni-bencher + bump: minor + diff --git a/substrate/bin/node/cli/src/command.rs b/substrate/bin/node/cli/src/command.rs index 51fbf0904cf8..2910002e5b27 100644 --- a/substrate/bin/node/cli/src/command.rs +++ b/substrate/bin/node/cli/src/command.rs @@ -136,11 +136,12 @@ pub fn run() -> Result<()> { let ext_builder = RemarkBuilder::new(partial.client.clone()); cmd.run( - config, + config.chain_spec.name().into(), partial.client, inherent_benchmark_data()?, Vec::new(), &ext_builder, + false, ) }, BenchmarkCmd::Extrinsic(cmd) => { diff --git a/substrate/client/chain-spec/src/genesis_block.rs b/substrate/client/chain-spec/src/genesis_block.rs index 3c7b9f64dcd6..3c5bf47c3fe8 100644 --- a/substrate/client/chain-spec/src/genesis_block.rs +++ b/substrate/client/chain-spec/src/genesis_block.rs @@ -108,6 +108,16 @@ impl, E: RuntimeVersionOf> GenesisBlockBuilder< ) -> sp_blockchain::Result { let genesis_storage = build_genesis_storage.build_storage().map_err(sp_blockchain::Error::Storage)?; + Self::new_with_storage(genesis_storage, commit_genesis_state, backend, executor) + } + + /// Constructs a new instance of [`GenesisBlockBuilder`] using provided storage. + pub fn new_with_storage( + genesis_storage: Storage, + commit_genesis_state: bool, + backend: Arc, + executor: E, + ) -> sp_blockchain::Result { Ok(Self { genesis_storage, commit_genesis_state, diff --git a/substrate/utils/frame/benchmarking-cli/Cargo.toml b/substrate/utils/frame/benchmarking-cli/Cargo.toml index ee5522f5bc04..8a4a06b1b40a 100644 --- a/substrate/utils/frame/benchmarking-cli/Cargo.toml +++ b/substrate/utils/frame/benchmarking-cli/Cargo.toml @@ -41,6 +41,7 @@ sc-cli = { workspace = true } sc-client-api = { workspace = true, default-features = true } sc-client-db = { workspace = true } sc-executor = { workspace = true, default-features = true } +sc-executor-common = { workspace = true } sc-service = { workspace = true } sc-sysinfo = { workspace = true, default-features = true } sp-api = { workspace = true, default-features = true } @@ -51,13 +52,30 @@ sp-externalities = { workspace = true, default-features = true } sp-genesis-builder = { workspace = true, default-features = true } sp-inherents = { workspace = true, default-features = true } sp-keystore = { workspace = true, default-features = true } +sp-crypto-hashing = { workspace = true, default-features = true } sp-runtime = { workspace = true, default-features = true } sp-state-machine = { workspace = true, default-features = true } sp-storage = { workspace = true, default-features = true } sp-trie = { workspace = true, default-features = true } +sp-block-builder = { workspace = true, default-features = true } +sp-transaction-pool = { workspace = true, default-features = true } +sp-version = { workspace = true, default-features = true } +sp-timestamp = { workspace = true, default-features = true } sp-io = { workspace = true, default-features = true } sp-wasm-interface = { workspace = true, default-features = true } +subxt = { workspace = true, features = ["native"] } +subxt-signer = { workspace = true, features = ["unstable-eth"] } +cumulus-primitives-proof-size-hostfunction = { workspace = true, default-features = true } +cumulus-client-parachain-inherent = { workspace = true, default-features = true } +polkadot-parachain-primitives = { workspace = true, default-features = true } +polkadot-primitives = { workspace = true, default-features = true } gethostname = { workspace = true } +hex = { workspace = true, default-features = true } + +[dev-dependencies] +cumulus-test-runtime = { workspace = true, default-features = true } +substrate-test-runtime = { workspace = true, default-features = true } +westend-runtime = { workspace = true, default-features = true } [features] default = ["rocksdb"] @@ -65,8 +83,11 @@ runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", + "polkadot-parachain-primitives/runtime-benchmarks", + "polkadot-primitives/runtime-benchmarks", "sc-client-db/runtime-benchmarks", "sc-service/runtime-benchmarks", "sp-runtime/runtime-benchmarks", + "westend-runtime/runtime-benchmarks", ] rocksdb = ["sc-cli/rocksdb", "sc-client-db/rocksdb"] diff --git a/substrate/utils/frame/benchmarking-cli/src/extrinsic/bench.rs b/substrate/utils/frame/benchmarking-cli/src/extrinsic/bench.rs index f0a7436dc729..0693db0dbbdd 100644 --- a/substrate/utils/frame/benchmarking-cli/src/extrinsic/bench.rs +++ b/substrate/utils/frame/benchmarking-cli/src/extrinsic/bench.rs @@ -17,7 +17,7 @@ //! Contains the core benchmarking logic. -use sc_block_builder::{BlockBuilderApi, BlockBuilderBuilder}; +use sc_block_builder::{BlockBuilderApi, BlockBuilderBuilder, BuiltBlock}; use sc_cli::{Error, Result}; use sc_client_api::UsageProvider; use sp_api::{ApiExt, CallApiAt, Core, ProvideRuntimeApi}; @@ -31,14 +31,15 @@ use sp_runtime::{ Digest, DigestItem, OpaqueExtrinsic, }; +use super::ExtrinsicBuilder; +use crate::shared::{StatSelect, Stats}; use clap::Args; +use codec::Encode; use log::info; use serde::Serialize; +use sp_trie::proof_size_extension::ProofSizeExt; use std::{marker::PhantomData, sync::Arc, time::Instant}; -use super::ExtrinsicBuilder; -use crate::shared::{StatSelect, Stats}; - /// Parameters to configure an *overhead* benchmark. #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] pub struct BenchmarkParams { @@ -66,6 +67,7 @@ pub(crate) struct Benchmark { params: BenchmarkParams, inherent_data: sp_inherents::InherentData, digest_items: Vec, + record_proof: bool, _p: PhantomData, } @@ -84,15 +86,19 @@ where params: BenchmarkParams, inherent_data: sp_inherents::InherentData, digest_items: Vec, + record_proof: bool, ) -> Self { - Self { client, params, inherent_data, digest_items, _p: PhantomData } + Self { client, params, inherent_data, digest_items, record_proof, _p: PhantomData } } /// Benchmark a block with only inherents. - pub fn bench_block(&self) -> Result { - let (block, _) = self.build_block(None)?; + /// + /// Returns the Ref time stats and the proof size. + pub fn bench_block(&self) -> Result<(Stats, u64)> { + let (block, _, proof_size) = self.build_block(None)?; let record = self.measure_block(&block)?; - Stats::new(&record) + + Ok((Stats::new(&record)?, proof_size)) } /// Benchmark the time of an extrinsic in a full block. @@ -100,13 +106,14 @@ where /// First benchmarks an empty block, analogous to `bench_block` and use it as baseline. /// Then benchmarks a full block built with the given `ext_builder` and subtracts the baseline /// from the result. - /// This is necessary to account for the time the inherents use. - pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result { - let (block, _) = self.build_block(None)?; + /// This is necessary to account for the time the inherents use. Returns ref time stats and the + /// proof size. + pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result<(Stats, u64)> { + let (block, _, base_proof_size) = self.build_block(None)?; let base = self.measure_block(&block)?; let base_time = Stats::new(&base)?.select(StatSelect::Average); - let (block, num_ext) = self.build_block(Some(ext_builder))?; + let (block, num_ext, proof_size) = self.build_block(Some(ext_builder))?; let num_ext = num_ext.ok_or_else(|| Error::Input("Block was empty".into()))?; let mut records = self.measure_block(&block)?; @@ -117,23 +124,24 @@ where *r = ((*r as f64) / (num_ext as f64)).ceil() as u64; } - Stats::new(&records) + Ok((Stats::new(&records)?, proof_size.saturating_sub(base_proof_size))) } /// Builds a block with some optional extrinsics. /// /// Returns the block and the number of extrinsics in the block - /// that are not inherents. + /// that are not inherents together with the proof size. /// Returns a block with only inherents if `ext_builder` is `None`. fn build_block( &self, ext_builder: Option<&dyn ExtrinsicBuilder>, - ) -> Result<(Block, Option)> { + ) -> Result<(Block, Option, u64)> { let chain = self.client.usage_info().chain; let mut builder = BlockBuilderBuilder::new(&*self.client) .on_parent_block(chain.best_hash) .with_parent_block_number(chain.best_number) .with_inherent_digests(Digest { logs: self.digest_items.clone() }) + .with_proof_recording(self.record_proof) .build()?; // Create and insert the inherents. @@ -142,34 +150,42 @@ where builder.push(inherent)?; } - // Return early if `ext_builder` is `None`. - let ext_builder = if let Some(ext_builder) = ext_builder { - ext_builder - } else { - return Ok((builder.build()?.block, None)) + let num_ext = match ext_builder { + Some(ext_builder) => { + // Put as many extrinsics into the block as possible and count them. + info!("Building block, this takes some time..."); + let mut num_ext = 0; + for nonce in 0..self.max_ext_per_block() { + let ext = ext_builder.build(nonce)?; + match builder.push(ext.clone()) { + Ok(()) => {}, + Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( + InvalidTransaction::ExhaustsResources, + )))) => break, // Block is full + Err(e) => return Err(Error::Client(e)), + } + num_ext += 1; + } + if num_ext == 0 { + return Err("A Block must hold at least one extrinsic".into()) + } + info!("Extrinsics per block: {}", num_ext); + Some(num_ext) + }, + None => None, }; - // Put as many extrinsics into the block as possible and count them. - info!("Building block, this takes some time..."); - let mut num_ext = 0; - for nonce in 0..self.max_ext_per_block() { - let ext = ext_builder.build(nonce)?; - match builder.push(ext.clone()) { - Ok(()) => {}, - Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( - InvalidTransaction::ExhaustsResources, - )))) => break, // Block is full - Err(e) => return Err(Error::Client(e)), - } - num_ext += 1; - } - if num_ext == 0 { - return Err("A Block must hold at least one extrinsic".into()) - } - info!("Extrinsics per block: {}", num_ext); - let block = builder.build()?.block; - - Ok((block, Some(num_ext))) + let BuiltBlock { block, proof, .. } = builder.build()?; + + Ok(( + block, + num_ext, + proof + .map(|p| p.encoded_size()) + .unwrap_or(0) + .try_into() + .map_err(|_| "Proof size is too large".to_string())?, + )) } /// Measures the time that it take to execute a block or an extrinsic. @@ -177,27 +193,35 @@ where let mut record = BenchRecord::new(); let genesis = self.client.info().genesis_hash; + let measure_block = || -> Result { + let block = block.clone(); + let mut runtime_api = self.client.runtime_api(); + if self.record_proof { + runtime_api.record_proof(); + let recorder = runtime_api + .proof_recorder() + .expect("Proof recording is enabled in the line above; qed."); + runtime_api.register_extension(ProofSizeExt::new(recorder)); + } + let start = Instant::now(); + + runtime_api + .execute_block(genesis, block) + .map_err(|e| Error::Client(RuntimeApiError(e)))?; + + Ok(start.elapsed().as_nanos()) + }; + info!("Running {} warmups...", self.params.warmup); for _ in 0..self.params.warmup { - self.client - .runtime_api() - .execute_block(genesis, block.clone()) - .map_err(|e| Error::Client(RuntimeApiError(e)))?; + let _ = measure_block()?; } info!("Executing block {} times", self.params.repeat); // Interesting part here: // Execute a block multiple times and record each execution time. for _ in 0..self.params.repeat { - let block = block.clone(); - let runtime_api = self.client.runtime_api(); - let start = Instant::now(); - - runtime_api - .execute_block(genesis, block) - .map_err(|e| Error::Client(RuntimeApiError(e)))?; - - let elapsed = start.elapsed().as_nanos(); + let elapsed = measure_block()?; record.push(elapsed as u64); } diff --git a/substrate/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs b/substrate/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs index 99c0230617cb..949b8211556a 100644 --- a/substrate/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs +++ b/substrate/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs @@ -118,7 +118,8 @@ impl ExtrinsicCmd { return Err("Unknown pallet or extrinsic. Use --list for a complete list.".into()), }; - let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, digest_items); + let bench = + Benchmark::new(client, self.params.bench.clone(), inherent_data, digest_items, false); let stats = bench.bench_extrinsic(ext_builder)?; info!( "Executing a {}::{} extrinsic takes[ns]:\n{:?}", diff --git a/substrate/utils/frame/benchmarking-cli/src/lib.rs b/substrate/utils/frame/benchmarking-cli/src/lib.rs index 0ef2c299de63..1e8642e54d70 100644 --- a/substrate/utils/frame/benchmarking-cli/src/lib.rs +++ b/substrate/utils/frame/benchmarking-cli/src/lib.rs @@ -28,7 +28,11 @@ mod storage; pub use block::BlockCmd; pub use extrinsic::{ExtrinsicBuilder, ExtrinsicCmd, ExtrinsicFactory}; pub use machine::{MachineCmd, SUBSTRATE_REFERENCE_HARDWARE}; -pub use overhead::OverheadCmd; +pub use overhead::{ + remark_builder::{DynamicRemarkBuilder, SubstrateRemarkBuilder}, + runtime_utilities::fetch_latest_metadata_from_code_blob, + OpaqueBlock, OverheadCmd, +}; pub use pallet::PalletCmd; pub use sc_service::BasePath; pub use storage::StorageCmd; diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/cmd.rs b/substrate/utils/frame/benchmarking-cli/src/overhead/cmd.rs deleted file mode 100644 index 4fa8cecf2f7d..000000000000 --- a/substrate/utils/frame/benchmarking-cli/src/overhead/cmd.rs +++ /dev/null @@ -1,175 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Contains the [`OverheadCmd`] as entry point for the CLI to execute -//! the *overhead* benchmarks. - -use sc_block_builder::BlockBuilderApi; -use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams}; -use sc_client_api::UsageProvider; -use sc_service::Configuration; -use sp_api::{ApiExt, CallApiAt, ProvideRuntimeApi}; -use sp_runtime::{traits::Block as BlockT, DigestItem, OpaqueExtrinsic}; - -use clap::{Args, Parser}; -use log::info; -use serde::Serialize; -use std::{fmt::Debug, path::PathBuf, sync::Arc}; - -use crate::{ - extrinsic::{ - bench::{Benchmark, BenchmarkParams as ExtrinsicBenchmarkParams}, - ExtrinsicBuilder, - }, - overhead::template::TemplateData, - shared::{HostInfoParams, WeightParams}, -}; - -/// Benchmark the execution overhead per-block and per-extrinsic. -#[derive(Debug, Parser)] -pub struct OverheadCmd { - #[allow(missing_docs)] - #[clap(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[clap(flatten)] - pub import_params: ImportParams, - - #[allow(missing_docs)] - #[clap(flatten)] - pub params: OverheadParams, -} - -/// Configures the benchmark, the post-processing and weight generation. -#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] -pub struct OverheadParams { - #[allow(missing_docs)] - #[clap(flatten)] - pub weight: WeightParams, - - #[allow(missing_docs)] - #[clap(flatten)] - pub bench: ExtrinsicBenchmarkParams, - - #[allow(missing_docs)] - #[clap(flatten)] - pub hostinfo: HostInfoParams, - - /// Add a header to the generated weight output file. - /// - /// Good for adding LICENSE headers. - #[arg(long, value_name = "PATH")] - pub header: Option, - - /// Enable the Trie cache. - /// - /// This should only be used for performance analysis and not for final results. - #[arg(long)] - pub enable_trie_cache: bool, -} - -/// Type of a benchmark. -#[derive(Serialize, Clone, PartialEq, Copy)] -pub(crate) enum BenchmarkType { - /// Measure the per-extrinsic execution overhead. - Extrinsic, - /// Measure the per-block execution overhead. - Block, -} - -impl OverheadCmd { - /// Measure the per-block and per-extrinsic execution overhead. - /// - /// Writes the results to console and into two instances of the - /// `weights.hbs` template, one for each benchmark. - pub fn run( - &self, - cfg: Configuration, - client: Arc, - inherent_data: sp_inherents::InherentData, - digest_items: Vec, - ext_builder: &dyn ExtrinsicBuilder, - ) -> Result<()> - where - Block: BlockT, - C: ProvideRuntimeApi - + CallApiAt - + UsageProvider - + sp_blockchain::HeaderBackend, - C::Api: ApiExt + BlockBuilderApi, - { - if ext_builder.pallet() != "system" || ext_builder.extrinsic() != "remark" { - return Err(format!("The extrinsic builder is required to build `System::Remark` extrinsics but builds `{}` extrinsics instead", ext_builder.name()).into()); - } - let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, digest_items); - - // per-block execution overhead - { - let stats = bench.bench_block()?; - info!("Per-block execution overhead [ns]:\n{:?}", stats); - let template = TemplateData::new(BenchmarkType::Block, &cfg, &self.params, &stats)?; - template.write(&self.params.weight.weight_path)?; - } - // per-extrinsic execution overhead - { - let stats = bench.bench_extrinsic(ext_builder)?; - info!("Per-extrinsic execution overhead [ns]:\n{:?}", stats); - let template = TemplateData::new(BenchmarkType::Extrinsic, &cfg, &self.params, &stats)?; - template.write(&self.params.weight.weight_path)?; - } - - Ok(()) - } -} - -impl BenchmarkType { - /// Short name of the benchmark type. - pub(crate) fn short_name(&self) -> &'static str { - match self { - Self::Extrinsic => "extrinsic", - Self::Block => "block", - } - } - - /// Long name of the benchmark type. - pub(crate) fn long_name(&self) -> &'static str { - match self { - Self::Extrinsic => "ExtrinsicBase", - Self::Block => "BlockExecution", - } - } -} - -// Boilerplate -impl CliConfiguration for OverheadCmd { - fn shared_params(&self) -> &SharedParams { - &self.shared_params - } - - fn import_params(&self) -> Option<&ImportParams> { - Some(&self.import_params) - } - - fn trie_cache_maximum_size(&self) -> Result> { - if self.params.enable_trie_cache { - Ok(self.import_params().map(|x| x.trie_cache_maximum_size()).unwrap_or_default()) - } else { - Ok(None) - } - } -} diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/command.rs b/substrate/utils/frame/benchmarking-cli/src/overhead/command.rs new file mode 100644 index 000000000000..8102f14b4f4b --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/command.rs @@ -0,0 +1,774 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the [`OverheadCmd`] as entry point for the CLI to execute +//! the *overhead* benchmarks. + +use super::runtime_utilities::*; +use crate::{ + extrinsic::{ + bench::{Benchmark, BenchmarkParams as ExtrinsicBenchmarkParams}, + ExtrinsicBuilder, + }, + overhead::{ + command::ChainType::{Parachain, Relaychain, Unknown}, + fake_runtime_api, + remark_builder::SubstrateRemarkBuilder, + template::TemplateData, + }, + shared::{ + genesis_state, + genesis_state::{GenesisStateHandler, SpecGenesisSource}, + HostInfoParams, WeightParams, + }, +}; +use clap::{error::ErrorKind, Args, CommandFactory, Parser}; +use codec::Encode; +use cumulus_client_parachain_inherent::MockValidationDataInherentDataProvider; +use fake_runtime_api::RuntimeApi as FakeRuntimeApi; +use frame_support::Deserialize; +use genesis_state::WARN_SPEC_GENESIS_CTOR; +use log::info; +use polkadot_parachain_primitives::primitives::Id as ParaId; +use sc_block_builder::BlockBuilderApi; +use sc_chain_spec::{ChainSpec, ChainSpecExtension, GenesisBlockBuilder}; +use sc_cli::{CliConfiguration, Database, ImportParams, Result, SharedParams}; +use sc_client_api::{execution_extensions::ExecutionExtensions, UsageProvider}; +use sc_client_db::{BlocksPruning, DatabaseSettings}; +use sc_executor::WasmExecutor; +use sc_service::{new_client, new_db_backend, BasePath, ClientConfig, TFullClient, TaskManager}; +use serde::Serialize; +use serde_json::{json, Value}; +use sp_api::{ApiExt, CallApiAt, Core, ProvideRuntimeApi}; +use sp_blockchain::HeaderBackend; +use sp_core::H256; +use sp_inherents::{InherentData, InherentDataProvider}; +use sp_runtime::{ + generic, + traits::{BlakeTwo256, Block as BlockT}, + DigestItem, OpaqueExtrinsic, +}; +use sp_storage::Storage; +use sp_wasm_interface::HostFunctions; +use std::{ + fmt::{Debug, Display, Formatter}, + fs, + path::PathBuf, + sync::Arc, +}; +use subxt::{client::RuntimeVersion, ext::futures, Metadata}; + +const DEFAULT_PARA_ID: u32 = 100; +const LOG_TARGET: &'static str = "polkadot_sdk_frame::benchmark::overhead"; + +/// Benchmark the execution overhead per-block and per-extrinsic. +#[derive(Debug, Parser)] +pub struct OverheadCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub import_params: ImportParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: OverheadParams, +} + +/// Configures the benchmark, the post-processing and weight generation. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct OverheadParams { + #[allow(missing_docs)] + #[clap(flatten)] + pub weight: WeightParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub bench: ExtrinsicBenchmarkParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub hostinfo: HostInfoParams, + + /// Add a header to the generated weight output file. + /// + /// Good for adding LICENSE headers. + #[arg(long, value_name = "PATH")] + pub header: Option, + + /// Enable the Trie cache. + /// + /// This should only be used for performance analysis and not for final results. + #[arg(long)] + pub enable_trie_cache: bool, + + /// Optional runtime blob to use instead of the one from the genesis config. + #[arg( + long, + value_name = "PATH", + conflicts_with = "chain", + required_if_eq("genesis_builder", "runtime") + )] + pub runtime: Option, + + /// The preset that we expect to find in the GenesisBuilder runtime API. + /// + /// This can be useful when a runtime has a dedicated benchmarking preset instead of using the + /// default one. + #[arg(long, default_value = sp_genesis_builder::DEV_RUNTIME_PRESET)] + pub genesis_builder_preset: String, + + /// How to construct the genesis state. + /// + /// Can be used together with `--chain` to determine whether the + /// genesis state should be initialized with the values from the + /// provided chain spec or a runtime-provided genesis preset. + #[arg(long, value_enum, alias = "genesis-builder-policy")] + pub genesis_builder: Option, + + /// Parachain Id to use for parachains. If not specified, the benchmark code will choose + /// a para-id and patch the state accordingly. + #[arg(long)] + pub para_id: Option, +} + +/// How the genesis state for benchmarking should be built. +#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy, Serialize)] +#[clap(rename_all = "kebab-case")] +pub enum GenesisBuilderPolicy { + /// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API. + /// This will use the `development` preset by default. + Runtime, + /// Use the runtime from the Spec file to build the genesis state. + SpecRuntime, + /// Use the spec file to build the genesis state. This fails when there is no spec. + #[value(alias = "spec")] + SpecGenesis, +} + +/// Type of a benchmark. +#[derive(Serialize, Clone, PartialEq, Copy)] +pub(crate) enum BenchmarkType { + /// Measure the per-extrinsic execution overhead. + Extrinsic, + /// Measure the per-block execution overhead. + Block, +} + +/// Hostfunctions that are typically used by parachains. +pub type ParachainHostFunctions = ( + cumulus_primitives_proof_size_hostfunction::storage_proof_size::HostFunctions, + sp_io::SubstrateHostFunctions, +); + +pub type BlockNumber = u32; + +/// Typical block header. +pub type Header = generic::Header; + +/// Typical block type using `OpaqueExtrinsic`. +pub type OpaqueBlock = generic::Block; + +/// Client type used throughout the benchmarking code. +type OverheadClient = TFullClient>; + +/// Creates inherent data for a given parachain ID. +/// +/// This function constructs the inherent data required for block execution, +/// including the relay chain state and validation data. Not all of these +/// inherents are required for every chain. The runtime will pick the ones +/// it requires based on their identifier. +fn create_inherent_data + HeaderBackend, Block: BlockT>( + client: &Arc, + chain_type: &ChainType, +) -> InherentData { + let genesis = client.usage_info().chain.best_hash; + let header = client.header(genesis).unwrap().unwrap(); + + let mut inherent_data = InherentData::new(); + + // Para inherent can only makes sense when we are handling a parachain. + if let Parachain(para_id) = chain_type { + let parachain_validation_data_provider = MockValidationDataInherentDataProvider::<()> { + para_id: ParaId::from(*para_id), + current_para_block_head: Some(header.encode().into()), + relay_offset: 1, + ..Default::default() + }; + let _ = futures::executor::block_on( + parachain_validation_data_provider.provide_inherent_data(&mut inherent_data), + ); + } + + // Parachain inherent that is used on relay chains to perform parachain validation. + let para_inherent = polkadot_primitives::InherentData { + bitfields: Vec::new(), + backed_candidates: Vec::new(), + disputes: Vec::new(), + parent_header: header, + }; + + // Timestamp inherent that is very common in substrate chains. + let timestamp = sp_timestamp::InherentDataProvider::new(std::time::Duration::default().into()); + + let _ = futures::executor::block_on(timestamp.provide_inherent_data(&mut inherent_data)); + let _ = + inherent_data.put_data(polkadot_primitives::PARACHAINS_INHERENT_IDENTIFIER, ¶_inherent); + + inherent_data +} + +/// Identifies what kind of chain we are dealing with. +/// +/// Chains containing the `ParachainSystem` and `ParachainInfo` pallet are considered parachains. +/// Chains containing the `ParaInherent` pallet are considered relay chains. +fn identify_chain(metadata: &Metadata, para_id: Option) -> ChainType { + let parachain_info_exists = metadata.pallet_by_name("ParachainInfo").is_some(); + let parachain_system_exists = metadata.pallet_by_name("ParachainSystem").is_some(); + let para_inherent_exists = metadata.pallet_by_name("ParaInherent").is_some(); + + log::debug!("{} ParachainSystem", if parachain_system_exists { "✅" } else { "❌" }); + log::debug!("{} ParachainInfo", if parachain_info_exists { "✅" } else { "❌" }); + log::debug!("{} ParaInherent", if para_inherent_exists { "✅" } else { "❌" }); + + let chain_type = if parachain_system_exists && parachain_info_exists { + Parachain(para_id.unwrap_or(DEFAULT_PARA_ID)) + } else if para_inherent_exists { + Relaychain + } else { + Unknown + }; + + log::info!(target: LOG_TARGET, "Identified Chain type from metadata: {}", chain_type); + + chain_type +} + +#[derive(Deserialize, Serialize, Clone, ChainSpecExtension)] +pub struct ParachainExtension { + /// The id of the Parachain. + pub para_id: Option, +} + +impl OverheadCmd { + fn state_handler_from_cli( + &self, + chain_spec_from_api: Option>, + ) -> Result<(GenesisStateHandler, Option)> { + let genesis_builder_to_source = || match self.params.genesis_builder { + Some(GenesisBuilderPolicy::Runtime) | Some(GenesisBuilderPolicy::SpecRuntime) => + SpecGenesisSource::Runtime(self.params.genesis_builder_preset.clone()), + Some(GenesisBuilderPolicy::SpecGenesis) | None => { + log::warn!(target: LOG_TARGET, "{WARN_SPEC_GENESIS_CTOR}"); + SpecGenesisSource::SpecJson + }, + }; + + // First handle chain-spec passed in via API parameter. + if let Some(chain_spec) = chain_spec_from_api { + log::debug!(target: LOG_TARGET, "Initializing state handler with chain-spec from API: {:?}", chain_spec); + + let source = genesis_builder_to_source(); + return Ok((GenesisStateHandler::ChainSpec(chain_spec, source), self.params.para_id)) + }; + + // Handle chain-spec passed in via CLI. + if let Some(chain_spec_path) = &self.shared_params.chain { + log::debug!(target: LOG_TARGET, + "Initializing state handler with chain-spec from path: {:?}", + chain_spec_path + ); + let (chain_spec, para_id_from_chain_spec) = + genesis_state::chain_spec_from_path::(chain_spec_path.to_string().into())?; + + let source = genesis_builder_to_source(); + + return Ok(( + GenesisStateHandler::ChainSpec(chain_spec, source), + self.params.para_id.or(para_id_from_chain_spec), + )) + }; + + // Check for runtimes. In general, we make sure that `--runtime` and `--chain` are + // incompatible on the CLI level. + if let Some(runtime_path) = &self.params.runtime { + log::debug!(target: LOG_TARGET, "Initializing state handler with runtime from path: {:?}", runtime_path); + + let runtime_blob = fs::read(runtime_path)?; + return Ok(( + GenesisStateHandler::Runtime( + runtime_blob, + Some(self.params.genesis_builder_preset.clone()), + ), + self.params.para_id, + )) + }; + + Err("Neither a runtime nor a chain-spec were specified".to_string().into()) + } + + fn check_args( + &self, + chain_spec: &Option>, + ) -> std::result::Result<(), (ErrorKind, String)> { + if chain_spec.is_none() && + self.params.runtime.is_none() && + self.shared_params.chain.is_none() + { + return Err(( + ErrorKind::MissingRequiredArgument, + "Provide either a runtime via `--runtime` or a chain spec via `--chain`" + .to_string(), + )) + } + + match self.params.genesis_builder { + Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::SpecRuntime) => + if chain_spec.is_none() && self.shared_params.chain.is_none() { + return Err(( + ErrorKind::MissingRequiredArgument, + "Provide a chain spec via `--chain`.".to_string(), + )) + }, + _ => {}, + }; + Ok(()) + } + + /// Run the overhead benchmark with the default extrinsic builder. + /// + /// This will use [SubstrateRemarkBuilder] to build the extrinsic. It is + /// designed to match common configurations found in substrate chains. + pub fn run_with_default_builder_and_spec( + &self, + chain_spec: Option>, + ) -> Result<()> + where + Block: BlockT, + ExtraHF: HostFunctions, + { + self.run_with_extrinsic_builder_and_spec::( + Box::new(|metadata, hash, version| { + let genesis = subxt::utils::H256::from(hash.to_fixed_bytes()); + Box::new(SubstrateRemarkBuilder::new(metadata, genesis, version)) as Box<_> + }), + chain_spec, + ) + } + + /// Run the benchmark overhead command. + /// + /// The provided [ExtrinsicBuilder] will be used to build extrinsics for + /// block-building. It is expected that the provided implementation builds + /// a `System::remark` extrinsic. + pub fn run_with_extrinsic_builder_and_spec( + &self, + ext_builder_provider: Box< + dyn FnOnce(Metadata, Block::Hash, RuntimeVersion) -> Box, + >, + chain_spec: Option>, + ) -> Result<()> + where + Block: BlockT, + ExtraHF: HostFunctions, + { + if let Err((error_kind, msg)) = self.check_args(&chain_spec) { + let mut cmd = OverheadCmd::command(); + cmd.error(error_kind, msg).exit(); + }; + + let (state_handler, para_id) = + self.state_handler_from_cli::<(ParachainHostFunctions, ExtraHF)>(chain_spec)?; + + let executor = WasmExecutor::<(ParachainHostFunctions, ExtraHF)>::builder() + .with_allow_missing_host_functions(true) + .build(); + + let metadata = + fetch_latest_metadata_from_code_blob(&executor, state_handler.get_code_bytes()?)?; + + // At this point we know what kind of chain we are dealing with. + let chain_type = identify_chain(&metadata, para_id); + + // If we are dealing with a parachain, make sure that the para id in genesis will + // match what we expect. + let genesis_patcher = match chain_type { + Parachain(para_id) => + Some(Box::new(move |value| patch_genesis(value, Some(para_id))) as Box<_>), + _ => None, + }; + + let client = self.build_client_components::( + state_handler.build_storage::<(ParachainHostFunctions, ExtraHF)>(genesis_patcher)?, + executor, + &chain_type, + )?; + + let inherent_data = create_inherent_data(&client, &chain_type); + + let (ext_builder, runtime_name) = { + let genesis = client.usage_info().chain.best_hash; + let version = client.runtime_api().version(genesis).unwrap(); + let runtime_name = version.spec_name; + let runtime_version = RuntimeVersion { + spec_version: version.spec_version, + transaction_version: version.transaction_version, + }; + + (ext_builder_provider(metadata, genesis, runtime_version), runtime_name) + }; + + self.run( + runtime_name.to_string(), + client, + inherent_data, + Default::default(), + &*ext_builder, + chain_type.requires_proof_recording(), + ) + } + + /// Run the benchmark overhead command. + pub fn run_with_extrinsic_builder( + &self, + ext_builder_provider: Box< + dyn FnOnce(Metadata, Block::Hash, RuntimeVersion) -> Box, + >, + ) -> Result<()> + where + Block: BlockT, + ExtraHF: HostFunctions, + { + self.run_with_extrinsic_builder_and_spec::(ext_builder_provider, None) + } + + fn build_client_components( + &self, + genesis_storage: Storage, + executor: WasmExecutor, + chain_type: &ChainType, + ) -> Result>> + where + Block: BlockT, + HF: HostFunctions, + { + let extensions = ExecutionExtensions::new(None, Arc::new(executor.clone())); + + let base_path = match &self.shared_params.base_path { + None => BasePath::new_temp_dir()?, + Some(path) => BasePath::from(path.clone()), + }; + + let database_source = self.database_config( + &base_path.path().to_path_buf(), + self.database_cache_size()?.unwrap_or(1024), + self.database()?.unwrap_or(Database::RocksDb), + )?; + + let backend = new_db_backend(DatabaseSettings { + trie_cache_maximum_size: self.trie_cache_maximum_size()?, + state_pruning: None, + blocks_pruning: BlocksPruning::KeepAll, + source: database_source, + })?; + + let genesis_block_builder = GenesisBlockBuilder::new_with_storage( + genesis_storage, + true, + backend.clone(), + executor.clone(), + )?; + + let tokio_runtime = sc_cli::build_runtime()?; + let task_manager = TaskManager::new(tokio_runtime.handle().clone(), None) + .map_err(|_| "Unable to build task manager")?; + + let client: Arc> = Arc::new(new_client( + backend.clone(), + executor, + genesis_block_builder, + Default::default(), + Default::default(), + extensions, + Box::new(task_manager.spawn_handle()), + None, + None, + ClientConfig { + offchain_worker_enabled: false, + offchain_indexing_api: false, + wasm_runtime_overrides: None, + no_genesis: false, + wasm_runtime_substitutes: Default::default(), + enable_import_proof_recording: chain_type.requires_proof_recording(), + }, + )?); + + Ok(client) + } + + /// Measure the per-block and per-extrinsic execution overhead. + /// + /// Writes the results to console and into two instances of the + /// `weights.hbs` template, one for each benchmark. + pub fn run( + &self, + chain_name: String, + client: Arc, + inherent_data: sp_inherents::InherentData, + digest_items: Vec, + ext_builder: &dyn ExtrinsicBuilder, + should_record_proof: bool, + ) -> Result<()> + where + Block: BlockT, + C: ProvideRuntimeApi + + CallApiAt + + UsageProvider + + sp_blockchain::HeaderBackend, + C::Api: ApiExt + BlockBuilderApi, + { + if ext_builder.pallet() != "system" || ext_builder.extrinsic() != "remark" { + return Err(format!("The extrinsic builder is required to build `System::Remark` extrinsics but builds `{}` extrinsics instead", ext_builder.name()).into()); + } + + let bench = Benchmark::new( + client, + self.params.bench.clone(), + inherent_data, + digest_items, + should_record_proof, + ); + + // per-block execution overhead + { + let (stats, proof_size) = bench.bench_block()?; + info!(target: LOG_TARGET, "Per-block execution overhead [ns]:\n{:?}", stats); + let template = TemplateData::new( + BenchmarkType::Block, + &chain_name, + &self.params, + &stats, + proof_size, + )?; + template.write(&self.params.weight.weight_path)?; + } + // per-extrinsic execution overhead + { + let (stats, proof_size) = bench.bench_extrinsic(ext_builder)?; + info!(target: LOG_TARGET, "Per-extrinsic execution overhead [ns]:\n{:?}", stats); + let template = TemplateData::new( + BenchmarkType::Extrinsic, + &chain_name, + &self.params, + &stats, + proof_size, + )?; + template.write(&self.params.weight.weight_path)?; + } + + Ok(()) + } +} + +impl BenchmarkType { + /// Short name of the benchmark type. + pub(crate) fn short_name(&self) -> &'static str { + match self { + Self::Extrinsic => "extrinsic", + Self::Block => "block", + } + } + + /// Long name of the benchmark type. + pub(crate) fn long_name(&self) -> &'static str { + match self { + Self::Extrinsic => "ExtrinsicBase", + Self::Block => "BlockExecution", + } + } +} + +#[derive(Clone, PartialEq, Debug)] +enum ChainType { + Parachain(u32), + Relaychain, + Unknown, +} + +impl Display for ChainType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ChainType::Parachain(id) => write!(f, "Parachain(paraid = {})", id), + ChainType::Relaychain => write!(f, "Relaychain"), + ChainType::Unknown => write!(f, "Unknown"), + } + } +} + +impl ChainType { + fn requires_proof_recording(&self) -> bool { + match self { + Parachain(_) => true, + Relaychain => false, + Unknown => false, + } + } +} + +/// Patch the parachain id into the genesis config. This is necessary since the inherents +/// also contain a parachain id and they need to match. +fn patch_genesis(mut input_value: Value, para_id: Option) -> Value { + // If we identified a parachain we should patch a parachain id into the genesis config. + // This ensures compatibility with the inherents that we provide to successfully build a + // block. + if let Some(para_id) = para_id { + sc_chain_spec::json_patch::merge( + &mut input_value, + json!({ + "parachainInfo": { + "parachainId": para_id, + } + }), + ); + log::debug!(target: LOG_TARGET, "Genesis Config Json"); + log::debug!(target: LOG_TARGET, "{}", input_value); + } + input_value +} + +// Boilerplate +impl CliConfiguration for OverheadCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } + + fn base_path(&self) -> Result> { + Ok(Some(BasePath::new_temp_dir()?)) + } + + fn trie_cache_maximum_size(&self) -> Result> { + if self.params.enable_trie_cache { + Ok(self.import_params().map(|x| x.trie_cache_maximum_size()).unwrap_or_default()) + } else { + Ok(None) + } + } +} + +#[cfg(test)] +mod tests { + use crate::{ + overhead::command::{identify_chain, ChainType, ParachainHostFunctions, DEFAULT_PARA_ID}, + OverheadCmd, + }; + use clap::Parser; + use sc_executor::WasmExecutor; + + #[test] + fn test_chain_type_relaychain() { + let executor: WasmExecutor = WasmExecutor::builder().build(); + let code_bytes = westend_runtime::WASM_BINARY + .expect("To run this test, build the wasm binary of westend-runtime") + .to_vec(); + let metadata = + super::fetch_latest_metadata_from_code_blob(&executor, code_bytes.into()).unwrap(); + let chain_type = identify_chain(&metadata, None); + assert_eq!(chain_type, ChainType::Relaychain); + assert_eq!(chain_type.requires_proof_recording(), false); + } + + #[test] + fn test_chain_type_parachain() { + let executor: WasmExecutor = WasmExecutor::builder().build(); + let code_bytes = cumulus_test_runtime::WASM_BINARY + .expect("To run this test, build the wasm binary of cumulus-test-runtime") + .to_vec(); + let metadata = + super::fetch_latest_metadata_from_code_blob(&executor, code_bytes.into()).unwrap(); + let chain_type = identify_chain(&metadata, Some(100)); + assert_eq!(chain_type, ChainType::Parachain(100)); + assert!(chain_type.requires_proof_recording()); + assert_eq!(identify_chain(&metadata, None), ChainType::Parachain(DEFAULT_PARA_ID)); + } + + #[test] + fn test_chain_type_custom() { + let executor: WasmExecutor = WasmExecutor::builder().build(); + let code_bytes = substrate_test_runtime::WASM_BINARY + .expect("To run this test, build the wasm binary of substrate-test-runtime") + .to_vec(); + let metadata = + super::fetch_latest_metadata_from_code_blob(&executor, code_bytes.into()).unwrap(); + let chain_type = identify_chain(&metadata, None); + assert_eq!(chain_type, ChainType::Unknown); + assert_eq!(chain_type.requires_proof_recording(), false); + } + + fn cli_succeed(args: &[&str]) -> Result<(), clap::Error> { + let cmd = OverheadCmd::try_parse_from(args)?; + assert!(cmd.check_args(&None).is_ok()); + Ok(()) + } + + fn cli_fail(args: &[&str]) { + let cmd = OverheadCmd::try_parse_from(args); + if let Ok(cmd) = cmd { + assert!(cmd.check_args(&None).is_err()); + } + } + + #[test] + fn test_cli_conflicts() -> Result<(), clap::Error> { + // Runtime tests + cli_succeed(&["test", "--runtime", "path/to/runtime", "--genesis-builder", "runtime"])?; + cli_succeed(&["test", "--runtime", "path/to/runtime"])?; + cli_succeed(&[ + "test", + "--runtime", + "path/to/runtime", + "--genesis-builder-preset", + "preset", + ])?; + cli_fail(&["test", "--runtime", "path/to/spec", "--genesis-builder", "spec"]); + cli_fail(&["test", "--runtime", "path/to/spec", "--genesis-builder", "spec-genesis"]); + cli_fail(&["test", "--runtime", "path/to/spec", "--genesis-builder", "spec-runtime"]); + + // Spec tests + cli_succeed(&["test", "--chain", "path/to/spec"])?; + cli_succeed(&["test", "--chain", "path/to/spec", "--genesis-builder", "spec"])?; + cli_succeed(&["test", "--chain", "path/to/spec", "--genesis-builder", "spec-genesis"])?; + cli_succeed(&["test", "--chain", "path/to/spec", "--genesis-builder", "spec-runtime"])?; + cli_fail(&["test", "--chain", "path/to/spec", "--genesis-builder", "none"]); + cli_fail(&["test", "--chain", "path/to/spec", "--genesis-builder", "runtime"]); + cli_fail(&[ + "test", + "--chain", + "path/to/spec", + "--genesis-builder", + "runtime", + "--genesis-builder-preset", + "preset", + ]); + Ok(()) + } +} diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/fake_runtime_api.rs b/substrate/utils/frame/benchmarking-cli/src/overhead/fake_runtime_api.rs new file mode 100644 index 000000000000..653908a5a205 --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/fake_runtime_api.rs @@ -0,0 +1,109 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! A fake runtime struct that allows us to instantiate a client. +//! Has all the required runtime APIs implemented to satisfy trait bounds, +//! but the methods are never called since we use WASM exclusively. + +use sp_core::OpaqueMetadata; +use sp_runtime::{ + generic, + traits::{BlakeTwo256, Block as BlockT}, + transaction_validity::{TransactionSource, TransactionValidity}, + ApplyExtrinsicResult, OpaqueExtrinsic, +}; + +/// Block number +type BlockNumber = u32; +/// Opaque block header type. +type Header = generic::Header; +/// Opaque block type. +type Block = generic::Block; + +#[allow(unused)] +pub struct Runtime; + +sp_api::impl_runtime_apis! { + impl sp_api::Core for Runtime { + fn version() -> sp_version::RuntimeVersion { + unimplemented!() + } + + fn execute_block(_: Block) { + unimplemented!() + } + + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { + unimplemented!() + } + } + + impl sp_api::Metadata for Runtime { + fn metadata() -> OpaqueMetadata { + unimplemented!() + } + + fn metadata_at_version(_: u32) -> Option { + unimplemented!() + } + + fn metadata_versions() -> Vec { + unimplemented!() + } + } + impl sp_block_builder::BlockBuilder for Runtime { + fn apply_extrinsic(_: ::Extrinsic) -> ApplyExtrinsicResult { + unimplemented!() + } + + fn finalize_block() -> ::Header { + unimplemented!() + } + + fn inherent_extrinsics(_: sp_inherents::InherentData) -> Vec<::Extrinsic> { + unimplemented!() + } + + fn check_inherents(_: Block, _: sp_inherents::InherentData) -> sp_inherents::CheckInherentsResult { + unimplemented!() + } + } + + impl sp_transaction_pool::runtime_api::TaggedTransactionQueue for Runtime { + fn validate_transaction( + _: TransactionSource, + _: ::Extrinsic, + _: ::Hash, + ) -> TransactionValidity { + unimplemented!() + } + } + + impl sp_genesis_builder::GenesisBuilder for Runtime { + fn build_state(_: Vec) -> sp_genesis_builder::Result { + unimplemented!() + } + + fn get_preset(_id: &Option) -> Option> { + unimplemented!() + } + + fn preset_names() -> Vec { + unimplemented!() + } + } +} diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/mod.rs b/substrate/utils/frame/benchmarking-cli/src/overhead/mod.rs index 00cde66fd722..89c23d1fb6c1 100644 --- a/substrate/utils/frame/benchmarking-cli/src/overhead/mod.rs +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/mod.rs @@ -15,7 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod cmd; +pub mod command; pub mod template; -pub use cmd::OverheadCmd; +mod fake_runtime_api; +pub mod remark_builder; +pub mod runtime_utilities; + +pub use command::{OpaqueBlock, OverheadCmd}; diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/remark_builder.rs b/substrate/utils/frame/benchmarking-cli/src/overhead/remark_builder.rs new file mode 100644 index 000000000000..a1d5f282d9f8 --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/remark_builder.rs @@ -0,0 +1,122 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::extrinsic::ExtrinsicBuilder; +use codec::Decode; +use sc_client_api::UsageProvider; +use sp_api::{ApiExt, Core, Metadata, ProvideRuntimeApi}; +use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic}; +use std::sync::Arc; +use subxt::{ + client::RuntimeVersion as SubxtRuntimeVersion, + config::substrate::SubstrateExtrinsicParamsBuilder, Config, OfflineClient, SubstrateConfig, +}; + +pub type SubstrateRemarkBuilder = DynamicRemarkBuilder; + +/// Remark builder that can be used to build simple extrinsics for +/// FRAME-based runtimes. +pub struct DynamicRemarkBuilder { + offline_client: OfflineClient, +} + +impl> DynamicRemarkBuilder { + /// Initializes a new remark builder from a client. + /// + /// This will first fetch metadata and runtime version from the runtime and then + /// construct an offline client that provides the extrinsics. + pub fn new_from_client(client: Arc) -> sc_cli::Result + where + Block: BlockT, + Client: UsageProvider + ProvideRuntimeApi, + Client::Api: Metadata + Core, + { + let genesis = client.usage_info().chain.best_hash; + let api = client.runtime_api(); + + let Ok(Some(metadata_api_version)) = api.api_version::>(genesis) else { + return Err("Unable to fetch metadata runtime API version.".to_string().into()); + }; + + log::debug!("Found metadata API version {}.", metadata_api_version); + let opaque_metadata = if metadata_api_version > 1 { + let Ok(mut supported_metadata_versions) = api.metadata_versions(genesis) else { + return Err("Unable to fetch metadata versions".to_string().into()); + }; + + let latest = supported_metadata_versions + .pop() + .ok_or("No metadata version supported".to_string())?; + + api.metadata_at_version(genesis, latest) + .map_err(|e| format!("Unable to fetch metadata: {:?}", e))? + .ok_or("Unable to decode metadata".to_string())? + } else { + // Fall back to using the non-versioned metadata API. + api.metadata(genesis) + .map_err(|e| format!("Unable to fetch metadata: {:?}", e))? + }; + + let version = api.version(genesis).unwrap(); + let runtime_version = SubxtRuntimeVersion { + spec_version: version.spec_version, + transaction_version: version.transaction_version, + }; + let metadata = subxt::Metadata::decode(&mut (*opaque_metadata).as_slice())?; + let genesis = subxt::utils::H256::from(genesis.to_fixed_bytes()); + + Ok(Self { offline_client: OfflineClient::new(genesis, runtime_version, metadata) }) + } +} + +impl DynamicRemarkBuilder { + /// Constructs a new remark builder. + pub fn new( + metadata: subxt::Metadata, + genesis_hash: C::Hash, + runtime_version: SubxtRuntimeVersion, + ) -> Self { + Self { offline_client: OfflineClient::new(genesis_hash, runtime_version, metadata) } + } +} + +impl ExtrinsicBuilder for DynamicRemarkBuilder { + fn pallet(&self) -> &str { + "system" + } + + fn extrinsic(&self) -> &str { + "remark" + } + + fn build(&self, nonce: u32) -> std::result::Result { + let signer = subxt_signer::sr25519::dev::alice(); + let dynamic_tx = subxt::dynamic::tx("System", "remark", vec![Vec::::new()]); + + let params = SubstrateExtrinsicParamsBuilder::new().nonce(nonce.into()).build(); + + // Default transaction parameters assume a nonce of 0. + let transaction = self + .offline_client + .tx() + .create_signed_offline(&dynamic_tx, &signer, params) + .unwrap(); + let mut encoded = transaction.into_encoded(); + + OpaqueExtrinsic::from_bytes(&mut encoded).map_err(|_| "Unable to construct OpaqueExtrinsic") + } +} diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/runtime_utilities.rs b/substrate/utils/frame/benchmarking-cli/src/overhead/runtime_utilities.rs new file mode 100644 index 000000000000..c498da38afb0 --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/runtime_utilities.rs @@ -0,0 +1,141 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use codec::{Decode, Encode}; +use sc_executor::WasmExecutor; +use sp_core::{ + traits::{CallContext, CodeExecutor, FetchRuntimeCode, RuntimeCode}, + OpaqueMetadata, +}; +use sp_state_machine::BasicExternalities; +use sp_wasm_interface::HostFunctions; +use std::borrow::Cow; + +/// Fetches the latest metadata from the given runtime blob. +pub fn fetch_latest_metadata_from_code_blob( + executor: &WasmExecutor, + code_bytes: Cow<[u8]>, +) -> sc_cli::Result { + let runtime_caller = RuntimeCaller::new(executor, code_bytes); + let version_result = runtime_caller.call("Metadata_metadata_versions", ()); + + let opaque_metadata: OpaqueMetadata = match version_result { + Ok(supported_versions) => { + let latest_version = Vec::::decode(&mut supported_versions.as_slice()) + .map_err(|e| format!("Unable to decode version list: {e}"))? + .pop() + .ok_or("No metadata versions supported".to_string())?; + + let encoded = runtime_caller + .call("Metadata_metadata_at_version", latest_version) + .map_err(|_| "Unable to fetch metadata from blob".to_string())?; + Option::::decode(&mut encoded.as_slice())? + .ok_or_else(|| "Metadata not found".to_string())? + }, + Err(_) => { + let encoded = runtime_caller + .call("Metadata_metadata", ()) + .map_err(|_| "Unable to fetch metadata from blob".to_string())?; + Decode::decode(&mut encoded.as_slice())? + }, + }; + + Ok(subxt::Metadata::decode(&mut (*opaque_metadata).as_slice())?) +} + +struct BasicCodeFetcher<'a> { + code: Cow<'a, [u8]>, + hash: Vec, +} + +impl<'a> FetchRuntimeCode for BasicCodeFetcher<'a> { + fn fetch_runtime_code(&self) -> Option> { + Some(self.code.as_ref().into()) + } +} + +impl<'a> BasicCodeFetcher<'a> { + pub fn new(code: Cow<'a, [u8]>) -> Self { + Self { hash: sp_crypto_hashing::blake2_256(&code).to_vec(), code } + } + + pub fn runtime_code(&'a self) -> RuntimeCode<'a> { + RuntimeCode { + code_fetcher: self as &'a dyn FetchRuntimeCode, + heap_pages: None, + hash: self.hash.clone(), + } + } +} + +/// Simple utility that is used to call into the runtime. +struct RuntimeCaller<'a, 'b, HF: HostFunctions> { + executor: &'b WasmExecutor, + code_fetcher: BasicCodeFetcher<'a>, +} + +impl<'a, 'b, HF: HostFunctions> RuntimeCaller<'a, 'b, HF> { + pub fn new(executor: &'b WasmExecutor, code_bytes: Cow<'a, [u8]>) -> Self { + Self { executor, code_fetcher: BasicCodeFetcher::new(code_bytes) } + } + + fn call(&self, method: &str, data: impl Encode) -> sc_executor_common::error::Result> { + let mut ext = BasicExternalities::default(); + self.executor + .call( + &mut ext, + &self.code_fetcher.runtime_code(), + method, + &data.encode(), + CallContext::Offchain, + ) + .0 + } +} + +#[cfg(test)] +mod tests { + use crate::overhead::command::ParachainHostFunctions; + use codec::Decode; + use sc_executor::WasmExecutor; + use sp_version::RuntimeVersion; + + #[test] + fn test_fetch_latest_metadata_from_blob_fetches_metadata() { + let executor: WasmExecutor = WasmExecutor::builder().build(); + let code_bytes = cumulus_test_runtime::WASM_BINARY + .expect("To run this test, build the wasm binary of cumulus-test-runtime") + .to_vec(); + let metadata = + super::fetch_latest_metadata_from_code_blob(&executor, code_bytes.into()).unwrap(); + assert!(metadata.pallet_by_name("ParachainInfo").is_some()); + } + + #[test] + fn test_runtime_caller_can_call_into_runtime() { + let executor: WasmExecutor = WasmExecutor::builder().build(); + let code_bytes = cumulus_test_runtime::WASM_BINARY + .expect("To run this test, build the wasm binary of cumulus-test-runtime") + .to_vec(); + let runtime_caller = super::RuntimeCaller::new(&executor, code_bytes.into()); + let runtime_version = runtime_caller + .call("Core_version", ()) + .expect("Should be able to call runtime_version"); + let _runtime_version: RuntimeVersion = Decode::decode(&mut runtime_version.as_slice()) + .expect("Should be able to decode runtime version"); + } +} diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/template.rs b/substrate/utils/frame/benchmarking-cli/src/overhead/template.rs index 7c8c92b07d74..08227607951b 100644 --- a/substrate/utils/frame/benchmarking-cli/src/overhead/template.rs +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/template.rs @@ -19,7 +19,6 @@ //! it into the `weights.hbs` template. use sc_cli::Result; -use sc_service::Configuration; use handlebars::Handlebars; use log::info; @@ -27,7 +26,7 @@ use serde::Serialize; use std::{env, fs, path::PathBuf}; use crate::{ - overhead::cmd::{BenchmarkType, OverheadParams}, + overhead::command::{BenchmarkType, OverheadParams}, shared::{Stats, UnderscoreHelper}, }; @@ -59,19 +58,22 @@ pub(crate) struct TemplateData { params: OverheadParams, /// Stats about the benchmark result. stats: Stats, - /// The resulting weight in ns. - weight: u64, + /// The resulting ref time weight. + ref_time: u64, + /// The size of the proof weight. + proof_size: u64, } impl TemplateData { /// Returns a new [`Self`] from the given params. pub(crate) fn new( t: BenchmarkType, - cfg: &Configuration, + chain_name: &String, params: &OverheadParams, stats: &Stats, + proof_size: u64, ) -> Result { - let weight = params.weight.calc_weight(stats)?; + let ref_time = params.weight.calc_weight(stats)?; let header = params .header .as_ref() @@ -82,7 +84,7 @@ impl TemplateData { Ok(TemplateData { short_name: t.short_name().into(), long_name: t.long_name().into(), - runtime_name: cfg.chain_spec.name().into(), + runtime_name: chain_name.to_owned(), version: VERSION.into(), date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(), hostname: params.hostinfo.hostname(), @@ -91,7 +93,8 @@ impl TemplateData { args: env::args().collect::>(), params: params.clone(), stats: stats.clone(), - weight, + ref_time, + proof_size, }) } diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/weights.hbs b/substrate/utils/frame/benchmarking-cli/src/overhead/weights.hbs index 6e364facc12f..1596bb57a41a 100644 --- a/substrate/utils/frame/benchmarking-cli/src/overhead/weights.hbs +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/weights.hbs @@ -18,9 +18,9 @@ use sp_weights::{constants::WEIGHT_REF_TIME_PER_NANOS, Weight}; parameter_types! { {{#if (eq short_name "block")}} - /// Time to execute an empty block. + /// Weight of executing an empty block. {{else}} - /// Time to execute a NO-OP extrinsic, for example `System::remark`. + /// Weight of executing a NO-OP extrinsic, for example `System::remark`. {{/if}} /// Calculated by multiplying the *{{params.weight.weight_metric}}* with `{{params.weight.weight_mul}}` and adding `{{params.weight.weight_add}}`. /// @@ -35,7 +35,7 @@ parameter_types! { /// 95th: {{underscore stats.p95}} /// 75th: {{underscore stats.p75}} pub const {{long_name}}Weight: Weight = - Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul({{underscore weight}}), 0); + Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul({{underscore ref_time}}), {{underscore proof_size}}); } #[cfg(test)] diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs index f33348190577..6f7e79f16384 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -19,7 +19,14 @@ use super::{ types::{ComponentRange, ComponentRangeMap}, writer, ListOutput, PalletCmd, }; -use crate::pallet::{types::FetchedCode, GenesisBuilderPolicy}; +use crate::{ + pallet::{types::FetchedCode, GenesisBuilderPolicy}, + shared::{ + genesis_state, + genesis_state::{GenesisStateHandler, SpecGenesisSource, WARN_SPEC_GENESIS_CTOR}, + }, +}; +use clap::{error::ErrorKind, CommandFactory}; use codec::{Decode, Encode}; use frame_benchmarking::{ Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter, @@ -27,7 +34,6 @@ use frame_benchmarking::{ }; use frame_support::traits::StorageInfo; use linked_hash_map::LinkedHashMap; -use sc_chain_spec::GenesisConfigBuilderRuntimeCaller; use sc_cli::{execution_method_from_cli, ChainSpec, CliConfiguration, Result, SharedParams}; use sc_client_db::BenchmarkingState; use sc_executor::{HeapAllocStrategy, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY}; @@ -43,7 +49,6 @@ use sp_externalities::Extensions; use sp_keystore::{testing::MemoryKeystore, KeystoreExt}; use sp_runtime::traits::Hash; use sp_state_machine::StateMachine; -use sp_storage::{well_known_keys::CODE, Storage}; use sp_trie::{proof_size_extension::ProofSizeExt, recorder::Recorder}; use sp_wasm_interface::HostFunctions; use std::{ @@ -58,6 +63,8 @@ use std::{ /// Logging target const LOG_TARGET: &'static str = "polkadot_sdk_frame::benchmark::pallet"; +type SubstrateAndExtraHF = + (sp_io::SubstrateHostFunctions, frame_benchmarking::benchmarking::HostFunctions, T); /// How the PoV size of a storage item should be estimated. #[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy)] pub enum PovEstimationMode { @@ -150,18 +157,6 @@ This could mean that you either did not build the node correctly with the \ `--features runtime-benchmarks` flag, or the chain spec that you are using was \ not created by a node that was compiled with the flag"; -/// When the runtime could not build the genesis storage. -const ERROR_CANNOT_BUILD_GENESIS: &str = "The runtime returned \ -an error when trying to build the genesis storage. Please ensure that all pallets \ -define a genesis config that can be built. This can be tested with: \ -https://github.com/paritytech/polkadot-sdk/pull/3412"; - -/// Warn when using the chain spec to generate the genesis state. -const WARN_SPEC_GENESIS_CTOR: &'static str = "Using the chain spec instead of the runtime to \ -generate the genesis state is deprecated. Please remove the `--chain`/`--dev`/`--local` argument, \ -point `--runtime` to your runtime blob and set `--genesis-builder=runtime`. This warning may \ -become a hard error any time after December 2024."; - impl PalletCmd { /// Runs the command and benchmarks a pallet. #[deprecated( @@ -177,6 +172,61 @@ impl PalletCmd { self.run_with_spec::(Some(config.chain_spec)) } + fn state_handler_from_cli( + &self, + chain_spec_from_api: Option>, + ) -> Result { + let genesis_builder_to_source = || match self.genesis_builder { + Some(GenesisBuilderPolicy::Runtime) | Some(GenesisBuilderPolicy::SpecRuntime) => + SpecGenesisSource::Runtime(self.genesis_builder_preset.clone()), + Some(GenesisBuilderPolicy::SpecGenesis) | None => { + log::warn!(target: LOG_TARGET, "{WARN_SPEC_GENESIS_CTOR}"); + SpecGenesisSource::SpecJson + }, + Some(GenesisBuilderPolicy::None) => SpecGenesisSource::None, + }; + + // First handle chain-spec passed in via API parameter. + if let Some(chain_spec) = chain_spec_from_api { + log::debug!("Initializing state handler with chain-spec from API: {:?}", chain_spec); + + let source = genesis_builder_to_source(); + return Ok(GenesisStateHandler::ChainSpec(chain_spec, source)) + }; + + // Handle chain-spec passed in via CLI. + if let Some(chain_spec_path) = &self.shared_params.chain { + log::debug!( + "Initializing state handler with chain-spec from path: {:?}", + chain_spec_path + ); + let (chain_spec, _) = + genesis_state::chain_spec_from_path::(chain_spec_path.to_string().into())?; + + let source = genesis_builder_to_source(); + + return Ok(GenesisStateHandler::ChainSpec(chain_spec, source)) + }; + + // Check for runtimes. In general, we make sure that `--runtime` and `--chain` are + // incompatible on the CLI level. + if let Some(runtime_path) = &self.runtime { + log::debug!("Initializing state handler with runtime from path: {:?}", runtime_path); + + let runtime_blob = fs::read(runtime_path)?; + return if let Some(GenesisBuilderPolicy::None) = self.genesis_builder { + Ok(GenesisStateHandler::Runtime(runtime_blob, None)) + } else { + Ok(GenesisStateHandler::Runtime( + runtime_blob, + Some(self.genesis_builder_preset.clone()), + )) + } + }; + + Err("Neither a runtime nor a chain-spec were specified".to_string().into()) + } + /// Runs the pallet benchmarking command. pub fn run_with_spec( &self, @@ -186,7 +236,11 @@ impl PalletCmd { Hasher: Hash, ExtraHostFunctions: HostFunctions, { - self.check_args()?; + if let Err((error_kind, msg)) = self.check_args(&chain_spec) { + let mut cmd = PalletCmd::command(); + cmd.error(error_kind, msg).exit(); + }; + let _d = self.execution.as_ref().map(|exec| { // We print the error at the end, since there is often A LOT of output. sp_core::defer::DeferGuard::new(move || { @@ -211,7 +265,10 @@ impl PalletCmd { return self.output_from_results(&batches) } - let genesis_storage = self.genesis_storage::(&chain_spec)?; + let state_handler = + self.state_handler_from_cli::>(chain_spec)?; + let genesis_storage = + state_handler.build_storage::>(None)?; let cache_size = Some(self.database_cache_size as usize); let state_with_tracking = BenchmarkingState::::new( @@ -240,18 +297,14 @@ impl PalletCmd { let runtime_code = runtime.code()?; let alloc_strategy = self.alloc_strategy(runtime_code.heap_pages); - let executor = WasmExecutor::<( - sp_io::SubstrateHostFunctions, - frame_benchmarking::benchmarking::HostFunctions, - ExtraHostFunctions, - )>::builder() - .with_execution_method(method) - .with_allow_missing_host_functions(self.allow_missing_host_functions) - .with_onchain_heap_alloc_strategy(alloc_strategy) - .with_offchain_heap_alloc_strategy(alloc_strategy) - .with_max_runtime_instances(2) - .with_runtime_cache_size(2) - .build(); + let executor = WasmExecutor::>::builder() + .with_execution_method(method) + .with_allow_missing_host_functions(self.allow_missing_host_functions) + .with_onchain_heap_alloc_strategy(alloc_strategy) + .with_offchain_heap_alloc_strategy(alloc_strategy) + .with_max_runtime_instances(2) + .with_runtime_cache_size(2) + .build(); let (list, storage_info): (Vec, Vec) = Self::exec_state_machine( @@ -564,97 +617,6 @@ impl PalletCmd { included && !excluded } - /// Build the genesis storage by either the Genesis Builder API, chain spec or nothing. - /// - /// Behaviour can be controlled by the `--genesis-builder` flag. - fn genesis_storage( - &self, - chain_spec: &Option>, - ) -> Result { - Ok(match (self.genesis_builder, self.runtime.as_ref()) { - (Some(GenesisBuilderPolicy::None), _) => Storage::default(), - (Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::Spec), Some(_)) => - return Err("Cannot use `--genesis-builder=spec-genesis` with `--runtime` since the runtime would be ignored.".into()), - (Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::Spec), None) | (None, None) => { - log::warn!(target: LOG_TARGET, "{WARN_SPEC_GENESIS_CTOR}"); - let Some(chain_spec) = chain_spec else { - return Err("No chain spec specified to generate the genesis state".into()); - }; - - let storage = chain_spec - .build_storage() - .map_err(|e| format!("{ERROR_CANNOT_BUILD_GENESIS}\nError: {e}"))?; - - storage - }, - (Some(GenesisBuilderPolicy::SpecRuntime), Some(_)) => - return Err("Cannot use `--genesis-builder=spec` with `--runtime` since the runtime would be ignored.".into()), - (Some(GenesisBuilderPolicy::SpecRuntime), None) => { - let Some(chain_spec) = chain_spec else { - return Err("No chain spec specified to generate the genesis state".into()); - }; - - self.genesis_from_spec_runtime::(chain_spec.as_ref())? - }, - (Some(GenesisBuilderPolicy::Runtime), None) => return Err("Cannot use `--genesis-builder=runtime` without `--runtime`".into()), - (Some(GenesisBuilderPolicy::Runtime), Some(runtime)) | (None, Some(runtime)) => { - log::info!(target: LOG_TARGET, "Loading WASM from {}", runtime.display()); - - let code = fs::read(&runtime).map_err(|e| { - format!( - "Could not load runtime file from path: {}, error: {}", - runtime.display(), - e - ) - })?; - - self.genesis_from_code::(&code)? - } - }) - } - - /// Setup the genesis state by calling the runtime APIs of the chain-specs genesis runtime. - fn genesis_from_spec_runtime( - &self, - chain_spec: &dyn ChainSpec, - ) -> Result { - log::info!(target: LOG_TARGET, "Building genesis state from chain spec runtime"); - let storage = chain_spec - .build_storage() - .map_err(|e| format!("{ERROR_CANNOT_BUILD_GENESIS}\nError: {e}"))?; - - let code: &Vec = - storage.top.get(CODE).ok_or("No runtime code in the genesis storage")?; - - self.genesis_from_code::(code) - } - - fn genesis_from_code(&self, code: &[u8]) -> Result { - let genesis_config_caller = GenesisConfigBuilderRuntimeCaller::<( - sp_io::SubstrateHostFunctions, - frame_benchmarking::benchmarking::HostFunctions, - EHF, - )>::new(code); - let preset = Some(&self.genesis_builder_preset); - - let mut storage = - genesis_config_caller.get_storage_for_named_preset(preset).inspect_err(|e| { - let presets = genesis_config_caller.preset_names().unwrap_or_default(); - log::error!( - target: LOG_TARGET, - "Please pick one of the available presets with \ - `--genesis-builder-preset=` or use a different `--genesis-builder-policy`. Available presets ({}): {:?}. Error: {:?}", - presets.len(), - presets, - e - ); - })?; - - storage.top.insert(CODE.into(), code.into()); - - Ok(storage) - } - /// Execute a state machine and decode its return value as `R`. fn exec_state_machine( mut machine: StateMachine, H, Exec>, @@ -948,35 +910,61 @@ impl PalletCmd { } /// Sanity check the CLI arguments. - fn check_args(&self) -> Result<()> { + fn check_args( + &self, + chain_spec: &Option>, + ) -> std::result::Result<(), (ErrorKind, String)> { if self.runtime.is_some() && self.shared_params.chain.is_some() { unreachable!("Clap should not allow both `--runtime` and `--chain` to be provided.") } + if chain_spec.is_none() && self.runtime.is_none() && self.shared_params.chain.is_none() { + return Err(( + ErrorKind::MissingRequiredArgument, + "Provide either a runtime via `--runtime` or a chain spec via `--chain`" + .to_string(), + )) + } + + match self.genesis_builder { + Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::SpecRuntime) => + if chain_spec.is_none() && self.shared_params.chain.is_none() { + return Err(( + ErrorKind::MissingRequiredArgument, + "Provide a chain spec via `--chain`.".to_string(), + )) + }, + _ => {}, + } + if let Some(output_path) = &self.output { if !output_path.is_dir() && output_path.file_name().is_none() { - return Err(format!( - "Output path is neither a directory nor a file: {output_path:?}" - ) - .into()) + return Err(( + ErrorKind::InvalidValue, + format!("Output path is neither a directory nor a file: {output_path:?}"), + )); } } if let Some(header_file) = &self.header { if !header_file.is_file() { - return Err(format!("Header file could not be found: {header_file:?}").into()) + return Err(( + ErrorKind::InvalidValue, + format!("Header file could not be found: {header_file:?}"), + )); }; } if let Some(handlebars_template_file) = &self.template { if !handlebars_template_file.is_file() { - return Err(format!( - "Handlebars template file could not be found: {handlebars_template_file:?}" - ) - .into()) + return Err(( + ErrorKind::InvalidValue, + format!( + "Handlebars template file could not be found: {handlebars_template_file:?}" + ), + )); }; } - Ok(()) } } @@ -1031,3 +1019,166 @@ fn list_benchmark( }, } } +#[cfg(test)] +mod tests { + use crate::pallet::PalletCmd; + use clap::Parser; + + fn cli_succeed(args: &[&str]) -> Result<(), clap::Error> { + let cmd = PalletCmd::try_parse_from(args)?; + assert!(cmd.check_args(&None).is_ok()); + Ok(()) + } + + fn cli_fail(args: &[&str]) { + let cmd = PalletCmd::try_parse_from(args); + if let Ok(cmd) = cmd { + assert!(cmd.check_args(&None).is_err()); + } + } + + #[test] + fn test_cli_conflicts() -> Result<(), clap::Error> { + // Runtime tests + cli_succeed(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--runtime", + "path/to/runtime", + "--genesis-builder", + "runtime", + ])?; + cli_succeed(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--runtime", + "path/to/runtime", + "--genesis-builder", + "none", + ])?; + cli_succeed(&["test", "--extrinsic", "", "--pallet", "", "--runtime", "path/to/runtime"])?; + cli_succeed(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--runtime", + "path/to/runtime", + "--genesis-builder-preset", + "preset", + ])?; + cli_fail(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--runtime", + "path/to/runtime", + "--genesis-builder", + "spec", + ]); + cli_fail(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--runtime", + "path/to/spec", + "--genesis-builder", + "spec-genesis", + ]); + cli_fail(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--runtime", + "path/to/spec", + "--genesis-builder", + "spec-runtime", + ]); + cli_fail(&["test", "--runtime", "path/to/spec", "--genesis-builder", "spec-genesis"]); + + // Spec tests + cli_succeed(&["test", "--extrinsic", "", "--pallet", "", "--chain", "path/to/spec"])?; + cli_succeed(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--chain", + "path/to/spec", + "--genesis-builder", + "spec", + ])?; + cli_succeed(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--chain", + "path/to/spec", + "--genesis-builder", + "spec-genesis", + ])?; + cli_succeed(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--chain", + "path/to/spec", + "--genesis-builder", + "spec-runtime", + ])?; + cli_succeed(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--chain", + "path/to/spec", + "--genesis-builder", + "none", + ])?; + cli_fail(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--chain", + "path/to/spec", + "--genesis-builder", + "runtime", + ]); + cli_fail(&[ + "test", + "--extrinsic", + "", + "--pallet", + "", + "--chain", + "path/to/spec", + "--genesis-builder", + "runtime", + "--genesis-builder-preset", + "preset", + ]); + Ok(()) + } +} diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs index 412a1a86cb8e..54a055d4a33f 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -19,8 +19,9 @@ mod command; mod types; mod writer; -use crate::{pallet::types::GenesisBuilderPolicy, shared::HostInfoParams}; +use crate::shared::HostInfoParams; use clap::ValueEnum; +use frame_support::Serialize; use sc_cli::{ WasmExecutionMethod, WasmtimeInstantiationStrategy, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, DEFAULT_WASM_EXECUTION_METHOD, @@ -172,7 +173,7 @@ pub struct PalletCmd { pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy, /// Optional runtime blob to use instead of the one from the genesis config. - #[arg(long, conflicts_with = "chain")] + #[arg(long, conflicts_with = "chain", required_if_eq("genesis_builder", "runtime"))] pub runtime: Option, /// Do not fail if there are unknown but also unused host functions in the runtime. @@ -181,8 +182,7 @@ pub struct PalletCmd { /// How to construct the genesis state. /// - /// Uses `GenesisBuilderPolicy::Spec` by default and `GenesisBuilderPolicy::Runtime` if - /// `runtime` is set. + /// Uses `GenesisBuilderPolicy::Spec` by default. #[arg(long, value_enum, alias = "genesis-builder-policy")] pub genesis_builder: Option, @@ -265,3 +265,22 @@ pub struct PalletCmd { #[arg(long)] disable_proof_recording: bool, } + +/// How the genesis state for benchmarking should be built. +#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy, Serialize)] +#[clap(rename_all = "kebab-case")] +pub enum GenesisBuilderPolicy { + /// Do not provide any genesis state. + /// + /// Benchmarks are advised to function with this, since they should setup their own required + /// state. However, to keep backwards compatibility, this is not the default. + None, + /// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API. + /// This will use the `development` preset by default. + Runtime, + /// Use the runtime from the Spec file to build the genesis state. + SpecRuntime, + /// Use the spec file to build the genesis state. This fails when there is no spec. + #[value(alias = "spec")] + SpecGenesis, +} diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs index a4799dc92369..4cfcc60907d9 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs @@ -21,25 +21,6 @@ use sc_cli::Result; use sp_core::traits::{RuntimeCode, WrappedRuntimeCode}; use sp_runtime::traits::Hash; -/// How the genesis state for benchmarking should be build. -#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy)] -#[clap(rename_all = "kebab-case")] -pub enum GenesisBuilderPolicy { - /// Do not provide any genesis state. - /// - /// Benchmarks are advised to function with this, since they should setup their own required - /// state. However, to keep backwards compatibility, this is not the default. - None, - /// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API. - Runtime, - // Use the runtime from the Spec file to build the genesis state. - SpecRuntime, - /// Use the spec file to build the genesis state. This fails when there is no spec. - SpecGenesis, - /// Same as `SpecGenesis` - only here for backwards compatibility. - Spec, -} - /// A runtime blob that was either fetched from genesis storage or loaded from a file. // NOTE: This enum is only needed for the annoying lifetime bounds on `RuntimeCode`. Otherwise we // could just directly return the blob. diff --git a/substrate/utils/frame/benchmarking-cli/src/shared/genesis_state.rs b/substrate/utils/frame/benchmarking-cli/src/shared/genesis_state.rs new file mode 100644 index 000000000000..1ca3e36d25ad --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/shared/genesis_state.rs @@ -0,0 +1,141 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::overhead::command::ParachainExtension; +use sc_chain_spec::{ChainSpec, GenericChainSpec, GenesisConfigBuilderRuntimeCaller}; +use sc_cli::Result; +use serde_json::Value; +use sp_storage::{well_known_keys::CODE, Storage}; +use sp_wasm_interface::HostFunctions; +use std::{borrow::Cow, path::PathBuf}; + +/// When the runtime could not build the genesis storage. +const ERROR_CANNOT_BUILD_GENESIS: &str = "The runtime returned \ +an error when trying to build the genesis storage. Please ensure that all pallets \ +define a genesis config that can be built. This can be tested with: \ +https://github.com/paritytech/polkadot-sdk/pull/3412"; + +/// Warn when using the chain spec to generate the genesis state. +pub const WARN_SPEC_GENESIS_CTOR: &'static str = "Using the chain spec instead of the runtime to \ +generate the genesis state is deprecated. Please remove the `--chain`/`--dev`/`--local` argument, \ +point `--runtime` to your runtime blob and set `--genesis-builder=runtime`. This warning may \ +become a hard error any time after December 2024."; + +/// Defines how the chain specification shall be used to build the genesis storage. +pub enum SpecGenesisSource { + /// Use preset provided by the runtime embedded in the chain specification. + Runtime(String), + /// Use provided chain-specification JSON file. + SpecJson, + /// Use default storage. + None, +} + +/// Defines how the genesis storage shall be built. +pub enum GenesisStateHandler { + ChainSpec(Box, SpecGenesisSource), + Runtime(Vec, Option), +} + +impl GenesisStateHandler { + /// Populate the genesis storage. + /// + /// If the raw storage is derived from a named genesis preset, `json_patcher` is can be used to + /// inject values into the preset. + pub fn build_storage( + &self, + json_patcher: Option Value + 'static>>, + ) -> Result { + match self { + GenesisStateHandler::ChainSpec(chain_spec, source) => match source { + SpecGenesisSource::Runtime(preset) => { + let mut storage = chain_spec.build_storage()?; + let code_bytes = storage + .top + .remove(CODE) + .ok_or("chain spec genesis does not contain code")?; + genesis_from_code::(code_bytes.as_slice(), preset, json_patcher) + }, + SpecGenesisSource::SpecJson => chain_spec + .build_storage() + .map_err(|e| format!("{ERROR_CANNOT_BUILD_GENESIS}\nError: {e}").into()), + SpecGenesisSource::None => Ok(Storage::default()), + }, + GenesisStateHandler::Runtime(code_bytes, Some(preset)) => + genesis_from_code::(code_bytes.as_slice(), preset, json_patcher), + GenesisStateHandler::Runtime(_, None) => Ok(Storage::default()), + } + } + + /// Get the runtime code blob. + pub fn get_code_bytes(&self) -> Result> { + match self { + GenesisStateHandler::ChainSpec(chain_spec, _) => { + let mut storage = chain_spec.build_storage()?; + storage + .top + .remove(CODE) + .map(|code| Cow::from(code)) + .ok_or("chain spec genesis does not contain code".into()) + }, + GenesisStateHandler::Runtime(code_bytes, _) => Ok(code_bytes.into()), + } + } +} + +pub fn chain_spec_from_path( + chain: PathBuf, +) -> Result<(Box, Option)> { + let spec = GenericChainSpec::::from_json_file(chain) + .map_err(|e| format!("Unable to load chain spec: {:?}", e))?; + + let para_id_from_chain_spec = spec.extensions().para_id; + Ok((Box::new(spec), para_id_from_chain_spec)) +} + +fn genesis_from_code( + code: &[u8], + genesis_builder_preset: &String, + storage_patcher: Option Value>>, +) -> Result { + let genesis_config_caller = GenesisConfigBuilderRuntimeCaller::<( + sp_io::SubstrateHostFunctions, + frame_benchmarking::benchmarking::HostFunctions, + EHF, + )>::new(code); + + let mut preset_json = genesis_config_caller.get_named_preset(Some(genesis_builder_preset))?; + if let Some(patcher) = storage_patcher { + preset_json = patcher(preset_json); + } + + let mut storage = + genesis_config_caller.get_storage_for_patch(preset_json).inspect_err(|e| { + let presets = genesis_config_caller.preset_names().unwrap_or_default(); + log::error!( + "Please pick one of the available presets with \ + `--genesis-builder-preset=`. Available presets ({}): {:?}. Error: {:?}", + presets.len(), + presets, + e + ); + })?; + + storage.top.insert(CODE.into(), code.into()); + + Ok(storage) +} diff --git a/substrate/utils/frame/benchmarking-cli/src/shared/mod.rs b/substrate/utils/frame/benchmarking-cli/src/shared/mod.rs index f8aa49b867f7..6c9c74e0312c 100644 --- a/substrate/utils/frame/benchmarking-cli/src/shared/mod.rs +++ b/substrate/utils/frame/benchmarking-cli/src/shared/mod.rs @@ -17,6 +17,7 @@ //! Code that is shared among all benchmarking sub-commands. +pub mod genesis_state; pub mod record; pub mod stats; pub mod weight_params; diff --git a/substrate/utils/frame/omni-bencher/Cargo.toml b/substrate/utils/frame/omni-bencher/Cargo.toml index e2ffca8b4714..345a7288d45b 100644 --- a/substrate/utils/frame/omni-bencher/Cargo.toml +++ b/substrate/utils/frame/omni-bencher/Cargo.toml @@ -20,3 +20,11 @@ sp-runtime = { workspace = true, default-features = true } sp-statement-store = { workspace = true, default-features = true } tracing-subscriber = { workspace = true } log = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +assert_cmd = { workspace = true } +cumulus-test-runtime = { workspace = true } +sp-tracing = { workspace = true, default-features = true } +sp-genesis-builder = { workspace = true, default-features = true } +sc-chain-spec = { workspace = true } diff --git a/substrate/utils/frame/omni-bencher/src/command.rs b/substrate/utils/frame/omni-bencher/src/command.rs index 19177ed549b7..f5796d05e339 100644 --- a/substrate/utils/frame/omni-bencher/src/command.rs +++ b/substrate/utils/frame/omni-bencher/src/command.rs @@ -16,7 +16,7 @@ // limitations under the License. use clap::Parser; -use frame_benchmarking_cli::BenchmarkCmd; +use frame_benchmarking_cli::{BenchmarkCmd, OpaqueBlock}; use sc_cli::Result; use sp_runtime::traits::BlakeTwo256; @@ -129,27 +129,28 @@ impl Command { } } } - impl V1SubCommand { pub fn run(self) -> Result<()> { - let pallet = match self { + match self { V1SubCommand::Benchmark(V1BenchmarkCommand { sub }) => match sub { - BenchmarkCmd::Pallet(pallet) => pallet, + BenchmarkCmd::Pallet(pallet) => { + if let Some(spec) = pallet.shared_params.chain { + return Err(format!( + "Chain specs are not supported. Please remove `--chain={spec}` and use \ + `--runtime=` instead" + ) + .into()); + } + + pallet.run_with_spec::(None) + }, + BenchmarkCmd::Overhead(overhead_cmd) => + overhead_cmd.run_with_default_builder_and_spec::(None), _ => return Err( - "Only the `v1 benchmark pallet` command is currently supported".into() + "Only the `v1 benchmark pallet` and `v1 benchmark overhead` command is currently supported".into() ), }, - }; - - if let Some(spec) = pallet.shared_params.chain { - return Err(format!( - "Chain specs are not supported. Please remove `--chain={spec}` and use \ - `--runtime=` instead" - ) - .into()) } - - pallet.run_with_spec::(None) } } diff --git a/substrate/utils/frame/omni-bencher/src/main.rs b/substrate/utils/frame/omni-bencher/src/main.rs index ef3450add8e4..7d8aa891dc4a 100644 --- a/substrate/utils/frame/omni-bencher/src/main.rs +++ b/substrate/utils/frame/omni-bencher/src/main.rs @@ -31,7 +31,16 @@ fn main() -> Result<()> { /// Setup logging with `info` as default level. Can be set via `RUST_LOG` env. fn setup_logger() { - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + // Disable these log targets because they are spammy. + let unwanted_targets = + &["cranelift_codegen", "wasm_cranelift", "wasmtime_jit", "wasmtime_cranelift", "wasm_jit"]; + + let mut env_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + + for target in unwanted_targets { + env_filter = env_filter.add_directive(format!("{}=off", target).parse().unwrap()); + } tracing_subscriber::fmt() .with_env_filter(env_filter) diff --git a/substrate/utils/frame/omni-bencher/tests/benchmark_works.rs b/substrate/utils/frame/omni-bencher/tests/benchmark_works.rs new file mode 100644 index 000000000000..fb1687639639 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/tests/benchmark_works.rs @@ -0,0 +1,167 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use assert_cmd::cargo::cargo_bin; +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, ExitStatus}, +}; + +#[test] +fn benchmark_overhead_runtime_works() -> std::result::Result<(), String> { + let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir."); + let base_path = tmp_dir.path(); + let wasm = cumulus_test_runtime::WASM_BINARY.ok_or("WASM binary not available".to_string())?; + let runtime_path = base_path.join("runtime.wasm"); + let _ = + fs::write(&runtime_path, wasm).map_err(|e| format!("Unable to write runtime file: {}", e)); + + // Invoke `benchmark overhead` with all options to make sure that they are valid. + let status = std::process::Command::new(cargo_bin("frame-omni-bencher")) + .args(["v1", "benchmark", "overhead", "--runtime", runtime_path.to_str().unwrap()]) + .arg("-d") + .arg(base_path) + .arg("--weight-path") + .arg(base_path) + .args(["--warmup", "5", "--repeat", "5"]) + // Exotic para id to see that we are actually patching. + .args(["--para-id", "666"]) + .args(["--add", "100", "--mul", "1.2", "--metric", "p75"]) + // Only put 5 extrinsics into the block otherwise it takes forever to build it + // especially for a non-release builds. + .args(["--max-ext-per-block", "5"]) + .status() + .map_err(|e| format!("command failed: {:?}", e))?; + + assert_benchmark_success(status, base_path) +} +#[test] +fn benchmark_overhead_chain_spec_works() -> std::result::Result<(), String> { + let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir."); + let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), false)?; + + let status = create_benchmark_spec_command(&base_path, &chain_spec_path) + .args(["--genesis-builder-policy", "spec-runtime"]) + .args(["--para-id", "666"]) + .status() + .map_err(|e| format!("command failed: {:?}", e))?; + + assert_benchmark_success(status, &base_path) +} + +#[test] +fn benchmark_overhead_chain_spec_works_plain_spec() -> std::result::Result<(), String> { + let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir."); + let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), false)?; + + let status = create_benchmark_spec_command(&base_path, &chain_spec_path) + .args(["--genesis-builder-policy", "spec"]) + .args(["--para-id", "100"]) + .status() + .map_err(|e| format!("command failed: {:?}", e))?; + + assert_benchmark_success(status, &base_path) +} + +#[test] +fn benchmark_overhead_chain_spec_works_raw() -> std::result::Result<(), String> { + let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir."); + let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), true)?; + + let status = create_benchmark_spec_command(&base_path, &chain_spec_path) + .args(["--genesis-builder-policy", "spec"]) + .args(["--para-id", "100"]) + .status() + .map_err(|e| format!("command failed: {:?}", e))?; + + assert_benchmark_success(status, &base_path) +} + +#[test] +fn benchmark_overhead_chain_spec_fails_wrong_para_id() -> std::result::Result<(), String> { + let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir."); + let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), false)?; + + let status = create_benchmark_spec_command(&base_path, &chain_spec_path) + .args(["--genesis-builder-policy", "spec"]) + .args(["--para-id", "666"]) + .status() + .map_err(|e| format!("command failed: {:?}", e))?; + + if status.success() { + return Err("Command should have failed!".into()) + } + + // Weight files should not have been created + assert!(!base_path.join("block_weights.rs").exists()); + assert!(!base_path.join("extrinsic_weights.rs").exists()); + Ok(()) +} + +/// Sets up a temporary directory and creates a chain spec file +fn setup_chain_spec(tmp_dir: &Path, raw: bool) -> Result<(PathBuf, PathBuf), String> { + let base_path = tmp_dir.to_path_buf(); + let chain_spec_path = base_path.join("chain_spec.json"); + + let wasm = cumulus_test_runtime::WASM_BINARY.ok_or("WASM binary not available".to_string())?; + + let mut properties = sc_chain_spec::Properties::new(); + properties.insert("tokenSymbol".into(), "UNIT".into()); + properties.insert("tokenDecimals".into(), 12.into()); + + let chain_spec = sc_chain_spec::GenericChainSpec::<()>::builder(wasm, Default::default()) + .with_name("some-chain") + .with_id("some-id") + .with_properties(properties) + .with_chain_type(sc_chain_spec::ChainType::Development) + .with_genesis_config_preset_name(sp_genesis_builder::LOCAL_TESTNET_RUNTIME_PRESET) + .build(); + + let json = chain_spec.as_json(raw).unwrap(); + fs::write(&chain_spec_path, json) + .map_err(|e| format!("Unable to write chain-spec file: {}", e))?; + + Ok((base_path, chain_spec_path)) +} + +/// Creates a Command for the benchmark with common arguments +fn create_benchmark_spec_command(base_path: &Path, chain_spec_path: &Path) -> Command { + let mut cmd = Command::new(cargo_bin("frame-omni-bencher")); + cmd.args(["v1", "benchmark", "overhead", "--chain", chain_spec_path.to_str().unwrap()]) + .arg("-d") + .arg(base_path) + .arg("--weight-path") + .arg(base_path) + .args(["--warmup", "5", "--repeat", "5"]) + .args(["--add", "100", "--mul", "1.2", "--metric", "p75"]) + // Only put 5 extrinsics into the block otherwise it takes forever to build it + .args(["--max-ext-per-block", "5"]); + cmd +} + +/// Checks if the benchmark completed successfully and created weight files +fn assert_benchmark_success(status: ExitStatus, base_path: &Path) -> Result<(), String> { + if !status.success() { + return Err("Command failed".into()) + } + + // Weight files have been created + assert!(base_path.join("block_weights.rs").exists()); + assert!(base_path.join("extrinsic_weights.rs").exists()); + Ok(()) +} diff --git a/templates/solochain/node/src/command.rs b/templates/solochain/node/src/command.rs index e2c7657c95cc..1c23e395ede9 100644 --- a/templates/solochain/node/src/command.rs +++ b/templates/solochain/node/src/command.rs @@ -144,11 +144,12 @@ pub fn run() -> sc_cli::Result<()> { let ext_builder = RemarkBuilder::new(client.clone()); cmd.run( - config, + config.chain_spec.name().into(), client, inherent_benchmark_data()?, Vec::new(), &ext_builder, + false, ) }, BenchmarkCmd::Extrinsic(cmd) => {