diff --git a/bin/node-template/node/src/command_helper.rs b/bin/node-template/node/src/benchmarking.rs similarity index 71% rename from bin/node-template/node/src/command_helper.rs rename to bin/node-template/node/src/benchmarking.rs index 287e81b1e96bd..f0e32104cd3ee 100644 --- a/bin/node-template/node/src/command_helper.rs +++ b/bin/node-template/node/src/benchmarking.rs @@ -16,13 +16,14 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Contains code to setup the command invocations in [`super::command`] which would -//! otherwise bloat that module. +//! Setup code for [`super::command`] which would otherwise bloat that module. +//! +//! Should only be used for benchmarking as it may break in other contexts. use crate::service::FullClient; use node_template_runtime as runtime; -use runtime::SystemCall; +use runtime::{AccountId, Balance, BalancesCall, SystemCall}; use sc_cli::Result; use sc_client_api::BlockBackend; use sp_core::{Encode, Pair}; @@ -35,19 +36,27 @@ use std::{sync::Arc, time::Duration}; /// Generates extrinsics for the `benchmark overhead` command. /// /// Note: Should only be used for benchmarking. -pub struct BenchmarkExtrinsicBuilder { +pub struct RemarkBuilder { client: Arc, } -impl BenchmarkExtrinsicBuilder { +impl RemarkBuilder { /// Creates a new [`Self`] from the given client. pub fn new(client: Arc) -> Self { Self { client } } } -impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { - fn remark(&self, nonce: u32) -> std::result::Result { +impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder { + fn pallet(&self) -> &str { + "system" + } + + fn extrinsic(&self) -> &str { + "remark" + } + + fn build(&self, nonce: u32) -> std::result::Result { let acc = Sr25519Keyring::Bob.pair(); let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic( self.client.as_ref(), @@ -61,6 +70,49 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { } } +/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks. +/// +/// Note: Should only be used for benchmarking. +pub struct TransferKeepAliveBuilder { + client: Arc, + dest: AccountId, + value: Balance, +} + +impl TransferKeepAliveBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc, dest: AccountId, value: Balance) -> Self { + Self { client, dest, value } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder { + fn pallet(&self) -> &str { + "balances" + } + + fn extrinsic(&self) -> &str { + "transfer_keep_alive" + } + + fn build(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic( + self.client.as_ref(), + acc, + BalancesCall::transfer_keep_alive { + dest: self.dest.clone().into(), + value: self.value.into(), + } + .into(), + nonce, + ) + .into(); + + Ok(extrinsic) + } +} + /// Create a transaction using the given `call`. /// /// Note: Should only be used for benchmarking. diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index e3e10007929e6..142f0b40c325e 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -1,14 +1,14 @@ use crate::{ + benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder}, chain_spec, cli::{Cli, Subcommand}, - command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}, service, }; -use frame_benchmarking_cli::{BenchmarkCmd, SUBSTRATE_REFERENCE_HARDWARE}; -use node_template_runtime::Block; +use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; +use node_template_runtime::{Block, EXISTENTIAL_DEPOSIT}; use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; -use std::sync::Arc; +use sp_keyring::Sr25519Keyring; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -137,9 +137,23 @@ pub fn run() -> sc_cli::Result<()> { }, BenchmarkCmd::Overhead(cmd) => { let PartialComponents { client, .. } = service::new_partial(&config)?; - let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + let ext_builder = RemarkBuilder::new(client.clone()); - cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + cmd.run(config, client, inherent_benchmark_data()?, &ext_builder) + }, + BenchmarkCmd::Extrinsic(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; + // Register the *Remark* and *TKA* builders. + let ext_factory = ExtrinsicFactory(vec![ + Box::new(RemarkBuilder::new(client.clone())), + Box::new(TransferKeepAliveBuilder::new( + client.clone(), + Sr25519Keyring::Alice.to_account_id(), + EXISTENTIAL_DEPOSIT, + )), + ]); + + cmd.run(client, inherent_benchmark_data()?, &ext_factory) }, BenchmarkCmd::Machine(cmd) => cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 0f2fbd5a909c6..426cbabb6fbf7 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -4,9 +4,9 @@ mod chain_spec; #[macro_use] mod service; +mod benchmarking; mod cli; mod command; -mod command_helper; mod rpc; fn main() -> sc_cli::Result<()> { diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index c514cdf6c25fd..a2f595f571a90 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -234,6 +234,9 @@ impl pallet_timestamp::Config for Runtime { type WeightInfo = (); } +/// Existential deposit. +pub const EXISTENTIAL_DEPOSIT: u128 = 500; + impl pallet_balances::Config for Runtime { type MaxLocks = ConstU32<50>; type MaxReserves = (); @@ -243,7 +246,7 @@ impl pallet_balances::Config for Runtime { /// The ubiquitous event type. type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ConstU128<500>; + type ExistentialDeposit = ConstU128; type AccountStore = System; type WeightInfo = pallet_balances::weights::SubstrateWeight; } diff --git a/bin/node/cli/src/command_helper.rs b/bin/node/cli/src/benchmarking.rs similarity index 52% rename from bin/node/cli/src/command_helper.rs rename to bin/node/cli/src/benchmarking.rs index 84d85ee367cab..52657016c046a 100644 --- a/bin/node/cli/src/command_helper.rs +++ b/bin/node/cli/src/benchmarking.rs @@ -16,12 +16,14 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Contains code to setup the command invocations in [`super::command`] which would -//! otherwise bloat that module. +//! Setup code for [`super::command`] which would otherwise bloat that module. +//! +//! Should only be used for benchmarking as it may break in other contexts. use crate::service::{create_extrinsic, FullClient}; -use node_runtime::SystemCall; +use node_primitives::{AccountId, Balance}; +use node_runtime::{BalancesCall, SystemCall}; use sc_cli::Result; use sp_inherents::{InherentData, InherentDataProvider}; use sp_keyring::Sr25519Keyring; @@ -29,20 +31,30 @@ use sp_runtime::OpaqueExtrinsic; use std::{sync::Arc, time::Duration}; -/// Generates extrinsics for the `benchmark overhead` command. -pub struct BenchmarkExtrinsicBuilder { +/// Generates `System::Remark` extrinsics for the benchmarks. +/// +/// Note: Should only be used for benchmarking. +pub struct RemarkBuilder { client: Arc, } -impl BenchmarkExtrinsicBuilder { +impl RemarkBuilder { /// Creates a new [`Self`] from the given client. pub fn new(client: Arc) -> Self { Self { client } } } -impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { - fn remark(&self, nonce: u32) -> std::result::Result { +impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder { + fn pallet(&self) -> &str { + "system" + } + + fn extrinsic(&self) -> &str { + "remark" + } + + fn build(&self, nonce: u32) -> std::result::Result { let acc = Sr25519Keyring::Bob.pair(); let extrinsic: OpaqueExtrinsic = create_extrinsic( self.client.as_ref(), @@ -56,6 +68,48 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { } } +/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks. +/// +/// Note: Should only be used for benchmarking. +pub struct TransferKeepAliveBuilder { + client: Arc, + dest: AccountId, + value: Balance, +} + +impl TransferKeepAliveBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc, dest: AccountId, value: Balance) -> Self { + Self { client, dest, value } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder { + fn pallet(&self) -> &str { + "balances" + } + + fn extrinsic(&self) -> &str { + "transfer_keep_alive" + } + + fn build(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_extrinsic( + self.client.as_ref(), + acc, + BalancesCall::transfer_keep_alive { + dest: self.dest.clone().into(), + value: self.value.into(), + }, + Some(nonce), + ) + .into(); + + Ok(extrinsic) + } +} + /// Generates inherent data for the `benchmark overhead` command. pub fn inherent_benchmark_data() -> Result { let mut inherent_data = InherentData::new(); diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index b17a26fa02935..df3d5a891e2ab 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use super::command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}; +use super::benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder}; use crate::{ chain_spec, service, service::{new_partial, FullClient}, @@ -25,9 +25,10 @@ use crate::{ use frame_benchmarking_cli::*; use node_executor::ExecutorDispatch; use node_primitives::Block; -use node_runtime::RuntimeApi; +use node_runtime::{ExistentialDeposit, RuntimeApi}; use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use sp_keyring::Sr25519Keyring; use std::sync::Arc; @@ -126,9 +127,23 @@ pub fn run() -> Result<()> { }, BenchmarkCmd::Overhead(cmd) => { let PartialComponents { client, .. } = new_partial(&config)?; - let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + let ext_builder = RemarkBuilder::new(client.clone()); - cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + cmd.run(config, client, inherent_benchmark_data()?, &ext_builder) + }, + BenchmarkCmd::Extrinsic(cmd) => { + let PartialComponents { client, .. } = service::new_partial(&config)?; + // Register the *Remark* and *TKA* builders. + let ext_factory = ExtrinsicFactory(vec![ + Box::new(RemarkBuilder::new(client.clone())), + Box::new(TransferKeepAliveBuilder::new( + client.clone(), + Sr25519Keyring::Alice.to_account_id(), + ExistentialDeposit::get(), + )), + ]); + + cmd.run(client, inherent_benchmark_data()?, &ext_factory) }, BenchmarkCmd::Machine(cmd) => cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index 06c0bcccbc296..13c074268e50f 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -35,11 +35,11 @@ pub mod chain_spec; #[macro_use] pub mod service; #[cfg(feature = "cli")] +mod benchmarking; +#[cfg(feature = "cli")] mod cli; #[cfg(feature = "cli")] mod command; -#[cfg(feature = "cli")] -mod command_helper; #[cfg(feature = "cli")] pub use cli::*; diff --git a/bin/node/cli/tests/benchmark_extrinsic_works.rs b/bin/node/cli/tests/benchmark_extrinsic_works.rs new file mode 100644 index 0000000000000..69800ad3c6c13 --- /dev/null +++ b/bin/node/cli/tests/benchmark_extrinsic_works.rs @@ -0,0 +1,46 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +/// Tests that the `benchmark extrinsic` command works for +/// remark and transfer_keep_alive within the substrate dev runtime. +#[test] +fn benchmark_extrinsic_works() { + benchmark_extrinsic("system", "remark"); + benchmark_extrinsic("balances", "transfer_keep_alive"); +} + +/// Checks that the `benchmark extrinsic` command works for the given pallet and extrinsic. +fn benchmark_extrinsic(pallet: &str, extrinsic: &str) { + let base_dir = tempdir().expect("could not create a temp dir"); + + let status = Command::new(cargo_bin("substrate")) + .args(&["benchmark", "extrinsic", "--dev"]) + .arg("-d") + .arg(base_dir.path()) + .args(&["--pallet", pallet, "--extrinsic", extrinsic]) + // Run with low repeats for faster execution. + .args(["--warmup=10", "--repeat=10", "--max-ext-per-block=10"]) + .status() + .unwrap(); + + assert!(status.success()); +} diff --git a/utils/frame/benchmarking-cli/src/overhead/bench.rs b/utils/frame/benchmarking-cli/src/extrinsic/bench.rs similarity index 70% rename from utils/frame/benchmarking-cli/src/overhead/bench.rs rename to utils/frame/benchmarking-cli/src/extrinsic/bench.rs index be7dac24021cf..27fc40fb34774 100644 --- a/utils/frame/benchmarking-cli/src/overhead/bench.rs +++ b/utils/frame/benchmarking-cli/src/extrinsic/bench.rs @@ -36,8 +36,8 @@ use log::info; use serde::Serialize; use std::{marker::PhantomData, sync::Arc, time::Instant}; -use super::cmd::ExtrinsicBuilder; -use crate::shared::Stats; +use super::ExtrinsicBuilder; +use crate::shared::{StatSelect, Stats}; /// Parameters to configure an *overhead* benchmark. #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] @@ -60,21 +60,11 @@ pub struct BenchmarkParams { /// The results of multiple runs in nano seconds. pub(crate) type BenchRecord = Vec; -/// Type of a benchmark. -#[derive(Serialize, Clone, PartialEq, Copy)] -pub(crate) enum BenchmarkType { - /// Measure the per-extrinsic execution overhead. - Extrinsic, - /// Measure the per-block execution overhead. - Block, -} - /// Holds all objects needed to run the *overhead* benchmarks. pub(crate) struct Benchmark { client: Arc, params: BenchmarkParams, inherent_data: sp_inherents::InherentData, - ext_builder: Arc, _p: PhantomData<(Block, BA)>, } @@ -90,23 +80,51 @@ where client: Arc, params: BenchmarkParams, inherent_data: sp_inherents::InherentData, - ext_builder: Arc, ) -> Self { - Self { client, params, inherent_data, ext_builder, _p: PhantomData } + Self { client, params, inherent_data, _p: PhantomData } } - /// Run the specified benchmark. - pub fn bench(&self, bench_type: BenchmarkType) -> Result { - let (block, num_ext) = self.build_block(bench_type)?; - let record = self.measure_block(&block, num_ext, bench_type)?; + /// Benchmark a block with only inherents. + pub fn bench_block(&self) -> Result { + let (block, _) = self.build_block(None)?; + let record = self.measure_block(&block)?; Stats::new(&record) } - /// Builds a block for the given benchmark type. + /// Benchmark the time of an extrinsic in a full block. + /// + /// First benchmarks an empty block, analogous to `bench_block` and use it as baseline. + /// Then benchmarks a full block built with the given `ext_builder` and subtracts the baseline + /// from the result. + /// This is necessary to account for the time the inherents use. + pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result { + let (block, _) = self.build_block(None)?; + let base = self.measure_block(&block)?; + let base_time = Stats::new(&base)?.select(StatSelect::Average); + + let (block, num_ext) = self.build_block(Some(ext_builder))?; + let num_ext = num_ext.ok_or_else(|| Error::Input("Block was empty".into()))?; + let mut records = self.measure_block(&block)?; + + for r in &mut records { + // Subtract the base time. + *r = r.saturating_sub(base_time); + // Divide by the number of extrinsics in the block. + *r = ((*r as f64) / (num_ext as f64)).ceil() as u64; + } + + Stats::new(&records) + } + + /// Builds a block with some optional extrinsics. /// /// Returns the block and the number of extrinsics in the block /// that are not inherents. - fn build_block(&self, bench_type: BenchmarkType) -> Result<(Block, u64)> { + /// Returns a block with only inherents if `ext_builder` is `None`. + fn build_block( + &self, + ext_builder: Option<&dyn ExtrinsicBuilder>, + ) -> Result<(Block, Option)> { let mut builder = self.client.new_block(Default::default())?; // Create and insert the inherents. let inherents = builder.create_inherents(self.inherent_data.clone())?; @@ -114,16 +132,18 @@ where builder.push(inherent)?; } - // Return early if we just want a block with inherents and no additional extrinsics. - if bench_type == BenchmarkType::Block { - return Ok((builder.build()?.block, 0)) - } + // Return early if `ext_builder` is `None`. + let ext_builder = if let Some(ext_builder) = ext_builder { + ext_builder + } else { + return Ok((builder.build()?.block, None)) + }; // Put as many extrinsics into the block as possible and count them. info!("Building block, this takes some time..."); let mut num_ext = 0; for nonce in 0..self.max_ext_per_block() { - let ext = self.ext_builder.remark(nonce)?; + let ext = ext_builder.build(nonce)?; match builder.push(ext.clone()) { Ok(()) => {}, Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( @@ -139,20 +159,12 @@ where info!("Extrinsics per block: {}", num_ext); let block = builder.build()?.block; - Ok((block, num_ext)) + Ok((block, Some(num_ext))) } /// Measures the time that it take to execute a block or an extrinsic. - fn measure_block( - &self, - block: &Block, - num_ext: u64, - bench_type: BenchmarkType, - ) -> Result { + fn measure_block(&self, block: &Block) -> Result { let mut record = BenchRecord::new(); - if bench_type == BenchmarkType::Extrinsic && num_ext == 0 { - return Err("Cannot measure the extrinsic time of an empty block".into()) - } let genesis = BlockId::Number(Zero::zero()); info!("Running {} warmups...", self.params.warmup); @@ -176,12 +188,7 @@ where .map_err(|e| Error::Client(RuntimeApiError(e)))?; let elapsed = start.elapsed().as_nanos(); - if bench_type == BenchmarkType::Extrinsic { - // Checked for non-zero div above. - record.push((elapsed as f64 / num_ext as f64).ceil() as u64); - } else { - record.push(elapsed as u64); - } + record.push(elapsed as u64); } Ok(record) @@ -191,21 +198,3 @@ where self.params.max_ext_per_block.unwrap_or(u32::MAX) } } - -impl BenchmarkType { - /// Short name of the benchmark type. - pub(crate) fn short_name(&self) -> &'static str { - match self { - Self::Extrinsic => "extrinsic", - Self::Block => "block", - } - } - - /// Long name of the benchmark type. - pub(crate) fn long_name(&self) -> &'static str { - match self { - Self::Extrinsic => "ExtrinsicBase", - Self::Block => "BlockExecution", - } - } -} diff --git a/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs b/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs new file mode 100644 index 0000000000000..6b4fd0fad6638 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs @@ -0,0 +1,134 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams}; +use sc_client_api::Backend as ClientBackend; +use sp_api::{ApiExt, ProvideRuntimeApi}; +use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic}; + +use clap::{Args, Parser}; +use log::info; +use serde::Serialize; +use std::{fmt::Debug, sync::Arc}; + +use super::{ + bench::{Benchmark, BenchmarkParams}, + extrinsic_factory::ExtrinsicFactory, +}; + +/// Benchmark the execution time of different extrinsics. +/// +/// This is calculated by filling a block with a specific extrinsic and executing the block. +/// The result time is then divided by the number of extrinsics in that block. +/// +/// NOTE: The BlockExecutionWeight is ignored in this case since it +// is very small compared to the total block execution time. +#[derive(Debug, Parser)] +pub struct ExtrinsicCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub import_params: ImportParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: ExtrinsicParams, +} + +/// The params for the [`ExtrinsicCmd`]. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct ExtrinsicParams { + #[clap(flatten)] + pub bench: BenchmarkParams, + + /// List all available pallets and extrinsics. + /// + /// The format is CSV with header `pallet, extrinsic`. + #[clap(long)] + pub list: bool, + + /// Pallet name of the extrinsic to benchmark. + #[clap(long, value_name = "PALLET", required_unless_present = "list")] + pub pallet: Option, + + /// Extrinsic to benchmark. + #[clap(long, value_name = "EXTRINSIC", required_unless_present = "list")] + pub extrinsic: Option, +} + +impl ExtrinsicCmd { + /// Benchmark the execution time of a specific type of extrinsic. + /// + /// The output will be printed to console. + pub fn run( + &self, + client: Arc, + inherent_data: sp_inherents::InherentData, + ext_factory: &ExtrinsicFactory, + ) -> Result<()> + where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + ProvideRuntimeApi, + C::Api: ApiExt + BlockBuilderApi, + { + // Short circuit if --list was specified. + if self.params.list { + let list: Vec = ext_factory.0.iter().map(|b| b.name()).collect(); + info!( + "Listing available extrinsics ({}):\npallet, extrinsic\n{}", + list.len(), + list.join("\n") + ); + return Ok(()) + } + + let pallet = self.params.pallet.clone().unwrap_or_default(); + let extrinsic = self.params.extrinsic.clone().unwrap_or_default(); + let ext_builder = match ext_factory.try_get(&pallet, &extrinsic) { + Some(ext_builder) => ext_builder, + None => + return Err("Unknown pallet or extrinsic. Use --list for a complete list.".into()), + }; + + let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data); + let stats = bench.bench_extrinsic(ext_builder)?; + info!( + "Executing a {}::{} extrinsic takes[ns]:\n{:?}", + ext_builder.pallet(), + ext_builder.extrinsic(), + stats + ); + + Ok(()) + } +} + +// Boilerplate +impl CliConfiguration for ExtrinsicCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } +} diff --git a/utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs b/utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs new file mode 100644 index 0000000000000..7e1a22ccd1419 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs @@ -0,0 +1,70 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Provides the [`ExtrinsicFactory`] and the [`ExtrinsicBuilder`] types. +//! Is used by the *overhead* and *extrinsic* benchmarks to build extrinsics. + +use sp_runtime::OpaqueExtrinsic; + +/// Helper to manage [`ExtrinsicBuilder`] instances. +#[derive(Default)] +pub struct ExtrinsicFactory(pub Vec>); + +impl ExtrinsicFactory { + /// Returns a builder for a pallet and extrinsic name. + /// + /// Is case in-sensitive. + pub fn try_get(&self, pallet: &str, extrinsic: &str) -> Option<&dyn ExtrinsicBuilder> { + let pallet = pallet.to_lowercase(); + let extrinsic = extrinsic.to_lowercase(); + + self.0 + .iter() + .find(|b| b.pallet() == pallet && b.extrinsic() == extrinsic) + .map(|b| b.as_ref()) + } +} + +/// Used by the benchmark to build signed extrinsics. +/// +/// The built extrinsics only need to be valid in the first block +/// who's parent block is the genesis block. +/// This assumption simplifies the generation of the extrinsics. +/// The signer should be one of the pre-funded dev accounts. +pub trait ExtrinsicBuilder { + /// Name of the pallet this builder is for. + /// + /// Should be all lowercase. + fn pallet(&self) -> &str; + + /// Name of the extrinsic this builder is for. + /// + /// Should be all lowercase. + fn extrinsic(&self) -> &str; + + /// Builds an extrinsic. + /// + /// Will be called multiple times with increasing nonces. + fn build(&self, nonce: u32) -> std::result::Result; +} + +impl dyn ExtrinsicBuilder + '_ { + /// Name of this builder in CSV format: `pallet, extrinsic`. + pub fn name(&self) -> String { + format!("{}, {}", self.pallet(), self.extrinsic()) + } +} diff --git a/utils/frame/benchmarking-cli/src/extrinsic/mod.rs b/utils/frame/benchmarking-cli/src/extrinsic/mod.rs new file mode 100644 index 0000000000000..12a09c3b1af63 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/extrinsic/mod.rs @@ -0,0 +1,27 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Benchmark the time it takes to execute a specific extrinsic. +//! This is a generalization of the *overhead* benchmark which can only measure `System::Remark` +//! extrinsics. + +pub mod bench; +pub mod cmd; +pub mod extrinsic_factory; + +pub use cmd::ExtrinsicCmd; +pub use extrinsic_factory::{ExtrinsicBuilder, ExtrinsicFactory}; diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index d0eee3d2939fc..96c7a997895d4 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -18,6 +18,7 @@ //! Contains the root [`BenchmarkCmd`] command and exports its sub-commands. mod block; +mod extrinsic; mod machine; mod overhead; mod pallet; @@ -25,8 +26,9 @@ mod shared; mod storage; pub use block::BlockCmd; +pub use extrinsic::{ExtrinsicBuilder, ExtrinsicCmd, ExtrinsicFactory}; pub use machine::{MachineCmd, Requirements, SUBSTRATE_REFERENCE_HARDWARE}; -pub use overhead::{ExtrinsicBuilder, OverheadCmd}; +pub use overhead::OverheadCmd; pub use pallet::PalletCmd; pub use storage::StorageCmd; @@ -41,8 +43,8 @@ pub enum BenchmarkCmd { Storage(StorageCmd), Overhead(OverheadCmd), Block(BlockCmd), - #[clap(hide = true)] // Hidden until fully completed. Machine(MachineCmd), + Extrinsic(ExtrinsicCmd), } /// Unwraps a [`BenchmarkCmd`] into its concrete sub-command. @@ -58,6 +60,7 @@ macro_rules! unwrap_cmd { BenchmarkCmd::Overhead($cmd) => $code, BenchmarkCmd::Block($cmd) => $code, BenchmarkCmd::Machine($cmd) => $code, + BenchmarkCmd::Extrinsic($cmd) => $code, } } } diff --git a/utils/frame/benchmarking-cli/src/overhead/cmd.rs b/utils/frame/benchmarking-cli/src/overhead/cmd.rs index 357c89d97a5ac..28ceea63f1572 100644 --- a/utils/frame/benchmarking-cli/src/overhead/cmd.rs +++ b/utils/frame/benchmarking-cli/src/overhead/cmd.rs @@ -31,10 +31,11 @@ use serde::Serialize; use std::{fmt::Debug, sync::Arc}; use crate::{ - overhead::{ - bench::{Benchmark, BenchmarkParams, BenchmarkType}, - template::TemplateData, + extrinsic::{ + bench::{Benchmark, BenchmarkParams as ExtrinsicBenchmarkParams}, + ExtrinsicBuilder, }, + overhead::template::TemplateData, shared::{HostInfoParams, WeightParams}, }; @@ -63,20 +64,20 @@ pub struct OverheadParams { #[allow(missing_docs)] #[clap(flatten)] - pub bench: BenchmarkParams, + pub bench: ExtrinsicBenchmarkParams, #[allow(missing_docs)] #[clap(flatten)] pub hostinfo: HostInfoParams, } -/// Used by the benchmark to build signed extrinsics. -/// -/// The built extrinsics only need to be valid in the first block -/// who's parent block is the genesis block. -pub trait ExtrinsicBuilder { - /// Build a `System::remark` extrinsic. - fn remark(&self, nonce: u32) -> std::result::Result; +/// Type of a benchmark. +#[derive(Serialize, Clone, PartialEq, Copy)] +pub(crate) enum BenchmarkType { + /// Measure the per-extrinsic execution overhead. + Extrinsic, + /// Measure the per-block execution overhead. + Block, } impl OverheadCmd { @@ -89,7 +90,7 @@ impl OverheadCmd { cfg: Configuration, client: Arc, inherent_data: sp_inherents::InherentData, - ext_builder: Arc, + ext_builder: &dyn ExtrinsicBuilder, ) -> Result<()> where Block: BlockT, @@ -97,18 +98,21 @@ impl OverheadCmd { C: BlockBuilderProvider + ProvideRuntimeApi, C::Api: ApiExt + BlockBuilderApi, { - let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, ext_builder); + if ext_builder.pallet() != "system" || ext_builder.extrinsic() != "remark" { + return Err(format!("The extrinsic builder is required to build `System::Remark` extrinsics but builds `{}` extrinsics instead", ext_builder.name()).into()); + } + let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data); // per-block execution overhead { - let stats = bench.bench(BenchmarkType::Block)?; + let stats = bench.bench_block()?; info!("Per-block execution overhead [ns]:\n{:?}", stats); let template = TemplateData::new(BenchmarkType::Block, &cfg, &self.params, &stats)?; template.write(&self.params.weight.weight_path)?; } // per-extrinsic execution overhead { - let stats = bench.bench(BenchmarkType::Extrinsic)?; + let stats = bench.bench_extrinsic(ext_builder)?; info!("Per-extrinsic execution overhead [ns]:\n{:?}", stats); let template = TemplateData::new(BenchmarkType::Extrinsic, &cfg, &self.params, &stats)?; template.write(&self.params.weight.weight_path)?; @@ -118,6 +122,24 @@ impl OverheadCmd { } } +impl BenchmarkType { + /// Short name of the benchmark type. + pub(crate) fn short_name(&self) -> &'static str { + match self { + Self::Extrinsic => "extrinsic", + Self::Block => "block", + } + } + + /// Long name of the benchmark type. + pub(crate) fn long_name(&self) -> &'static str { + match self { + Self::Extrinsic => "ExtrinsicBase", + Self::Block => "BlockExecution", + } + } +} + // Boilerplate impl CliConfiguration for OverheadCmd { fn shared_params(&self) -> &SharedParams { diff --git a/utils/frame/benchmarking-cli/src/overhead/mod.rs b/utils/frame/benchmarking-cli/src/overhead/mod.rs index abdeac22b7898..fc3db912f7a30 100644 --- a/utils/frame/benchmarking-cli/src/overhead/mod.rs +++ b/utils/frame/benchmarking-cli/src/overhead/mod.rs @@ -15,8 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod bench; pub mod cmd; -mod template; +pub mod template; -pub use cmd::{ExtrinsicBuilder, OverheadCmd}; +pub use cmd::OverheadCmd; diff --git a/utils/frame/benchmarking-cli/src/overhead/template.rs b/utils/frame/benchmarking-cli/src/overhead/template.rs index 33c2c7999039a..aa82e45cf6db9 100644 --- a/utils/frame/benchmarking-cli/src/overhead/template.rs +++ b/utils/frame/benchmarking-cli/src/overhead/template.rs @@ -27,7 +27,7 @@ use serde::Serialize; use std::{env, fs, path::PathBuf}; use crate::{ - overhead::{bench::BenchmarkType, cmd::OverheadParams}, + overhead::cmd::{BenchmarkType, OverheadParams}, shared::{Stats, UnderscoreHelper}, };