From d1300d610db59f97ab2d9478e8b4faf7a610236c Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Fri, 8 Jul 2022 14:50:00 +0000 Subject: [PATCH] Improve base weights consistency and make sure they're never zero --- Cargo.lock | 314 ++++++++++++++---- Cargo.toml | 1 - frame/benchmarking/Cargo.toml | 8 +- frame/benchmarking/src/analysis.rs | 148 ++++++--- frame/benchmarking/src/lib.rs | 2 +- .../benchmarking-cli/src/pallet/writer.rs | 18 +- 6 files changed, 362 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d68330cfb15c2..8259e81cd998f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,15 @@ version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" +[[package]] +name = "approx" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2a05fd1bd10b2527e20a2cd32d8873d115b8b39fe219ee25f42a8aca6ba278" +dependencies = [ + "num-traits", +] + [[package]] name = "approx" version = "0.5.0" @@ -112,6 +121,32 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "698b65a961a9d730fb45b6b0327e20207810c9f61ee421b082b27ba003f49e2b" +[[package]] +name = "argmin" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc077a0240b05e5df4e658e4ad8a3d42b856e3136d4a05ac8330e0a9170d39e" +dependencies = [ + "anyhow", + "approx 0.5.0", + "bincode", + "instant", + "ndarray", + "ndarray-rand", + "num", + "num-complex", + "paste 1.0.6", + "rand 0.8.4", + "rand_xorshift 0.3.0", + "serde", + "serde_json", + "slog", + "slog-async", + "slog-json", + "slog-term", + "thiserror", +] + [[package]] name = "arrayref" version = "0.3.6" @@ -916,7 +951,7 @@ dependencies = [ "libc", "num-integer", "num-traits", - "time", + "time 0.1.44", "winapi", ] @@ -1657,6 +1692,16 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.3.6" @@ -2106,8 +2151,10 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", - "linregress", + "linfa", + "linfa-linear", "log", + "ndarray", "parity-scale-codec", "paste 1.0.6", "scale-info", @@ -4083,6 +4130,47 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linfa" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20345c27f1041e3d3c6457d92530688a9134c28cabaf4cf9a6e0d90a91ac7075" +dependencies = [ + "anyhow", + "approx 0.4.0", + "ndarray", + "num-traits", + "rand 0.8.4", + "serde", + "thiserror", +] + +[[package]] +name = "linfa-linalg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e7562b41c8876d3367897067013bb2884cc78e6893f092ecd26b305176ac82" +dependencies = [ + "ndarray", + "num-traits", + "thiserror", +] + +[[package]] +name = "linfa-linear" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8bff3f1b2cd598bc65b68e2ad2adf91e537e03e11cf5d954be53e85dda0ef90" +dependencies = [ + "argmin", + "linfa", + "linfa-linalg", + "ndarray", + "num-traits", + "serde", + "thiserror", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -4098,16 +4186,6 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "linregress" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c601a85f5ecd1aba625247bca0031585fb1c446461b142878a16f8245ddeb8" -dependencies = [ - "nalgebra", - "statrs", -] - [[package]] name = "linux-raw-sys" version = "0.0.42" @@ -4254,9 +4332,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "matrixmultiply" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a8a15b776d9dfaecd44b03c5828c2199cddff5247215858aac14624f8d6b741" +checksum = "add85d4dd35074e6fedc608f8c8f513a3548619a9024b751949ef0e8e45a4d84" dependencies = [ "rawpointer", ] @@ -4468,41 +4546,38 @@ dependencies = [ ] [[package]] -name = "nalgebra" -version = "0.27.1" +name = "names" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "462fffe4002f4f2e1f6a9dcf12cc1a6fc0e15989014efc02a941d3e0f5dc2120" +checksum = "e7d66043b25d4a6cccb23619d10c19c25304b355a7dccd4a8e11423dd2382146" dependencies = [ - "approx", - "matrixmultiply", - "nalgebra-macros", - "num-complex", - "num-rational 0.4.0", - "num-traits", "rand 0.8.4", - "rand_distr", - "simba", - "typenum", ] [[package]] -name = "nalgebra-macros" -version = "0.1.0" +name = "ndarray" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01fcc0b8149b4632adc89ac3b7b31a12fb6099a0317a4eb2ebff574ef7de7218" +checksum = "dec23e6762830658d2b3d385a75aa212af2f67a4586d4442907144f3bb6a1ca8" dependencies = [ - "proc-macro2", - "quote", - "syn", + "approx 0.4.0", + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "rawpointer", + "serde", ] [[package]] -name = "names" -version = "0.13.0" +name = "ndarray-rand" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d66043b25d4a6cccb23619d10c19c25304b355a7dccd4a8e11423dd2382146" +checksum = "65608f937acc725f5b164dcf40f4f0bc5d67dc268ab8a649d3002606718c4588" dependencies = [ + "ndarray", "rand 0.8.4", + "rand_distr", ] [[package]] @@ -5058,6 +5133,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +dependencies = [ + "num-bigint 0.4.3", + "num-complex", + "num-integer", + "num-iter", + "num-rational 0.4.0", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.2.6" @@ -5069,6 +5158,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.0" @@ -5098,6 +5198,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg 1.0.1", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.2.4" @@ -5105,7 +5216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ "autocfg 1.0.1", - "num-bigint", + "num-bigint 0.2.6", "num-integer", "num-traits", ] @@ -5117,6 +5228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" dependencies = [ "autocfg 1.0.1", + "num-bigint 0.4.3", "num-integer", "num-traits", ] @@ -5141,6 +5253,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.27.1" @@ -7326,7 +7447,7 @@ dependencies = [ "rand_jitter", "rand_os", "rand_pcg 0.1.2", - "rand_xorshift", + "rand_xorshift 0.1.1", "winapi", ] @@ -7354,6 +7475,7 @@ dependencies = [ "rand_chacha 0.3.0", "rand_core 0.6.2", "rand_hc 0.3.0", + "serde", ] [[package]] @@ -7417,6 +7539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ "getrandom 0.2.3", + "serde", ] [[package]] @@ -7527,6 +7650,16 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.2", + "serde", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -8250,7 +8383,7 @@ dependencies = [ "futures", "log", "merlin", - "num-bigint", + "num-bigint 0.2.6", "num-rational 0.2.4", "num-traits", "parity-scale-codec", @@ -9565,18 +9698,6 @@ dependencies = [ "rand_core 0.6.2", ] -[[package]] -name = "simba" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e82063457853d00243beda9952e910b82593e4b07ae9f721b9278a99a0d3d5c" -dependencies = [ - "approx", - "num-complex", - "num-traits", - "paste 1.0.6", -] - [[package]] name = "slab" version = "0.4.2" @@ -9589,6 +9710,49 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-async" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-json" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" +dependencies = [ + "serde", + "serde_json", + "slog", + "time 0.3.11", +] + +[[package]] +name = "slog-term" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" +dependencies = [ + "atty", + "slog", + "term", + "thread_local", + "time 0.3.11", +] + [[package]] name = "smallvec" version = "1.8.0" @@ -9751,7 +9915,7 @@ name = "sp-arithmetic-fuzzer" version = "2.0.0" dependencies = [ "honggfuzz", - "num-bigint", + "num-bigint 0.2.6", "primitive-types", "sp-arithmetic", ] @@ -10514,19 +10678,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "statrs" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05bdbb8e4e78216a85785a85d3ec3183144f98d0097b9281802c019bb07a6f05" -dependencies = [ - "approx", - "lazy_static", - "nalgebra", - "num-traits", - "rand 0.8.4", -] - [[package]] name = "strsim" version = "0.10.0" @@ -10871,6 +11022,12 @@ dependencies = [ "libc", ] +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + [[package]] name = "tap" version = "1.0.1" @@ -10897,6 +11054,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -10987,6 +11155,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" +dependencies = [ + "itoa 1.0.1", + "libc", + "num_threads", + "time-macros", +] + +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tiny-bip39" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 9909e6f893877..37b0d0ebbc8a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -272,7 +272,6 @@ librocksdb-sys = { opt-level = 3 } libsecp256k1 = { opt-level = 3 } libz-sys = { opt-level = 3 } mio = { opt-level = 3 } -nalgebra = { opt-level = 3 } num-bigint = { opt-level = 3 } parking_lot = { opt-level = 3 } parking_lot_core = { opt-level = 3 } diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml index 4205274b5dbc3..299a31f0346aa 100644 --- a/frame/benchmarking/Cargo.toml +++ b/frame/benchmarking/Cargo.toml @@ -14,7 +14,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } -linregress = { version = "0.4.4", optional = true } +ndarray = { version = "0.15.0", optional = true } +linfa-linear = { version = "0.6.0", optional = true } +linfa = { version = "0.6.0", optional = true } log = { version = "0.4.17", default-features = false } paste = "1.0" scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } @@ -39,7 +41,9 @@ std = [ "codec/std", "frame-support/std", "frame-system/std", - "linregress", + "linfa-linear", + "linfa", + "ndarray", "log/std", "scale-info/std", "serde", diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index c19af781234f8..e4a1d92c364de 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -18,17 +18,14 @@ //! Tools for analyzing the benchmark results. use crate::BenchmarkResult; -use linregress::{FormulaRegressionBuilder, RegressionDataBuilder}; use std::collections::BTreeMap; -pub use linregress::RegressionModel; - pub struct Analysis { pub base: u128, pub slopes: Vec, pub names: Vec, pub value_dists: Option, u128, u128)>>, - pub model: Option, + pub errors: Option>, } #[derive(Clone, Copy)] @@ -72,6 +69,82 @@ impl TryFrom> for AnalysisChoice { } } +fn raw_linear_regression( + xs: Vec, + ys: Vec, + x_vars: usize, + with_intercept: bool, +) -> Option<(f64, Vec)> { + use linfa::prelude::*; + + let records = ndarray::Array1::from_vec(ys); + let targets = ndarray::Array2::from_shape_vec((xs.len() / x_vars, x_vars), xs).ok()?; + let dataset: linfa::DatasetView = + (targets.view(), records.view()).into(); + let model = linfa_linear::LinearRegression::default() + .with_intercept(with_intercept) + .fit(&dataset) + .ok()?; + Some((model.intercept(), model.params().to_vec())) +} + +fn linear_regression(xs: Vec, mut ys: Vec, x_vars: usize) -> Option<(f64, Vec)> { + let mut min = ys[0]; + for &value in &ys { + if value < min { + min = value; + } + } + + for value in &mut ys { + *value -= min; + } + + let (intercept, params) = raw_linear_regression(xs, ys, x_vars, false)?; + Some((intercept + min, params)) +} + +#[test] +fn test_linear_regression() { + let ys = vec![ + 3797981.0, + 37857779.0, + 70569402.0, + 104004114.0, + 137233924.0, + 169826237.0, + 203521133.0, + 237552333.0, + 271082065.0, + 305554637.0, + 335218347.0, + 371759065.0, + 405086197.0, + 438353555.0, + 472891417.0, + 505339532.0, + 527784778.0, + 562590596.0, + 635291991.0, + 673027090.0, + 708119408.0, + ]; + let xs = vec![ + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, + 17.0, 18.0, 19.0, 20.0, + ]; + + let (intercept, params) = raw_linear_regression(xs.clone(), ys.clone(), 1, true).unwrap(); + assert_eq!(intercept as i64, -2712997); + assert_eq!(params.len(), 1); + assert_eq!(params[0] as i64, 34444926); + + let (intercept, params) = linear_regression(xs, ys, 1).unwrap(); + assert_eq!(intercept as i64, 3797981); + assert_eq!(params.len(), 1); + assert_eq!(params[0] as i64, 33968513); +} + impl Analysis { // Useful for when there are no components, and we just need an median value of the benchmark // results. Note: We choose the median value because it is more robust to outliers. @@ -99,7 +172,7 @@ impl Analysis { slopes: Vec::new(), names: Vec::new(), value_dists: None, - model: None, + errors: None, }) } @@ -194,7 +267,7 @@ impl Analysis { slopes, names: results.into_iter().map(|x| x.0).collect::>(), value_dists: None, - model: None, + errors: None, }) } @@ -221,36 +294,7 @@ impl Analysis { *rs = rs[ql..rs.len() - ql].to_vec(); } - let mut data = - vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())]; - let names = r[0].components.iter().map(|x| format!("{:?}", x.0)).collect::>(); - data.extend(names.iter().enumerate().map(|(i, p)| { - ( - p.as_str(), - results - .iter() - .flat_map(|x| Some(x.0[i] as f64).into_iter().cycle().take(x.1.len())) - .collect::>(), - ) - })); - - let data = RegressionDataBuilder::new().build_from(data).ok()?; - - let model = FormulaRegressionBuilder::new() - .data(&data) - .formula(format!("Y ~ {}", names.join(" + "))) - .fit() - .ok()?; - - let slopes = model - .parameters - .regressor_values - .iter() - .enumerate() - .map(|(_, x)| (*x + 0.5) as u128) - .collect(); - let value_dists = results .iter() .map(|(p, vs)| { @@ -269,12 +313,21 @@ impl Analysis { }) .collect::>(); + let mut ys: Vec = Vec::new(); + let mut xs: Vec = Vec::new(); + for result in results { + xs.extend(result.0.iter().map(|value| *value as f64)); + ys.push(result.1[0] as f64); + } + + let (intercept, slopes) = linear_regression(xs, ys, r[0].components.len())?; + Some(Self { - base: (model.parameters.intercept_value + 0.5) as u128, - slopes, + base: (intercept + 0.5) as u128, + slopes: slopes.into_iter().map(|value| (value + 0.5) as u128).collect(), names, value_dists: Some(value_dists), - model: Some(model), + errors: None, // TODO }) } @@ -304,9 +357,9 @@ impl Analysis { .for_each(|(a, b)| assert!(a == b, "benchmark results not in the same order")); let names = median_slopes.names; let value_dists = min_squares.value_dists; - let model = min_squares.model; + let errors = min_squares.errors; - Some(Self { base, slopes, names, value_dists, model }) + Some(Self { base, slopes, names, value_dists, errors }) } } @@ -363,13 +416,6 @@ impl std::fmt::Display for Analysis { } } } - if let Some(ref model) = self.model { - writeln!(f, "\nQuality and confidence:")?; - writeln!(f, "param error")?; - for (p, se) in self.names.iter().zip(model.se.regressor_values.iter()) { - writeln!(f, "{} {:>8}", p, ms(*se as u128))?; - } - } writeln!(f, "\nModel:")?; writeln!(f, "Time ~= {:>8}", ms(self.base))?; @@ -553,15 +599,15 @@ mod tests { let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap(); - assert_eq!(extrinsic_time.base, 10_000_000); - assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]); + assert_eq!(extrinsic_time.base, 11_500_000); + assert_eq!(extrinsic_time.slopes, vec![630636, 23699]); let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap(); - assert_eq!(reads.base, 2); + assert_eq!(reads.base, 3); assert_eq!(reads.slopes, vec![1, 0]); let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap(); - assert_eq!(writes.base, 0); + assert_eq!(writes.base, 2); assert_eq!(writes.slopes, vec![0, 2]); } } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index afd53915cc397..3915c365363c9 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -30,7 +30,7 @@ mod utils; pub mod baseline; #[cfg(feature = "std")] -pub use analysis::{Analysis, AnalysisChoice, BenchmarkSelector, RegressionModel}; +pub use analysis::{Analysis, AnalysisChoice, BenchmarkSelector}; #[doc(hidden)] pub use frame_support; #[doc(hidden)] diff --git a/utils/frame/benchmarking-cli/src/pallet/writer.rs b/utils/frame/benchmarking-cli/src/pallet/writer.rs index 42a237fcf3ce3..7e628daf2d24e 100644 --- a/utils/frame/benchmarking-cli/src/pallet/writer.rs +++ b/utils/frame/benchmarking-cli/src/pallet/writer.rs @@ -29,7 +29,6 @@ use serde::Serialize; use crate::{pallet::command::ComponentRange, shared::UnderscoreHelper, PalletCmd}; use frame_benchmarking::{ Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector, - RegressionModel, }; use frame_support::traits::StorageInfo; use sp_core::hexdisplay::HexDisplay; @@ -145,12 +144,11 @@ fn map_results( Ok(all_benchmarks) } -// Get an iterator of errors from a model. If the model is `None` all errors are zero. -fn extract_errors(model: &Option) -> impl Iterator + '_ { - let mut errors = model.as_ref().map(|m| m.se.regressor_values.iter()); - std::iter::from_fn(move || match &mut errors { - Some(model) => model.next().map(|val| *val as u128), - _ => Some(0), +// Get an iterator of errors. +fn extract_errors(errors: &Option>) -> impl Iterator + '_ { + std::iter::from_fn(move || { + // TODO + Some(0) }) } @@ -190,7 +188,7 @@ fn get_benchmark_data( .slopes .into_iter() .zip(extrinsic_time.names.iter()) - .zip(extract_errors(&extrinsic_time.model)) + .zip(extract_errors(&extrinsic_time.errors)) .for_each(|((slope, name), error)| { if !slope.is_zero() { if !used_components.contains(&name) { @@ -207,7 +205,7 @@ fn get_benchmark_data( .slopes .into_iter() .zip(reads.names.iter()) - .zip(extract_errors(&reads.model)) + .zip(extract_errors(&reads.errors)) .for_each(|((slope, name), error)| { if !slope.is_zero() { if !used_components.contains(&name) { @@ -220,7 +218,7 @@ fn get_benchmark_data( .slopes .into_iter() .zip(writes.names.iter()) - .zip(extract_errors(&writes.model)) + .zip(extract_errors(&writes.errors)) .for_each(|((slope, name), error)| { if !slope.is_zero() { if !used_components.contains(&name) {