From 979598ee75a0fd97e07b8b25960ccf55aab429a6 Mon Sep 17 00:00:00 2001 From: Jordi Chauzi Date: Sun, 28 Feb 2021 10:22:21 +0100 Subject: [PATCH 1/3] Export exit codes to JSON output --- CHANGELOG.md | 1 + src/hyperfine/benchmark.rs | 36 +++++++++++++++++++++++++++----- src/hyperfine/export/asciidoc.rs | 4 ++++ src/hyperfine/export/csv.rs | 4 +++- src/hyperfine/export/markdown.rs | 8 +++++++ src/hyperfine/internal.rs | 1 + src/hyperfine/types.rs | 5 +++++ 7 files changed, 53 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0599f1bc..0d8354a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # unreleased - Add command exit code to output if it fails, see #342 (@KaindlJulian) +- Export command exit code to JSON output, see #371 (@JordiChauzi) ## Features diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index 446d75de5..e3444f890 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -1,6 +1,6 @@ use std::cmp; use std::io; -use std::process::Stdio; +use std::process::{ExitStatus, Stdio}; use colored::*; use statistical::{mean, median, standard_deviation}; @@ -46,7 +46,7 @@ pub fn time_shell_command( show_output: bool, failure_action: CmdFailureAction, shell_spawning_time: Option, -) -> io::Result<(TimingResult, bool)> { +) -> io::Result<(TimingResult, ExitStatus)> { let (stdout, stderr) = if show_output { (Stdio::inherit(), Stdio::inherit()) } else { @@ -88,7 +88,7 @@ pub fn time_shell_command( time_user, time_system, }, - result.status.success(), + result.status, )) } @@ -201,6 +201,26 @@ fn run_cleanup_command( run_intermediate_command(shell, command, show_output, error_output) } +#[cfg(unix)] +fn extract_exit_code(status: ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + + /* From the ExitStatus::code documentation: + "On Unix, this will return None if the process was terminated by a signal." + In that case, ExitStatusExt::signal should never return None. + */ + status.code().unwrap_or_else(|| status.signal().unwrap()) +} + +#[cfg(not(unix))] +fn extract_exit_code(status: ExitStatus) -> i32 { + /* From the ExitStatus::code documentation: + "On Unix, this will return None if the process was terminated by a signal." + On the other configurations, ExitStatus::code should never return None. + */ + status.code().unwrap() +} + /// Run the benchmark for a single shell command pub fn run_benchmark( num: usize, @@ -228,6 +248,7 @@ pub fn run_benchmark( let mut times_real: Vec = vec![]; let mut times_user: Vec = vec![]; let mut times_system: Vec = vec![]; + let mut exit_codes: Vec = vec![]; let mut all_succeeded = true; // Run init command @@ -280,13 +301,14 @@ pub fn run_benchmark( let prepare_res = run_preparation_command(&options.shell, &prepare_cmd, options.show_output)?; // Initial timing run - let (res, success) = time_shell_command( + let (res, status) = time_shell_command( &options.shell, cmd, options.show_output, options.failure_action, Some(shell_spawning_time), )?; + let success = status.success(); // Determine number of benchmark runs let runs_in_min_time = (options.min_time_sec @@ -310,6 +332,7 @@ pub fn run_benchmark( times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -328,17 +351,19 @@ pub fn run_benchmark( progress_bar.as_ref().map(|bar| bar.set_message(&msg)); - let (res, success) = time_shell_command( + let (res, status) = time_shell_command( &options.shell, cmd, options.show_output, options.failure_action, Some(shell_spawning_time), )?; + let success = status.success(); times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -438,6 +463,7 @@ pub fn run_benchmark( t_min, t_max, times_real, + exit_codes, cmd.get_parameters() .iter() .map(|(name, value)| ((*name).to_string(), value.to_string())) diff --git a/src/hyperfine/export/asciidoc.rs b/src/hyperfine/export/asciidoc.rs index cae6075f9..0a7bd06b3 100644 --- a/src/hyperfine/export/asciidoc.rs +++ b/src/hyperfine/export/asciidoc.rs @@ -99,6 +99,7 @@ fn test_asciidoc_table_row() { 0.10745223440000001, 0.10697327940000001, ], + vec![0, 0, 0], // exit codes BTreeMap::new(), // param ); @@ -151,6 +152,7 @@ fn test_asciidoc_table_row_command_escape() { 0.10745223440000001, 0.10697327940000001, ], + vec![0, 0, 0], // exit codes BTreeMap::new(), // param ); let exps = format!( @@ -185,6 +187,7 @@ fn test_asciidoc() { 5.0, 6.0, vec![7.0, 8.0, 9.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); @@ -202,6 +205,7 @@ fn test_asciidoc() { 15.0, 16.0, vec![17.0, 18.0, 19.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); diff --git a/src/hyperfine/export/csv.rs b/src/hyperfine/export/csv.rs index 28593a44a..ef2c3821d 100644 --- a/src/hyperfine/export/csv.rs +++ b/src/hyperfine/export/csv.rs @@ -17,7 +17,7 @@ impl Exporter for CsvExporter { { let mut headers: Vec> = [ - // The list of times cannot be exported to the CSV file - omit it. + // The list of times and exit codes cannot be exported to the CSV file - omit them. "command", "mean", "stddev", "median", "user", "system", "min", "max", ] .iter() @@ -68,6 +68,7 @@ fn test_csv() { 5.0, 6.0, vec![7.0, 8.0, 9.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); @@ -85,6 +86,7 @@ fn test_csv() { 15.0, 16.5, vec![17.0, 18.0, 19.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); diff --git a/src/hyperfine/export/markdown.rs b/src/hyperfine/export/markdown.rs index 40b3a6abe..a21a81597 100644 --- a/src/hyperfine/export/markdown.rs +++ b/src/hyperfine/export/markdown.rs @@ -102,6 +102,7 @@ fn test_markdown_format_ms() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -115,6 +116,7 @@ fn test_markdown_format_ms() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -150,6 +152,7 @@ fn test_markdown_format_s() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -163,6 +166,7 @@ fn test_markdown_format_s() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -197,6 +201,7 @@ fn test_markdown_format_time_unit_s() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -210,6 +215,7 @@ fn test_markdown_format_time_unit_s() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -250,6 +256,7 @@ fn test_markdown_format_time_unit_ms() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -263,6 +270,7 @@ fn test_markdown_format_time_unit_ms() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); diff --git a/src/hyperfine/internal.rs b/src/hyperfine/internal.rs index adeff63b3..72ea70033 100644 --- a/src/hyperfine/internal.rs +++ b/src/hyperfine/internal.rs @@ -157,6 +157,7 @@ fn create_result(name: &str, mean: Scalar) -> BenchmarkResult { min: mean, max: mean, times: None, + exit_codes: Vec::new(), parameters: BTreeMap::new(), } } diff --git a/src/hyperfine/types.rs b/src/hyperfine/types.rs index 98e5ea0b3..dd792e3ef 100644 --- a/src/hyperfine/types.rs +++ b/src/hyperfine/types.rs @@ -268,6 +268,9 @@ pub struct BenchmarkResult { #[serde(skip_serializing_if = "Option::is_none")] pub times: Option>, + /// All run exit codes + pub exit_codes: Vec, + /// Any parameter values used #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, @@ -285,6 +288,7 @@ impl BenchmarkResult { min: Second, max: Second, times: Vec, + exit_codes: Vec, parameters: BTreeMap, ) -> Self { BenchmarkResult { @@ -297,6 +301,7 @@ impl BenchmarkResult { min, max, times: Some(times), + exit_codes, parameters, } } From 59bb1387eb1c621f8ea508d2a45dcae2791e6d63 Mon Sep 17 00:00:00 2001 From: Jordi Chauzi Date: Sun, 9 May 2021 18:20:59 +0200 Subject: [PATCH 2/3] Store exit codes as Options (and remove unwrapping) --- src/hyperfine/benchmark.rs | 14 +-- src/hyperfine/export/asciidoc.rs | 12 +-- src/hyperfine/export/csv.rs | 4 +- src/hyperfine/export/markdown.rs | 160 +++++++++++++++---------------- src/hyperfine/types.rs | 4 +- 5 files changed, 95 insertions(+), 99 deletions(-) diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index e3444f890..116920875 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -202,23 +202,19 @@ fn run_cleanup_command( } #[cfg(unix)] -fn extract_exit_code(status: ExitStatus) -> i32 { +fn extract_exit_code(status: ExitStatus) -> Option { use std::os::unix::process::ExitStatusExt; /* From the ExitStatus::code documentation: "On Unix, this will return None if the process was terminated by a signal." In that case, ExitStatusExt::signal should never return None. */ - status.code().unwrap_or_else(|| status.signal().unwrap()) + status.code().or_else(|| status.signal()) } #[cfg(not(unix))] -fn extract_exit_code(status: ExitStatus) -> i32 { - /* From the ExitStatus::code documentation: - "On Unix, this will return None if the process was terminated by a signal." - On the other configurations, ExitStatus::code should never return None. - */ - status.code().unwrap() +fn extract_exit_code(status: ExitStatus) -> Option { + status.code() } /// Run the benchmark for a single shell command @@ -248,7 +244,7 @@ pub fn run_benchmark( let mut times_real: Vec = vec![]; let mut times_user: Vec = vec![]; let mut times_system: Vec = vec![]; - let mut exit_codes: Vec = vec![]; + let mut exit_codes: Vec> = vec![]; let mut all_succeeded = true; // Run init command diff --git a/src/hyperfine/export/asciidoc.rs b/src/hyperfine/export/asciidoc.rs index 0a7bd06b3..08623bfe6 100644 --- a/src/hyperfine/export/asciidoc.rs +++ b/src/hyperfine/export/asciidoc.rs @@ -99,8 +99,8 @@ fn test_asciidoc_table_row() { 0.10745223440000001, 0.10697327940000001, ], - vec![0, 0, 0], // exit codes - BTreeMap::new(), // param + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // param ); let expms = format!( @@ -152,8 +152,8 @@ fn test_asciidoc_table_row_command_escape() { 0.10745223440000001, 0.10697327940000001, ], - vec![0, 0, 0], // exit codes - BTreeMap::new(), // param + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // param ); let exps = format!( "| `sleep 1\\|`\n\ @@ -187,7 +187,7 @@ fn test_asciidoc() { 5.0, 6.0, vec![7.0, 8.0, 9.0], - vec![0, 0, 0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); @@ -205,7 +205,7 @@ fn test_asciidoc() { 15.0, 16.0, vec![17.0, 18.0, 19.0], - vec![0, 0, 0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); diff --git a/src/hyperfine/export/csv.rs b/src/hyperfine/export/csv.rs index ef2c3821d..ef2cc2328 100644 --- a/src/hyperfine/export/csv.rs +++ b/src/hyperfine/export/csv.rs @@ -68,7 +68,7 @@ fn test_csv() { 5.0, 6.0, vec![7.0, 8.0, 9.0], - vec![0, 0, 0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); @@ -86,7 +86,7 @@ fn test_csv() { 15.0, 16.5, vec![17.0, 18.0, 19.0], - vec![0, 0, 0], + vec![Some(0), Some(0), Some(0)], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); diff --git a/src/hyperfine/export/markdown.rs b/src/hyperfine/export/markdown.rs index a21a81597..3d7a2e25e 100644 --- a/src/hyperfine/export/markdown.rs +++ b/src/hyperfine/export/markdown.rs @@ -94,30 +94,30 @@ fn test_markdown_format_ms() { timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap(); @@ -144,30 +144,30 @@ fn test_markdown_format_s() { timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap(); @@ -193,30 +193,30 @@ fn test_markdown_format_time_unit_s() { timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8( @@ -248,30 +248,30 @@ fn test_markdown_format_time_unit_ms() { timing_results.push(BenchmarkResult::new( String::from("sleep 2"), - 2.0050, // mean - 0.0020, // std dev - 2.0050, // median - 0.0009, // user_mean - 0.0012, // system_mean - 2.0020, // min - 2.0080, // max - vec![2.0, 2.0, 2.0], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 2.0050, // mean + 0.0020, // std dev + 2.0050, // median + 0.0009, // user_mean + 0.0012, // system_mean + 2.0020, // min + 2.0080, // max + vec![2.0, 2.0, 2.0], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); timing_results.push(BenchmarkResult::new( String::from("sleep 0.1"), - 0.1057, // mean - 0.0016, // std dev - 0.1057, // median - 0.0009, // user_mean - 0.0011, // system_mean - 0.1023, // min - 0.1080, // max - vec![0.1, 0.1, 0.1], // times - vec![0, 0, 0], // exit codes - BTreeMap::new(), // parameter + 0.1057, // mean + 0.0016, // std dev + 0.1057, // median + 0.0009, // user_mean + 0.0011, // system_mean + 0.1023, // min + 0.1080, // max + vec![0.1, 0.1, 0.1], // times + vec![Some(0), Some(0), Some(0)], // exit codes + BTreeMap::new(), // parameter )); let formatted = String::from_utf8( diff --git a/src/hyperfine/types.rs b/src/hyperfine/types.rs index dd792e3ef..850a5415b 100644 --- a/src/hyperfine/types.rs +++ b/src/hyperfine/types.rs @@ -269,7 +269,7 @@ pub struct BenchmarkResult { pub times: Option>, /// All run exit codes - pub exit_codes: Vec, + pub exit_codes: Vec>, /// Any parameter values used #[serde(skip_serializing_if = "BTreeMap::is_empty")] @@ -288,7 +288,7 @@ impl BenchmarkResult { min: Second, max: Second, times: Vec, - exit_codes: Vec, + exit_codes: Vec>, parameters: BTreeMap, ) -> Self { BenchmarkResult { From 153f34d77e0b7b698ffed7f9da9d05dca439372c Mon Sep 17 00:00:00 2001 From: Jordi Chauzi Date: Mon, 10 May 2021 11:07:59 +0200 Subject: [PATCH 3/3] Avoid confusion between exit code and signal values --- src/hyperfine/benchmark.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index 116920875..60ac8f5de 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -209,7 +209,12 @@ fn extract_exit_code(status: ExitStatus) -> Option { "On Unix, this will return None if the process was terminated by a signal." In that case, ExitStatusExt::signal should never return None. */ - status.code().or_else(|| status.signal()) + status.code().or_else(|| + /* To differentiate between "normal" exit codes and signals, we are using + something similar to bash exit codes (https://tldp.org/LDP/abs/html/exitcodes.html) + by adding 128 to a signal integer value. + */ + status.signal().map(|s| 128 + s)) } #[cfg(not(unix))]