Support matrices of independent parameter lists

This patch permits the `-L`/`--parameter-list` argument to appear
multiple times. Each additional parameter increases the dimensionality
of the parameter matrix. One benchmark will be run for each combination
of parameters (i.e., the benchmark space is the Cartesian product of the
parameter spaces).

For now, `--parameter-list` and `--parameter-scan` are still mutually
exclusive. If desired, a follow-up change could similarly permit
multiple occurrences of `--parameter-scan` and also permit use of both
listed and scanned parameters together. (After all, `--parameter-scan`
can be thought of as syntactic sugar for a parameter list.)

This implementation is a little profligate with memory usage for
`BenchmarkResult`s: each result contains a separate copy of all
parameter names (as `String`s). This could be done away with by
threading lifetimes through the `BenchmarkResult`s, or by ref-counting
the names.

Because `BenchmarkResult`s now have maps with arbitrary key sets, we can
no longer use `serde` to serialize them to CSV. But a CSV writer is easy
to write by hand, so we just do that.

Fixes sharkdp#253.

Test Plan:
Some unit tests included. As an integration test, try:

```
cargo run --release -- --runs 10 \
    -L foo foo1,foo2 -L bar bar9,bar8 \
    'echo {foo} {bar}' \
    'printf "%s\n" {foo} {bar}' \
    --export-csv /tmp/out
```

with all the export formats.

wchargin-branch: param-list-matrix
wchargin-source: 11051ab222c9de0fbc6ac6a080fbca48e05996c2

src/hyperfine/app.rs

@@ -3,9 +3,10 @@ use atty::Stream;
 use clap::{crate_version, App, AppSettings, Arg, ArgMatches};
 use std::ffi::OsString;
 
-pub fn get_arg_matches<T>(args: T) -> ArgMatches<'static>
+pub fn get_arg_matches<I, T>(args: I) -> ArgMatches<'static>
 where
-    T: Iterator<Item = OsString>,
+    I: IntoIterator<Item = T>,
+    T: Into<OsString> + Clone,
 {
     let app = build_app();
     app.get_matches_from(args)
@@ -134,6 +135,7 @@ fn build_app() -> App<'static, 'static> {
                 .long("parameter-list")
                 .short("L")
                 .takes_value(true)
+                .multiple(true)
                 .allow_hyphen_values(true)
                 .value_names(&["VAR", "VALUES"])
                 .conflicts_with_all(&["parameter-scan", "parameter-step-size"])

src/hyperfine/benchmark.rs

@@ -223,12 +223,7 @@ pub fn run_benchmark(
         } else {
             &values[num]
         };
-        match cmd.get_parameter() {
-            Some((param, value)) => {
-                Command::new_parametrized(preparation_command, param, value.clone())
-            }
-            None => Command::new(preparation_command),
-        }
+        Command::new_parametrized(preparation_command, cmd.get_parameters().clone())
     });
 
     // Warmup phase
@@ -414,16 +409,9 @@ pub fn run_benchmark(
     }
 
     // Run cleanup command
-    let cleanup_cmd =
-        options
-            .cleanup_command
-            .as_ref()
-            .map(|cleanup_command| match cmd.get_parameter() {
-                Some((param, value)) => {
-                    Command::new_parametrized(cleanup_command, param, value.clone())
-                }
-                None => Command::new(cleanup_command),
-            });
+    let cleanup_cmd = options.cleanup_command.as_ref().map(|cleanup_command| {
+        Command::new_parametrized(cleanup_command, cmd.get_parameters().clone())
+    });
     run_cleanup_command(&options.shell, &cleanup_cmd, options.show_output)?;
 
     Ok(BenchmarkResult::new(
@@ -436,6 +424,9 @@ pub fn run_benchmark(
         t_min,
         t_max,
         times_real,
-        cmd.get_parameter().as_ref().map(|p| p.1.to_string()),
+        cmd.get_parameters()
+            .iter()
+            .map(|(name, value)| ((*name).to_string(), value.to_string()))
+            .collect(),
     ))
 }

src/hyperfine/export/asciidoc.rs

@@ -83,6 +83,7 @@ fn test_asciidoc_header() {
 /// Ensure each table row is generated properly
 #[test]
 fn test_asciidoc_table_row() {
+    use std::collections::BTreeMap;
     let result = BenchmarkResult::new(
         String::from("sleep 1"), // command
         0.10491992406666667,     // mean
@@ -98,7 +99,7 @@ fn test_asciidoc_table_row() {
             0.10745223440000001,
             0.10697327940000001,
         ],
-        None, // param
+        BTreeMap::new(), // param
     );
 
     let expms = format!(
@@ -134,6 +135,7 @@ fn test_asciidoc_table_row() {
 /// Ensure commands get properly escaped
 #[test]
 fn test_asciidoc_table_row_command_escape() {
+    use std::collections::BTreeMap;
     let result = BenchmarkResult::new(
         String::from("sleep 1|"), // command
         0.10491992406666667,      // mean
@@ -149,7 +151,7 @@ fn test_asciidoc_table_row_command_escape() {
             0.10745223440000001,
             0.10697327940000001,
         ],
-        None, // param
+        BTreeMap::new(), // param
     );
     let exps = format!(
         "| `sleep 1\\|`\n\
@@ -169,11 +171,12 @@ fn test_asciidoc_table_row_command_escape() {
 /// Integration test
 #[test]
 fn test_asciidoc() {
+    use std::collections::BTreeMap;
     let exporter = AsciidocExporter::default();
     // NOTE: results are fabricated, unlike above
     let results = vec![
         BenchmarkResult::new(
-            String::from("command | 1"),
+            String::from("FOO=1 BAR=2 command | 1"),
             1.0,
             2.0,
             1.0,
@@ -182,10 +185,15 @@ fn test_asciidoc() {
             5.0,
             6.0,
             vec![7.0, 8.0, 9.0],
-            None,
+            {
+                let mut params = BTreeMap::new();
+                params.insert("foo".into(), "1".into());
+                params.insert("bar".into(), "2".into());
+                params
+            },
         ),
         BenchmarkResult::new(
-            String::from("command | 2"),
+            String::from("FOO=1 BAR=7 command | 2"),
             11.0,
             12.0,
             11.0,
@@ -194,7 +202,12 @@ fn test_asciidoc() {
             15.0,
             16.0,
             vec![17.0, 18.0, 19.0],
-            None,
+            {
+                let mut params = BTreeMap::new();
+                params.insert("foo".into(), "1".into());
+                params.insert("bar".into(), "7".into());
+                params
+            },
         ),
     ];
     // NOTE: only testing with s, s/ms is tested elsewhere
@@ -203,11 +216,11 @@ fn test_asciidoc() {
          |===\n\
          | Command | Mean [s] | Min…Max [s]\n\
          \n\
-         | `command \\| 1`\n\
+         | `FOO=1 BAR=2 command \\| 1`\n\
          | 1.000 ± 2.000\n\
          | 5.000…6.000\n\
          \n\
-         | `command \\| 2`\n\
+         | `FOO=1 BAR=7 command \\| 2`\n\
          | 11.000 ± 12.000\n\
          | 15.000…16.000\n\
          |===\n\

src/hyperfine/export/csv.rs

@@ -3,6 +3,7 @@ use super::Exporter;
 use crate::hyperfine::types::BenchmarkResult;
 use crate::hyperfine::units::Unit;
 
+use std::borrow::Cow;
 use std::io::{Error, ErrorKind, Result};
 
 use csv::WriterBuilder;
@@ -13,16 +14,93 @@ pub struct CsvExporter {}
 impl Exporter for CsvExporter {
     fn serialize(&self, results: &[BenchmarkResult], _unit: Option<Unit>) -> Result<Vec<u8>> {
         let mut writer = WriterBuilder::new().from_writer(vec![]);
-        for res in results {
-            // The list of times cannot be exported to the CSV file - remove it:
-            let mut result = res.clone();
-            result.times = None;
 
-            writer.serialize(result)?;
+        {
+            let mut headers: Vec<String> = [
+                // The list of times cannot be exported to the CSV file - omit it.
+                "command", "mean", "stddev", "median", "user", "system", "min", "max",
+            ]
+            .iter()
+            .map(|x| (*x).to_string())
+            .collect();
+            if let Some(res) = results.first() {
+                for param_name in res.parameters.keys() {
+                    headers.push(format!("parameter_{}", param_name));
+                }
+            }
+            writer.write_record(headers)?;
+        }
+
+        for res in results {
+            let mut fields = Vec::new();
+            fields.push(Cow::Borrowed(res.command.as_bytes()));
+            for f in &[
+                res.mean, res.stddev, res.median, res.user, res.system, res.min, res.max,
+            ] {
+                fields.push(Cow::Owned(f.to_string().into_bytes()))
+            }
+            for v in res.parameters.values() {
+                fields.push(Cow::Borrowed(v.as_bytes()))
+            }
+            writer.write_record(fields)?;
         }
 
         writer
             .into_inner()
             .map_err(|e| Error::new(ErrorKind::Other, e))
     }
 }
+
+#[test]
+fn test_csv() {
+    use std::collections::BTreeMap;
+    let exporter = CsvExporter::default();
+
+    // NOTE: results are fabricated
+    let results = vec![
+        BenchmarkResult::new(
+            String::from("FOO=one BAR=two command | 1"),
+            1.0,
+            2.0,
+            1.0,
+            3.0,
+            4.0,
+            5.0,
+            6.0,
+            vec![7.0, 8.0, 9.0],
+            {
+                let mut params = BTreeMap::new();
+                params.insert("foo".into(), "one".into());
+                params.insert("bar".into(), "two".into());
+                params
+            },
+        ),
+        BenchmarkResult::new(
+            String::from("FOO=one BAR=seven command | 2"),
+            11.0,
+            12.0,
+            11.0,
+            13.0,
+            14.0,
+            15.0,
+            16.5,
+            vec![17.0, 18.0, 19.0],
+            {
+                let mut params = BTreeMap::new();
+                params.insert("foo".into(), "one".into());
+                params.insert("bar".into(), "seven".into());
+                params
+            },
+        ),
+    ];
+    let exps: String = String::from(
+        "command,mean,stddev,median,user,system,min,max,parameter_bar,parameter_foo\n\
+        FOO=one BAR=two command | 1,1,2,1,3,4,5,6,two,one\n\
+        FOO=one BAR=seven command | 2,11,12,11,13,14,15,16.5,seven,one\n\
+        ",
+    );
+    let gens =
+        String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap();
+
+    assert_eq!(exps, gens);
+}

src/hyperfine/export/markdown.rs

@@ -82,6 +82,7 @@ fn add_table_row(dest: &mut Vec<u8>, entry: &BenchmarkResultWithRelativeSpeed, u
 /// the units for all entries when the time unit is not given.
 #[test]
 fn test_markdown_format_ms() {
+    use std::collections::BTreeMap;
     let exporter = MarkdownExporter::default();
 
     let mut timing_results = vec![];
@@ -96,7 +97,7 @@ fn test_markdown_format_ms() {
         0.1023,              // min
         0.1080,              // max
         vec![0.1, 0.1, 0.1], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     timing_results.push(BenchmarkResult::new(
@@ -109,7 +110,7 @@ fn test_markdown_format_ms() {
         2.0020,              // min
         2.0080,              // max
         vec![2.0, 2.0, 2.0], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     let formatted = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap();
@@ -129,6 +130,7 @@ fn test_markdown_format_ms() {
 /// the units for all entries when the time unit is not given.
 #[test]
 fn test_markdown_format_s() {
+    use std::collections::BTreeMap;
     let exporter = MarkdownExporter::default();
 
     let mut timing_results = vec![];
@@ -143,7 +145,7 @@ fn test_markdown_format_s() {
         2.0020,              // min
         2.0080,              // max
         vec![2.0, 2.0, 2.0], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     timing_results.push(BenchmarkResult::new(
@@ -156,7 +158,7 @@ fn test_markdown_format_s() {
         0.1023,              // min
         0.1080,              // max
         vec![0.1, 0.1, 0.1], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     let formatted = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap();
@@ -175,6 +177,7 @@ fn test_markdown_format_s() {
 /// The given time unit (s) is used to set the units for all entries.
 #[test]
 fn test_markdown_format_time_unit_s() {
+    use std::collections::BTreeMap;
     let exporter = MarkdownExporter::default();
 
     let mut timing_results = vec![];
@@ -189,7 +192,7 @@ fn test_markdown_format_time_unit_s() {
         0.1023,              // min
         0.1080,              // max
         vec![0.1, 0.1, 0.1], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     timing_results.push(BenchmarkResult::new(
@@ -202,7 +205,7 @@ fn test_markdown_format_time_unit_s() {
         2.0020,              // min
         2.0080,              // max
         vec![2.0, 2.0, 2.0], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     let formatted = String::from_utf8(
@@ -227,6 +230,7 @@ fn test_markdown_format_time_unit_s() {
 /// the units for all entries.
 #[test]
 fn test_markdown_format_time_unit_ms() {
+    use std::collections::BTreeMap;
     let exporter = MarkdownExporter::default();
 
     let mut timing_results = vec![];
@@ -241,7 +245,7 @@ fn test_markdown_format_time_unit_ms() {
         2.0020,              // min
         2.0080,              // max
         vec![2.0, 2.0, 2.0], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     timing_results.push(BenchmarkResult::new(
@@ -254,7 +258,7 @@ fn test_markdown_format_time_unit_ms() {
         0.1023,              // min
         0.1080,              // max
         vec![0.1, 0.1, 0.1], // times
-        None,                // parameter
+        BTreeMap::new(),     // parameter
     ));
 
     let formatted = String::from_utf8(

src/hyperfine/internal.rs

@@ -128,6 +128,7 @@ fn test_max() {
 #[test]
 fn test_compute_relative_speed() {
     use approx::assert_relative_eq;
+    use std::collections::BTreeMap;
 
     let create_result = |name: &str, mean| BenchmarkResult {
         command: name.into(),
@@ -139,7 +140,7 @@ fn test_compute_relative_speed() {
         min: mean,
         max: mean,
         times: None,
-        parameter: None,
+        parameters: BTreeMap::new(),
     };
 
     let results = vec![

src/hyperfine/parameter_range.rs

@@ -94,8 +94,7 @@ fn build_parameterized_commands<'a, T: Numeric>(
         for cmd in &command_strings {
             commands.push(Command::new_parametrized(
                 cmd,
-                param_name,
-                ParameterValue::Numeric(value.into()),
+                vec![(param_name, ParameterValue::Numeric(value.into()))],
             ));
         }
     }