feat: add --aggregate-parameter-runs to combine parameter scan stats (fixes #778)

src/benchmark/benchmark_result.rs

@@ -4,6 +4,10 @@ use serde::Serialize;
 
 use crate::util::units::Second;
 
+use statistical::{mean, median, standard_deviation};
+
+use crate::util::min_max::{max, min};
+
 /// Set of values that will be exported.
 // NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
 // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
@@ -53,3 +57,57 @@ pub struct BenchmarkResult {
     #[serde(skip_serializing_if = "BTreeMap::is_empty")]
     pub parameters: BTreeMap<String, String>,
 }
+
+/// Combine individual parameter benchmark runs into a single aggregated result.
+pub fn merge_parameter_benchmark_results(
+    results: Vec<BenchmarkResult>,
+    template: &str,
+) -> BenchmarkResult {
+    let mut times = Vec::new();
+    let mut memory_usage_byte = Vec::new();
+    let mut exit_codes = Vec::new();
+    let mut user_values = Vec::new();
+    let mut system_values = Vec::new();
+
+    for result in &results {
+        if let Some(result_times) = &result.times {
+            times.extend(result_times);
+        }
+        if let Some(memory) = &result.memory_usage_byte {
+            memory_usage_byte.extend(memory);
+        }
+        exit_codes.extend(result.exit_codes.iter().copied());
+        user_values.push(result.user);
+        system_values.push(result.system);
+    }
+
+    let t_mean = mean(&times);
+    let t_stddev = if times.len() > 1 {
+        Some(standard_deviation(&times, Some(t_mean)))
+    } else {
+        None
+    };
+
+    BenchmarkResult {
+        command: template.to_string(),
+        command_with_unused_parameters: format!(
+            "{template} (aggregated over {} parameter values)",
+            results.len()
+        ),
+        mean: t_mean,
+        stddev: t_stddev,
+        median: median(&times),
+        user: mean(&user_values),
+        system: mean(&system_values),
+        min: min(&times),
+        max: max(&times),
+        times: Some(times),
+        memory_usage_byte: if memory_usage_byte.is_empty() {
+            None
+        } else {
+            Some(memory_usage_byte)
+        },
+        exit_codes,
+        parameters: BTreeMap::new(),
+    }
+}

src/benchmark/scheduler.rs

@@ -1,4 +1,4 @@
-use super::benchmark_result::BenchmarkResult;
+use super::benchmark_result::{merge_parameter_benchmark_results, BenchmarkResult};
 use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
 use super::{relative_speed, Benchmark};
 use colored::*;
@@ -46,6 +46,35 @@ impl<'a> Scheduler<'a> {
 
         executor.calibrate()?;
 
+        if self.options.aggregate_parameter_runs {
+            let template = self
+                .commands
+                .template_expression()
+                .expect("validated in Options::validate_against_command_list");
+
+            if let Some(ref_cmd) = reference.as_ref() {
+                self.results
+                    .push(Benchmark::new(0, ref_cmd, self.options, &*executor).run()?);
+            }
+
+            let start_index = if reference.is_some() { 1 } else { 0 };
+            let mut parameter_results = Vec::with_capacity(self.commands.num_commands(false));
+
+            for (offset, cmd) in self.commands.iter().enumerate() {
+                parameter_results.push(
+                    Benchmark::new(start_index + offset, cmd, self.options, &*executor).run()?,
+                );
+            }
+
+            self.results.push(merge_parameter_benchmark_results(
+                parameter_results,
+                template,
+            ));
+            self.export_manager.write_results(&self.results, true)?;
+
+            return Ok(());
+        }
+
         for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
             self.results
                 .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
@@ -226,3 +255,42 @@ fn scheduler_basic() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn scheduler_aggregate_parameter_runs() -> Result<()> {
+    insta::assert_yaml_snapshot!(
+        generate_results(&[
+            "--runs=1",
+            "--aggregate-parameter-runs",
+            "--parameter-scan",
+            "index",
+            "1",
+            "3",
+            "sleep {index}.123",
+        ])?,
+        @r#"
+    - command: "sleep {index}.123"
+      mean: 2.123
+      stddev: 1
+      median: 2.123
+      user: 0
+      system: 0
+      min: 1.123
+      max: 3.123
+      times:
+        - 1.123
+        - 2.123
+        - 3.123
+      memory_usage_byte:
+        - 0
+        - 0
+        - 0
+      exit_codes:
+        - 0
+        - 0
+        - 0
+    "#
+    );
+
+    Ok(())
+}

src/cli.rs

@@ -200,6 +200,16 @@ fn build_command() -> Command {
                      possible parameter combinations.\n"
                 ),
         )
+        .arg(
+            Arg::new("aggregate-parameter-runs")
+                .long("aggregate-parameter-runs")
+                .action(ArgAction::SetTrue)
+                .help(
+                    "Combine benchmark results from all parameter values into a single summary. \
+                     Useful when each parameter value should contribute one sample to the same \
+                     benchmark, e.g. timing transfers of different files once each.",
+                ),
+        )
         .arg(
             Arg::new("shell")
                 .long("shell")

src/command.rs

@@ -78,6 +78,10 @@ impl<'a> Command<'a> {
         self.replace_parameters_in(self.expression)
     }
 
+    pub fn expression(&self) -> &'a str {
+        self.expression
+    }
+
     pub fn get_command(&self) -> Result<std::process::Command> {
         let command_line = self.get_command_line();
         let mut tokens = shell_words::split(&command_line)
@@ -255,6 +259,26 @@ impl<'a> Commands<'a> {
         self.0.len() + if has_reference_command { 1 } else { 0 }
     }
 
+    /// Whether all commands share the same template expression (parameter scan/list).
+    pub fn supports_parameter_aggregation(&self) -> bool {
+        if self.0.len() <= 1 {
+            return false;
+        }
+
+        let template = self.0[0].expression();
+        self.0
+            .iter()
+            .all(|command| command.expression() == template)
+    }
+
+    pub fn template_expression(&self) -> Option<&'a str> {
+        if self.supports_parameter_aggregation() {
+            Some(self.0[0].expression())
+        } else {
+            None
+        }
+    }
+
     /// Finds all the strings that appear multiple times in the input iterator, returning them in
     /// sorted order. If no string appears more than once, the result is an empty vector.
     fn find_duplicates<'b, I: IntoIterator<Item = &'b str>>(i: I) -> Vec<&'b str> {

src/options.rs

@@ -246,6 +246,9 @@ pub struct Options {
 
     /// Which time unit to use when displaying results
     pub time_unit: Option<Unit>,
+
+    /// Combine results from parameter scan/list commands into one summary
+    pub aggregate_parameter_runs: bool,
 }
 
 impl Default for Options {
@@ -268,6 +271,7 @@ impl Default for Options {
             command_output_policies: vec![CommandOutputPolicy::Null],
             time_unit: None,
             command_input_policy: CommandInputPolicy::Null,
+            aggregate_parameter_runs: false,
         }
     }
 }
@@ -464,6 +468,8 @@ impl Options {
             CommandInputPolicy::Null
         };
 
+        options.aggregate_parameter_runs = matches.get_flag("aggregate-parameter-runs");
+
         Ok(options)
     }
 
@@ -498,6 +504,18 @@ impl Options {
             );
         }
 
+        if self.aggregate_parameter_runs {
+            ensure!(
+                !has_reference_command,
+                "The '--aggregate-parameter-runs' option cannot be combined with '--reference'."
+            );
+            ensure!(
+                commands.supports_parameter_aggregation(),
+                "The '--aggregate-parameter-runs' option requires multiple commands that share \
+                 the same template (from --parameter-scan or --parameter-list)."
+            );
+        }
+
         Ok(())
     }
 }