From b006ca6d4e70134a36935afa3025858b6b393822 Mon Sep 17 00:00:00 2001 From: wuyangfan Date: Tue, 26 May 2026 17:43:27 +0800 Subject: [PATCH] feat: add --aggregate-parameter-runs to combine parameter scan stats (fixes #778) When benchmarking one sample per parameter value, pool all timed runs into a single mean/stddev summary instead of reporting separate benchmarks for each substituted command. Co-authored-by: Cursor --- src/benchmark/benchmark_result.rs | 58 +++++++++++++++++++++++++ src/benchmark/scheduler.rs | 70 ++++++++++++++++++++++++++++++- src/cli.rs | 10 +++++ src/command.rs | 24 +++++++++++ src/options.rs | 18 ++++++++ 5 files changed, 179 insertions(+), 1 deletion(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 287c73bef..ca4d870c8 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -4,6 +4,10 @@ use serde::Serialize; use crate::util::units::Second; +use statistical::{mean, median, standard_deviation}; + +use crate::util::min_max::{max, min}; + /// Set of values that will be exported. // NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate. @@ -53,3 +57,57 @@ pub struct BenchmarkResult { #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, } + +/// Combine individual parameter benchmark runs into a single aggregated result. +pub fn merge_parameter_benchmark_results( + results: Vec, + template: &str, +) -> BenchmarkResult { + let mut times = Vec::new(); + let mut memory_usage_byte = Vec::new(); + let mut exit_codes = Vec::new(); + let mut user_values = Vec::new(); + let mut system_values = Vec::new(); + + for result in &results { + if let Some(result_times) = &result.times { + times.extend(result_times); + } + if let Some(memory) = &result.memory_usage_byte { + memory_usage_byte.extend(memory); + } + exit_codes.extend(result.exit_codes.iter().copied()); + user_values.push(result.user); + system_values.push(result.system); + } + + let t_mean = mean(×); + let t_stddev = if times.len() > 1 { + Some(standard_deviation(×, Some(t_mean))) + } else { + None + }; + + BenchmarkResult { + command: template.to_string(), + command_with_unused_parameters: format!( + "{template} (aggregated over {} parameter values)", + results.len() + ), + mean: t_mean, + stddev: t_stddev, + median: median(×), + user: mean(&user_values), + system: mean(&system_values), + min: min(×), + max: max(×), + times: Some(times), + memory_usage_byte: if memory_usage_byte.is_empty() { + None + } else { + Some(memory_usage_byte) + }, + exit_codes, + parameters: BTreeMap::new(), + } +} diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 242dd0e7d..dc083a758 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -1,4 +1,4 @@ -use super::benchmark_result::BenchmarkResult; +use super::benchmark_result::{merge_parameter_benchmark_results, BenchmarkResult}; use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor}; use super::{relative_speed, Benchmark}; use colored::*; @@ -46,6 +46,35 @@ impl<'a> Scheduler<'a> { executor.calibrate()?; + if self.options.aggregate_parameter_runs { + let template = self + .commands + .template_expression() + .expect("validated in Options::validate_against_command_list"); + + if let Some(ref_cmd) = reference.as_ref() { + self.results + .push(Benchmark::new(0, ref_cmd, self.options, &*executor).run()?); + } + + let start_index = if reference.is_some() { 1 } else { 0 }; + let mut parameter_results = Vec::with_capacity(self.commands.num_commands(false)); + + for (offset, cmd) in self.commands.iter().enumerate() { + parameter_results.push( + Benchmark::new(start_index + offset, cmd, self.options, &*executor).run()?, + ); + } + + self.results.push(merge_parameter_benchmark_results( + parameter_results, + template, + )); + self.export_manager.write_results(&self.results, true)?; + + return Ok(()); + } + for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() { self.results .push(Benchmark::new(number, cmd, self.options, &*executor).run()?); @@ -226,3 +255,42 @@ fn scheduler_basic() -> Result<()> { Ok(()) } + +#[test] +fn scheduler_aggregate_parameter_runs() -> Result<()> { + insta::assert_yaml_snapshot!( + generate_results(&[ + "--runs=1", + "--aggregate-parameter-runs", + "--parameter-scan", + "index", + "1", + "3", + "sleep {index}.123", + ])?, + @r#" + - command: "sleep {index}.123" + mean: 2.123 + stddev: 1 + median: 2.123 + user: 0 + system: 0 + min: 1.123 + max: 3.123 + times: + - 1.123 + - 2.123 + - 3.123 + memory_usage_byte: + - 0 + - 0 + - 0 + exit_codes: + - 0 + - 0 + - 0 + "# + ); + + Ok(()) +} diff --git a/src/cli.rs b/src/cli.rs index b12f6d34c..10ec02be1 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -200,6 +200,16 @@ fn build_command() -> Command { possible parameter combinations.\n" ), ) + .arg( + Arg::new("aggregate-parameter-runs") + .long("aggregate-parameter-runs") + .action(ArgAction::SetTrue) + .help( + "Combine benchmark results from all parameter values into a single summary. \ + Useful when each parameter value should contribute one sample to the same \ + benchmark, e.g. timing transfers of different files once each.", + ), + ) .arg( Arg::new("shell") .long("shell") diff --git a/src/command.rs b/src/command.rs index 5d35cfaf7..366350923 100644 --- a/src/command.rs +++ b/src/command.rs @@ -78,6 +78,10 @@ impl<'a> Command<'a> { self.replace_parameters_in(self.expression) } + pub fn expression(&self) -> &'a str { + self.expression + } + pub fn get_command(&self) -> Result { let command_line = self.get_command_line(); let mut tokens = shell_words::split(&command_line) @@ -255,6 +259,26 @@ impl<'a> Commands<'a> { self.0.len() + if has_reference_command { 1 } else { 0 } } + /// Whether all commands share the same template expression (parameter scan/list). + pub fn supports_parameter_aggregation(&self) -> bool { + if self.0.len() <= 1 { + return false; + } + + let template = self.0[0].expression(); + self.0 + .iter() + .all(|command| command.expression() == template) + } + + pub fn template_expression(&self) -> Option<&'a str> { + if self.supports_parameter_aggregation() { + Some(self.0[0].expression()) + } else { + None + } + } + /// Finds all the strings that appear multiple times in the input iterator, returning them in /// sorted order. If no string appears more than once, the result is an empty vector. fn find_duplicates<'b, I: IntoIterator>(i: I) -> Vec<&'b str> { diff --git a/src/options.rs b/src/options.rs index 7c83da1c5..65bb9679c 100644 --- a/src/options.rs +++ b/src/options.rs @@ -246,6 +246,9 @@ pub struct Options { /// Which time unit to use when displaying results pub time_unit: Option, + + /// Combine results from parameter scan/list commands into one summary + pub aggregate_parameter_runs: bool, } impl Default for Options { @@ -268,6 +271,7 @@ impl Default for Options { command_output_policies: vec![CommandOutputPolicy::Null], time_unit: None, command_input_policy: CommandInputPolicy::Null, + aggregate_parameter_runs: false, } } } @@ -464,6 +468,8 @@ impl Options { CommandInputPolicy::Null }; + options.aggregate_parameter_runs = matches.get_flag("aggregate-parameter-runs"); + Ok(options) } @@ -498,6 +504,18 @@ impl Options { ); } + if self.aggregate_parameter_runs { + ensure!( + !has_reference_command, + "The '--aggregate-parameter-runs' option cannot be combined with '--reference'." + ); + ensure!( + commands.supports_parameter_aggregation(), + "The '--aggregate-parameter-runs' option requires multiple commands that share \ + the same template (from --parameter-scan or --parameter-list)." + ); + } + Ok(()) } }