Add opt-in cross-thread statistics for multithreaded runs

bindings/python/google_benchmark/benchmark.cc

@@ -98,6 +98,8 @@ NB_MODULE(_benchmark, m) {
            nb::rv_policy::reference, nb::arg("value") = true)
       .def("display_aggregates_only", &Benchmark::DisplayAggregatesOnly,
            nb::rv_policy::reference, nb::arg("value") = true)
+      .def("report_thread_statistics", &Benchmark::ReportThreadStatistics,
+           nb::rv_policy::reference, nb::arg("value") = true)
       .def("measure_process_cpu_time", &Benchmark::MeasureProcessCPUTime,
            nb::rv_policy::reference)
       .def("use_real_time", &Benchmark::UseRealTime, nb::rv_policy::reference)

docs/user_guide.md

@@ -36,6 +36,8 @@
 
 [Custom Counters](#custom-counters)
 
+[Cross-thread statistics](#cross-thread-statistics)
+
 [Multithreaded Benchmarks](#multithreaded-benchmarks)
 
 [CPU Timers](#cpu-timers)
@@ -227,6 +229,17 @@ When enabled, only the mean, standard deviation, and other statistics are displa
 $ ./benchmark --benchmark_repetitions=5 --benchmark_display_aggregates_only
 ```
 
+#### `--benchmark_report_thread_statistics` (BENCHMARK_REPORT_THREAD_STATISTICS)
+
+When enabled, multithreaded benchmarks also report mean, median, standard deviation, and coefficient of variation **across threads** of that run (user counters and times). The usual summed result row is unchanged. See [Cross-thread statistics](#cross-thread-statistics).
+
+**Default:** `false`
+
+**Example:**
+```bash
+$ ./benchmark --benchmark_report_thread_statistics=true
+```
+
 #### `--benchmark_counters_tabular` (BENCHMARK_COUNTERS_TABULAR)
 
 Whether to use tabular format when printing user counters to the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0.
@@ -1118,6 +1131,40 @@ In multithreaded benchmarks, each counter is set on the calling thread only.
 When the benchmark finishes, the counters from each thread will be summed.
 Counters that are configured with `kIsRate`, will report the average rate across all threads, while `kAvgThreadsRate` counters will report the average rate per thread.
 
+<a name="cross-thread-statistics" />
+
+### Cross-thread statistics
+
+By default the library only reports that summed (then normalized) view. To also
+see how values are distributed across the N threads of **one** multithreaded
+run, enable thread statistics:
+
+```c++
+BENCHMARK(BM_Fairness)
+    ->Threads(8)
+    ->ReportThreadStatistics();
+```
+
+or pass `--benchmark_report_thread_statistics=true`. The per-benchmark setter
+overrides the flag.
+
+When enabled and the instance uses more than one thread, extra aggregate rows
+are printed after that run, named with a `thread_` prefix so they do not collide
+with repetition statistics:
+
+```
+BM_Fairness/threads:8
+BM_Fairness/threads:8_thread_mean
+BM_Fairness/threads:8_thread_median
+BM_Fairness/threads:8_thread_stddev
+BM_Fairness/threads:8_thread_cv
+```
+
+Each thread's counters are finished independently (`num_threads = 1`) before
+mean/median/stddev/cv are computed, so the extra rows describe per-thread
+contributions. Custom statistics registered with `ComputeStatistics` are
+included as `thread_<name>`.
+
 ### Counter Reporting
 
 When using the console reporter, by default, user counters are printed at

include/benchmark/benchmark_api.h

@@ -129,6 +129,7 @@ class BENCHMARK_EXPORT Benchmark {
   Benchmark* Repetitions(int n);
   Benchmark* ReportAggregatesOnly(bool value = true);
   Benchmark* DisplayAggregatesOnly(bool value = true);
+  Benchmark* ReportThreadStatistics(bool value = true);
   Benchmark* MeasureProcessCPUTime();
   Benchmark* UseRealTime();
   Benchmark* UseManualTime();
@@ -180,6 +181,8 @@ class BENCHMARK_EXPORT Benchmark {
   BigOFunc* complexity_lambda_;
   std::vector<internal::Statistics> statistics_;
   std::vector<int> thread_counts_;
+  bool report_thread_statistics_specified_;
+  bool report_thread_statistics_;
 
   callback_function setup_;
   callback_function teardown_;

src/benchmark.cc

@@ -126,6 +126,10 @@ BM_DEFINE_bool(benchmark_report_aggregates_only, false);
 // all the output.
 BM_DEFINE_bool(benchmark_display_aggregates_only, false);
 
+// If true, also report mean/median/stddev/cv of per-thread counters and times
+// for multithreaded benchmarks. Default output (the summed row) is unchanged.
+BM_DEFINE_bool(benchmark_report_thread_statistics, false);
+
 // The format to use for console output.
 // Valid values are 'console', 'json', or 'csv'.
 BM_DEFINE_string(benchmark_format, "console");
@@ -407,9 +411,16 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
     name_field_width =
         std::max<size_t>(name_field_width, benchmark.name().str().size());
     might_have_aggregates |= benchmark.repetitions() > 1;
+    const bool thread_stats =
+        benchmark.report_thread_statistics() && benchmark.threads() > 1;
+    might_have_aggregates |= thread_stats;
 
     for (const auto& Stat : benchmark.statistics()) {
-      stat_field_width = std::max<size_t>(stat_field_width, Stat.name_.size());
+      size_t name_size = Stat.name_.size();
+      if (thread_stats) {
+        name_size += sizeof("thread_") - 1;
+      }
+      stat_field_width = std::max<size_t>(stat_field_width, name_size);
     }
   }
   if (might_have_aggregates) {
@@ -775,6 +786,8 @@ void ParseCommandLineFlags(int* argc, char** argv) {
                       &FLAGS_benchmark_report_aggregates_only) ||
         ParseBoolFlag(argv[i], "benchmark_display_aggregates_only",
                       &FLAGS_benchmark_display_aggregates_only) ||
+        ParseBoolFlag(argv[i], "benchmark_report_thread_statistics",
+                      &FLAGS_benchmark_report_thread_statistics) ||
         ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) ||
         ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) ||
         ParseStringFlag(argv[i], "benchmark_out_format",
@@ -975,6 +988,7 @@ void PrintDefaultHelp() {
           "          [--benchmark_enable_random_interleaving={true|false}]\n"
           "          [--benchmark_report_aggregates_only={true|false}]\n"
           "          [--benchmark_display_aggregates_only={true|false}]\n"
+          "          [--benchmark_report_thread_statistics={true|false}]\n"
           "          [--benchmark_format=<console|json|csv>]\n"
           "          [--benchmark_out=<filename>]\n"
           "          [--benchmark_out_format=<json|console|csv>]\n"

src/benchmark_api_internal.cc

@@ -2,11 +2,19 @@
 
 #include <cinttypes>
 
+#include "commandlineflags.h"
 #include "string_util.h"
 
 namespace benchmark {
+BM_DECLARE_bool(benchmark_report_thread_statistics);
 namespace internal {
 
+bool BenchmarkInstance::report_thread_statistics() const {
+  return report_thread_statistics_specified_
+             ? report_thread_statistics_
+             : FLAGS_benchmark_report_thread_statistics;
+}
+
 BenchmarkInstance::BenchmarkInstance(benchmark::Benchmark* benchmark,
                                      int family_idx,
                                      int per_family_instance_idx,
@@ -29,6 +37,9 @@ BenchmarkInstance::BenchmarkInstance(benchmark::Benchmark* benchmark,
       min_warmup_time_(benchmark_.min_warmup_time_),
       iterations_(benchmark_.iterations_),
       threads_(thread_count),
+      report_thread_statistics_specified_(
+          benchmark_.report_thread_statistics_specified_),
+      report_thread_statistics_(benchmark_.report_thread_statistics_),
       setup_(benchmark_.setup_),
       teardown_(benchmark_.teardown_) {
   name_.function_name = benchmark_.name_;

src/benchmark_api_internal.h

@@ -41,6 +41,7 @@ class BenchmarkInstance {
   double min_warmup_time() const { return min_warmup_time_; }
   IterationCount iterations() const { return iterations_; }
   int threads() const { return threads_; }
+  bool report_thread_statistics() const;
   void Setup() const;
   void Teardown() const;
   const auto& GetUserThreadRunnerFactory() const {
@@ -72,6 +73,8 @@ class BenchmarkInstance {
   double min_warmup_time_;
   IterationCount iterations_;
   int threads_;  // Number of concurrent threads to us
+  bool report_thread_statistics_specified_;
+  bool report_thread_statistics_;
 
   callback_function setup_;
   callback_function teardown_;

src/benchmark_register.cc

@@ -233,7 +233,9 @@ Benchmark::Benchmark(const std::string& name)
       use_real_time_(false),
       use_manual_time_(false),
       complexity_(oNone),
-      complexity_lambda_(nullptr) {
+      complexity_lambda_(nullptr),
+      report_thread_statistics_specified_(false),
+      report_thread_statistics_(false) {
   ComputeStatistics("mean", StatisticsMean);
   ComputeStatistics("median", StatisticsMedian);
   ComputeStatistics("stddev", StatisticsStdDev);
@@ -426,6 +428,12 @@ Benchmark* Benchmark::DisplayAggregatesOnly(bool value) {
   return this;
 }
 
+Benchmark* Benchmark::ReportThreadStatistics(bool value) {
+  report_thread_statistics_specified_ = true;
+  report_thread_statistics_ = value;
+  return this;
+}
+
 Benchmark* Benchmark::MeasureProcessCPUTime() {
   // Can be used together with UseRealTime() / UseManualTime().
   measure_process_cpu_time_ = true;