diff --git a/bindings/python/google_benchmark/benchmark.cc b/bindings/python/google_benchmark/benchmark.cc index ccd7eb5a50..ceb0cd4c45 100644 --- a/bindings/python/google_benchmark/benchmark.cc +++ b/bindings/python/google_benchmark/benchmark.cc @@ -98,6 +98,8 @@ NB_MODULE(_benchmark, m) { nb::rv_policy::reference, nb::arg("value") = true) .def("display_aggregates_only", &Benchmark::DisplayAggregatesOnly, nb::rv_policy::reference, nb::arg("value") = true) + .def("report_thread_statistics", &Benchmark::ReportThreadStatistics, + nb::rv_policy::reference, nb::arg("value") = true) .def("measure_process_cpu_time", &Benchmark::MeasureProcessCPUTime, nb::rv_policy::reference) .def("use_real_time", &Benchmark::UseRealTime, nb::rv_policy::reference) diff --git a/docs/user_guide.md b/docs/user_guide.md index 3fed9261d7..06fca5414c 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -36,6 +36,8 @@ [Custom Counters](#custom-counters) +[Cross-thread statistics](#cross-thread-statistics) + [Multithreaded Benchmarks](#multithreaded-benchmarks) [CPU Timers](#cpu-timers) @@ -227,6 +229,17 @@ When enabled, only the mean, standard deviation, and other statistics are displa $ ./benchmark --benchmark_repetitions=5 --benchmark_display_aggregates_only ``` +#### `--benchmark_report_thread_statistics` (BENCHMARK_REPORT_THREAD_STATISTICS) + +When enabled, multithreaded benchmarks also report mean, median, standard deviation, and coefficient of variation **across threads** of that run (user counters and times). The usual summed result row is unchanged. See [Cross-thread statistics](#cross-thread-statistics). + +**Default:** `false` + +**Example:** +```bash +$ ./benchmark --benchmark_report_thread_statistics=true +``` + #### `--benchmark_counters_tabular` (BENCHMARK_COUNTERS_TABULAR) Whether to use tabular format when printing user counters to the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0. @@ -1118,6 +1131,40 @@ In multithreaded benchmarks, each counter is set on the calling thread only. When the benchmark finishes, the counters from each thread will be summed. Counters that are configured with `kIsRate`, will report the average rate across all threads, while `kAvgThreadsRate` counters will report the average rate per thread. + + +### Cross-thread statistics + +By default the library only reports that summed (then normalized) view. To also +see how values are distributed across the N threads of **one** multithreaded +run, enable thread statistics: + +```c++ +BENCHMARK(BM_Fairness) + ->Threads(8) + ->ReportThreadStatistics(); +``` + +or pass `--benchmark_report_thread_statistics=true`. The per-benchmark setter +overrides the flag. + +When enabled and the instance uses more than one thread, extra aggregate rows +are printed after that run, named with a `thread_` prefix so they do not collide +with repetition statistics: + +``` +BM_Fairness/threads:8 +BM_Fairness/threads:8_thread_mean +BM_Fairness/threads:8_thread_median +BM_Fairness/threads:8_thread_stddev +BM_Fairness/threads:8_thread_cv +``` + +Each thread's counters are finished independently (`num_threads = 1`) before +mean/median/stddev/cv are computed, so the extra rows describe per-thread +contributions. Custom statistics registered with `ComputeStatistics` are +included as `thread_`. + ### Counter Reporting When using the console reporter, by default, user counters are printed at diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index 1bff47278c..0cbe67e5de 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -129,6 +129,7 @@ class BENCHMARK_EXPORT Benchmark { Benchmark* Repetitions(int n); Benchmark* ReportAggregatesOnly(bool value = true); Benchmark* DisplayAggregatesOnly(bool value = true); + Benchmark* ReportThreadStatistics(bool value = true); Benchmark* MeasureProcessCPUTime(); Benchmark* UseRealTime(); Benchmark* UseManualTime(); @@ -180,6 +181,8 @@ class BENCHMARK_EXPORT Benchmark { BigOFunc* complexity_lambda_; std::vector statistics_; std::vector thread_counts_; + bool report_thread_statistics_specified_; + bool report_thread_statistics_; callback_function setup_; callback_function teardown_; diff --git a/src/benchmark.cc b/src/benchmark.cc index 22a395d2a3..1becf2c0a2 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -126,6 +126,10 @@ BM_DEFINE_bool(benchmark_report_aggregates_only, false); // all the output. BM_DEFINE_bool(benchmark_display_aggregates_only, false); +// If true, also report mean/median/stddev/cv of per-thread counters and times +// for multithreaded benchmarks. Default output (the summed row) is unchanged. +BM_DEFINE_bool(benchmark_report_thread_statistics, false); + // The format to use for console output. // Valid values are 'console', 'json', or 'csv'. BM_DEFINE_string(benchmark_format, "console"); @@ -407,9 +411,16 @@ void RunBenchmarks(const std::vector& benchmarks, name_field_width = std::max(name_field_width, benchmark.name().str().size()); might_have_aggregates |= benchmark.repetitions() > 1; + const bool thread_stats = + benchmark.report_thread_statistics() && benchmark.threads() > 1; + might_have_aggregates |= thread_stats; for (const auto& Stat : benchmark.statistics()) { - stat_field_width = std::max(stat_field_width, Stat.name_.size()); + size_t name_size = Stat.name_.size(); + if (thread_stats) { + name_size += sizeof("thread_") - 1; + } + stat_field_width = std::max(stat_field_width, name_size); } } if (might_have_aggregates) { @@ -775,6 +786,8 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_report_aggregates_only) || ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", &FLAGS_benchmark_display_aggregates_only) || + ParseBoolFlag(argv[i], "benchmark_report_thread_statistics", + &FLAGS_benchmark_report_thread_statistics) || ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) || ParseStringFlag(argv[i], "benchmark_out_format", @@ -975,6 +988,7 @@ void PrintDefaultHelp() { " [--benchmark_enable_random_interleaving={true|false}]\n" " [--benchmark_report_aggregates_only={true|false}]\n" " [--benchmark_display_aggregates_only={true|false}]\n" + " [--benchmark_report_thread_statistics={true|false}]\n" " [--benchmark_format=]\n" " [--benchmark_out=]\n" " [--benchmark_out_format=]\n" diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc index f9c4990ddf..d96d6a150b 100644 --- a/src/benchmark_api_internal.cc +++ b/src/benchmark_api_internal.cc @@ -2,11 +2,19 @@ #include +#include "commandlineflags.h" #include "string_util.h" namespace benchmark { +BM_DECLARE_bool(benchmark_report_thread_statistics); namespace internal { +bool BenchmarkInstance::report_thread_statistics() const { + return report_thread_statistics_specified_ + ? report_thread_statistics_ + : FLAGS_benchmark_report_thread_statistics; +} + BenchmarkInstance::BenchmarkInstance(benchmark::Benchmark* benchmark, int family_idx, int per_family_instance_idx, @@ -29,6 +37,9 @@ BenchmarkInstance::BenchmarkInstance(benchmark::Benchmark* benchmark, min_warmup_time_(benchmark_.min_warmup_time_), iterations_(benchmark_.iterations_), threads_(thread_count), + report_thread_statistics_specified_( + benchmark_.report_thread_statistics_specified_), + report_thread_statistics_(benchmark_.report_thread_statistics_), setup_(benchmark_.setup_), teardown_(benchmark_.teardown_) { name_.function_name = benchmark_.name_; diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h index 0dd950cbe5..052388ee33 100644 --- a/src/benchmark_api_internal.h +++ b/src/benchmark_api_internal.h @@ -41,6 +41,7 @@ class BenchmarkInstance { double min_warmup_time() const { return min_warmup_time_; } IterationCount iterations() const { return iterations_; } int threads() const { return threads_; } + bool report_thread_statistics() const; void Setup() const; void Teardown() const; const auto& GetUserThreadRunnerFactory() const { @@ -72,6 +73,8 @@ class BenchmarkInstance { double min_warmup_time_; IterationCount iterations_; int threads_; // Number of concurrent threads to us + bool report_thread_statistics_specified_; + bool report_thread_statistics_; callback_function setup_; callback_function teardown_; diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc index 560a762e9b..1625bfc2f2 100644 --- a/src/benchmark_register.cc +++ b/src/benchmark_register.cc @@ -233,7 +233,9 @@ Benchmark::Benchmark(const std::string& name) use_real_time_(false), use_manual_time_(false), complexity_(oNone), - complexity_lambda_(nullptr) { + complexity_lambda_(nullptr), + report_thread_statistics_specified_(false), + report_thread_statistics_(false) { ComputeStatistics("mean", StatisticsMean); ComputeStatistics("median", StatisticsMedian); ComputeStatistics("stddev", StatisticsStdDev); @@ -426,6 +428,12 @@ Benchmark* Benchmark::DisplayAggregatesOnly(bool value) { return this; } +Benchmark* Benchmark::ReportThreadStatistics(bool value) { + report_thread_statistics_specified_ = true; + report_thread_statistics_ = value; + return this; +} + Benchmark* Benchmark::MeasureProcessCPUTime() { // Can be used together with UseRealTime() / UseManualTime(). measure_process_cpu_time_ = true; diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc index 0d267a38d5..50599cf50f 100644 --- a/src/benchmark_runner.cc +++ b/src/benchmark_runner.cc @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -69,6 +70,7 @@ BM_DECLARE_double(benchmark_min_warmup_time); BM_DECLARE_int32(benchmark_repetitions); BM_DECLARE_bool(benchmark_report_aggregates_only); BM_DECLARE_bool(benchmark_display_aggregates_only); +BM_DECLARE_bool(benchmark_report_thread_statistics); BM_DECLARE_string(benchmark_perf_counters); namespace internal { @@ -139,6 +141,106 @@ BenchmarkReporter::Run CreateRunReport( return report; } +std::vector CreateThreadStatReports( + const BenchmarkInstance& b, const BenchmarkReporter::Run& base, + const std::vector& thread_results) { + std::vector out; + if (base.skipped != 0u || thread_results.size() < 2) { + return out; + } + + const size_t n = thread_results.size(); + std::vector finished; + finished.reserve(n); + std::vector real_times; + std::vector cpu_times; + real_times.reserve(n); + cpu_times.reserve(n); + + const IterationCount thread_iters = thread_results.front().iterations; + + for (const auto& tr : thread_results) { + double seconds = tr.cpu_time_used; + if (b.use_manual_time()) { + seconds = tr.manual_time_used; + } else if (b.use_real_time()) { + seconds = tr.real_time_used; + } + UserCounters counters = tr.counters; + internal::Finish(&counters, tr.iterations, seconds, /*num_threads=*/1.0); + finished.push_back(std::move(counters)); + + if (b.use_manual_time()) { + real_times.push_back(tr.manual_time_used); + } else { + real_times.push_back(tr.real_time_used); + } + cpu_times.push_back(tr.cpu_time_used); + } + + struct CounterStat { + Counter c; + std::vector s; + }; + std::map counter_stats; + for (const auto& fc : finished) { + for (const auto& cnt : fc) { + auto it = counter_stats.find(cnt.first); + if (it == counter_stats.end()) { + it = counter_stats + .emplace(cnt.first, CounterStat{cnt.second, std::vector{}}) + .first; + it->second.s.reserve(n); + } + } + } + for (const auto& fc : finished) { + for (auto& kv : counter_stats) { + auto it = fc.find(kv.first); + if (it != fc.end()) { + kv.second.s.push_back(it->second); + } + } + } + + const double iteration_rescale_factor = + static_cast(n) / + static_cast(thread_iters != 0 ? thread_iters : 1); + + for (const auto& Stat : b.statistics()) { + BenchmarkReporter::Run data; + data.run_name = base.run_name; + data.family_index = base.family_index; + data.per_family_instance_index = base.per_family_instance_index; + data.run_type = BenchmarkReporter::Run::RT_Aggregate; + data.threads = base.threads; + data.repetitions = base.repetitions; + data.repetition_index = base.repetition_index; + data.aggregate_name = "thread_" + Stat.name_; + data.aggregate_unit = Stat.unit_; + data.iterations = static_cast(n); + data.time_unit = base.time_unit; + data.statistics = base.statistics; + + data.real_accumulated_time = Stat.compute_(real_times); + data.cpu_accumulated_time = Stat.compute_(cpu_times); + if (data.aggregate_unit == StatisticUnit::kTime) { + data.real_accumulated_time *= iteration_rescale_factor; + data.cpu_accumulated_time *= iteration_rescale_factor; + } + for (const auto& kv : counter_stats) { + if (kv.second.s.size() != n) { + continue; + } + const auto uc_stat = Stat.compute_(kv.second.s); + data.counters[kv.first] = + Counter(uc_stat, kv.second.c.flags, kv.second.c.oneK); + } + out.push_back(std::move(data)); + } + return out; +} + // Execute one thread of benchmark b for the specified number of iterations. // Adds the stats collected for the thread into manager->results. void RunInThread(const BenchmarkInstance* b, IterationCount iters, @@ -166,6 +268,17 @@ void RunInThread(const BenchmarkInstance* b, IterationCount iters, results.manual_time_used += timer.manual_time_used(); results.complexity_n += st.complexity_length_n(); internal::Increment(&results.counters, st.counters); + if (!manager->per_thread_results.empty()) { + internal::ThreadManager::Result per_thread; + per_thread.iterations = st.iterations(); + per_thread.cpu_time_used = timer.cpu_time_used(); + per_thread.real_time_used = timer.real_time_used(); + per_thread.manual_time_used = timer.manual_time_used(); + per_thread.complexity_n = st.complexity_length_n(); + per_thread.counters = st.counters; + manager->per_thread_results[static_cast(thread_id)] = + std::move(per_thread); + } } manager->NotifyThreadComplete(); } @@ -305,6 +418,7 @@ BenchmarkRunner::BenchmarkRunner( has_explicit_iteration_count(b.iterations() != 0 || parsed_benchtime_flag.tag == BenchTimeType::ITERS), + report_thread_statistics(b.report_thread_statistics()), thread_runner( GetThreadRunner(b.GetUserThreadRunnerFactory(), b.threads())), iters(FLAGS_benchmark_dry_run @@ -335,7 +449,8 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { BM_VLOG(2) << "Running " << b.name().str() << " for " << iters << "\n"; std::unique_ptr manager; - manager.reset(new internal::ThreadManager(b.threads())); + manager.reset(new internal::ThreadManager(b.threads(), + report_thread_statistics)); thread_runner->RunThreads([&](int thread_idx) { RunInThread(&b, iters, thread_idx, manager.get(), @@ -347,6 +462,7 @@ BenchmarkRunner::IterationResults BenchmarkRunner::DoNIterations() { { MutexLock l(manager->GetBenchmarkMutex()); i.results = manager->results; + i.thread_results = manager->per_thread_results; } // And get rid of the manager. @@ -566,6 +682,8 @@ void BenchmarkRunner::DoOneRepetition() { } run_results.non_aggregates.push_back(report); + thread_stats_per_repetition_.push_back( + CreateThreadStatReports(b, report, i.thread_results)); ++num_repetitions_done; } @@ -576,6 +694,22 @@ RunResults&& BenchmarkRunner::GetResults() { // Calculate additional statistics over the repetitions of this instance. run_results.aggregates_only = ComputeStats(run_results.non_aggregates); + std::vector expanded; + expanded.reserve(run_results.non_aggregates.size() + + (thread_stats_per_repetition_.empty() + ? 0 + : thread_stats_per_repetition_.size() * + thread_stats_per_repetition_.front().size())); + for (size_t i = 0; i < run_results.non_aggregates.size(); ++i) { + expanded.push_back(std::move(run_results.non_aggregates[i])); + if (i < thread_stats_per_repetition_.size()) { + for (auto& thread_run : thread_stats_per_repetition_[i]) { + expanded.push_back(std::move(thread_run)); + } + } + } + run_results.non_aggregates = std::move(expanded); + return std::move(run_results); } diff --git a/src/benchmark_runner.h b/src/benchmark_runner.h index 9a2231a2a4..e705e13ea8 100644 --- a/src/benchmark_runner.h +++ b/src/benchmark_runner.h @@ -77,6 +77,8 @@ class BenchmarkRunner { private: RunResults run_results; + std::vector> + thread_stats_per_repetition_; const benchmark::internal::BenchmarkInstance& b; BenchmarkReporter::PerFamilyRunReports* reports_for_family; @@ -87,6 +89,7 @@ class BenchmarkRunner { bool warmup_done; const int repeats; const bool has_explicit_iteration_count; + const bool report_thread_statistics; int num_repetitions_done = 0; @@ -100,6 +103,7 @@ class BenchmarkRunner { struct IterationResults { internal::ThreadManager::Result results; + std::vector thread_results; IterationCount iters; double seconds; }; diff --git a/src/thread_manager.h b/src/thread_manager.h index 80252b6117..da26313a51 100644 --- a/src/thread_manager.h +++ b/src/thread_manager.h @@ -2,6 +2,7 @@ #define BENCHMARK_THREAD_MANAGER_H #include +#include #include "benchmark/counter.h" #include "benchmark/statistics.h" @@ -13,7 +14,12 @@ namespace internal { class ThreadManager { public: - explicit ThreadManager(int num_threads) : start_stop_barrier_(num_threads) {} + explicit ThreadManager(int num_threads, bool collect_per_thread = false) + : start_stop_barrier_(num_threads) { + if (collect_per_thread && num_threads > 1) { + per_thread_results.resize(static_cast(num_threads)); + } + } Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) { return benchmark_mutex_; @@ -35,6 +41,8 @@ class ThreadManager { UserCounters counters; }; GUARDED_BY(GetBenchmarkMutex()) Result results; + // Populated only when collect_per_thread is true and num_threads > 1. + GUARDED_BY(GetBenchmarkMutex()) std::vector per_thread_results; private: mutable Mutex benchmark_mutex_; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4384c30ec9..e76746b383 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -189,6 +189,9 @@ benchmark_add_test(NAME user_counters_test COMMAND user_counters_test --benchmar compile_output_test(user_counters_threads_test) benchmark_add_test(NAME user_counters_threads_test COMMAND user_counters_threads_test --benchmark_min_time=0.2s) +compile_output_test(thread_statistics_test) +benchmark_add_test(NAME thread_statistics_test COMMAND thread_statistics_test --benchmark_min_time=0.01s) + compile_output_test(perf_counters_test) benchmark_add_test(NAME perf_counters_test COMMAND perf_counters_test --benchmark_min_time=0.01s --benchmark_perf_counters=CYCLES,INSTRUCTIONS) diff --git a/test/thread_statistics_test.cc b/test/thread_statistics_test.cc new file mode 100644 index 0000000000..26608eb233 --- /dev/null +++ b/test/thread_statistics_test.cc @@ -0,0 +1,230 @@ + +#undef NDEBUG + +#include "benchmark/benchmark_api.h" +#include "benchmark/counter.h" +#include "benchmark/registration.h" +#include "benchmark/state.h" +#include "benchmark/utils.h" +#include "output_test.h" + +// clang-format off + +ADD_CASES(TC_ConsoleOut, + {{"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next}, + {"^[-]+$", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"%csv_header,\"work\""}}); + +// clang-format on + +namespace { +void BM_ThreadStats(benchmark::State& state) { + for (auto _ : state) { + } + state.counters["work"] = static_cast(state.thread_index() + 1); +} +BENCHMARK(BM_ThreadStats)->Threads(3)->ReportThreadStatistics(); + +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStats/threads:3 %console_report work=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStats/threads:3_thread_mean %console_report " + "work=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStats/threads:3_thread_median %console_report " + "work=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStats/threads:3_thread_stddev %console_report " + "work=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStats/threads:3_thread_cv %console_percentage_report " + "work=%percentage%$"}}); + +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ThreadStats/threads:3\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_ThreadStats/threads:3\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"repetition_index\": 0,$", MR_Next}, + {"\"threads\": 3,$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"work\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ThreadStats/threads:3_thread_mean\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_ThreadStats/threads:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"threads\": 3,$", MR_Next}, + {"\"aggregate_name\": \"thread_mean\",$", MR_Next}, + {"\"aggregate_unit\": \"time\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"work\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ThreadStats/threads:3_thread_median\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_ThreadStats/threads:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"threads\": 3,$", MR_Next}, + {"\"aggregate_name\": \"thread_median\",$", MR_Next}, + {"\"aggregate_unit\": \"time\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"work\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ThreadStats/threads:3_thread_stddev\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_ThreadStats/threads:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"threads\": 3,$", MR_Next}, + {"\"aggregate_name\": \"thread_stddev\",$", MR_Next}, + {"\"aggregate_unit\": \"time\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"work\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_ThreadStats/threads:3_thread_cv\",$"}, + {"\"family_index\": 0,$", MR_Next}, + {"\"per_family_instance_index\": 0,$", MR_Next}, + {"\"run_name\": \"BM_ThreadStats/threads:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"repetitions\": 1,$", MR_Next}, + {"\"threads\": 3,$", MR_Next}, + {"\"aggregate_name\": \"thread_cv\",$", MR_Next}, + {"\"aggregate_unit\": \"percentage\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"work\": %float$", MR_Next}, + {"}", MR_Next}}); + +ADD_CASES(TC_CSVOut, + {{"^\"BM_ThreadStats/threads:3\",%csv_report,%float$"}}); +ADD_CASES(TC_CSVOut, + {{"^\"BM_ThreadStats/threads:3_thread_mean\",%csv_report,%float$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ThreadStats/threads:3_thread_median\",%csv_report," + "%float$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ThreadStats/threads:3_thread_stddev\",%csv_report," + "%float$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_ThreadStats/threads:3_thread_cv\",%csv_cv_report," + "%float$"}}); + +void CheckThreadStatsSum(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "work", EQ, 6); +} +void CheckThreadStatsMean(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "work", EQ, 2); +} +void CheckThreadStatsMedian(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "work", EQ, 2); +} +void CheckThreadStatsStdDev(Results const& e) { + CHECK_FLOAT_COUNTER_VALUE(e, "work", EQ, 1.0, 0.001); +} +void CheckThreadStatsCV(Results const& e) { + CHECK_FLOAT_COUNTER_VALUE(e, "work", EQ, 0.5, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_ThreadStats/threads:3$", &CheckThreadStatsSum); +CHECK_BENCHMARK_RESULTS("BM_ThreadStats/threads:3_thread_mean$", + &CheckThreadStatsMean); +CHECK_BENCHMARK_RESULTS("BM_ThreadStats/threads:3_thread_median$", + &CheckThreadStatsMedian); +CHECK_BENCHMARK_RESULTS("BM_ThreadStats/threads:3_thread_stddev$", + &CheckThreadStatsStdDev); +CHECK_BENCHMARK_RESULTS("BM_ThreadStats/threads:3_thread_cv$", + &CheckThreadStatsCV); +} // namespace + +namespace { +void BM_ThreadStatsAvg(benchmark::State& state) { + for (auto _ : state) { + } + state.counters["work"] = benchmark::Counter{ + static_cast(state.thread_index() + 1), + benchmark::Counter::kAvgThreads}; +} +BENCHMARK(BM_ThreadStatsAvg)->Threads(3)->ReportThreadStatistics(); + +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStatsAvg/threads:3 %console_report work=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStatsAvg/threads:3_thread_mean %console_report " + "work=%hrfloat$"}}); + +void CheckThreadStatsAvgSum(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "work", EQ, 2); +} +void CheckThreadStatsAvgMean(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "work", EQ, 2); +} +CHECK_BENCHMARK_RESULTS("BM_ThreadStatsAvg/threads:3$", + &CheckThreadStatsAvgSum); +CHECK_BENCHMARK_RESULTS("BM_ThreadStatsAvg/threads:3_thread_mean$", + &CheckThreadStatsAvgMean); +} // namespace + +namespace { +void BM_ThreadStatsOff(benchmark::State& state) { + for (auto _ : state) { + } + state.counters["work"] = static_cast(state.thread_index() + 1); +} +BENCHMARK(BM_ThreadStatsOff)->Threads(3); + +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStatsOff/threads:3 %console_report work=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, {{".*BM_ThreadStatsOff/threads:3_thread_", MR_Not}}); + +void CheckThreadStatsOff(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "work", EQ, 6); +} +CHECK_BENCHMARK_RESULTS("BM_ThreadStatsOff/threads:3$", &CheckThreadStatsOff); +} // namespace + +namespace { +void BM_ThreadStatsSingle(benchmark::State& state) { + for (auto _ : state) { + } + state.counters["work"] = 7; +} +BENCHMARK(BM_ThreadStatsSingle)->Threads(1)->ReportThreadStatistics(); + +ADD_CASES(TC_ConsoleOut, + {{"^BM_ThreadStatsSingle/threads:1 %console_report work=%hrfloat$"}}); +ADD_CASES(TC_ConsoleOut, + {{".*BM_ThreadStatsSingle/threads:1_thread_", MR_Not}}); + +void CheckThreadStatsSingle(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "work", EQ, 7); +} +CHECK_BENCHMARK_RESULTS("BM_ThreadStatsSingle/threads:1$", + &CheckThreadStatsSingle); +} // namespace + +int main(int argc, char* argv[]) { + benchmark::MaybeReenterWithoutASLR(argc, argv); + RunOutputTests(argc, argv); +}