diff --git a/docs/release-notes.md b/docs/release-notes.md index bab645deb2..ba736cb51d 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,6 +2,7 @@ # Release notes **Contents**
+[3.16.1](#3161)
[3.16.0](#3160)
[3.15.3](#3153)
[3.15.2](#3152)
@@ -80,6 +81,12 @@ +## 3.16.1 + +### Fixes +* Fixed benchmark clock-resolution estimation potentially exhausting memory on fast or memory-constrained platforms. (#3180) + + ## 3.16.0 ### Fixes diff --git a/src/catch2/benchmark/detail/catch_estimate_clock.hpp b/src/catch2/benchmark/detail/catch_estimate_clock.hpp index 576fdb6835..cbb45b8fb6 100644 --- a/src/catch2/benchmark/detail/catch_estimate_clock.hpp +++ b/src/catch2/benchmark/detail/catch_estimate_clock.hpp @@ -49,6 +49,9 @@ namespace Catch { constexpr auto minimum_ticks = 1000; constexpr auto warmup_seed = 10000; constexpr auto clock_resolution_estimation_time = std::chrono::milliseconds(500); + // resolution() allocates O(n) TimePoints; without a sample cap a + // cheap high-resolution clock can exhaust memory within 500ms. (#3180) + constexpr auto clock_resolution_estimation_iteration_limit = 100000; constexpr auto clock_cost_estimation_time_limit = std::chrono::seconds(1); constexpr auto clock_cost_estimation_tick_limit = 100000; constexpr auto clock_cost_estimation_time = std::chrono::milliseconds(10); @@ -56,13 +59,19 @@ namespace Catch { template int warmup() { - return run_for_at_least(warmup_time, warmup_seed, &resolution) + return run_for_at_least(warmup_time, + warmup_seed, + &resolution, + clock_resolution_estimation_iteration_limit) .iterations; } template EnvironmentEstimate estimate_clock_resolution(int iterations) { - auto r = run_for_at_least(clock_resolution_estimation_time, iterations, &resolution) - .result; + auto r = run_for_at_least(clock_resolution_estimation_time, + iterations, + &resolution, + clock_resolution_estimation_iteration_limit) + .result; return { FDuration(mean(r.data(), r.data() + r.size())), classify_outliers(r.data(), r.data() + r.size()), diff --git a/src/catch2/benchmark/detail/catch_run_for_at_least.hpp b/src/catch2/benchmark/detail/catch_run_for_at_least.hpp index 4dfa8bbbb6..07e51ee458 100644 --- a/src/catch2/benchmark/detail/catch_run_for_at_least.hpp +++ b/src/catch2/benchmark/detail/catch_run_for_at_least.hpp @@ -42,19 +42,28 @@ namespace Catch { [[noreturn]] void throw_optimized_away_error(); + constexpr auto run_for_at_least_max_iterations = 1 << 30; + template TimingOf> run_for_at_least(IDuration how_long, const int initial_iterations, - Fun&& fun) { + Fun&& fun, + int max_iterations = run_for_at_least_max_iterations) { auto iters = initial_iterations; - while (iters < (1 << 30)) { + if (iters > max_iterations) { + iters = max_iterations; + } + while (iters < run_for_at_least_max_iterations) { auto&& Timing = measure_one(fun, iters, is_callable()); - if (Timing.elapsed >= how_long) { + if (Timing.elapsed >= how_long || iters >= max_iterations) { return { Timing.elapsed, CATCH_MOVE(Timing.result), iters }; } iters *= 2; + if (iters > max_iterations) { + iters = max_iterations; + } } throw_optimized_away_error(); } diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index a25bc52a68..48880c88da 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -356,6 +356,7 @@ Message from section two :test-result: PASS checkedIf :test-result: FAIL checkedIf, failing :test-result: PASS classify_outliers +:test-result: PASS clock resolution estimation is iteration-capped :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits @@ -406,6 +407,7 @@ b1! :test-result: PASS replaceInPlace :test-result: PASS request an unknown %-starting stream fails :test-result: PASS resolution +:test-result: PASS run_for_at_least respects max_iterations :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 3f846b7b45..fcdea52c4e 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -349,6 +349,7 @@ :test-result: PASS checkedIf :test-result: FAIL checkedIf, failing :test-result: PASS classify_outliers +:test-result: PASS clock resolution estimation is iteration-capped :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables :test-result: PASS convertToBits @@ -395,6 +396,7 @@ :test-result: PASS replaceInPlace :test-result: PASS request an unknown %-starting stream fails :test-result: PASS resolution +:test-result: PASS run_for_at_least respects max_iterations :test-result: PASS run_for_at_least, chronometer :test-result: PASS run_for_at_least, int :test-result: FAIL second tag diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 09c0bc6194..7805fdb919 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -2647,6 +2647,12 @@ InternalBenchmark.tests.cpp:: passed: o.low_mild == lom for: 0 == 0 InternalBenchmark.tests.cpp:: passed: o.high_mild == him for: 1 == 1 InternalBenchmark.tests.cpp:: passed: o.high_severe == his for: 0 == 0 InternalBenchmark.tests.cpp:: passed: o.total() == los + lom + him + his for: 2 == 2 +InternalBenchmark.tests.cpp:: passed: iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +InternalBenchmark.tests.cpp:: passed: iterations > 0 for: 100000 (0x) > 0 +InternalBenchmark.tests.cpp:: passed: res.mean.count() == 1 for: 1.0 == 1 +InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:: passed: res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +InternalBenchmark.tests.cpp:: passed: res.outliers.samples_seen > 0 for: 100000 (0x) > 0 Condition.tests.cpp:: passed: unsigned_char_var == 1 for: 1 == 1 Condition.tests.cpp:: passed: unsigned_short_var == 1 for: 1 == 1 Condition.tests.cpp:: passed: unsigned_int_var == 1 for: 1 == 1 @@ -2680,6 +2686,7 @@ InternalBenchmark.tests.cpp:: passed: erfc_inv(0.050000) == Approx( Approx( 1.38590382434967796 ) InternalBenchmark.tests.cpp:: passed: res.mean.count() == rate for: 2000.0 == 2000 (0x) InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:: passed: res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: @@ -2819,6 +2826,8 @@ InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == 1000 (0x) InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == 1000 (0x) InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == 1000 (0x) +InternalBenchmark.tests.cpp:: passed: Timing.iterations == 16 for: 16 == 16 +InternalBenchmark.tests.cpp:: passed: Timing.elapsed < time for: 1 ns < 1000000 ns InternalBenchmark.tests.cpp:: passed: meter.runs() >= old_runs for: 1 >= 1 InternalBenchmark.tests.cpp:: passed: meter.runs() >= old_runs for: 2 >= 1 InternalBenchmark.tests.cpp:: passed: meter.runs() >= old_runs for: 4 >= 2 @@ -3008,14 +3017,14 @@ Misc.tests.cpp:: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:: passed: v.capacity() >= 5 for: 5 >= 5 Misc.tests.cpp:: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:: passed: v.capacity() >= 5 for: 5 >= 5 -InternalBenchmark.tests.cpp:: passed: (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 160000000 (0x) > 100 -InternalBenchmark.tests.cpp:: passed: (end - start) > Catch::Benchmark::Detail::warmup_time for: 310016000 ns > 100 ms +InternalBenchmark.tests.cpp:: passed: (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 100000000 (0x) > 100 +InternalBenchmark.tests.cpp:: passed: (end - start) > Catch::Benchmark::Detail::warmup_time for: 250016000 ns > 100 ms InternalBenchmark.tests.cpp:: passed: q1 == 14.5 for: 14.5 == 14.5 InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 454 | 334 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2423 | 2222 passed | 158 failed | 43 failed as expected +test cases: 456 | 336 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2432 | 2231 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 5a2a6e037c..744d94aa12 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -2640,6 +2640,12 @@ InternalBenchmark.tests.cpp:: passed: o.low_mild == lom for: 0 == 0 InternalBenchmark.tests.cpp:: passed: o.high_mild == him for: 1 == 1 InternalBenchmark.tests.cpp:: passed: o.high_severe == his for: 0 == 0 InternalBenchmark.tests.cpp:: passed: o.total() == los + lom + him + his for: 2 == 2 +InternalBenchmark.tests.cpp:: passed: iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +InternalBenchmark.tests.cpp:: passed: iterations > 0 for: 100000 (0x) > 0 +InternalBenchmark.tests.cpp:: passed: res.mean.count() == 1 for: 1.0 == 1 +InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:: passed: res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +InternalBenchmark.tests.cpp:: passed: res.outliers.samples_seen > 0 for: 100000 (0x) > 0 Condition.tests.cpp:: passed: unsigned_char_var == 1 for: 1 == 1 Condition.tests.cpp:: passed: unsigned_short_var == 1 for: 1 == 1 Condition.tests.cpp:: passed: unsigned_int_var == 1 for: 1 == 1 @@ -2673,6 +2679,7 @@ InternalBenchmark.tests.cpp:: passed: erfc_inv(0.050000) == Approx( Approx( 1.38590382434967796 ) InternalBenchmark.tests.cpp:: passed: res.mean.count() == rate for: 2000.0 == 2000 (0x) InternalBenchmark.tests.cpp:: passed: res.outliers.total() == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:: passed: res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: @@ -2808,6 +2815,8 @@ InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == 1000 (0x) InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == 1000 (0x) InternalBenchmark.tests.cpp:: passed: res[i] == rate for: 1000.0 == 1000 (0x) +InternalBenchmark.tests.cpp:: passed: Timing.iterations == 16 for: 16 == 16 +InternalBenchmark.tests.cpp:: passed: Timing.elapsed < time for: 1 ns < 1000000 ns InternalBenchmark.tests.cpp:: passed: meter.runs() >= old_runs for: 1 >= 1 InternalBenchmark.tests.cpp:: passed: meter.runs() >= old_runs for: 2 >= 1 InternalBenchmark.tests.cpp:: passed: meter.runs() >= old_runs for: 4 >= 2 @@ -2997,14 +3006,14 @@ Misc.tests.cpp:: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:: passed: v.capacity() >= 5 for: 5 >= 5 Misc.tests.cpp:: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:: passed: v.capacity() >= 5 for: 5 >= 5 -InternalBenchmark.tests.cpp:: passed: (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 160000000 (0x) > 100 -InternalBenchmark.tests.cpp:: passed: (end - start) > Catch::Benchmark::Detail::warmup_time for: 310016000 ns > 100 ms +InternalBenchmark.tests.cpp:: passed: (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 100000000 (0x) > 100 +InternalBenchmark.tests.cpp:: passed: (end - start) > Catch::Benchmark::Detail::warmup_time for: 250016000 ns > 100 ms InternalBenchmark.tests.cpp:: passed: q1 == 14.5 for: 14.5 == 14.5 InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 454 | 334 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2423 | 2222 passed | 158 failed | 43 failed as expected +test cases: 456 | 336 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2432 | 2231 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 344c5b032a..2f04095749 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1743,6 +1743,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 454 | 352 passed | 76 failed | 7 skipped | 19 failed as expected -assertions: 2401 | 2222 passed | 136 failed | 43 failed as expected +test cases: 456 | 354 passed | 76 failed | 7 skipped | 19 failed as expected +assertions: 2410 | 2231 passed | 136 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index f53249472f..9d973bfb80 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -17580,6 +17580,50 @@ InternalBenchmark.tests.cpp:: PASSED: with expansion: 2 == 2 +------------------------------------------------------------------------------- +clock resolution estimation is iteration-capped + warmup +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp: +............................................................................... + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit ) +with expansion: + 100000 (0x) <= 100000 (0x) + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( iterations > 0 ) +with expansion: + 100000 (0x) > 0 + +------------------------------------------------------------------------------- +clock resolution estimation is iteration-capped + estimate_clock_resolution +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp: +............................................................................... + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.mean.count() == 1 ) +with expansion: + 1.0 == 1 + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.total() == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit ) +with expansion: + 100000 (0x) <= 100000 (0x) + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.samples_seen > 0 ) +with expansion: + 100000 (0x) > 0 + ------------------------------------------------------------------------------- comparisons between const int variables ------------------------------------------------------------------------------- @@ -17752,6 +17796,11 @@ InternalBenchmark.tests.cpp:: PASSED: with expansion: 0 == 0 +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit ) +with expansion: + 100000 (0x) <= 100000 (0x) + ------------------------------------------------------------------------------- even more nested SECTION tests c @@ -18853,6 +18902,22 @@ InternalBenchmark.tests.cpp:: PASSED: with expansion: 1000.0 == 1000 (0x) +------------------------------------------------------------------------------- +run_for_at_least respects max_iterations +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp: +............................................................................... + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( Timing.iterations == 16 ) +with expansion: + 16 == 16 + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( Timing.elapsed < time ) +with expansion: + 1 ns < 1000000 ns + ------------------------------------------------------------------------------- run_for_at_least, chronometer ------------------------------------------------------------------------------- @@ -20177,12 +20242,12 @@ InternalBenchmark.tests.cpp: InternalBenchmark.tests.cpp:: PASSED: REQUIRE( (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() ) with expansion: - 160000000 (0x) > 100 + 100000000 (0x) > 100 InternalBenchmark.tests.cpp:: PASSED: REQUIRE( (end - start) > Catch::Benchmark::Detail::warmup_time ) with expansion: - 310016000 ns > 100 ms + 250016000 ns > 100 ms ------------------------------------------------------------------------------- weighted_average_quantile @@ -20226,6 +20291,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 454 | 334 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2423 | 2222 passed | 158 failed | 43 failed as expected +test cases: 456 | 336 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2432 | 2231 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 9dba61236d..2819bdb57e 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -17573,6 +17573,50 @@ InternalBenchmark.tests.cpp:: PASSED: with expansion: 2 == 2 +------------------------------------------------------------------------------- +clock resolution estimation is iteration-capped + warmup +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp: +............................................................................... + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit ) +with expansion: + 100000 (0x) <= 100000 (0x) + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( iterations > 0 ) +with expansion: + 100000 (0x) > 0 + +------------------------------------------------------------------------------- +clock resolution estimation is iteration-capped + estimate_clock_resolution +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp: +............................................................................... + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.mean.count() == 1 ) +with expansion: + 1.0 == 1 + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.total() == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit ) +with expansion: + 100000 (0x) <= 100000 (0x) + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.samples_seen > 0 ) +with expansion: + 100000 (0x) > 0 + ------------------------------------------------------------------------------- comparisons between const int variables ------------------------------------------------------------------------------- @@ -17745,6 +17789,11 @@ InternalBenchmark.tests.cpp:: PASSED: with expansion: 0 == 0 +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit ) +with expansion: + 100000 (0x) <= 100000 (0x) + ------------------------------------------------------------------------------- even more nested SECTION tests c @@ -18842,6 +18891,22 @@ InternalBenchmark.tests.cpp:: PASSED: with expansion: 1000.0 == 1000 (0x) +------------------------------------------------------------------------------- +run_for_at_least respects max_iterations +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp: +............................................................................... + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( Timing.iterations == 16 ) +with expansion: + 16 == 16 + +InternalBenchmark.tests.cpp:: PASSED: + REQUIRE( Timing.elapsed < time ) +with expansion: + 1 ns < 1000000 ns + ------------------------------------------------------------------------------- run_for_at_least, chronometer ------------------------------------------------------------------------------- @@ -20166,12 +20231,12 @@ InternalBenchmark.tests.cpp: InternalBenchmark.tests.cpp:: PASSED: REQUIRE( (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() ) with expansion: - 160000000 (0x) > 100 + 100000000 (0x) > 100 InternalBenchmark.tests.cpp:: PASSED: REQUIRE( (end - start) > Catch::Benchmark::Detail::warmup_time ) with expansion: - 310016000 ns > 100 ms + 250016000 ns > 100 ms ------------------------------------------------------------------------------- weighted_average_quantile @@ -20215,6 +20280,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 454 | 334 passed | 96 failed | 6 skipped | 18 failed as expected -assertions: 2423 | 2222 passed | 158 failed | 43 failed as expected +test cases: 456 | 336 passed | 96 failed | 6 skipped | 18 failed as expected +assertions: 2432 | 2231 passed | 158 failed | 43 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index f58412b65a..15b5a28aff 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1600,6 +1600,9 @@ at Misc.tests.cpp: + + + @@ -1797,6 +1800,7 @@ at Message.tests.cpp: + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index c80e5855ed..9aaa7387eb 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1599,6 +1599,9 @@ at Misc.tests.cpp: + + + @@ -1796,6 +1799,7 @@ at Message.tests.cpp: + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 17c26f362c..7aaf6b5d76 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -229,12 +229,16 @@ at AssertionHandler.tests.cpp: + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 4c1131891d..7f8ccf1adc 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -228,12 +228,16 @@ at AssertionHandler.tests.cpp: + + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 6fa5949efe..ea0b5d2bf0 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -4245,6 +4245,18 @@ ok {test-number} - o.high_mild == him for: 1 == 1 ok {test-number} - o.high_severe == his for: 0 == 0 # classify_outliers ok {test-number} - o.total() == los + lom + him + his for: 2 == 2 +# clock resolution estimation is iteration-capped +ok {test-number} - iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +# clock resolution estimation is iteration-capped +ok {test-number} - iterations > 0 for: 100000 (0x) > 0 +# clock resolution estimation is iteration-capped +ok {test-number} - res.mean.count() == 1 for: 1.0 == 1 +# clock resolution estimation is iteration-capped +ok {test-number} - res.outliers.total() == 0 for: 0 == 0 +# clock resolution estimation is iteration-capped +ok {test-number} - res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +# clock resolution estimation is iteration-capped +ok {test-number} - res.outliers.samples_seen > 0 for: 100000 (0x) > 0 # comparisons between const int variables ok {test-number} - unsigned_char_var == 1 for: 1 == 1 # comparisons between const int variables @@ -4291,6 +4303,8 @@ ok {test-number} - erfc_inv(0.050000) == Approx(1.38590382434967796) for: 1.3859 ok {test-number} - res.mean.count() == rate for: 2000.0 == 2000 (0x) # estimate_clock_resolution ok {test-number} - res.outliers.total() == 0 for: 0 == 0 +# estimate_clock_resolution +ok {test-number} - res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) # even more nested SECTION tests ok {test-number} - # even more nested SECTION tests @@ -4545,6 +4559,10 @@ ok {test-number} - res[i] == rate for: 1000.0 == 1000 (0x) ok {test-number} - res[i] == rate for: 1000.0 == 1000 (0x) # resolution ok {test-number} - res[i] == rate for: 1000.0 == 1000 (0x) +# run_for_at_least respects max_iterations +ok {test-number} - Timing.iterations == 16 for: 16 == 16 +# run_for_at_least respects max_iterations +ok {test-number} - Timing.elapsed < time for: 1 ns < 1000000 ns # run_for_at_least, chronometer ok {test-number} - meter.runs() >= old_runs for: 1 >= 1 # run_for_at_least, chronometer @@ -4852,9 +4870,9 @@ ok {test-number} - v.size() == 5 for: 5 == 5 # vectors can be sized and resized ok {test-number} - v.capacity() >= 5 for: 5 >= 5 # warmup -ok {test-number} - (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 160000000 (0x) > 100 +ok {test-number} - (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 100000000 (0x) > 100 # warmup -ok {test-number} - (end - start) > Catch::Benchmark::Detail::warmup_time for: 310016000 ns > 100 ms +ok {test-number} - (end - start) > Catch::Benchmark::Detail::warmup_time for: 250016000 ns > 100 ms # weighted_average_quantile ok {test-number} - q1 == 14.5 for: 14.5 == 14.5 # weighted_average_quantile @@ -4865,5 +4883,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2435 +1..2444 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 0a7d381a95..5370ea6b9c 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -4238,6 +4238,18 @@ ok {test-number} - o.high_mild == him for: 1 == 1 ok {test-number} - o.high_severe == his for: 0 == 0 # classify_outliers ok {test-number} - o.total() == los + lom + him + his for: 2 == 2 +# clock resolution estimation is iteration-capped +ok {test-number} - iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +# clock resolution estimation is iteration-capped +ok {test-number} - iterations > 0 for: 100000 (0x) > 0 +# clock resolution estimation is iteration-capped +ok {test-number} - res.mean.count() == 1 for: 1.0 == 1 +# clock resolution estimation is iteration-capped +ok {test-number} - res.outliers.total() == 0 for: 0 == 0 +# clock resolution estimation is iteration-capped +ok {test-number} - res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) +# clock resolution estimation is iteration-capped +ok {test-number} - res.outliers.samples_seen > 0 for: 100000 (0x) > 0 # comparisons between const int variables ok {test-number} - unsigned_char_var == 1 for: 1 == 1 # comparisons between const int variables @@ -4284,6 +4296,8 @@ ok {test-number} - erfc_inv(0.050000) == Approx(1.38590382434967796) for: 1.3859 ok {test-number} - res.mean.count() == rate for: 2000.0 == 2000 (0x) # estimate_clock_resolution ok {test-number} - res.outliers.total() == 0 for: 0 == 0 +# estimate_clock_resolution +ok {test-number} - res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit for: 100000 (0x) <= 100000 (0x) # even more nested SECTION tests ok {test-number} - # even more nested SECTION tests @@ -4534,6 +4548,10 @@ ok {test-number} - res[i] == rate for: 1000.0 == 1000 (0x) ok {test-number} - res[i] == rate for: 1000.0 == 1000 (0x) # resolution ok {test-number} - res[i] == rate for: 1000.0 == 1000 (0x) +# run_for_at_least respects max_iterations +ok {test-number} - Timing.iterations == 16 for: 16 == 16 +# run_for_at_least respects max_iterations +ok {test-number} - Timing.elapsed < time for: 1 ns < 1000000 ns # run_for_at_least, chronometer ok {test-number} - meter.runs() >= old_runs for: 1 >= 1 # run_for_at_least, chronometer @@ -4841,9 +4859,9 @@ ok {test-number} - v.size() == 5 for: 5 == 5 # vectors can be sized and resized ok {test-number} - v.capacity() >= 5 for: 5 >= 5 # warmup -ok {test-number} - (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 160000000 (0x) > 100 +ok {test-number} - (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 100000000 (0x) > 100 # warmup -ok {test-number} - (end - start) > Catch::Benchmark::Detail::warmup_time for: 310016000 ns > 100 ms +ok {test-number} - (end - start) > Catch::Benchmark::Detail::warmup_time for: 250016000 ns > 100 ms # weighted_average_quantile ok {test-number} - q1 == 14.5 for: 14.5 == 14.5 # weighted_average_quantile @@ -4854,5 +4872,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2435 +1..2444 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index 16fe73f92b..1e8d4035a0 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -864,6 +864,8 @@ ##teamcity[testFinished name='checkedIf, failing' duration="{duration}"] ##teamcity[testStarted name='classify_outliers'] ##teamcity[testFinished name='classify_outliers' duration="{duration}"] +##teamcity[testStarted name='clock resolution estimation is iteration-capped'] +##teamcity[testFinished name='clock resolution estimation is iteration-capped' duration="{duration}"] ##teamcity[testStarted name='comparisons between const int variables'] ##teamcity[testFinished name='comparisons between const int variables' duration="{duration}"] ##teamcity[testStarted name='comparisons between int variables'] @@ -983,6 +985,8 @@ loose text artifact ##teamcity[testFinished name='request an unknown %-starting stream fails' duration="{duration}"] ##teamcity[testStarted name='resolution'] ##teamcity[testFinished name='resolution' duration="{duration}"] +##teamcity[testStarted name='run_for_at_least respects max_iterations'] +##teamcity[testFinished name='run_for_at_least respects max_iterations' duration="{duration}"] ##teamcity[testStarted name='run_for_at_least, chronometer'] ##teamcity[testFinished name='run_for_at_least, chronometer' duration="{duration}"] ##teamcity[testStarted name='run_for_at_least, int'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index acc777f282..87f8ee9ae2 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -864,6 +864,8 @@ ##teamcity[testFinished name='checkedIf, failing' duration="{duration}"] ##teamcity[testStarted name='classify_outliers'] ##teamcity[testFinished name='classify_outliers' duration="{duration}"] +##teamcity[testStarted name='clock resolution estimation is iteration-capped'] +##teamcity[testFinished name='clock resolution estimation is iteration-capped' duration="{duration}"] ##teamcity[testStarted name='comparisons between const int variables'] ##teamcity[testFinished name='comparisons between const int variables' duration="{duration}"] ##teamcity[testStarted name='comparisons between int variables'] @@ -982,6 +984,8 @@ ##teamcity[testFinished name='request an unknown %-starting stream fails' duration="{duration}"] ##teamcity[testStarted name='resolution'] ##teamcity[testFinished name='resolution' duration="{duration}"] +##teamcity[testStarted name='run_for_at_least respects max_iterations'] +##teamcity[testFinished name='run_for_at_least respects max_iterations' duration="{duration}"] ##teamcity[testStarted name='run_for_at_least, chronometer'] ##teamcity[testFinished name='run_for_at_least, chronometer' duration="{duration}"] ##teamcity[testStarted name='run_for_at_least, int'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 81df5a5356..2cf3591b4a 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -20600,6 +20600,63 @@ Approx( 1.23999999999999999 ) + +
+ + + iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit + + + 100000 (0x) <= 100000 (0x) + + + + + iterations > 0 + + + 100000 (0x) > 0 + + + +
+
+ + + res.mean.count() == 1 + + + 1.0 == 1 + + + + + res.outliers.total() == 0 + + + 0 == 0 + + + + + res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit + + + 100000 (0x) <= 100000 (0x) + + + + + res.outliers.samples_seen > 0 + + + 100000 (0x) > 0 + + + +
+ + @@ -20795,6 +20852,14 @@ Approx( 1.38590382434967796 ) 0 == 0 + + + res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit + + + 100000 (0x) <= 100000 (0x) + + @@ -21937,6 +22002,25 @@ Approx( -1.95996398454005449 ) + + + + Timing.iterations == 16 + + + 16 == 16 + + + + + Timing.elapsed < time + + + 1 ns < 1000000 ns + + + + @@ -23427,7 +23511,7 @@ Approx( -1.95996398454005449 ) (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() - 160000000 (0x) > 100 + 100000000 (0x) > 100 @@ -23435,7 +23519,7 @@ Approx( -1.95996398454005449 ) (end - start) > Catch::Benchmark::Detail::warmup_time - 310016000 ns > 100 ms + 250016000 ns > 100 ms @@ -23476,6 +23560,6 @@ Approx( -1.95996398454005449 ) - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 5888529cf2..8a0cd43a94 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -20600,6 +20600,63 @@ Approx( 1.23999999999999999 ) + +
+ + + iterations <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit + + + 100000 (0x) <= 100000 (0x) + + + + + iterations > 0 + + + 100000 (0x) > 0 + + + +
+
+ + + res.mean.count() == 1 + + + 1.0 == 1 + + + + + res.outliers.total() == 0 + + + 0 == 0 + + + + + res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit + + + 100000 (0x) <= 100000 (0x) + + + + + res.outliers.samples_seen > 0 + + + 100000 (0x) > 0 + + + +
+ + @@ -20795,6 +20852,14 @@ Approx( 1.38590382434967796 ) 0 == 0 + + + res.outliers.samples_seen <= Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit + + + 100000 (0x) <= 100000 (0x) + + @@ -21936,6 +22001,25 @@ Approx( -1.95996398454005449 ) + + + + Timing.iterations == 16 + + + 16 == 16 + + + + + Timing.elapsed < time + + + 1 ns < 1000000 ns + + + + @@ -23426,7 +23510,7 @@ Approx( -1.95996398454005449 ) (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() - 160000000 (0x) > 100 + 100000000 (0x) > 100 @@ -23434,7 +23518,7 @@ Approx( -1.95996398454005449 ) (end - start) > Catch::Benchmark::Detail::warmup_time - 310016000 ns > 100 ms + 250016000 ns > 100 ms @@ -23475,6 +23559,6 @@ Approx( -1.95996398454005449 ) - - + + diff --git a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp index b7ec17b857..d0cf7758f4 100644 --- a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp @@ -114,6 +114,35 @@ TEST_CASE("estimate_clock_resolution", "[benchmark]") { REQUIRE(res.mean.count() == rate); REQUIRE(res.outliers.total() == 0); + REQUIRE(res.outliers.samples_seen <= + Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit); +} + +TEST_CASE("clock resolution estimation is iteration-capped", "[benchmark]") { + // 1ns per now() would otherwise demand hundreds of millions of samples + // to fill the 500ms window, exhausting memory. (#3180) + counting_clock::set_rate(1); + + SECTION("warmup") { + auto iterations = Catch::Benchmark::Detail::warmup(); + REQUIRE(iterations <= + Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit); + REQUIRE(iterations > 0); + } + SECTION("estimate_clock_resolution") { + int iters = + Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit * + 4; + auto res = + Catch::Benchmark::Detail::estimate_clock_resolution( + iters); + + REQUIRE(res.mean.count() == 1); + REQUIRE(res.outliers.total() == 0); + REQUIRE(res.outliers.samples_seen <= + Catch::Benchmark::Detail::clock_resolution_estimation_iteration_limit); + REQUIRE(res.outliers.samples_seen > 0); + } } TEST_CASE("benchmark function call", "[benchmark]") { @@ -372,6 +401,22 @@ TEST_CASE("run_for_at_least, int", "[benchmark]") { REQUIRE(Timing.iterations >= time.count()); } +TEST_CASE("run_for_at_least respects max_iterations", "[benchmark]") { + manual_clock::duration time(1'000'000); + + auto Timing = Catch::Benchmark::Detail::run_for_at_least( + time, + 1, + [](int x) -> int { + manual_clock::advance(1); + return x; + }, + 16); + + REQUIRE(Timing.iterations == 16); + REQUIRE(Timing.elapsed < time); +} + TEST_CASE("run_for_at_least, chronometer", "[benchmark]") { manual_clock::duration time(100); @@ -425,7 +470,9 @@ TEST_CASE("run benchmark", "[benchmark][approvals]") { bench.run(); auto end = counting_clock::now(); - CHECK((end - start).count() == 2867251000); + // Exact fake-clock ticks for this counting_clock setup (environment probe + // is sample-capped). Update if warmup/resolution iteration limits change. + CHECK((end - start).count() == 1787245000); } TEST_CASE("Failing benchmarks", "[!benchmark][.approvals]") {