From 3938bd85810ab6bf16fed54a20e20d22e81184ca Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Sat, 22 Aug 2026 16:57:58 +0200 Subject: [PATCH 1/6] Ignore skipped benchmark when computing geomean Skipped benchmarks have their real and CPU times set to zero. If they are part of the data for which the geomean is calculated, these zeros cause the computed geomean to be zero. Hence, a single skipped benchmark will effectively disable the computation of a geomean. This seems unfortunate as in many cases there are lots of other benchmarks that still ran normally and for which one might still be interested in the overall geomean change. --- tools/gbench/report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/gbench/report.py b/tools/gbench/report.py index e143e45a71..fab8852043 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -191,6 +191,8 @@ def calculate_geomean(json): for benchmark in json["benchmarks"]: if "run_type" in benchmark and benchmark["run_type"] == "aggregate": continue + if "skip_message" in benchmark: + continue times.append( [ get_timedelta_field_as_seconds(benchmark, "real_time"), From 08f072a2b24fce3b8cef4de88d4cdae908dd1e34 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Mon, 24 Aug 2026 08:11:11 +0200 Subject: [PATCH 2/6] Use 'skipped' attribute instead of 'skip_message' --- tools/gbench/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gbench/report.py b/tools/gbench/report.py index fab8852043..fc7a40e958 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -191,7 +191,7 @@ def calculate_geomean(json): for benchmark in json["benchmarks"]: if "run_type" in benchmark and benchmark["run_type"] == "aggregate": continue - if "skip_message" in benchmark: + if benchmark.get("skipped", False): continue times.append( [ From 7aa16c40f3bc2e6a472b8a2c5ce234bdc0378363 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Mon, 24 Aug 2026 08:11:34 +0200 Subject: [PATCH 3/6] Remove skipped benchmarks from partitions --- tools/gbench/report.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/gbench/report.py b/tools/gbench/report.py index fc7a40e958..8098ffc697 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -161,13 +161,17 @@ def partition_benchmarks(json1, json2): lhs = [ x for x in json1["benchmarks"] - if x["name"] == name and x["time_unit"] == time_unit + if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) ] rhs = [ x for x in json2["benchmarks"] - if x["name"] == name and x["time_unit"] == time_unit + if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) ] + + if len(lhs) == 0 or len(rhs) == 0: + continue + partitions.append([lhs, rhs]) return partitions From 694d3428948de708d0e40fb01683342c5d22e863 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Mon, 24 Aug 2026 08:11:52 +0200 Subject: [PATCH 4/6] Add skipped benchmark to test data --- tools/gbench/Inputs/test1_run1.json | 10 ++++++++++ tools/gbench/Inputs/test1_run2.json | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/tools/gbench/Inputs/test1_run1.json b/tools/gbench/Inputs/test1_run1.json index 9daed0bcc6..fd52df8b06 100644 --- a/tools/gbench/Inputs/test1_run1.json +++ b/tools/gbench/Inputs/test1_run1.json @@ -122,6 +122,16 @@ "real_time": 1, "cpu_time": 1, "time_unit": "s" + }, + { + "name": "BM_skipped", + "label": "a label", + "iterations": 1, + "real_time": 0, + "cpu_time": 0, + "time_unit": "s", + "skipped": true, + "skip_message": "Some message" } ] } diff --git a/tools/gbench/Inputs/test1_run2.json b/tools/gbench/Inputs/test1_run2.json index dc52970abf..aad9d86f7d 100644 --- a/tools/gbench/Inputs/test1_run2.json +++ b/tools/gbench/Inputs/test1_run2.json @@ -122,6 +122,16 @@ "real_time": 1, "cpu_time": 1, "time_unit": "s" + }, + { + "name": "BM_skipped", + "label": "a label", + "iterations": 1, + "real_time": 0, + "cpu_time": 0, + "time_unit": "s", + "skipped": true, + "skip_message": "Some message" } ] } From d207cabc6e9a589675bf31143e0986089242f59e Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Mon, 24 Aug 2026 08:17:34 +0200 Subject: [PATCH 5/6] Also skip/ignore benchmarks with errors --- tools/gbench/Inputs/test1_run1.json | 9 +++++++++ tools/gbench/Inputs/test1_run2.json | 9 +++++++++ tools/gbench/report.py | 6 +++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tools/gbench/Inputs/test1_run1.json b/tools/gbench/Inputs/test1_run1.json index fd52df8b06..b4871eb562 100644 --- a/tools/gbench/Inputs/test1_run1.json +++ b/tools/gbench/Inputs/test1_run1.json @@ -132,6 +132,15 @@ "time_unit": "s", "skipped": true, "skip_message": "Some message" + }, + { + "name": "BM_error", + "iterations": 1, + "real_time": 0, + "cpu_time": 0, + "time_unit": "s", + "error_occurred": true, + "error_message": "Something happened" } ] } diff --git a/tools/gbench/Inputs/test1_run2.json b/tools/gbench/Inputs/test1_run2.json index aad9d86f7d..75e4708f1f 100644 --- a/tools/gbench/Inputs/test1_run2.json +++ b/tools/gbench/Inputs/test1_run2.json @@ -132,6 +132,15 @@ "time_unit": "s", "skipped": true, "skip_message": "Some message" + }, + { + "name": "BM_error", + "iterations": 1, + "real_time": 0, + "cpu_time": 0, + "time_unit": "s", + "error_occurred": true, + "error_message": "Something happened" } ] } diff --git a/tools/gbench/report.py b/tools/gbench/report.py index 8098ffc697..e24fcb8909 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -161,12 +161,12 @@ def partition_benchmarks(json1, json2): lhs = [ x for x in json1["benchmarks"] - if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) + if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) and not x.get("error_occurred", False) ] rhs = [ x for x in json2["benchmarks"] - if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) + if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) and not x.get("error_occurred", False) ] if len(lhs) == 0 or len(rhs) == 0: @@ -195,7 +195,7 @@ def calculate_geomean(json): for benchmark in json["benchmarks"]: if "run_type" in benchmark and benchmark["run_type"] == "aggregate": continue - if benchmark.get("skipped", False): + if benchmark.get("skipped", False) or benchmark.get("error_occurred", False): continue times.append( [ From 29ab7f676ec8fdd7213df19fc54101e63d7c29ca Mon Sep 17 00:00:00 2001 From: Roman Lebedev Date: Fri, 28 Aug 2026 03:07:31 +0300 Subject: [PATCH 6/6] Pacify python pre-commit check --- tools/gbench/report.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/gbench/report.py b/tools/gbench/report.py index e24fcb8909..16b674ab09 100644 --- a/tools/gbench/report.py +++ b/tools/gbench/report.py @@ -161,15 +161,21 @@ def partition_benchmarks(json1, json2): lhs = [ x for x in json1["benchmarks"] - if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) and not x.get("error_occurred", False) + if x["name"] == name + and x["time_unit"] == time_unit + and not x.get("skipped", False) + and not x.get("error_occurred", False) ] rhs = [ x for x in json2["benchmarks"] - if x["name"] == name and x["time_unit"] == time_unit and not x.get("skipped", False) and not x.get("error_occurred", False) + if x["name"] == name + and x["time_unit"] == time_unit + and not x.get("skipped", False) + and not x.get("error_occurred", False) ] - if len(lhs) == 0 or len(rhs) == 0: + if len(lhs) == 0 or len(rhs) == 0: continue partitions.append([lhs, rhs]) @@ -195,7 +201,9 @@ def calculate_geomean(json): for benchmark in json["benchmarks"]: if "run_type" in benchmark and benchmark["run_type"] == "aggregate": continue - if benchmark.get("skipped", False) or benchmark.get("error_occurred", False): + if benchmark.get("skipped", False) or benchmark.get( + "error_occurred", False + ): continue times.append( [