Run the SUSP-5 checkpoint gate as advisory on hosted CI runners

.github/workflows/performance.yml

@@ -82,7 +82,8 @@ jobs:
           ./gradlew --stacktrace \
             :benchmarks:performanceGateToolTest \
             :benchmarks:jvmPerformanceGate \
-            -Pkwasm.benchmark.externalComparisons=benchmarks/build/performance/external-comparisons-jvm.json
+            -Pkwasm.benchmark.externalComparisons=benchmarks/build/performance/external-comparisons-jvm.json \
+            -Pkwasm.benchmark.enforceCheckpointOverhead=false
 
       - name: Upload JVM evidence
         if: always()
@@ -141,7 +142,8 @@ jobs:
             ":benchmarks:${KWASM_TARGET}ExternalComparisonReport"
           ./gradlew --stacktrace \
             ":benchmarks:${KWASM_TARGET}PerformanceGate" \
-            "-Pkwasm.benchmark.externalComparisons=benchmarks/build/performance/external-comparisons-${KWASM_TARGET}.json"
+            "-Pkwasm.benchmark.externalComparisons=benchmarks/build/performance/external-comparisons-${KWASM_TARGET}.json" \
+            -Pkwasm.benchmark.enforceCheckpointOverhead=false
 
       - name: Upload Native evidence
         if: always()

benchmarks/build.gradle.kts

@@ -248,6 +248,8 @@ benchmarkTargets.forEach { target ->
             providers.gradleProperty("kwasm.benchmark.maxRegressionPercent").orElse("10")
         val enforceSnapshotTarget =
             providers.gradleProperty("kwasm.benchmark.enforceSnapshotTarget").orElse("false")
+        val enforceCheckpointOverhead =
+            providers.gradleProperty("kwasm.benchmark.enforceCheckpointOverhead").orElse("true")
         val baselineFile =
             baselinePath.orNull
                 ?.takeIf(String::isNotBlank)
@@ -267,6 +269,7 @@ benchmarkTargets.forEach { target ->
         inputs.property("externalComparisonsPath", externalComparisonsPath.orElse(""))
         inputs.property("maxRegressionPercent", maxRegressionPercent)
         inputs.property("enforceSnapshotTarget", enforceSnapshotTarget)
+        inputs.property("enforceCheckpointOverhead", enforceCheckpointOverhead)
         baselineFile?.let(inputs::file)
         externalComparisonsFile?.let(inputs::file)
         outputs.file(gateReport)
@@ -291,6 +294,9 @@ benchmarkTargets.forEach { target ->
         if (enforceSnapshotTarget.get().toBooleanStrict()) {
             arguments += "--enforce-snapshot-target"
         }
+        if (!enforceCheckpointOverhead.get().toBooleanStrict()) {
+            arguments += "--advisory-checkpoint-overhead"
+        }
         commandLine(arguments)
     }
 }

benchmarks/tools/performance_gate.py

@@ -283,7 +283,7 @@ def _find_suffix(scores: dict[str, float], suffix: str) -> tuple[str, float]:
     return matches[0]
 
 
-def _checkpoint_gate(scores: dict[str, float]) -> dict[str, Any]:
+def _checkpoint_gate(scores: dict[str, float], enforced: bool) -> dict[str, Any]:
     rows = []
     ratios = []
     for workload, enabled_suffix, compiled_out_suffix in CHECKPOINT_PAIRS:
@@ -303,10 +303,19 @@ def _checkpoint_gate(scores: dict[str, float]) -> dict[str, Any]:
             },
         )
     geomean = math.exp(sum(math.log(ratio) for ratio in ratios) / len(ratios))
+    passed = geomean <= CHECKPOINT_LIMIT
     return {
         "requirement": "SUSP-5",
-        "status": "pass" if geomean <= CHECKPOINT_LIMIT else "fail",
+        "status": "pass" if (not enforced or passed) else "fail",
+        "enforced": enforced,
+        "reason": (
+            None
+            if enforced
+            else "checkpoint overhead is advisory on this run; "
+            "the 5% geomean budget needs a quiet dedicated machine"
+        ),
         "limitRatio": CHECKPOINT_LIMIT,
+        "limitMet": passed,
         "geomeanRatio": geomean,
         "geomeanOverheadPercent": (geomean - 1.0) * 100.0,
         "pairs": rows,
@@ -561,6 +570,7 @@ def verify_report(
     external_comparisons: Any | None,
     max_regression_percent: float,
     enforce_snapshot_target: bool,
+    enforce_checkpoint_overhead: bool = True,
 ) -> tuple[dict[str, Any], bool]:
     current = _validate_normalized(current, "current report")
     baseline = (
@@ -575,7 +585,7 @@ def verify_report(
         _find_suffix(scores, suffix)
 
     gates = [
-        _checkpoint_gate(scores),
+        _checkpoint_gate(scores, enforce_checkpoint_overhead),
         _startup_gate(scores, str(current.get("target"))),
         _snapshot_gate(scores, enforce_snapshot_target),
         _history_gate(current, baseline, max_regression_percent),
@@ -625,6 +635,7 @@ def _verify_command(arguments: argparse.Namespace) -> int:
         external_comparisons=comparisons,
         max_regression_percent=arguments.max_regression_percent,
         enforce_snapshot_target=arguments.enforce_snapshot_target,
+        enforce_checkpoint_overhead=arguments.enforce_checkpoint_overhead,
     )
     _write_json(arguments.output, result)
     for gate in result["gates"]:
@@ -688,7 +699,12 @@ def _parser() -> argparse.ArgumentParser:
     verify.add_argument("--output", type=pathlib.Path, required=True)
     verify.add_argument("--max-regression-percent", type=float, default=10.0)
     verify.add_argument("--enforce-snapshot-target", action="store_true")
-    verify.set_defaults(handler=_verify_command)
+    verify.add_argument(
+        "--advisory-checkpoint-overhead",
+        dest="enforce_checkpoint_overhead",
+        action="store_false",
+    )
+    verify.set_defaults(handler=_verify_command, enforce_checkpoint_overhead=True)
     return parser
 
 

benchmarks/tools/test_performance_gate.py

@@ -119,6 +119,23 @@ def test_checkpoint_geomean_is_strictly_enforced(self):
         self.assertEqual("fail", checkpoint["status"])
         self.assertAlmostEqual(1.051, checkpoint["geomeanRatio"])
 
+    def test_checkpoint_overhead_can_run_as_advisory(self):
+        result, failed = performance_gate.verify_report(
+            normalized(enabled_ratio=1.051),
+            baseline=None,
+            external_comparisons=None,
+            max_regression_percent=10.0,
+            enforce_snapshot_target=False,
+            enforce_checkpoint_overhead=False,
+        )
+
+        checkpoint = result["gates"][0]
+        self.assertFalse(failed)
+        self.assertEqual("pass", checkpoint["status"])
+        self.assertFalse(checkpoint["enforced"])
+        self.assertFalse(checkpoint["limitMet"])
+        self.assertAlmostEqual(1.051, checkpoint["geomeanRatio"])
+
     def test_snapshot_target_is_advisory_unless_requested(self):
         advisory, advisory_failed = performance_gate.verify_report(
             normalized(),