From aaa00f4d51be1b406dbead15bec69a0da080fe94 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Tue, 29 Sep 2026 01:06:33 +0000
Subject: [PATCH] Time only real updates in compare_prequential's p95 latency

Masked-out batches were timed as zero-cost updates, so with a sparse
observe_mask p95_update_ms collapsed toward zero (0.006 ms vs 0.76 ms
with 2% of labels observed) and select(max_update_ms=...) could accept
configurations that miss the budget. The delayed-label path already
timed only real updates; the batch path now matches it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011pXaJ5DLex7BrdUyfTqjL4
---
 CHANGELOG.md                |  6 ++++++
 src/streamcal/evaluation.py |  8 +++++---
 tests/test_tradeoffs.py     | 25 +++++++++++++++++++++++++
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4801be9..2717a22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,12 @@ Release tags match the version committed in `pyproject.toml`.
 
 ## [Unreleased]
 
+### Fixed
+
+- `compare_prequential` no longer times batches with no observed labels as
+  zero-cost updates, which understated `p95_update_ms` under a sparse
+  `observe_mask`.
+
 ## [0.2.0] - 2026-09-20
 
 This release breaks the experimental 0.1 API. Predict with `calibrate` before
diff --git a/src/streamcal/evaluation.py b/src/streamcal/evaluation.py
index fe5d6d9..7ed524e 100644
--- a/src/streamcal/evaluation.py
+++ b/src/streamcal/evaluation.py
@@ -352,14 +352,16 @@ def compare_prequential(
             cumulative_observations += calibrated.size
             cumulative_scores.append(cumulative_squared_error / cumulative_observations)
 
-            before = perf_counter_ns()
+            # Only calls that update are timed: counting masked-out batches as
+            # zero-cost updates would drag the p95 toward zero.
             batch_mask = mask[start:stop]
             if np.any(batch_mask):
+                before = perf_counter_ns()
                 calibrator.update(
                     batch_probabilities[batch_mask], batch_outcomes[batch_mask]
                 )
-            update_calls.append(perf_counter_ns() - before)
-            update_ns += update_calls[-1]
+                update_calls.append(perf_counter_ns() - before)
+                update_ns += update_calls[-1]
 
         if times is not None:
             calibrated_all, predict_calls, update_calls = _run_delayed(
diff --git a/tests/test_tradeoffs.py b/tests/test_tradeoffs.py
index a30c106..47dfa8c 100644
--- a/tests/test_tradeoffs.py
+++ b/tests/test_tradeoffs.py
@@ -1,5 +1,6 @@
 """Expected outcomes for scheduling, constrained selection and uncertainty."""
 
+import time
 from dataclasses import replace
 from functools import partial
 
@@ -132,6 +133,30 @@ def test_methods_receive_identical_available_labels_and_sparse_mask():
     assert [expected, expected] == TRACE_LOGS
 
 
+class SlowUpdater(RawReference):
+    def update(self, probabilities, outcomes):
+        time.sleep(0.002)
+        return self
+
+
+@pytest.mark.parametrize(
+    "timing",
+    [{"batch_size": 1}, {"batch_size": 1, "prediction_times": np.arange(40)}],
+)
+def test_update_latency_ignores_masked_batches(timing):
+    # One observed label in forty: skipped batches must not count as updates.
+    mask = np.zeros(40, dtype=bool)
+    mask[20] = True
+    report = compare_prequential(
+        np.full(40, 0.5),
+        np.tile([0, 1], 20),
+        {"slow": SlowUpdater},
+        observe_mask=mask,
+        **timing,
+    )
+    assert report.results[0].p95_update_ms >= 2.0
+
+
 def test_paired_bootstrap_preserves_constant_expected_difference():
     report = compare_prequential(
         np.full(15, 0.5),