diff --git a/CHANGELOG.md b/CHANGELOG.md index 4801be9..2717a22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ Release tags match the version committed in `pyproject.toml`. ## [Unreleased] +### Fixed + +- `compare_prequential` no longer times batches with no observed labels as + zero-cost updates, which understated `p95_update_ms` under a sparse + `observe_mask`. + ## [0.2.0] - 2026-09-20 This release breaks the experimental 0.1 API. Predict with `calibrate` before diff --git a/src/streamcal/evaluation.py b/src/streamcal/evaluation.py index fe5d6d9..7ed524e 100644 --- a/src/streamcal/evaluation.py +++ b/src/streamcal/evaluation.py @@ -352,14 +352,16 @@ def compare_prequential( cumulative_observations += calibrated.size cumulative_scores.append(cumulative_squared_error / cumulative_observations) - before = perf_counter_ns() + # Only calls that update are timed: counting masked-out batches as + # zero-cost updates would drag the p95 toward zero. batch_mask = mask[start:stop] if np.any(batch_mask): + before = perf_counter_ns() calibrator.update( batch_probabilities[batch_mask], batch_outcomes[batch_mask] ) - update_calls.append(perf_counter_ns() - before) - update_ns += update_calls[-1] + update_calls.append(perf_counter_ns() - before) + update_ns += update_calls[-1] if times is not None: calibrated_all, predict_calls, update_calls = _run_delayed( diff --git a/tests/test_tradeoffs.py b/tests/test_tradeoffs.py index a30c106..47dfa8c 100644 --- a/tests/test_tradeoffs.py +++ b/tests/test_tradeoffs.py @@ -1,5 +1,6 @@ """Expected outcomes for scheduling, constrained selection and uncertainty.""" +import time from dataclasses import replace from functools import partial @@ -132,6 +133,30 @@ def test_methods_receive_identical_available_labels_and_sparse_mask(): assert [expected, expected] == TRACE_LOGS +class SlowUpdater(RawReference): + def update(self, probabilities, outcomes): + time.sleep(0.002) + return self + + +@pytest.mark.parametrize( + "timing", + [{"batch_size": 1}, {"batch_size": 1, "prediction_times": np.arange(40)}], +) +def test_update_latency_ignores_masked_batches(timing): + # One observed label in forty: skipped batches must not count as updates. + mask = np.zeros(40, dtype=bool) + mask[20] = True + report = compare_prequential( + np.full(40, 0.5), + np.tile([0, 1], 20), + {"slow": SlowUpdater}, + observe_mask=mask, + **timing, + ) + assert report.results[0].p95_update_ms >= 2.0 + + def test_paired_bootstrap_preserves_constant_expected_difference(): report = compare_prequential( np.full(15, 0.5),