diff --git a/docs/explanation/benchmark_results.txt b/docs/explanation/benchmark_results.txt new file mode 100644 index 0000000..91056a8 --- /dev/null +++ b/docs/explanation/benchmark_results.txt @@ -0,0 +1,11 @@ +Synthetic donor pool: 4000 rows, positive rate 0.378 (major donors), features=['total_gift_amount', 'years_active', 'event_attendance_count'] +Stratified 75/25 split, test n=1000, seeds=[42, 43, 44, 45, 46] +Each cell is mean (min-max) across the seeds. + +model precision recall f1 roc_auc +---------------------------------------------------------------------------------------------------------------------- +PropensityScorer (baseline) 0.000 (0.000-0.000) 0.000 (0.000-0.000) 0.000 (0.000-0.000) 0.500 (0.500-0.500) +DonorPropensityModel 0.669 (0.625-0.707) 0.607 (0.588-0.632) 0.636 (0.623-0.645) 0.810 (0.802-0.815) +MajorGiftClassifier 0.732 (0.693-0.752) 0.565 (0.544-0.593) 0.638 (0.614-0.663) 0.828 (0.819-0.832) +LapsePredictor 0.669 (0.625-0.707) 0.607 (0.588-0.632) 0.636 (0.623-0.645) 0.810 (0.802-0.815) +PlannedGivingIntentScorer 0.730 (0.717-0.740) 0.605 (0.576-0.636) 0.662 (0.643-0.683) 0.840 (0.833-0.844) diff --git a/docs/explanation/benchmarks.md b/docs/explanation/benchmarks.md index 125af29..d3eb55e 100644 --- a/docs/explanation/benchmarks.md +++ b/docs/explanation/benchmarks.md @@ -70,6 +70,10 @@ Five seeds rather than one on purpose. A single three-decimal score reads as a claim about the method when it is mostly a claim about the split; the spread below is the honest resolution of these numbers. +CI runs `scripts/benchmark_models.py` on every push and diffs the output against +the committed golden file `docs/explanation/benchmark_results.txt` (generated on +Linux so BLAS rounding matches the runner). + ## Results Synthetic pool: 4,000 rows, positive rate 0.378; test split 1,000 rows. diff --git a/scripts/benchmark_models.py b/scripts/benchmark_models.py index 5a33e7b..d0ecec1 100644 --- a/scripts/benchmark_models.py +++ b/scripts/benchmark_models.py @@ -36,7 +36,6 @@ PropensityScorer, ) -RANDOM_STATE = 42 # Five seeds, not one: a three-decimal score from a single split reads as a # claim about the method when it is mostly a claim about the split. SEEDS = (42, 43, 44, 45, 46) diff --git a/tests/test_benchmark_models.py b/tests/test_benchmark_models.py new file mode 100644 index 0000000..6f1c55f --- /dev/null +++ b/tests/test_benchmark_models.py @@ -0,0 +1,54 @@ +"""Lock the benchmark table to scripts/benchmark_models.py output.""" + +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +GOLDEN = REPO_ROOT / "docs" / "explanation" / "benchmark_results.txt" +SCRIPT = REPO_ROOT / "scripts" / "benchmark_models.py" + +# The metrics come from fitted sklearn models, so they move with the +# BLAS/sklearn build. CI showed means drifting up to ~0.006 and seed range +# endpoints up to ~0.015 across Python versions on the same commit; anything +# past these bands means the benchmark itself broke. +MEAN_TOLERANCE = 0.02 +RANGE_TOLERANCE = 0.05 +DECIMAL = re.compile(r"\d+\.\d+") +CELL = re.compile(r"(\d+\.\d+) \((\d+\.\d+)-(\d+\.\d+)\)") + + +def test_benchmark_table_is_reproducible(): + result = subprocess.run( + [sys.executable, str(SCRIPT)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + golden_lines = GOLDEN.read_text().splitlines() + actual_lines = result.stdout.splitlines() + assert len(actual_lines) == len(golden_lines), ( + "benchmark output changed shape; regenerate " + "docs/explanation/benchmark_results.txt and commit verbatim" + ) + for actual, golden in zip(actual_lines, golden_lines): + assert DECIMAL.sub("#", actual) == DECIMAL.sub("#", golden), ( + f"non-metric part of the benchmark table drifted:\n" + f"expected: {golden}\nactual: {actual}" + ) + actual_cells = CELL.findall(actual) + golden_cells = CELL.findall(golden) + assert len(actual_cells) == len(golden_cells), ( + f"metric cell count drifted:\nexpected: {golden}\nactual: {actual}" + ) + for actual_cell, golden_cell in zip(actual_cells, golden_cells): + for actual_value, golden_value, tolerance in zip( + actual_cell, golden_cell, (MEAN_TOLERANCE,) + (RANGE_TOLERANCE,) * 2 + ): + assert abs(float(actual_value) - float(golden_value)) <= tolerance, ( + f"benchmark metric drifted beyond {tolerance}: " + f"expected {golden_value}, got {actual_value}\n" + f"line: {actual}" + )