From 481c25f2ed401c24bfb91025f6d7df48464c4131 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 9 Nov 2026 15:00:36 -0500 Subject: [PATCH 1/2] feat: publish calibrated scoring probability as stat (ranking unchanged) --- .../scripts/backtrack/run_daily_picks.py | 69 ++++++++++++++----- smartscore/scripts/backtrack/train_export.py | 26 +++++++ tests/unit/test_picks_db_nan_behaviors.py | 8 +-- tests/unit/test_predict_picks.py | 2 +- 4 files changed, 84 insertions(+), 21 deletions(-) diff --git a/smartscore/scripts/backtrack/run_daily_picks.py b/smartscore/scripts/backtrack/run_daily_picks.py index 95075cf..f37cf0b 100644 --- a/smartscore/scripts/backtrack/run_daily_picks.py +++ b/smartscore/scripts/backtrack/run_daily_picks.py @@ -390,6 +390,16 @@ def _pick(candidates, required): names = ("hgb", "lgbm", "logreg") members = {n: (models.get(n) if contract else _pick([n], required=False)) for n in names} absent = [n for n, m in members.items() if m is None] + + # Calibrator (optional): a monotonic map from the scoring probability to a + # calibrated probability. Absent on older artifacts -> inference falls back + # to the raw mean member probability. + calibrator = None + if isinstance(bundle, dict): + calibrator = bundle.get("calibrator") + if calibrator is None and isinstance(models, dict): + calibrator = models.get("calibrator") + bundle["calibrator"] = calibrator if absent: raise RuntimeError( f"artifact {key} ({source} design) is malformed: missing ensemble " @@ -1130,6 +1140,22 @@ def ensemble_scores(bundle, frame): sim[["hgb", "lgbm", "logreg"]] = sim[["hgb", "lgbm", "logreg"]].fillna(0.0) ranks = sim.groupby(["date", "tims"])[["hgb", "lgbm", "logreg"]].rank(pct=True) sim["ensemble"] = ranks.mean(axis=1) + # Actual scoring probability: mean of the members' native probabilities. + # The ensemble percentile above remains the ranking key, so picks are + # unchanged; only the published value moves from a rank to a probability. + sim["prob"] = sim[["hgb", "lgbm", "logreg"]].mean(axis=1) + calibrator = bundle.get("calibrator") if isinstance(bundle, dict) else None + if calibrator is not None: + try: + if hasattr(calibrator, "predict_proba"): + sim["prob"] = np.asarray(calibrator.predict_proba(sim[["prob"]])[:, 1], dtype=float) + elif callable(calibrator): + sim["prob"] = np.asarray(calibrator(sim["prob"]), dtype=float) + elif hasattr(calibrator, "transform"): + sim["prob"] = np.asarray(calibrator.transform(sim["prob"]), dtype=float) + except Exception: # noqa: BLE001 + pass + sim["prob"] = sim["prob"].clip(lower=0.0, upper=1.0) return sim @@ -1147,19 +1173,23 @@ def validate_picks(picks, on_date): ) -def stat_value(ensemble): - """The Picks `stat` DISPLAY convention, shared by the SMARTSCORE_ML_PICKS - top-3 output (pick_rows_to_picks_db's stat cell) and the stat_update - computation stage: the ensemble's within-(date,tims) rank percentile - scaled to percent, 1 decimal - None when the prediction is not finite - (a NaN must never PATCH a number into the column). The DB WRITE boundary - only (stat_update_picks_db) carries the 0-1 decimal form: the PATCHed - value is stat_value(ensemble) / 100, full precision - the log/picks - display stays on the 0-100 scale.""" - stat = float(ensemble) - if not np.isfinite(stat): +def stat_value(prob): + """The Picks `stat` value: an actual probability of the player scoring in + that game (0-1), shared by the SMARTSCORE_ML_PICKS top-3 output + (pick_rows_to_picks_db's stat cell) and the stat_update computation + stage. Returns None when the prediction is not finite (a NaN must never + PATCH a number into the column). + + The ranking is deliberately NOT derived from this value: picks are selected + on the ensemble rank percentile (see ``ensemble_scores``), so publishing a + probability here cannot change which players are picked. The DB WRITE + boundary (stat_update_picks_db) stores this value verbatim (full + precision) - no /100 scaling, because the value is already a 0-1 + probability.""" + value = float(prob) + if not np.isfinite(value): return None - return round(stat * 100, 1) + return float(np.clip(value, 0.0, 1.0)) def pick_rows_to_picks_db(picks): @@ -1176,7 +1206,8 @@ def pick_rows_to_picks_db(picks): write_keys = [key for key in PICK_DB_COLUMNS if key not in REMOVE_KEYS] rows = [] for _, row in picks.iterrows(): - stat = stat_value(row.get("ensemble", np.nan)) + prob = row.get("prob", row.get("ensemble", np.nan)) + stat = stat_value(prob) values = [] for key in write_keys: if key == "stat": @@ -1287,9 +1318,15 @@ def stat_update_picks_db(sim, on_date, env=None): if slate is None: return {"slate_rows": 0, "filled": 0, "no_frame": 0, "not_on_slate": 0, "skipped": True} + if "prob" in getattr(sim, "columns", []): + prob_series = sim["prob"] + elif all(col in getattr(sim, "columns", []) for col in ("hgb", "lgbm", "logreg")): + prob_series = sim[["hgb", "lgbm", "logreg"]].mean(axis=1) + else: + prob_series = sim["ensemble"] predictions = { - int(player_id): (stat / 100 if (stat := stat_value(ensemble)) is not None else None) - for player_id, ensemble in zip(sim["player_id"], sim["ensemble"]) + int(player_id): (stat if (stat := stat_value(prob)) is not None else None) + for player_id, prob in zip(sim["player_id"], prob_series) if pd.notna(player_id) } slate_ids = set() @@ -1669,7 +1706,7 @@ def run(event=None, context=None): ], "stat_column": stat_payload, "mean_max_prob": mean_max_prob, - "ensemble_scale": "within-(date,tims) rank percentile", + "stat_scale": "scoring probability (mean member predict_proba)", "write_env": ENV, "artifact": { "key": key, diff --git a/smartscore/scripts/backtrack/train_export.py b/smartscore/scripts/backtrack/train_export.py index 522a3b3..98c30be 100644 --- a/smartscore/scripts/backtrack/train_export.py +++ b/smartscore/scripts/backtrack/train_export.py @@ -92,6 +92,7 @@ import joblib # noqa: E402 import model # noqa: E402 +from sklearn.isotonic import IsotonicRegression # noqa: E402 # noqa: E402 import pandas as pd # noqa: E402 (model.py uses the global pd namespace) @@ -260,6 +261,29 @@ def fit(self, *fit_args, **fit_kwargs): # sklearn Pipeline is plain-picklable, so option (a) is used. logreg = model.make_pipeline(logreg_imputer, logreg_scaler, captured["LogisticRegression"]) + # Calibrated probability: monotonic isotonic fit of the mean member + # probability onto the binary `scored` label. Monotonicity guarantees the + # published probability cannot reorder picks (ranking stays on the ensemble + # percentile); the fit only rescales the probability to the observed rate. + calibrator = None + try: + from sklearn.isotonic import IsotonicRegression # noqa: PLC0415 + + probe = frame.iloc[:1] + x_train, _ = model.categorise(frame, probe) + raw_prob = pd.Series(0.0, index=frame.index) + for name, member, cols in ( + ("hgb", hgb, list(model.DATASET_FEATURES)), + ("lgbm", lgbm, list(model.DATASET_FEATURES)), + ("logreg", logreg, list(model.NUMERIC_FEATURES)), + ): + raw_prob = raw_prob + np.asarray(member.predict_proba(x_train[cols])[:, 1], dtype=float) / 3.0 + calibrator = IsotonicRegression(y_min=0.0, y_max=1.0, out_of_bounds="clip") + calibrator.fit(raw_prob.to_numpy(dtype=float), frame["scored"].astype(int).to_numpy()) + except Exception as exc: # noqa: BLE001 + print(f"WARNING: calibrator fit skipped: {exc}", flush=True) + calibrator = None + bundle = { # --- ARTIFACT CONTRACT keys (sibling-agreed; the daily pass reads both # --- designs via normalise_bundle) ------------------------------ @@ -297,7 +321,9 @@ def fit(self, *fit_args, **fit_kwargs): "lgbm": lgbm, "logreg": logreg, "ranker": ranker, + "calibrator": calibrator, }, + "calibrator": calibrator, # --- legacy envelope keys (same objects; the legacy reader path # --- and any older reader keep working) -------------------------- "hgb": hgb, diff --git a/tests/unit/test_picks_db_nan_behaviors.py b/tests/unit/test_picks_db_nan_behaviors.py index 4b62295..c663ef1 100644 --- a/tests/unit/test_picks_db_nan_behaviors.py +++ b/tests/unit/test_picks_db_nan_behaviors.py @@ -268,8 +268,8 @@ def test_stat_value_convention_matches_the_legacy_percent(): the DB write divides it by 100 at the stat_update PATCH boundary, so the column holds a 0-1 fraction. None on a non-finite prediction (a NaN must never patch a number into the column).""" - assert mod.stat_value(0.12345) == 12.3 - assert mod.stat_value(0.999) == 99.9 + assert mod.stat_value(0.12345) == 0.12345 + assert mod.stat_value(0.999) == 0.999 assert mod.stat_value(np.nan) is None assert mod.stat_value(float("inf")) is None @@ -297,8 +297,8 @@ def test_stat_update_patches_only_stat_and_never_inserts(monkeypatch): assert len(table) == 2 # row count unchanged (99 was never inserted) assert sorted(row["id"] for row in table) == [1, 2] # ids untouched by_id = {row["id"]: row for row in table} - assert by_id[1] == {**prior[0], "stat": 99.9 / 100} # 0.999 - the 0-1 decimal write form - assert by_id[2] == {**prior[1], "stat": 11.1 / 100} # 0.111 + assert by_id[1] == {**prior[0], "stat": 0.999} # probability - the 0-1 write form + assert by_id[2] == {**prior[1], "stat": 0.111} # 0.111 assert result["filled"] == 2 and result["slate_rows"] == 2 and result["not_on_slate"] == 1 diff --git a/tests/unit/test_predict_picks.py b/tests/unit/test_predict_picks.py index 71d3b69..5573038 100644 --- a/tests/unit/test_predict_picks.py +++ b/tests/unit/test_predict_picks.py @@ -1098,7 +1098,7 @@ def test_pick_payload_whitelist(): "player_id": 1, "name": "a", "team_name": "T", - "stat": 12.3, # ensemble ranked-pct -> percent, 1 decimal + "stat": 0.12345, # scoring probability (mean member proba) "tims": 1, "gpg": 0.5, "hgpg": 0.1, From 1aeef3424a70c617a8ef6f49f3e399fd73c4d84d Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 9 Nov 2026 15:05:59 -0500 Subject: [PATCH 2/2] fix: lint (numpy import, calibrator error logging) --- smartscore/scripts/backtrack/run_daily_picks.py | 4 ++-- smartscore/scripts/backtrack/train_export.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/smartscore/scripts/backtrack/run_daily_picks.py b/smartscore/scripts/backtrack/run_daily_picks.py index f37cf0b..9caddf0 100644 --- a/smartscore/scripts/backtrack/run_daily_picks.py +++ b/smartscore/scripts/backtrack/run_daily_picks.py @@ -1153,8 +1153,8 @@ def ensemble_scores(bundle, frame): sim["prob"] = np.asarray(calibrator(sim["prob"]), dtype=float) elif hasattr(calibrator, "transform"): sim["prob"] = np.asarray(calibrator.transform(sim["prob"]), dtype=float) - except Exception: # noqa: BLE001 - pass + except Exception as exc: # noqa: BLE001 + print(f"WARNING: calibrator apply failed: {exc}", flush=True) sim["prob"] = sim["prob"].clip(lower=0.0, upper=1.0) return sim diff --git a/smartscore/scripts/backtrack/train_export.py b/smartscore/scripts/backtrack/train_export.py index 98c30be..6c26e3a 100644 --- a/smartscore/scripts/backtrack/train_export.py +++ b/smartscore/scripts/backtrack/train_export.py @@ -86,13 +86,14 @@ import time from pathlib import Path +import numpy as np # noqa: E402 + ROOT = Path(__file__).resolve().parents[3] sys.path.insert(0, str(ROOT / "smartscore")) sys.path.insert(1, Path(__file__).resolve().parent) import joblib # noqa: E402 import model # noqa: E402 -from sklearn.isotonic import IsotonicRegression # noqa: E402 # noqa: E402 import pandas as pd # noqa: E402 (model.py uses the global pd namespace)