Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 53 additions & 16 deletions smartscore/scripts/backtrack/run_daily_picks.py
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,16 @@ def _pick(candidates, required):
names = ("hgb", "lgbm", "logreg")
members = {n: (models.get(n) if contract else _pick([n], required=False)) for n in names}
absent = [n for n, m in members.items() if m is None]

# Calibrator (optional): a monotonic map from the scoring probability to a
# calibrated probability. Absent on older artifacts -> inference falls back
# to the raw mean member probability.
calibrator = None
if isinstance(bundle, dict):
calibrator = bundle.get("calibrator")
if calibrator is None and isinstance(models, dict):
calibrator = models.get("calibrator")
bundle["calibrator"] = calibrator
if absent:
raise RuntimeError(
f"artifact {key} ({source} design) is malformed: missing ensemble "
Expand Down Expand Up @@ -1130,6 +1140,22 @@ def ensemble_scores(bundle, frame):
sim[["hgb", "lgbm", "logreg"]] = sim[["hgb", "lgbm", "logreg"]].fillna(0.0)
ranks = sim.groupby(["date", "tims"])[["hgb", "lgbm", "logreg"]].rank(pct=True)
sim["ensemble"] = ranks.mean(axis=1)
# Actual scoring probability: mean of the members' native probabilities.
# The ensemble percentile above remains the ranking key, so picks are
# unchanged; only the published value moves from a rank to a probability.
sim["prob"] = sim[["hgb", "lgbm", "logreg"]].mean(axis=1)
calibrator = bundle.get("calibrator") if isinstance(bundle, dict) else None
if calibrator is not None:
try:
if hasattr(calibrator, "predict_proba"):
sim["prob"] = np.asarray(calibrator.predict_proba(sim[["prob"]])[:, 1], dtype=float)
elif callable(calibrator):
sim["prob"] = np.asarray(calibrator(sim["prob"]), dtype=float)
elif hasattr(calibrator, "transform"):
sim["prob"] = np.asarray(calibrator.transform(sim["prob"]), dtype=float)
except Exception as exc: # noqa: BLE001
print(f"WARNING: calibrator apply failed: {exc}", flush=True)
sim["prob"] = sim["prob"].clip(lower=0.0, upper=1.0)
return sim


Expand All @@ -1147,19 +1173,23 @@ def validate_picks(picks, on_date):
)


def stat_value(ensemble):
"""The Picks `stat` DISPLAY convention, shared by the SMARTSCORE_ML_PICKS
top-3 output (pick_rows_to_picks_db's stat cell) and the stat_update
computation stage: the ensemble's within-(date,tims) rank percentile
scaled to percent, 1 decimal - None when the prediction is not finite
(a NaN must never PATCH a number into the column). The DB WRITE boundary
only (stat_update_picks_db) carries the 0-1 decimal form: the PATCHed
value is stat_value(ensemble) / 100, full precision - the log/picks
display stays on the 0-100 scale."""
stat = float(ensemble)
if not np.isfinite(stat):
def stat_value(prob):
"""The Picks `stat` value: an actual probability of the player scoring in
that game (0-1), shared by the SMARTSCORE_ML_PICKS top-3 output
(pick_rows_to_picks_db's stat cell) and the stat_update computation
stage. Returns None when the prediction is not finite (a NaN must never
PATCH a number into the column).

The ranking is deliberately NOT derived from this value: picks are selected
on the ensemble rank percentile (see ``ensemble_scores``), so publishing a
probability here cannot change which players are picked. The DB WRITE
boundary (stat_update_picks_db) stores this value verbatim (full
precision) - no /100 scaling, because the value is already a 0-1
probability."""
value = float(prob)
if not np.isfinite(value):
return None
return round(stat * 100, 1)
return float(np.clip(value, 0.0, 1.0))


def pick_rows_to_picks_db(picks):
Expand All @@ -1176,7 +1206,8 @@ def pick_rows_to_picks_db(picks):
write_keys = [key for key in PICK_DB_COLUMNS if key not in REMOVE_KEYS]
rows = []
for _, row in picks.iterrows():
stat = stat_value(row.get("ensemble", np.nan))
prob = row.get("prob", row.get("ensemble", np.nan))
stat = stat_value(prob)
values = []
for key in write_keys:
if key == "stat":
Expand Down Expand Up @@ -1287,9 +1318,15 @@ def stat_update_picks_db(sim, on_date, env=None):
if slate is None:
return {"slate_rows": 0, "filled": 0, "no_frame": 0, "not_on_slate": 0, "skipped": True}

if "prob" in getattr(sim, "columns", []):
prob_series = sim["prob"]
elif all(col in getattr(sim, "columns", []) for col in ("hgb", "lgbm", "logreg")):
prob_series = sim[["hgb", "lgbm", "logreg"]].mean(axis=1)
else:
prob_series = sim["ensemble"]
predictions = {
int(player_id): (stat / 100 if (stat := stat_value(ensemble)) is not None else None)
for player_id, ensemble in zip(sim["player_id"], sim["ensemble"])
int(player_id): (stat if (stat := stat_value(prob)) is not None else None)
for player_id, prob in zip(sim["player_id"], prob_series)
if pd.notna(player_id)
}
slate_ids = set()
Expand Down Expand Up @@ -1669,7 +1706,7 @@ def run(event=None, context=None):
],
"stat_column": stat_payload,
"mean_max_prob": mean_max_prob,
"ensemble_scale": "within-(date,tims) rank percentile",
"stat_scale": "scoring probability (mean member predict_proba)",
"write_env": ENV,
"artifact": {
"key": key,
Expand Down
27 changes: 27 additions & 0 deletions smartscore/scripts/backtrack/train_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@
import time
from pathlib import Path

import numpy as np # noqa: E402

ROOT = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT / "smartscore"))
sys.path.insert(1, Path(__file__).resolve().parent)
Expand Down Expand Up @@ -260,6 +262,29 @@ def fit(self, *fit_args, **fit_kwargs):
# sklearn Pipeline is plain-picklable, so option (a) is used.
logreg = model.make_pipeline(logreg_imputer, logreg_scaler, captured["LogisticRegression"])

# Calibrated probability: monotonic isotonic fit of the mean member
# probability onto the binary `scored` label. Monotonicity guarantees the
# published probability cannot reorder picks (ranking stays on the ensemble
# percentile); the fit only rescales the probability to the observed rate.
calibrator = None
try:
from sklearn.isotonic import IsotonicRegression # noqa: PLC0415

probe = frame.iloc[:1]
x_train, _ = model.categorise(frame, probe)
raw_prob = pd.Series(0.0, index=frame.index)
for name, member, cols in (
("hgb", hgb, list(model.DATASET_FEATURES)),
("lgbm", lgbm, list(model.DATASET_FEATURES)),
("logreg", logreg, list(model.NUMERIC_FEATURES)),
):
raw_prob = raw_prob + np.asarray(member.predict_proba(x_train[cols])[:, 1], dtype=float) / 3.0
calibrator = IsotonicRegression(y_min=0.0, y_max=1.0, out_of_bounds="clip")
calibrator.fit(raw_prob.to_numpy(dtype=float), frame["scored"].astype(int).to_numpy())
except Exception as exc: # noqa: BLE001
print(f"WARNING: calibrator fit skipped: {exc}", flush=True)
calibrator = None

bundle = {
# --- ARTIFACT CONTRACT keys (sibling-agreed; the daily pass reads both
# --- designs via normalise_bundle) ------------------------------
Expand Down Expand Up @@ -297,7 +322,9 @@ def fit(self, *fit_args, **fit_kwargs):
"lgbm": lgbm,
"logreg": logreg,
"ranker": ranker,
"calibrator": calibrator,
},
"calibrator": calibrator,
# --- legacy envelope keys (same objects; the legacy reader path
# --- and any older reader keep working) --------------------------
"hgb": hgb,
Expand Down
8 changes: 4 additions & 4 deletions tests/unit/test_picks_db_nan_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,8 +268,8 @@ def test_stat_value_convention_matches_the_legacy_percent():
the DB write divides it by 100 at the stat_update PATCH boundary, so
the column holds a 0-1 fraction. None on a non-finite prediction (a
NaN must never patch a number into the column)."""
assert mod.stat_value(0.12345) == 12.3
assert mod.stat_value(0.999) == 99.9
assert mod.stat_value(0.12345) == 0.12345
assert mod.stat_value(0.999) == 0.999
assert mod.stat_value(np.nan) is None
assert mod.stat_value(float("inf")) is None

Expand Down Expand Up @@ -297,8 +297,8 @@ def test_stat_update_patches_only_stat_and_never_inserts(monkeypatch):
assert len(table) == 2 # row count unchanged (99 was never inserted)
assert sorted(row["id"] for row in table) == [1, 2] # ids untouched
by_id = {row["id"]: row for row in table}
assert by_id[1] == {**prior[0], "stat": 99.9 / 100} # 0.999 - the 0-1 decimal write form
assert by_id[2] == {**prior[1], "stat": 11.1 / 100} # 0.111
assert by_id[1] == {**prior[0], "stat": 0.999} # probability - the 0-1 write form
assert by_id[2] == {**prior[1], "stat": 0.111} # 0.111
assert result["filled"] == 2 and result["slate_rows"] == 2 and result["not_on_slate"] == 1


Expand Down
2 changes: 1 addition & 1 deletion tests/unit/test_predict_picks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1098,7 +1098,7 @@ def test_pick_payload_whitelist():
"player_id": 1,
"name": "a",
"team_name": "T",
"stat": 12.3, # ensemble ranked-pct -> percent, 1 decimal
"stat": 0.12345, # scoring probability (mean member proba)
"tims": 1,
"gpg": 0.5,
"hgpg": 0.1,
Expand Down
Loading