From f122cba8ea2e174aaf8a60cba0f9efa1d2279d10 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:10:54 +0300 Subject: [PATCH 01/13] fix calibration multi-population contract --- .../calibration/population_calibration.py | 381 ++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 src/rtichoke/calibration/population_calibration.py diff --git a/src/rtichoke/calibration/population_calibration.py b/src/rtichoke/calibration/population_calibration.py new file mode 100644 index 00000000..5b589214 --- /dev/null +++ b/src/rtichoke/calibration/population_calibration.py @@ -0,0 +1,381 @@ +"""Public calibration API with a consistent population data contract.""" + +from typing import Dict, List, Union + +import numpy as np +import polars as pl +from plotly.graph_objs._figure import Figure + +from .calibration import ( + _add_hover_text_to_calibration_data, + _apply_heuristics_and_censoring, + _build_initial_df_for_times, + _check_performance_type_by_probs_and_reals, + _create_colors_dictionary_for_calibration, + _create_plotly_curve_from_calibration_curve_list, + _create_plotly_curve_from_calibration_curve_list_times, + _create_reference_data_for_calibration_curve, + _define_limits_for_calibration_plot, +) + +_DEFAULT_COLORS = [ + "#1b9e77", + "#d95f02", + "#7570b3", + "#e7298a", + "#07004D", + "#E6AB02", + "#FE5F55", + "#54494B", + "#006E90", + "#BC96E6", + "#52050A", + "#1F271B", + "#BE7C4D", + "#63768D", + "#08A045", + "#320A28", + "#82FF9E", + "#2176FF", + "#D1603D", + "#585123", +] + + +def _build_binary_calibration_df( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], +) -> pl.DataFrame: + """Normalize binary calibration inputs to one long data frame.""" + frames: list[pl.DataFrame] = [] + + if isinstance(reals, dict): + if probs.keys() == reals.keys(): + for population in reals: + p = np.asarray(probs[population]).ravel() + y = np.asarray(reals[population]).ravel() + if p.shape[0] != y.shape[0]: + raise ValueError( + f"Length mismatch for population '{population}': " + f"probs has length {p.shape[0]} but reals has length {y.shape[0]}." + ) + frames.append( + pl.DataFrame( + { + "reference_group": population, + "model": population, + "prob": p.astype(float, copy=False), + "real": y.astype(float, copy=False), + } + ) + ) + elif len(probs) == 1: + model, p_all = next(iter(probs.items())) + p_all = np.asarray(p_all).ravel() + populations = list(reals.keys()) + outcomes = [np.asarray(reals[population]).ravel() for population in populations] + lengths = [len(y) for y in outcomes] + n_total = sum(lengths) + if p_all.shape[0] != n_total: + raise ValueError( + f"probs['{model}'] length={p_all.shape[0]} does not match " + f"sum of population sizes={n_total}." + ) + + start = 0 + for population, y in zip(populations, outcomes): + end = start + len(y) + frames.append( + pl.DataFrame( + { + "reference_group": population, + "model": model, + "prob": p_all[start:end].astype(float, copy=False), + "real": y.astype(float, copy=False), + } + ) + ) + start = end + else: + raise ValueError( + "When probs and reals are dictionaries, use matching population keys, " + "or provide one concatenated probability vector for all populations." + ) + else: + y = np.asarray(reals).ravel() + for model, prob_array in probs.items(): + p = np.asarray(prob_array).ravel() + if p.shape[0] != y.shape[0]: + raise ValueError( + f"probs['{model}'] length={p.shape[0]} does not match " + f"reals length={y.shape[0]}." + ) + frames.append( + pl.DataFrame( + { + "reference_group": model, + "model": model, + "prob": p.astype(float, copy=False), + "real": y.astype(float, copy=False), + } + ) + ) + + return pl.concat(frames, how="vertical") + + +def _make_deciles_from_df(df: pl.DataFrame, n_bins: int = 10) -> pl.DataFrame: + prepared = df.with_columns( + [ + pl.col("prob").cast(pl.Float64), + pl.col("real").cast(pl.Float64), + ( + (pl.col("prob").rank("ordinal").over(["reference_group", "model"]) - 1) + * n_bins + // pl.len().over(["reference_group", "model"]) + + 1 + ).alias("decile"), + ] + ) + return ( + prepared.group_by(["reference_group", "model", "decile"]) + .agg( + [ + pl.len().alias("n"), + pl.mean("prob").alias("x"), + pl.mean("real").alias("y"), + pl.sum("real").alias("n_reals"), + ] + ) + .sort(["reference_group", "model", "decile"]) + ) + + +def _make_smooth_from_df(df: pl.DataFrame) -> pl.DataFrame: + from statsmodels.nonparametric.smoothers_lowess import lowess + + frames: list[pl.DataFrame] = [] + for group in df.partition_by(["reference_group", "model"], maintain_order=True): + reference_group = str(group["reference_group"][0]) + p = group["prob"].to_numpy() + y = group["real"].to_numpy() + if len(np.unique(p)) == 1: + frames.append( + pl.DataFrame( + { + "x": [float(p[0])], + "y": [float(np.mean(y))], + "reference_group": [reference_group], + } + ) + ) + continue + + smoothed = lowess(y, p, it=0) + xout = np.linspace(0, 1, 101) + yout = np.interp(xout, smoothed[:, 0], smoothed[:, 1]) + frames.append( + pl.DataFrame( + { + "x": xout, + "y": yout, + "reference_group": [reference_group] * len(xout), + } + ) + ) + + return pl.concat(frames, how="vertical") + + +def _make_histogram_from_df(df: pl.DataFrame) -> pl.DataFrame: + frames: list[pl.DataFrame] = [] + for group in df.partition_by("reference_group", maintain_order=True): + reference_group = str(group["reference_group"][0]) + counts, mids = np.histogram( + group["prob"].to_numpy(), bins=np.arange(0, 1.01, 0.01) + ) + hist = pl.DataFrame( + { + "mids": mids[:-1] + 0.005, + "counts": counts, + "reference_group": reference_group, + } + ).with_columns( + ( + pl.col("counts").cast(str) + + " observations in [" + + (pl.col("mids") - 0.005).round(3).cast(str) + + ", " + + (pl.col("mids") + 0.005).round(3).cast(str) + + "]" + ).alias("text") + ) + frames.append(hist) + return pl.concat(frames, how="vertical") + + +def _reference_groups(df: pl.DataFrame) -> list[str]: + return [str(value) for value in df["reference_group"].unique(maintain_order=True)] + + +def create_calibration_curve( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + calibration_type: str = "discrete", + size: int = 600, + color_values: List[str] = _DEFAULT_COLORS, +) -> Figure: + """Create a calibration curve. + + Parameters + ---------- + probs : dict[str, numpy.ndarray] + Predicted probabilities. When ``reals`` is a dictionary with matching keys, + each key is treated as an independent population and may have its own sample size. + reals : numpy.ndarray or dict[str, numpy.ndarray] + Observed binary outcomes. Matching dictionary keys are paired population-by-population. + calibration_type : str, default="discrete" + Calibration rendering type, either ``"discrete"`` or ``"smooth"``. + size : int, default=600 + Figure width and height in pixels. + color_values : list[str] + Colors used for population or model traces. + + Returns + ------- + plotly.graph_objs.Figure + Interactive calibration figure. + """ + df = _build_binary_calibration_df(probs, reals) + performance_type = _check_performance_type_by_probs_and_reals(probs, reals) + deciles = _make_deciles_from_df(df) + smooth = _make_smooth_from_df(df) + deciles, smooth = _add_hover_text_to_calibration_data( + deciles, smooth, performance_type + ) + groups = _reference_groups(df) + colors = _create_colors_dictionary_for_calibration( + groups, color_values, performance_type + ) + limits = _define_limits_for_calibration_plot(deciles) + + curve_data = { + "deciles_dat": deciles, + "smooth_dat": smooth, + "reference_data": _create_reference_data_for_calibration_curve(), + "histogram_for_calibration": _make_histogram_from_df(df), + "axes_ranges": {"xaxis": limits, "yaxis": limits}, + "colors_dictionary": colors, + "performance_type": [performance_type], + "size": [(size, size)], + } + return _create_plotly_curve_from_calibration_curve_list( + curve_data, calibration_type=calibration_type + ) + + +def create_calibration_curve_times( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + times: Union[np.ndarray, Dict[str, np.ndarray]], + fixed_time_horizons: List[float], + heuristics_sets: List[Dict[str, str]], + calibration_type: str = "discrete", + size: int = 600, + color_values: List[str] = _DEFAULT_COLORS, +) -> Figure: + """Create time-dependent calibration curves. + + Parameters + ---------- + probs : dict[str, numpy.ndarray] + Predicted probabilities. Matching dictionary keys identify populations. + reals : numpy.ndarray or dict[str, numpy.ndarray] + Observed event indicators. + times : numpy.ndarray or dict[str, numpy.ndarray] + Observed event or censoring times. + fixed_time_horizons : list[float] + Time horizons to display. + heuristics_sets : list[dict[str, str]] + Censoring and competing-risk heuristic combinations. + calibration_type : str, default="discrete" + Calibration rendering type. + size : int, default=600 + Figure width and height in pixels. + color_values : list[str] + Colors used for population or model traces. + + Returns + ------- + plotly.graph_objs.Figure + Interactive time-dependent calibration figure. + """ + initial_df = _build_initial_df_for_times(probs, reals, times) + performance_type = _check_performance_type_by_probs_and_reals(probs, reals) + reference_groups = _reference_groups(initial_df) + + all_deciles: list[pl.DataFrame] = [] + all_smooth: list[pl.DataFrame] = [] + all_histograms: list[pl.DataFrame] = [] + + for horizon in fixed_time_horizons: + for heuristics in heuristics_sets: + censoring_heuristic = heuristics["censoring_heuristic"] + competing_heuristic = heuristics["competing_heuristic"] + if ( + censoring_heuristic == "adjusted" + or competing_heuristic == "adjusted_as_censored" + ): + continue + + adjusted = _apply_heuristics_and_censoring( + initial_df, horizon, censoring_heuristic, competing_heuristic + ) + if adjusted.height == 0: + continue + + all_deciles.append( + _make_deciles_from_df(adjusted).with_columns( + pl.lit(horizon).alias("fixed_time_horizon") + ) + ) + all_smooth.append( + _make_smooth_from_df(adjusted).with_columns( + pl.lit(horizon).alias("fixed_time_horizon") + ) + ) + all_histograms.append( + _make_histogram_from_df(adjusted).with_columns( + pl.lit(horizon).alias("fixed_time_horizon") + ) + ) + + if not all_deciles: + raise ValueError("No data remaining after applying heuristics and time horizons.") + + deciles = pl.concat(all_deciles) + smooth = pl.concat(all_smooth) + histograms = pl.concat(all_histograms) + deciles, smooth = _add_hover_text_to_calibration_data( + deciles, smooth, performance_type + ) + colors = _create_colors_dictionary_for_calibration( + reference_groups, color_values, performance_type + ) + limits = _define_limits_for_calibration_plot(deciles) + + curve_data = { + "deciles_dat": deciles, + "smooth_dat": smooth, + "reference_data": _create_reference_data_for_calibration_curve(), + "histogram_for_calibration": histograms, + "axes_ranges": {"xaxis": limits, "yaxis": limits}, + "colors_dictionary": colors, + "performance_type": [performance_type], + "size": [(size, size)], + "fixed_time_horizons": fixed_time_horizons, + "reference_group_keys": reference_groups, + } + return _create_plotly_curve_from_calibration_curve_list_times( + curve_data, calibration_type=calibration_type + ) From ebade8d62230e27fcc8c537972b48b2f354fc998 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:11:04 +0300 Subject: [PATCH 02/13] route calibration through normalized population API --- src/rtichoke/calibration/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rtichoke/calibration/__init__.py b/src/rtichoke/calibration/__init__.py index 190e74e9..854e2fbe 100644 --- a/src/rtichoke/calibration/__init__.py +++ b/src/rtichoke/calibration/__init__.py @@ -2,6 +2,6 @@ Subpackage for Calibration """ -from .calibration import create_calibration_curve, create_calibration_curve_times +from .population_calibration import create_calibration_curve, create_calibration_curve_times __all__ = ["create_calibration_curve", "create_calibration_curve_times"] From e566c19fb50e0539caa518bdd468f98bf37b22e1 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:11:18 +0300 Subject: [PATCH 03/13] export normalized calibration API --- src/rtichoke/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rtichoke/__init__.py b/src/rtichoke/__init__.py index bc84b742..8dd5146d 100644 --- a/src/rtichoke/__init__.py +++ b/src/rtichoke/__init__.py @@ -30,7 +30,7 @@ ) from rtichoke.discrimination.gains import plot_gains_curve as plot_gains_curve -from rtichoke.calibration.calibration import ( +from rtichoke.calibration.population_calibration import ( create_calibration_curve as create_calibration_curve, create_calibration_curve_times as create_calibration_curve_times, ) @@ -45,7 +45,6 @@ prepare_performance_data as prepare_performance_data, prepare_binned_classification_data as prepare_binned_classification_data, ) - from rtichoke.performance_data.performance_data_times import ( prepare_performance_data_times as prepare_performance_data_times, prepare_binned_classification_data_times as prepare_binned_classification_data_times, From 09227a5972b8efe7745765bfd8a762102a1ca87b Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:11:58 +0300 Subject: [PATCH 04/13] add calibration population regression tests --- tests/test_calibration.py | 85 +++++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 12 deletions(-) diff --git a/tests/test_calibration.py b/tests/test_calibration.py index 4e79687f..82a8bb68 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -1,28 +1,89 @@ import numpy as np -from rtichoke.calibration.calibration import create_calibration_curve +import pytest + +from rtichoke import create_calibration_curve def test_create_calibration_curve_smooth(): probs = {"model_1": np.linspace(0, 1, 100)} - reals = np.random.randint(0, 2, 100) + reals = np.random.default_rng(1).integers(0, 2, 100) fig = create_calibration_curve(probs, reals, calibration_type="smooth") - # Check if the figure has the correct number of traces (smooth curve, histogram, and reference line) assert len(fig.data) == 3 - - # Check reference line data - reference_line = fig.data[0] - assert reference_line.name == "Perfectly Calibrated" + assert fig.data[0].name == "Perfectly Calibrated" def test_create_calibration_curve_smooth_single_point(): probs = {"model_1": np.array([0.5] * 100)} - reals = np.random.randint(0, 2, 100) + reals = np.random.default_rng(2).integers(0, 2, 100) fig = create_calibration_curve(probs, reals, calibration_type="smooth") - # Check that the plot mode is "lines+markers" assert fig.data[1].mode == "lines+markers" + assert fig.data[2].type == "bar" + + +def test_create_calibration_curve_equal_size_populations(): + probs = { + "Train": np.array([0.1, 0.9, 0.2, 0.8, 0.3, 0.7]), + "Test": np.array([0.2, 0.8, 0.3, 0.7, 0.4, 0.6]), + } + reals = { + "Train": np.array([0, 1, 0, 1, 0, 1]), + "Test": np.array([0, 1, 0, 1, 0, 0]), + } + + fig = create_calibration_curve(probs, reals) + + assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} + + +def test_create_calibration_curve_unequal_size_populations_discrete(): + probs = { + "Train": np.array([0.1, 0.9, 0.2, 0.8, 0.3, 0.7]), + "Test": np.array([0.2, 0.8, 0.3, 0.7]), + } + reals = { + "Train": np.array([0, 1, 0, 1, 0, 1]), + "Test": np.array([0, 1, 0, 0]), + } + + fig = create_calibration_curve(probs, reals, calibration_type="discrete") + + assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} + + +def test_create_calibration_curve_unequal_size_populations_smooth(): + probs = { + "Train": np.linspace(0.05, 0.95, 20), + "Test": np.linspace(0.1, 0.9, 12), + } + reals = { + "Train": np.array([0, 1] * 10), + "Test": np.array([0, 1] * 6), + } + + fig = create_calibration_curve(probs, reals, calibration_type="smooth") + + assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} + + +def test_create_calibration_curve_rejects_mismatched_population_keys(): + probs = {"Train": np.array([0.1, 0.9]), "Validation": np.array([0.2, 0.8])} + reals = {"Train": np.array([0, 1]), "Test": np.array([0, 1])} + + with pytest.raises(ValueError, match="matching population keys"): + create_calibration_curve(probs, reals) + + +def test_create_calibration_curve_rejects_within_population_length_mismatch(): + probs = { + "Train": np.array([0.1, 0.9, 0.2]), + "Test": np.array([0.2, 0.8]), + } + reals = { + "Train": np.array([0, 1]), + "Test": np.array([0, 1]), + } - # Check histogram data - histogram = fig.data[2] - assert histogram.type == "bar" + with pytest.raises(ValueError, match="population 'Train'"): + create_calibration_curve(probs, reals) From 6f2b88c5710b0037bd0afc0db9bd7ca9a1927d9a Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:12:21 +0300 Subject: [PATCH 05/13] test time calibration with unequal populations --- tests/test_calibration_times.py | 35 +++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/test_calibration_times.py b/tests/test_calibration_times.py index b6570c95..b05ba887 100644 --- a/tests/test_calibration_times.py +++ b/tests/test_calibration_times.py @@ -6,7 +6,6 @@ def test_create_calibration_curve_times(): probs = {"model_1": np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])} reals = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) times = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - fixed_time_horizons = [5, 10] heuristics_sets = [ {"censoring_heuristic": "excluded", "competing_heuristic": "excluded"} ] @@ -15,10 +14,42 @@ def test_create_calibration_curve_times(): probs, reals, times, - fixed_time_horizons=fixed_time_horizons, + fixed_time_horizons=[5.0, 10.0], heuristics_sets=heuristics_sets, ) assert fig is not None assert len(fig.data) > 0 assert len(fig.layout.sliders) > 0 + + +def test_create_calibration_curve_times_unequal_size_populations(): + probs = { + "Train": np.array([0.1, 0.9, 0.2, 0.8, 0.3, 0.7]), + "Test": np.array([0.2, 0.8, 0.3, 0.7]), + } + reals = { + "Train": np.array([0, 1, 0, 1, 0, 1]), + "Test": np.array([0, 1, 0, 0]), + } + times = { + "Train": np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + "Test": np.array([1.0, 2.0, 3.0, 4.0]), + } + heuristics_sets = [ + { + "censoring_heuristic": "excluded", + "competing_heuristic": "adjusted_as_negative", + } + ] + + fig = create_calibration_curve_times( + probs, + reals, + times, + fixed_time_horizons=[3.0, 6.0], + heuristics_sets=heuristics_sets, + ) + + assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} + assert len(fig.layout.sliders) == 1 From 2560fbf110c3d93aebf0745fe26cdd48cccae845 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:12:43 +0300 Subject: [PATCH 06/13] clarify multiple-population calibration example --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f5ad00d..f72bdfd0 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ pip install rtichoke To use `rtichoke`, you'll usually need two main inputs: * `probs`: A dictionary containing model-predicted probabilities. -* `reals`: A dictionary containing the observed outcomes. +* `reals`: Observed outcomes, provided either as one array or as a dictionary keyed by population. Here's a quick example of creating a ROC curve for a single model: @@ -54,6 +54,30 @@ fig = rk.create_roc_curve( fig.show() ``` +### Compare populations + +When predictions and outcomes are both dictionaries with the same keys, rtichoke pairs them population-by-population. The populations do **not** need to have the same sample size. + +```python +probs = { + "Train": np.array([0.10, 0.90, 0.20, 0.80, 0.30, 0.70]), + "Test": np.array([0.15, 0.85, 0.25, 0.75]), +} +reals = { + "Train": np.array([0, 1, 0, 1, 0, 1]), + "Test": np.array([0, 1, 0, 0]), +} + +fig = rk.create_calibration_curve( + probs=probs, + reals=reals, +) + +fig.show() +``` + +Here, `Train` contains six observations and `Test` contains four. Each probability vector only needs to match the outcome vector for its own population. + ## Key Features * **Simple API**: Create complex visualizations with a small amount of code. From 3c7ecfa84b4a1941e91e93a3f6c5aaaa32fdd126 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:28:35 +0300 Subject: [PATCH 07/13] keep calibration exports in existing module --- src/rtichoke/calibration/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rtichoke/calibration/__init__.py b/src/rtichoke/calibration/__init__.py index 854e2fbe..190e74e9 100644 --- a/src/rtichoke/calibration/__init__.py +++ b/src/rtichoke/calibration/__init__.py @@ -2,6 +2,6 @@ Subpackage for Calibration """ -from .population_calibration import create_calibration_curve, create_calibration_curve_times +from .calibration import create_calibration_curve, create_calibration_curve_times __all__ = ["create_calibration_curve", "create_calibration_curve_times"] From 195353697ba71d6090594cc4570a63d1be9727f3 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:28:49 +0300 Subject: [PATCH 08/13] keep calibration import path unchanged --- src/rtichoke/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rtichoke/__init__.py b/src/rtichoke/__init__.py index 8dd5146d..68b86701 100644 --- a/src/rtichoke/__init__.py +++ b/src/rtichoke/__init__.py @@ -30,7 +30,7 @@ ) from rtichoke.discrimination.gains import plot_gains_curve as plot_gains_curve -from rtichoke.calibration.population_calibration import ( +from rtichoke.calibration.calibration import ( create_calibration_curve as create_calibration_curve, create_calibration_curve_times as create_calibration_curve_times, ) From 56d80145d9a905d95ddab19e2bee1ebc6f6c4b00 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:28:59 +0300 Subject: [PATCH 09/13] remove calibration wrapper refactor --- .../calibration/population_calibration.py | 381 ------------------ 1 file changed, 381 deletions(-) delete mode 100644 src/rtichoke/calibration/population_calibration.py diff --git a/src/rtichoke/calibration/population_calibration.py b/src/rtichoke/calibration/population_calibration.py deleted file mode 100644 index 5b589214..00000000 --- a/src/rtichoke/calibration/population_calibration.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Public calibration API with a consistent population data contract.""" - -from typing import Dict, List, Union - -import numpy as np -import polars as pl -from plotly.graph_objs._figure import Figure - -from .calibration import ( - _add_hover_text_to_calibration_data, - _apply_heuristics_and_censoring, - _build_initial_df_for_times, - _check_performance_type_by_probs_and_reals, - _create_colors_dictionary_for_calibration, - _create_plotly_curve_from_calibration_curve_list, - _create_plotly_curve_from_calibration_curve_list_times, - _create_reference_data_for_calibration_curve, - _define_limits_for_calibration_plot, -) - -_DEFAULT_COLORS = [ - "#1b9e77", - "#d95f02", - "#7570b3", - "#e7298a", - "#07004D", - "#E6AB02", - "#FE5F55", - "#54494B", - "#006E90", - "#BC96E6", - "#52050A", - "#1F271B", - "#BE7C4D", - "#63768D", - "#08A045", - "#320A28", - "#82FF9E", - "#2176FF", - "#D1603D", - "#585123", -] - - -def _build_binary_calibration_df( - probs: Dict[str, np.ndarray], - reals: Union[np.ndarray, Dict[str, np.ndarray]], -) -> pl.DataFrame: - """Normalize binary calibration inputs to one long data frame.""" - frames: list[pl.DataFrame] = [] - - if isinstance(reals, dict): - if probs.keys() == reals.keys(): - for population in reals: - p = np.asarray(probs[population]).ravel() - y = np.asarray(reals[population]).ravel() - if p.shape[0] != y.shape[0]: - raise ValueError( - f"Length mismatch for population '{population}': " - f"probs has length {p.shape[0]} but reals has length {y.shape[0]}." - ) - frames.append( - pl.DataFrame( - { - "reference_group": population, - "model": population, - "prob": p.astype(float, copy=False), - "real": y.astype(float, copy=False), - } - ) - ) - elif len(probs) == 1: - model, p_all = next(iter(probs.items())) - p_all = np.asarray(p_all).ravel() - populations = list(reals.keys()) - outcomes = [np.asarray(reals[population]).ravel() for population in populations] - lengths = [len(y) for y in outcomes] - n_total = sum(lengths) - if p_all.shape[0] != n_total: - raise ValueError( - f"probs['{model}'] length={p_all.shape[0]} does not match " - f"sum of population sizes={n_total}." - ) - - start = 0 - for population, y in zip(populations, outcomes): - end = start + len(y) - frames.append( - pl.DataFrame( - { - "reference_group": population, - "model": model, - "prob": p_all[start:end].astype(float, copy=False), - "real": y.astype(float, copy=False), - } - ) - ) - start = end - else: - raise ValueError( - "When probs and reals are dictionaries, use matching population keys, " - "or provide one concatenated probability vector for all populations." - ) - else: - y = np.asarray(reals).ravel() - for model, prob_array in probs.items(): - p = np.asarray(prob_array).ravel() - if p.shape[0] != y.shape[0]: - raise ValueError( - f"probs['{model}'] length={p.shape[0]} does not match " - f"reals length={y.shape[0]}." - ) - frames.append( - pl.DataFrame( - { - "reference_group": model, - "model": model, - "prob": p.astype(float, copy=False), - "real": y.astype(float, copy=False), - } - ) - ) - - return pl.concat(frames, how="vertical") - - -def _make_deciles_from_df(df: pl.DataFrame, n_bins: int = 10) -> pl.DataFrame: - prepared = df.with_columns( - [ - pl.col("prob").cast(pl.Float64), - pl.col("real").cast(pl.Float64), - ( - (pl.col("prob").rank("ordinal").over(["reference_group", "model"]) - 1) - * n_bins - // pl.len().over(["reference_group", "model"]) - + 1 - ).alias("decile"), - ] - ) - return ( - prepared.group_by(["reference_group", "model", "decile"]) - .agg( - [ - pl.len().alias("n"), - pl.mean("prob").alias("x"), - pl.mean("real").alias("y"), - pl.sum("real").alias("n_reals"), - ] - ) - .sort(["reference_group", "model", "decile"]) - ) - - -def _make_smooth_from_df(df: pl.DataFrame) -> pl.DataFrame: - from statsmodels.nonparametric.smoothers_lowess import lowess - - frames: list[pl.DataFrame] = [] - for group in df.partition_by(["reference_group", "model"], maintain_order=True): - reference_group = str(group["reference_group"][0]) - p = group["prob"].to_numpy() - y = group["real"].to_numpy() - if len(np.unique(p)) == 1: - frames.append( - pl.DataFrame( - { - "x": [float(p[0])], - "y": [float(np.mean(y))], - "reference_group": [reference_group], - } - ) - ) - continue - - smoothed = lowess(y, p, it=0) - xout = np.linspace(0, 1, 101) - yout = np.interp(xout, smoothed[:, 0], smoothed[:, 1]) - frames.append( - pl.DataFrame( - { - "x": xout, - "y": yout, - "reference_group": [reference_group] * len(xout), - } - ) - ) - - return pl.concat(frames, how="vertical") - - -def _make_histogram_from_df(df: pl.DataFrame) -> pl.DataFrame: - frames: list[pl.DataFrame] = [] - for group in df.partition_by("reference_group", maintain_order=True): - reference_group = str(group["reference_group"][0]) - counts, mids = np.histogram( - group["prob"].to_numpy(), bins=np.arange(0, 1.01, 0.01) - ) - hist = pl.DataFrame( - { - "mids": mids[:-1] + 0.005, - "counts": counts, - "reference_group": reference_group, - } - ).with_columns( - ( - pl.col("counts").cast(str) - + " observations in [" - + (pl.col("mids") - 0.005).round(3).cast(str) - + ", " - + (pl.col("mids") + 0.005).round(3).cast(str) - + "]" - ).alias("text") - ) - frames.append(hist) - return pl.concat(frames, how="vertical") - - -def _reference_groups(df: pl.DataFrame) -> list[str]: - return [str(value) for value in df["reference_group"].unique(maintain_order=True)] - - -def create_calibration_curve( - probs: Dict[str, np.ndarray], - reals: Union[np.ndarray, Dict[str, np.ndarray]], - calibration_type: str = "discrete", - size: int = 600, - color_values: List[str] = _DEFAULT_COLORS, -) -> Figure: - """Create a calibration curve. - - Parameters - ---------- - probs : dict[str, numpy.ndarray] - Predicted probabilities. When ``reals`` is a dictionary with matching keys, - each key is treated as an independent population and may have its own sample size. - reals : numpy.ndarray or dict[str, numpy.ndarray] - Observed binary outcomes. Matching dictionary keys are paired population-by-population. - calibration_type : str, default="discrete" - Calibration rendering type, either ``"discrete"`` or ``"smooth"``. - size : int, default=600 - Figure width and height in pixels. - color_values : list[str] - Colors used for population or model traces. - - Returns - ------- - plotly.graph_objs.Figure - Interactive calibration figure. - """ - df = _build_binary_calibration_df(probs, reals) - performance_type = _check_performance_type_by_probs_and_reals(probs, reals) - deciles = _make_deciles_from_df(df) - smooth = _make_smooth_from_df(df) - deciles, smooth = _add_hover_text_to_calibration_data( - deciles, smooth, performance_type - ) - groups = _reference_groups(df) - colors = _create_colors_dictionary_for_calibration( - groups, color_values, performance_type - ) - limits = _define_limits_for_calibration_plot(deciles) - - curve_data = { - "deciles_dat": deciles, - "smooth_dat": smooth, - "reference_data": _create_reference_data_for_calibration_curve(), - "histogram_for_calibration": _make_histogram_from_df(df), - "axes_ranges": {"xaxis": limits, "yaxis": limits}, - "colors_dictionary": colors, - "performance_type": [performance_type], - "size": [(size, size)], - } - return _create_plotly_curve_from_calibration_curve_list( - curve_data, calibration_type=calibration_type - ) - - -def create_calibration_curve_times( - probs: Dict[str, np.ndarray], - reals: Union[np.ndarray, Dict[str, np.ndarray]], - times: Union[np.ndarray, Dict[str, np.ndarray]], - fixed_time_horizons: List[float], - heuristics_sets: List[Dict[str, str]], - calibration_type: str = "discrete", - size: int = 600, - color_values: List[str] = _DEFAULT_COLORS, -) -> Figure: - """Create time-dependent calibration curves. - - Parameters - ---------- - probs : dict[str, numpy.ndarray] - Predicted probabilities. Matching dictionary keys identify populations. - reals : numpy.ndarray or dict[str, numpy.ndarray] - Observed event indicators. - times : numpy.ndarray or dict[str, numpy.ndarray] - Observed event or censoring times. - fixed_time_horizons : list[float] - Time horizons to display. - heuristics_sets : list[dict[str, str]] - Censoring and competing-risk heuristic combinations. - calibration_type : str, default="discrete" - Calibration rendering type. - size : int, default=600 - Figure width and height in pixels. - color_values : list[str] - Colors used for population or model traces. - - Returns - ------- - plotly.graph_objs.Figure - Interactive time-dependent calibration figure. - """ - initial_df = _build_initial_df_for_times(probs, reals, times) - performance_type = _check_performance_type_by_probs_and_reals(probs, reals) - reference_groups = _reference_groups(initial_df) - - all_deciles: list[pl.DataFrame] = [] - all_smooth: list[pl.DataFrame] = [] - all_histograms: list[pl.DataFrame] = [] - - for horizon in fixed_time_horizons: - for heuristics in heuristics_sets: - censoring_heuristic = heuristics["censoring_heuristic"] - competing_heuristic = heuristics["competing_heuristic"] - if ( - censoring_heuristic == "adjusted" - or competing_heuristic == "adjusted_as_censored" - ): - continue - - adjusted = _apply_heuristics_and_censoring( - initial_df, horizon, censoring_heuristic, competing_heuristic - ) - if adjusted.height == 0: - continue - - all_deciles.append( - _make_deciles_from_df(adjusted).with_columns( - pl.lit(horizon).alias("fixed_time_horizon") - ) - ) - all_smooth.append( - _make_smooth_from_df(adjusted).with_columns( - pl.lit(horizon).alias("fixed_time_horizon") - ) - ) - all_histograms.append( - _make_histogram_from_df(adjusted).with_columns( - pl.lit(horizon).alias("fixed_time_horizon") - ) - ) - - if not all_deciles: - raise ValueError("No data remaining after applying heuristics and time horizons.") - - deciles = pl.concat(all_deciles) - smooth = pl.concat(all_smooth) - histograms = pl.concat(all_histograms) - deciles, smooth = _add_hover_text_to_calibration_data( - deciles, smooth, performance_type - ) - colors = _create_colors_dictionary_for_calibration( - reference_groups, color_values, performance_type - ) - limits = _define_limits_for_calibration_plot(deciles) - - curve_data = { - "deciles_dat": deciles, - "smooth_dat": smooth, - "reference_data": _create_reference_data_for_calibration_curve(), - "histogram_for_calibration": histograms, - "axes_ranges": {"xaxis": limits, "yaxis": limits}, - "colors_dictionary": colors, - "performance_type": [performance_type], - "size": [(size, size)], - "fixed_time_horizons": fixed_time_horizons, - "reference_group_keys": reference_groups, - } - return _create_plotly_curve_from_calibration_curve_list_times( - curve_data, calibration_type=calibration_type - ) From 4095ca04da5d2e15a33b05c075b915e98fed5a40 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:31:10 +0300 Subject: [PATCH 10/13] fix calibration population pairing --- src/rtichoke/calibration/calibration.py | 47 +++++++++++++++++++------ 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/src/rtichoke/calibration/calibration.py b/src/rtichoke/calibration/calibration.py index 567c05c4..2efbf57d 100644 --- a/src/rtichoke/calibration/calibration.py +++ b/src/rtichoke/calibration/calibration.py @@ -470,17 +470,39 @@ def _make_deciles_dat_binary( n_bins: int = 10, ) -> pl.DataFrame: if isinstance(reals, dict): - reference_groups_keys = list(reals.keys()) - y_list = [ - np.asarray(reals[str(reference_group)]).ravel() - for reference_group in reference_groups_keys - ] - lengths = np.array([len(y) for y in y_list], dtype=np.int64) - offsets = np.concatenate([np.array([0], dtype=np.int64), np.cumsum(lengths)]) - n_total = int(offsets[-1]) - frames: list[pl.DataFrame] = [] - for model, p_all in probs.items(): + + if probs.keys() == reals.keys(): + for population in reals: + p = np.asarray(probs[population]).ravel() + y = np.asarray(reals[population]).ravel() + if p.shape[0] != y.shape[0]: + raise ValueError( + f"Length mismatch for population '{population}': " + f"probs has length {p.shape[0]} but reals has length {y.shape[0]}." + ) + frames.append( + pl.DataFrame( + { + "reference_group": population, + "model": population, + "prob": p.astype(float, copy=False), + "real": y.astype(float, copy=False), + } + ) + ) + elif len(probs) == 1: + reference_groups_keys = list(reals.keys()) + y_list = [ + np.asarray(reals[str(reference_group)]).ravel() + for reference_group in reference_groups_keys + ] + lengths = np.array([len(y) for y in y_list], dtype=np.int64) + offsets = np.concatenate( + [np.array([0], dtype=np.int64), np.cumsum(lengths)] + ) + n_total = int(offsets[-1]) + model, p_all = next(iter(probs.items())) p_all = np.asarray(p_all).ravel() if p_all.shape[0] != n_total: raise ValueError( @@ -491,7 +513,6 @@ def _make_deciles_dat_binary( for i, pop in enumerate(reference_groups_keys): start = int(offsets[i]) end = int(offsets[i + 1]) - frames.append( pl.DataFrame( { @@ -502,6 +523,10 @@ def _make_deciles_dat_binary( } ) ) + else: + raise ValueError( + "When probs and reals are dictionaries, their population keys must match." + ) df = pl.concat(frames, how="vertical") From f9428d64dd088ff233e3de727aab297c25dccb1e Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:31:45 +0300 Subject: [PATCH 11/13] remove unrelated import formatting change --- src/rtichoke/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rtichoke/__init__.py b/src/rtichoke/__init__.py index 68b86701..bc84b742 100644 --- a/src/rtichoke/__init__.py +++ b/src/rtichoke/__init__.py @@ -45,6 +45,7 @@ prepare_performance_data as prepare_performance_data, prepare_binned_classification_data as prepare_binned_classification_data, ) + from rtichoke.performance_data.performance_data_times import ( prepare_performance_data_times as prepare_performance_data_times, prepare_binned_classification_data_times as prepare_binned_classification_data_times, From e6574f05c27d63f3f8ce4a1ad4df5bed1cbf599d Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:32:30 +0300 Subject: [PATCH 12/13] keep calibration regression tests focused --- tests/test_calibration.py | 80 +++++++++------------------------------ 1 file changed, 18 insertions(+), 62 deletions(-) diff --git a/tests/test_calibration.py b/tests/test_calibration.py index 82a8bb68..f17a2b91 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -1,43 +1,34 @@ import numpy as np -import pytest - -from rtichoke import create_calibration_curve +from rtichoke.calibration.calibration import create_calibration_curve def test_create_calibration_curve_smooth(): probs = {"model_1": np.linspace(0, 1, 100)} - reals = np.random.default_rng(1).integers(0, 2, 100) + reals = np.random.randint(0, 2, 100) fig = create_calibration_curve(probs, reals, calibration_type="smooth") + # Check if the figure has the correct number of traces (smooth curve, histogram, and reference line) assert len(fig.data) == 3 - assert fig.data[0].name == "Perfectly Calibrated" + + # Check reference line data + reference_line = fig.data[0] + assert reference_line.name == "Perfectly Calibrated" def test_create_calibration_curve_smooth_single_point(): probs = {"model_1": np.array([0.5] * 100)} - reals = np.random.default_rng(2).integers(0, 2, 100) + reals = np.random.randint(0, 2, 100) fig = create_calibration_curve(probs, reals, calibration_type="smooth") + # Check that the plot mode is "lines+markers" assert fig.data[1].mode == "lines+markers" - assert fig.data[2].type == "bar" - - -def test_create_calibration_curve_equal_size_populations(): - probs = { - "Train": np.array([0.1, 0.9, 0.2, 0.8, 0.3, 0.7]), - "Test": np.array([0.2, 0.8, 0.3, 0.7, 0.4, 0.6]), - } - reals = { - "Train": np.array([0, 1, 0, 1, 0, 1]), - "Test": np.array([0, 1, 0, 1, 0, 0]), - } - fig = create_calibration_curve(probs, reals) + # Check histogram data + histogram = fig.data[2] + assert histogram.type == "bar" - assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} - -def test_create_calibration_curve_unequal_size_populations_discrete(): +def test_create_calibration_curve_multiple_populations_unequal_sizes(): probs = { "Train": np.array([0.1, 0.9, 0.2, 0.8, 0.3, 0.7]), "Test": np.array([0.2, 0.8, 0.3, 0.7]), @@ -47,43 +38,8 @@ def test_create_calibration_curve_unequal_size_populations_discrete(): "Test": np.array([0, 1, 0, 0]), } - fig = create_calibration_curve(probs, reals, calibration_type="discrete") - - assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} - - -def test_create_calibration_curve_unequal_size_populations_smooth(): - probs = { - "Train": np.linspace(0.05, 0.95, 20), - "Test": np.linspace(0.1, 0.9, 12), - } - reals = { - "Train": np.array([0, 1] * 10), - "Test": np.array([0, 1] * 6), - } - - fig = create_calibration_curve(probs, reals, calibration_type="smooth") - - assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} - - -def test_create_calibration_curve_rejects_mismatched_population_keys(): - probs = {"Train": np.array([0.1, 0.9]), "Validation": np.array([0.2, 0.8])} - reals = {"Train": np.array([0, 1]), "Test": np.array([0, 1])} - - with pytest.raises(ValueError, match="matching population keys"): - create_calibration_curve(probs, reals) - - -def test_create_calibration_curve_rejects_within_population_length_mismatch(): - probs = { - "Train": np.array([0.1, 0.9, 0.2]), - "Test": np.array([0.2, 0.8]), - } - reals = { - "Train": np.array([0, 1]), - "Test": np.array([0, 1]), - } - - with pytest.raises(ValueError, match="population 'Train'"): - create_calibration_curve(probs, reals) + for calibration_type in ("discrete", "smooth"): + fig = create_calibration_curve( + probs, reals, calibration_type=calibration_type + ) + assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} From 36f5d8c23e5e3b3b965796b92ddcb3151ea2b17e Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Tue, 11 Aug 2026 18:32:49 +0300 Subject: [PATCH 13/13] keep time calibration regression test focused --- tests/test_calibration_times.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_calibration_times.py b/tests/test_calibration_times.py index b05ba887..8e73c63a 100644 --- a/tests/test_calibration_times.py +++ b/tests/test_calibration_times.py @@ -6,6 +6,7 @@ def test_create_calibration_curve_times(): probs = {"model_1": np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])} reals = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) times = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + fixed_time_horizons = [5, 10] heuristics_sets = [ {"censoring_heuristic": "excluded", "competing_heuristic": "excluded"} ] @@ -14,7 +15,7 @@ def test_create_calibration_curve_times(): probs, reals, times, - fixed_time_horizons=[5.0, 10.0], + fixed_time_horizons=fixed_time_horizons, heuristics_sets=heuristics_sets, ) @@ -52,4 +53,3 @@ def test_create_calibration_curve_times_unequal_size_populations(): ) assert {trace.name for trace in fig.data if trace.name} >= {"Train", "Test"} - assert len(fig.layout.sliders) == 1