From 59db38b3509e685b5620c1d645116d4100c288f0 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Wed, 19 Aug 2026 22:00:05 +0300 Subject: [PATCH 1/8] Fix time-dependent gains perfect reference --- src/rtichoke/discrimination/gains.py | 62 ++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/src/rtichoke/discrimination/gains.py b/src/rtichoke/discrimination/gains.py index 06fea386..51d506f4 100644 --- a/src/rtichoke/discrimination/gains.py +++ b/src/rtichoke/discrimination/gains.py @@ -8,11 +8,60 @@ _create_rtichoke_plotly_curve_times, _create_rtichoke_plotly_curve_binary, _plot_rtichoke_curve_binary, + _create_rtichoke_curve_list_times, + _create_plotly_curve_times, + _create_reference_lines_data, + _check_if_multiple_populations_are_being_validated_times, ) +from rtichoke.performance_data.performance_data_times import prepare_performance_data_times import numpy as np import polars as pl +def _get_gains_aj_estimates_times(performance_data: pl.DataFrame) -> pl.DataFrame: + """Return one horizon-specific event estimate per reference group. + + For a gains reference curve the required event probability is the overall + event probability at the horizon. At probability threshold 0 everyone is + classified positive, so ``real_positives / n`` gives that quantity without + mixing in the cutoff-specific estimate at threshold 1. + """ + return ( + performance_data.filter(pl.col("chosen_cutoff") == 0) + .select("reference_group", "fixed_time_horizon", "real_positives", "n") + .unique() + .with_columns((pl.col("real_positives") / pl.col("n")).alias("aj_estimate")) + .select("reference_group", "fixed_time_horizon", "aj_estimate") + .sort(["reference_group", "fixed_time_horizon"]) + ) + + +def _replace_gains_reference_data_times( + curve_list: dict, performance_data: pl.DataFrame +) -> dict: + """Replace time-dependent gains references with one AJ estimate per horizon.""" + aj_estimates = _get_gains_aj_estimates_times(performance_data) + references = [] + + for horizon in curve_list["fixed_time_horizons"]: + aj_horizon = aj_estimates.filter(pl.col("fixed_time_horizon") == horizon) + multiple_populations = _check_if_multiple_populations_are_being_validated_times( + aj_horizon + ) + references.append( + _create_reference_lines_data( + curve="gains", + aj_estimates_from_performance_data=aj_horizon, + multiple_populations=multiple_populations, + ).with_columns(pl.lit(horizon).alias("fixed_time_horizon")) + ) + + curve_list["reference_data"] = ( + pl.concat(references, how="vertical") if references else pl.DataFrame() + ) + return curve_list + + def create_gains_curve( probs: Dict[str, np.ndarray], reals: Union[np.ndarray, Dict[str, np.ndarray]], @@ -185,8 +234,7 @@ def create_gains_curve_times( Figure A Plotly ``Figure`` object for the time-dependent Gains curve. """ - - fig = _create_rtichoke_plotly_curve_times( + performance_data = prepare_performance_data_times( probs, reals, times, @@ -194,9 +242,15 @@ def create_gains_curve_times( heuristics_sets=heuristics_sets, by=by, stratified_by=stratified_by, + ) + + curve_list = _create_rtichoke_curve_list_times( + performance_data, + stratified_by=stratified_by[0], size=size, - color_values=color_values, + color_value=color_values, curve="gains", ) + curve_list = _replace_gains_reference_data_times(curve_list, performance_data) - return fig + return _create_plotly_curve_times(curve_list) From f51e6d53ab38af1cecbe3a2315560f7ee87757bf Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Wed, 19 Aug 2026 22:00:26 +0300 Subject: [PATCH 2/8] Add regression test for gains perfect reference --- tests/test_gains_times.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/test_gains_times.py diff --git a/tests/test_gains_times.py b/tests/test_gains_times.py new file mode 100644 index 00000000..c9a6de5f --- /dev/null +++ b/tests/test_gains_times.py @@ -0,0 +1,36 @@ +import polars as pl + +from rtichoke.discrimination.gains import ( + _get_gains_aj_estimates_times, + _replace_gains_reference_data_times, +) + + +def test_gains_reference_uses_cutoff_zero_event_probability(): + performance_data = pl.DataFrame( + { + "reference_group": ["model", "model"], + "fixed_time_horizon": [5.0, 5.0], + "chosen_cutoff": [0.0, 1.0], + "real_positives": [20.0, 5.0], + "n": [100.0, 100.0], + } + ) + + aj = _get_gains_aj_estimates_times(performance_data) + + assert aj.height == 1 + assert aj["aj_estimate"].item() == 0.2 + + curve_list = { + "fixed_time_horizons": [5.0], + "reference_data": pl.DataFrame(), + } + fixed = _replace_gains_reference_data_times(curve_list, performance_data) + perfect = fixed["reference_data"].filter( + pl.col("reference_group") == "perfect_model" + ) + + # For p=0.2, the perfect gains curve has y=x/p, so at x=0.1 y=0.5. + y_at_point_one = perfect.filter(pl.col("x") == 0.1)["y"].item() + assert y_at_point_one == 0.5 From 60f06d650c178c8535b192ee3f78ee9328910755 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Wed, 19 Aug 2026 22:07:21 +0300 Subject: [PATCH 3/8] Add rendered example for time-dependent gains fix --- .../06-time-dependent-gains-example.qmd | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 user_guide/06-time-dependent-gains-example.qmd diff --git a/user_guide/06-time-dependent-gains-example.qmd b/user_guide/06-time-dependent-gains-example.qmd new file mode 100644 index 00000000..55a57dee --- /dev/null +++ b/user_guide/06-time-dependent-gains-example.qmd @@ -0,0 +1,25 @@ +# Time-dependent gains example + +This example shows the time-dependent gains curve and its perfect-model reference at a fixed horizon. + +```{python} +import numpy as np +from rtichoke import create_gains_curve_times + +probs = { + "model": np.array([0.90, 0.80, 0.70, 0.60, 0.50, 0.40, 0.30, 0.20, 0.10, 0.05]) +} +reals = np.array([1, 1, 0, 1, 0, 0, 1, 0, 0, 0]) +times = np.array([1.0, 2.0, 8.0, 3.0, 9.0, 10.0, 4.0, 7.0, 6.0, 11.0]) + +fig = create_gains_curve_times( + probs=probs, + reals=reals, + times=times, + fixed_time_horizons=[5.0], + by=0.05, +) +fig.show() +``` + +At the five-unit horizon, the perfect-model reference is constructed from the single horizon-specific event estimate for the full population. The empirical gains curve should therefore not appear better than the perfect reference. From 3311aa51bdc980d33d39f0d61f9f19ae026bac88 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Wed, 19 Aug 2026 23:00:53 +0300 Subject: [PATCH 4/8] Trigger documentation preview for gains example --- user_guide/06-time-dependent-gains-example.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_guide/06-time-dependent-gains-example.qmd b/user_guide/06-time-dependent-gains-example.qmd index 55a57dee..c1e65f89 100644 --- a/user_guide/06-time-dependent-gains-example.qmd +++ b/user_guide/06-time-dependent-gains-example.qmd @@ -22,4 +22,4 @@ fig = create_gains_curve_times( fig.show() ``` -At the five-unit horizon, the perfect-model reference is constructed from the single horizon-specific event estimate for the full population. The empirical gains curve should therefore not appear better than the perfect reference. +At the five-unit horizon, the perfect-model reference is constructed from one horizon-specific event estimate for the full population. The empirical gains curve should therefore not appear better than the perfect reference. From 6ee7d9a4f63cfae28581eae13a2678e3bd2b4f84 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Wed, 19 Aug 2026 23:41:14 +0300 Subject: [PATCH 5/8] Retrigger docs preview after concurrency fix --- user_guide/06-time-dependent-gains-example.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_guide/06-time-dependent-gains-example.qmd b/user_guide/06-time-dependent-gains-example.qmd index c1e65f89..caf52526 100644 --- a/user_guide/06-time-dependent-gains-example.qmd +++ b/user_guide/06-time-dependent-gains-example.qmd @@ -22,4 +22,4 @@ fig = create_gains_curve_times( fig.show() ``` -At the five-unit horizon, the perfect-model reference is constructed from one horizon-specific event estimate for the full population. The empirical gains curve should therefore not appear better than the perfect reference. +At the five-unit horizon, the perfect-model reference is constructed from one horizon-specific event estimate for the full population. The empirical gains curve should therefore remain at or below the perfect reference. From bea04e0ef2043f8fe02b3f819d9d9d001c8f0105 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 08:39:14 +0300 Subject: [PATCH 6/8] Test gains references across populations and horizons --- tests/test_gains_times.py | 60 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/test_gains_times.py b/tests/test_gains_times.py index c9a6de5f..8cb697c6 100644 --- a/tests/test_gains_times.py +++ b/tests/test_gains_times.py @@ -1,4 +1,5 @@ import polars as pl +import pytest from rtichoke.discrimination.gains import ( _get_gains_aj_estimates_times, @@ -34,3 +35,62 @@ def test_gains_reference_uses_cutoff_zero_event_probability(): # For p=0.2, the perfect gains curve has y=x/p, so at x=0.1 y=0.5. y_at_point_one = perfect.filter(pl.col("x") == 0.1)["y"].item() assert y_at_point_one == 0.5 + + +def test_gains_reference_is_population_and_horizon_specific(): + performance_data = pl.DataFrame( + { + "reference_group": [ + "population_a", + "population_a", + "population_b", + "population_b", + "population_a", + "population_a", + "population_b", + "population_b", + ], + "fixed_time_horizon": [5.0, 5.0, 5.0, 5.0, 10.0, 10.0, 10.0, 10.0], + "chosen_cutoff": [0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0], + "real_positives": [20.0, 5.0, 40.0, 10.0, 30.0, 7.0, 60.0, 15.0], + "n": [100.0] * 8, + } + ) + + aj = _get_gains_aj_estimates_times(performance_data) + assert aj.height == 4 + + expected_aj = { + ("population_a", 5.0): 0.2, + ("population_b", 5.0): 0.4, + ("population_a", 10.0): 0.3, + ("population_b", 10.0): 0.6, + } + for (population, horizon), expected in expected_aj.items(): + value = aj.filter( + (pl.col("reference_group") == population) + & (pl.col("fixed_time_horizon") == horizon) + )["aj_estimate"].item() + assert value == pytest.approx(expected) + + curve_list = { + "fixed_time_horizons": [5.0, 10.0], + "reference_data": pl.DataFrame(), + } + reference_data = _replace_gains_reference_data_times( + curve_list, performance_data + )["reference_data"] + + expected_y_at_point_one = { + ("perfect_model_population_a", 5.0): 0.1 / 0.2, + ("perfect_model_population_b", 5.0): 0.1 / 0.4, + ("perfect_model_population_a", 10.0): 0.1 / 0.3, + ("perfect_model_population_b", 10.0): 0.1 / 0.6, + } + for (reference_group, horizon), expected in expected_y_at_point_one.items(): + y = reference_data.filter( + (pl.col("reference_group") == reference_group) + & (pl.col("fixed_time_horizon") == horizon) + & (pl.col("x") == 0.1) + )["y"].item() + assert y == pytest.approx(expected) From 0ad1a54b4be7e0e8dc7d12d7eca4ff7930314ce4 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 08:58:55 +0300 Subject: [PATCH 7/8] Expand gains preview to populations and horizons --- .../06-time-dependent-gains-example.qmd | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/user_guide/06-time-dependent-gains-example.qmd b/user_guide/06-time-dependent-gains-example.qmd index caf52526..550bb87f 100644 --- a/user_guide/06-time-dependent-gains-example.qmd +++ b/user_guide/06-time-dependent-gains-example.qmd @@ -1,25 +1,32 @@ # Time-dependent gains example -This example shows the time-dependent gains curve and its perfect-model reference at a fixed horizon. +This temporary example visually checks the perfect-model reference for multiple populations and multiple fixed time horizons. ```{python} import numpy as np from rtichoke import create_gains_curve_times probs = { - "model": np.array([0.90, 0.80, 0.70, 0.60, 0.50, 0.40, 0.30, 0.20, 0.10, 0.05]) + "population_a": np.array([0.90, 0.80, 0.70, 0.60, 0.50, 0.40, 0.30, 0.20, 0.10, 0.05]), + "population_b": np.array([0.85, 0.75, 0.65, 0.55, 0.45, 0.35, 0.25, 0.15, 0.08, 0.03]), +} +reals = { + "population_a": np.array([1, 1, 0, 1, 0, 0, 1, 0, 0, 0]), + "population_b": np.array([1, 0, 1, 1, 1, 0, 1, 0, 0, 0]), +} +times = { + "population_a": np.array([1.0, 2.0, 8.0, 3.0, 9.0, 10.0, 4.0, 7.0, 6.0, 11.0]), + "population_b": np.array([1.0, 7.0, 2.0, 3.0, 8.0, 9.0, 4.0, 10.0, 6.0, 11.0]), } -reals = np.array([1, 1, 0, 1, 0, 0, 1, 0, 0, 0]) -times = np.array([1.0, 2.0, 8.0, 3.0, 9.0, 10.0, 4.0, 7.0, 6.0, 11.0]) fig = create_gains_curve_times( probs=probs, reals=reals, times=times, - fixed_time_horizons=[5.0], + fixed_time_horizons=[5.0, 10.0], by=0.05, ) fig.show() ``` -At the five-unit horizon, the perfect-model reference is constructed from one horizon-specific event estimate for the full population. The empirical gains curve should therefore remain at or below the perfect reference. +The rendered figure contains two populations evaluated at horizons 5 and 10. Each population–horizon combination should use its own horizon-specific event probability for the perfect-model reference, and the empirical gains curve should remain at or below that corresponding reference. From 2366ae3e50ee55722a5f1de5138f144b62e1e444 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 09:11:35 +0300 Subject: [PATCH 8/8] Remove temporary gains preview page --- .../06-time-dependent-gains-example.qmd | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 user_guide/06-time-dependent-gains-example.qmd diff --git a/user_guide/06-time-dependent-gains-example.qmd b/user_guide/06-time-dependent-gains-example.qmd deleted file mode 100644 index 550bb87f..00000000 --- a/user_guide/06-time-dependent-gains-example.qmd +++ /dev/null @@ -1,32 +0,0 @@ -# Time-dependent gains example - -This temporary example visually checks the perfect-model reference for multiple populations and multiple fixed time horizons. - -```{python} -import numpy as np -from rtichoke import create_gains_curve_times - -probs = { - "population_a": np.array([0.90, 0.80, 0.70, 0.60, 0.50, 0.40, 0.30, 0.20, 0.10, 0.05]), - "population_b": np.array([0.85, 0.75, 0.65, 0.55, 0.45, 0.35, 0.25, 0.15, 0.08, 0.03]), -} -reals = { - "population_a": np.array([1, 1, 0, 1, 0, 0, 1, 0, 0, 0]), - "population_b": np.array([1, 0, 1, 1, 1, 0, 1, 0, 0, 0]), -} -times = { - "population_a": np.array([1.0, 2.0, 8.0, 3.0, 9.0, 10.0, 4.0, 7.0, 6.0, 11.0]), - "population_b": np.array([1.0, 7.0, 2.0, 3.0, 8.0, 9.0, 4.0, 10.0, 6.0, 11.0]), -} - -fig = create_gains_curve_times( - probs=probs, - reals=reals, - times=times, - fixed_time_horizons=[5.0, 10.0], - by=0.05, -) -fig.show() -``` - -The rendered figure contains two populations evaluated at horizons 5 and 10. Each population–horizon combination should use its own horizon-specific event probability for the perfect-model reference, and the empirical gains curve should remain at or below that corresponding reference.