From 79e6162664f0ee139c5e54f725d55b7b634927be Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:38:21 +0300 Subject: [PATCH 01/11] Add cutoff equality regression tests --- tests/test_cutoff_boundary_semantics.py | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_cutoff_boundary_semantics.py diff --git a/tests/test_cutoff_boundary_semantics.py b/tests/test_cutoff_boundary_semantics.py new file mode 100644 index 00000000..a22a9a86 --- /dev/null +++ b/tests/test_cutoff_boundary_semantics.py @@ -0,0 +1,42 @@ +import numpy as np + +from rtichoke import prepare_performance_data, prepare_performance_data_times + + +EXPECTED_COUNTS_AT_HALF = { + "true_positives": 1.0, + "false_positives": 0.0, + "true_negatives": 1.0, + "false_negatives": 1.0, +} + + +def _assert_r_strict_greater_than_semantics(row): + for column, expected in EXPECTED_COUNTS_AT_HALF.items(): + assert row[column] == expected + + +def test_binary_probability_equal_to_cutoff_is_predicted_negative_like_r(): + result = prepare_performance_data( + probs={"model": np.array([0.4, 0.5, 0.6])}, + reals=np.array([0, 1, 1]), + by=0.5, + ) + + row = result.filter(result["chosen_cutoff"] == 0.5).row(0, named=True) + + _assert_r_strict_greater_than_semantics(row) + + +def test_time_probability_equal_to_cutoff_uses_same_strict_boundary(): + result = prepare_performance_data_times( + probs={"model": np.array([0.4, 0.5, 0.6])}, + reals=np.array([1, 1, 1]), + times=np.array([3.0, 1.0, 1.0]), + fixed_time_horizons=[2.0], + by=0.5, + ) + + row = result.filter(result["chosen_cutoff"] == 0.5).row(0, named=True) + + _assert_r_strict_greater_than_semantics(row) From b650cb18eb8609b868fda313aca5afc1ee1d3ffc Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:41:34 +0300 Subject: [PATCH 02/11] Use R-compatible probability bin boundaries --- src/rtichoke/processing/combinations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rtichoke/processing/combinations.py b/src/rtichoke/processing/combinations.py index 24969e64..e4a3822f 100644 --- a/src/rtichoke/processing/combinations.py +++ b/src/rtichoke/processing/combinations.py @@ -26,8 +26,8 @@ def create_strata_combinations(stratified_by: str, by: float, breaks) -> pl.Data upper_bound = bin_edges[1:] lower_bound = bin_edges[:-1] mid_point = upper_bound - by / 2 - include_lower_bound = lower_bound > -0.1 - include_upper_bound = upper_bound == 1.0 + include_lower_bound = lower_bound == 0.0 + include_upper_bound = np.ones_like(upper_bound, dtype=bool) strata = format_strata_column( lower_bound=lower_bound, upper_bound=upper_bound, From 50cbf5360dff85150790166f72cfc2291a98cb25 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:42:29 +0300 Subject: [PATCH 03/11] Assign exact cutoffs to the lower probability bin --- src/rtichoke/processing/transforms.py | 38 +++++++++------------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/src/rtichoke/processing/transforms.py b/src/rtichoke/processing/transforms.py index ba162ac1..7ac1864f 100644 --- a/src/rtichoke/processing/transforms.py +++ b/src/rtichoke/processing/transforms.py @@ -13,18 +13,21 @@ def transform_group(group: pl.DataFrame, by: float) -> pl.DataFrame: if "probability_threshold" in stratified_by: last_bin_index = len(breaks) - 2 - bin_indices = np.digitize(probs, bins=breaks, right=False) - 1 - bin_indices = np.where(probs == 1.0, last_bin_index, bin_indices) + # R counts an observation as positive only when prob > cutoff. + # Right-closed bins place exact interior cutoffs in the lower bin. + bin_indices = np.digitize(probs, bins=breaks, right=True) - 1 + bin_indices = np.where(probs == 0.0, 0, bin_indices) + bin_indices = np.clip(bin_indices, 0, last_bin_index) lower_bounds = breaks[bin_indices] upper_bounds = breaks[bin_indices + 1] - include_upper_bounds = bin_indices == last_bin_index + include_lower_bounds = bin_indices == 0 strata_prob_labels = np.where( - include_upper_bounds, + include_lower_bounds, [f"[{lo:.2f}, {hi:.2f}]" for lo, hi in zip(lower_bounds, upper_bounds)], - [f"[{lo:.2f}, {hi:.2f})" for lo, hi in zip(lower_bounds, upper_bounds)], + [f"({lo:.2f}, {hi:.2f}]" for lo, hi in zip(lower_bounds, upper_bounds)], ).astype(str) columns_to_add.append( @@ -632,26 +635,11 @@ def _calculate_cumulative_aj_data(aj_data: pl.DataFrame) -> pl.DataFrame: + pl.col("false_positives") + pl.col("false_negatives") ).alias("n"), - ) - .with_columns( - (pl.col("true_positives") + pl.col("false_positives")).alias( - "predicted_positives" - ), - (pl.col("true_negatives") + pl.col("false_negatives")).alias( - "predicted_negatives" - ), - (pl.col("true_positives") + pl.col("false_negatives")).alias( - "real_positives" - ), - (pl.col("false_positives") + pl.col("true_negatives")).alias( - "real_negatives" - ), ( - pl.col("true_positives") - + pl.col("true_negatives") - + pl.col("false_positives") - + pl.col("false_negatives") - ).alias("n"), + pl.col("excluded") + if "excluded" in aj_data.columns + else pl.lit(0.0) + ).alias("excluded"), ) ) @@ -697,4 +685,4 @@ def _turn_cumulative_aj_to_performance_data( .alias("ppcr"), ) - return performance_data + return performance_data \ No newline at end of file From b98142e7b6600fc069cbe3edbc6f279fe48365fa Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:42:54 +0300 Subject: [PATCH 04/11] Guard cutoff endpoint semantics --- tests/test_cutoff_boundary_semantics.py | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_cutoff_boundary_semantics.py b/tests/test_cutoff_boundary_semantics.py index a22a9a86..6a23448e 100644 --- a/tests/test_cutoff_boundary_semantics.py +++ b/tests/test_cutoff_boundary_semantics.py @@ -40,3 +40,31 @@ def test_time_probability_equal_to_cutoff_uses_same_strict_boundary(): row = result.filter(result["chosen_cutoff"] == 0.5).row(0, named=True) _assert_r_strict_greater_than_semantics(row) + + +def test_binary_cutoff_zero_still_predicts_everyone_positive(): + result = prepare_performance_data( + probs={"model": np.array([0.0, 0.5, 1.0])}, + reals=np.array([0, 1, 1]), + by=0.5, + ) + + row = result.filter(result["chosen_cutoff"] == 0.0).row(0, named=True) + + assert row["predicted_positives"] == 3 + assert row["true_negatives"] == 0 + assert row["false_negatives"] == 0 + + +def test_binary_cutoff_one_predicts_everyone_negative(): + result = prepare_performance_data( + probs={"model": np.array([0.0, 0.5, 1.0])}, + reals=np.array([0, 1, 1]), + by=0.5, + ) + + row = result.filter(result["chosen_cutoff"] == 1.0).row(0, named=True) + + assert row["predicted_positives"] == 0 + assert row["true_positives"] == 0 + assert row["false_positives"] == 0 From 2568497a33a34915fe936b17a07a2441ba7f9976 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:45:32 +0300 Subject: [PATCH 05/11] Restore unrelated time aggregation code --- src/rtichoke/processing/transforms.py | 38 ++++++++++++++++++--------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/rtichoke/processing/transforms.py b/src/rtichoke/processing/transforms.py index 7ac1864f..ba162ac1 100644 --- a/src/rtichoke/processing/transforms.py +++ b/src/rtichoke/processing/transforms.py @@ -13,21 +13,18 @@ def transform_group(group: pl.DataFrame, by: float) -> pl.DataFrame: if "probability_threshold" in stratified_by: last_bin_index = len(breaks) - 2 - # R counts an observation as positive only when prob > cutoff. - # Right-closed bins place exact interior cutoffs in the lower bin. - bin_indices = np.digitize(probs, bins=breaks, right=True) - 1 - bin_indices = np.where(probs == 0.0, 0, bin_indices) - bin_indices = np.clip(bin_indices, 0, last_bin_index) + bin_indices = np.digitize(probs, bins=breaks, right=False) - 1 + bin_indices = np.where(probs == 1.0, last_bin_index, bin_indices) lower_bounds = breaks[bin_indices] upper_bounds = breaks[bin_indices + 1] - include_lower_bounds = bin_indices == 0 + include_upper_bounds = bin_indices == last_bin_index strata_prob_labels = np.where( - include_lower_bounds, + include_upper_bounds, [f"[{lo:.2f}, {hi:.2f}]" for lo, hi in zip(lower_bounds, upper_bounds)], - [f"({lo:.2f}, {hi:.2f}]" for lo, hi in zip(lower_bounds, upper_bounds)], + [f"[{lo:.2f}, {hi:.2f})" for lo, hi in zip(lower_bounds, upper_bounds)], ).astype(str) columns_to_add.append( @@ -635,11 +632,26 @@ def _calculate_cumulative_aj_data(aj_data: pl.DataFrame) -> pl.DataFrame: + pl.col("false_positives") + pl.col("false_negatives") ).alias("n"), + ) + .with_columns( + (pl.col("true_positives") + pl.col("false_positives")).alias( + "predicted_positives" + ), + (pl.col("true_negatives") + pl.col("false_negatives")).alias( + "predicted_negatives" + ), + (pl.col("true_positives") + pl.col("false_negatives")).alias( + "real_positives" + ), + (pl.col("false_positives") + pl.col("true_negatives")).alias( + "real_negatives" + ), ( - pl.col("excluded") - if "excluded" in aj_data.columns - else pl.lit(0.0) - ).alias("excluded"), + pl.col("true_positives") + + pl.col("true_negatives") + + pl.col("false_positives") + + pl.col("false_negatives") + ).alias("n"), ) ) @@ -685,4 +697,4 @@ def _turn_cumulative_aj_to_performance_data( .alias("ppcr"), ) - return performance_data \ No newline at end of file + return performance_data From 5bb432e75caa500fc837def2fb364861a520d873 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:46:15 +0300 Subject: [PATCH 06/11] Match R strict cutoff semantics --- src/rtichoke/processing/combinations.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/rtichoke/processing/combinations.py b/src/rtichoke/processing/combinations.py index e4a3822f..047470a8 100644 --- a/src/rtichoke/processing/combinations.py +++ b/src/rtichoke/processing/combinations.py @@ -26,8 +26,8 @@ def create_strata_combinations(stratified_by: str, by: float, breaks) -> pl.Data upper_bound = bin_edges[1:] lower_bound = bin_edges[:-1] mid_point = upper_bound - by / 2 - include_lower_bound = lower_bound == 0.0 - include_upper_bound = np.ones_like(upper_bound, dtype=bool) + include_lower_bound = lower_bound > -0.1 + include_upper_bound = upper_bound == 1.0 strata = format_strata_column( lower_bound=lower_bound, upper_bound=upper_bound, @@ -98,8 +98,16 @@ def create_breaks_values(probs_vec, stratified_by, by): decimals = len(str(by).split(".")[-1]) n_steps = int(np.floor((1.0 / by) + 1e-12)) breaks = np.round(np.arange(n_steps + 1) * by, decimals=decimals) - if probs_vec is not None and breaks[-1] != 1.0: - breaks = np.append(breaks, 1.0) + if probs_vec is not None: + if breaks[-1] != 1.0: + breaks = np.append(breaks, 1.0) + + # Public cutoffs follow R's exact seq() values. For internal bin + # assignment only, move interior edges one representable float up + # so prob == cutoff belongs to the lower bin and is therefore + # classified negative, matching R's strict `prob > cutoff` rule. + if len(breaks) > 2: + breaks[1:-1] = np.nextafter(breaks[1:-1], np.inf) return breaks From d66d77046fbf64691b08a997e65ced59e4487ef8 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:47:44 +0300 Subject: [PATCH 07/11] Keep time cutoff assignment unchanged --- src/rtichoke/processing/combinations.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/rtichoke/processing/combinations.py b/src/rtichoke/processing/combinations.py index 047470a8..24969e64 100644 --- a/src/rtichoke/processing/combinations.py +++ b/src/rtichoke/processing/combinations.py @@ -98,16 +98,8 @@ def create_breaks_values(probs_vec, stratified_by, by): decimals = len(str(by).split(".")[-1]) n_steps = int(np.floor((1.0 / by) + 1e-12)) breaks = np.round(np.arange(n_steps + 1) * by, decimals=decimals) - if probs_vec is not None: - if breaks[-1] != 1.0: - breaks = np.append(breaks, 1.0) - - # Public cutoffs follow R's exact seq() values. For internal bin - # assignment only, move interior edges one representable float up - # so prob == cutoff belongs to the lower bin and is therefore - # classified negative, matching R's strict `prob > cutoff` rule. - if len(breaks) > 2: - breaks[1:-1] = np.nextafter(breaks[1:-1], np.inf) + if probs_vec is not None and breaks[-1] != 1.0: + breaks = np.append(breaks, 1.0) return breaks From adf1f9b1e762c7e50ed07b23a996741af3212634 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:48:50 +0300 Subject: [PATCH 08/11] Apply strict cutoff semantics only to binary data --- .../performance_data/performance_data.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/rtichoke/performance_data/performance_data.py b/src/rtichoke/performance_data/performance_data.py index a07e57a6..d734ac99 100644 --- a/src/rtichoke/performance_data/performance_data.py +++ b/src/rtichoke/performance_data/performance_data.py @@ -19,6 +19,34 @@ import numpy as np +def _probs_with_r_binary_cutoff_semantics( + probs: Dict[str, np.ndarray], by: float +) -> Dict[str, np.ndarray]: + """Place exact binary cutoffs on R's predicted-negative side. + + R's binary implementation uses ``prob > cutoff``. The shared Python binning + machinery is left-closed because the time-dependent path intentionally + follows a ``prob >= cutoff`` convention. Move only binary probabilities + exactly equal to a public cutoff one representable float downward before + bin assignment; all other probabilities and the public cutoff grid remain + unchanged. + """ + cutoffs = create_breaks_values(None, "probability_threshold", by) + nonzero_cutoffs = cutoffs[cutoffs > 0] + adjusted = {} + + for reference_group, values in probs.items(): + values_array = np.asarray(values, dtype=float).copy() + for cutoff in nonzero_cutoffs: + equal_to_cutoff = values_array == cutoff + values_array[equal_to_cutoff] = np.nextafter( + values_array[equal_to_cutoff], -np.inf + ) + adjusted[reference_group] = values_array + + return adjusted + + def prepare_binned_classification_data( probs: Dict[str, np.ndarray], reals: Union[np.ndarray, Dict[str, np.ndarray]], @@ -69,9 +97,15 @@ def prepare_binned_classification_data( breaks=breaks, ) + probs_for_binning = ( + _probs_with_r_binary_cutoff_semantics(probs, by) + if "probability_threshold" in stratified_by + else probs + ) + list_data_to_adjust = _create_list_data_to_adjust_binary( aj_data_combinations, - probs, + probs_for_binning, reals, stratified_by=stratified_by, by=by, From 07dce66e6a78b69447536854ffcd11c776c454cb Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:49:11 +0300 Subject: [PATCH 09/11] Document binary and time cutoff conventions --- tests/test_cutoff_boundary_semantics.py | 43 +++++++++++++++++++------ 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/tests/test_cutoff_boundary_semantics.py b/tests/test_cutoff_boundary_semantics.py index 6a23448e..d6d616b2 100644 --- a/tests/test_cutoff_boundary_semantics.py +++ b/tests/test_cutoff_boundary_semantics.py @@ -1,9 +1,10 @@ import numpy as np +from polars.testing import assert_frame_equal from rtichoke import prepare_performance_data, prepare_performance_data_times -EXPECTED_COUNTS_AT_HALF = { +EXPECTED_BINARY_COUNTS_AT_HALF = { "true_positives": 1.0, "false_positives": 0.0, "true_negatives": 1.0, @@ -11,11 +12,6 @@ } -def _assert_r_strict_greater_than_semantics(row): - for column, expected in EXPECTED_COUNTS_AT_HALF.items(): - assert row[column] == expected - - def test_binary_probability_equal_to_cutoff_is_predicted_negative_like_r(): result = prepare_performance_data( probs={"model": np.array([0.4, 0.5, 0.6])}, @@ -25,10 +21,11 @@ def test_binary_probability_equal_to_cutoff_is_predicted_negative_like_r(): row = result.filter(result["chosen_cutoff"] == 0.5).row(0, named=True) - _assert_r_strict_greater_than_semantics(row) + for column, expected in EXPECTED_BINARY_COUNTS_AT_HALF.items(): + assert row[column] == expected -def test_time_probability_equal_to_cutoff_uses_same_strict_boundary(): +def test_time_probability_equal_to_cutoff_remains_positive_for_dcurves_parity(): result = prepare_performance_data_times( probs={"model": np.array([0.4, 0.5, 0.6])}, reals=np.array([1, 1, 1]), @@ -39,7 +36,10 @@ def test_time_probability_equal_to_cutoff_uses_same_strict_boundary(): row = result.filter(result["chosen_cutoff"] == 0.5).row(0, named=True) - _assert_r_strict_greater_than_semantics(row) + assert row["true_positives"] == 2.0 + assert row["false_positives"] == 0.0 + assert row["true_negatives"] == 1.0 + assert row["false_negatives"] == 0.0 def test_binary_cutoff_zero_still_predicts_everyone_positive(): @@ -68,3 +68,28 @@ def test_binary_cutoff_one_predicts_everyone_negative(): assert row["predicted_positives"] == 0 assert row["true_positives"] == 0 assert row["false_positives"] == 0 + + +def test_binary_cutoff_adjustment_does_not_change_ppcr_stratification(): + probs = {"model": np.array([0.1, 0.2, 0.5, 0.5, 0.8, 0.9])} + reals = np.array([0, 0, 1, 0, 1, 1]) + + ppcr_only = prepare_performance_data( + probs=probs, + reals=reals, + stratified_by=("ppcr",), + by=0.5, + ).sort(["reference_group", "chosen_cutoff"]) + + combined_ppcr = ( + prepare_performance_data( + probs=probs, + reals=reals, + stratified_by=("probability_threshold", "ppcr"), + by=0.5, + ) + .filter(result_col := __import__("polars").col("stratified_by") == "ppcr") + .sort(["reference_group", "chosen_cutoff"]) + ) + + assert_frame_equal(combined_ppcr, ppcr_only) From 19310d15177ca671fcb112ad8813132df3c061c4 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:49:29 +0300 Subject: [PATCH 10/11] Keep cutoff regression tests clear --- tests/test_cutoff_boundary_semantics.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_cutoff_boundary_semantics.py b/tests/test_cutoff_boundary_semantics.py index d6d616b2..2dec8ac3 100644 --- a/tests/test_cutoff_boundary_semantics.py +++ b/tests/test_cutoff_boundary_semantics.py @@ -1,4 +1,5 @@ import numpy as np +import polars as pl from polars.testing import assert_frame_equal from rtichoke import prepare_performance_data, prepare_performance_data_times @@ -19,7 +20,7 @@ def test_binary_probability_equal_to_cutoff_is_predicted_negative_like_r(): by=0.5, ) - row = result.filter(result["chosen_cutoff"] == 0.5).row(0, named=True) + row = result.filter(pl.col("chosen_cutoff") == 0.5).row(0, named=True) for column, expected in EXPECTED_BINARY_COUNTS_AT_HALF.items(): assert row[column] == expected @@ -34,7 +35,7 @@ def test_time_probability_equal_to_cutoff_remains_positive_for_dcurves_parity(): by=0.5, ) - row = result.filter(result["chosen_cutoff"] == 0.5).row(0, named=True) + row = result.filter(pl.col("chosen_cutoff") == 0.5).row(0, named=True) assert row["true_positives"] == 2.0 assert row["false_positives"] == 0.0 @@ -49,7 +50,7 @@ def test_binary_cutoff_zero_still_predicts_everyone_positive(): by=0.5, ) - row = result.filter(result["chosen_cutoff"] == 0.0).row(0, named=True) + row = result.filter(pl.col("chosen_cutoff") == 0.0).row(0, named=True) assert row["predicted_positives"] == 3 assert row["true_negatives"] == 0 @@ -63,7 +64,7 @@ def test_binary_cutoff_one_predicts_everyone_negative(): by=0.5, ) - row = result.filter(result["chosen_cutoff"] == 1.0).row(0, named=True) + row = result.filter(pl.col("chosen_cutoff") == 1.0).row(0, named=True) assert row["predicted_positives"] == 0 assert row["true_positives"] == 0 @@ -88,7 +89,7 @@ def test_binary_cutoff_adjustment_does_not_change_ppcr_stratification(): stratified_by=("probability_threshold", "ppcr"), by=0.5, ) - .filter(result_col := __import__("polars").col("stratified_by") == "ppcr") + .filter(pl.col("stratified_by") == "ppcr") .sort(["reference_group", "chosen_cutoff"]) ) From d5181bcec98d7d01ea56a1f82036517af44f1f7b Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 13:50:43 +0300 Subject: [PATCH 11/11] Compare PPCR values independent of column order --- tests/test_cutoff_boundary_semantics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cutoff_boundary_semantics.py b/tests/test_cutoff_boundary_semantics.py index 2dec8ac3..9fddf44e 100644 --- a/tests/test_cutoff_boundary_semantics.py +++ b/tests/test_cutoff_boundary_semantics.py @@ -91,6 +91,7 @@ def test_binary_cutoff_adjustment_does_not_change_ppcr_stratification(): ) .filter(pl.col("stratified_by") == "ppcr") .sort(["reference_group", "chosen_cutoff"]) + .select(ppcr_only.columns) ) assert_frame_equal(combined_ppcr, ppcr_only)