From f0a30f845c1e5198cadd207806b35906fa8865ac Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:05:33 +0300 Subject: [PATCH 01/12] Sort time-dependent performance data deterministically --- .../performance_data/performance_data_times.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/rtichoke/performance_data/performance_data_times.py b/src/rtichoke/performance_data/performance_data_times.py index 53678155..fdce7990 100644 --- a/src/rtichoke/performance_data/performance_data_times.py +++ b/src/rtichoke/performance_data/performance_data_times.py @@ -97,7 +97,16 @@ def prepare_performance_data_times( # 3. Turn AJ output into performance metrics performance_data = _turn_cumulative_aj_to_performance_data(cumulative_aj_data) - return performance_data + return performance_data.sort( + [ + "reference_group", + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + "stratified_by", + "chosen_cutoff", + ] + ) def prepare_binned_classification_data_times( From 756bbd7e32835671ba33215e0cf6693e68ce0cb5 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:05:43 +0300 Subject: [PATCH 02/12] Add regression test for time performance ordering --- tests/test_time_performance_order.py | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/test_time_performance_order.py diff --git a/tests/test_time_performance_order.py b/tests/test_time_performance_order.py new file mode 100644 index 00000000..db52d289 --- /dev/null +++ b/tests/test_time_performance_order.py @@ -0,0 +1,47 @@ +import numpy as np + +from rtichoke import prepare_performance_data_times + + +def test_prepare_performance_data_times_returns_deterministic_order(): + probs = { + "population_b": np.array([0.8, 0.2, 0.6, 0.4]), + "population_a": np.array([0.7, 0.1, 0.9, 0.3]), + } + reals = { + "population_b": np.array([1, 0, 1, 0]), + "population_a": np.array([0, 1, 1, 0]), + } + times = { + "population_b": np.array([2.0, 7.0, 4.0, 9.0]), + "population_a": np.array([8.0, 3.0, 5.0, 10.0]), + } + + result = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=[8.0, 5.0], + heuristics_sets=[ + { + "censoring_heuristic": "excluded", + "competing_heuristic": "excluded", + }, + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + }, + ], + by=0.2, + ) + + sort_columns = [ + "reference_group", + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + "stratified_by", + "chosen_cutoff", + ] + + assert result.equals(result.sort(sort_columns)) From fcc7f0ed2cacb7f8a51a91bafdaf45f0031f0c80 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:05:53 +0300 Subject: [PATCH 03/12] noop --- noop | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 noop diff --git a/noop b/noop new file mode 100644 index 00000000..e69de29b From d6828350f38a0235baf241fac4739938a1fed0c8 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:06:04 +0300 Subject: [PATCH 04/12] Remove accidental noop file --- noop | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 noop diff --git a/noop b/noop deleted file mode 100644 index e69de29b..00000000 From acdfcbe97128ba7c646600b57b68da4e23a4636b Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:06:11 +0300 Subject: [PATCH 05/12] noop2 --- noop2 | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 noop2 diff --git a/noop2 b/noop2 new file mode 100644 index 00000000..e69de29b From 12179325f6874ab1b4f802728f41a4633fd2ae01 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:06:19 +0300 Subject: [PATCH 06/12] Remove accidental noop2 file --- noop2 | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 noop2 diff --git a/noop2 b/noop2 deleted file mode 100644 index e69de29b..00000000 From b7aebf67547344cf6a38a78415fe4d421b7f4dbe Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:13:49 +0300 Subject: [PATCH 07/12] Preserve caller order in time performance data --- .../performance_data_times.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/rtichoke/performance_data/performance_data_times.py b/src/rtichoke/performance_data/performance_data_times.py index fdce7990..0b6e52e2 100644 --- a/src/rtichoke/performance_data/performance_data_times.py +++ b/src/rtichoke/performance_data/performance_data_times.py @@ -97,16 +97,26 @@ def prepare_performance_data_times( # 3. Turn AJ output into performance metrics performance_data = _turn_cumulative_aj_to_performance_data(cumulative_aj_data) - return performance_data.sort( - [ - "reference_group", - "fixed_time_horizon", - "censoring_heuristic", - "competing_heuristic", - "stratified_by", - "chosen_cutoff", - ] - ) + ordered_blocks = [] + for reference_group in probs: + for fixed_time_horizon in fixed_time_horizons: + for heuristics in heuristics_sets: + block = performance_data.filter( + (pl.col("reference_group") == reference_group) + & (pl.col("fixed_time_horizon") == float(fixed_time_horizon)) + & ( + pl.col("censoring_heuristic") + == heuristics["censoring_heuristic"] + ) + & ( + pl.col("competing_heuristic") + == heuristics["competing_heuristic"] + ) + ).sort(["stratified_by", "chosen_cutoff"]) + if block.height: + ordered_blocks.append(block) + + return pl.concat(ordered_blocks, how="vertical") def prepare_binned_classification_data_times( From 774618d5fd976eda22d7ce0225f2b4449e931452 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:14:00 +0300 Subject: [PATCH 08/12] Test R-style time performance ordering --- tests/test_time_performance_order.py | 67 +++++++++++++++++++--------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/tests/test_time_performance_order.py b/tests/test_time_performance_order.py index db52d289..38fead78 100644 --- a/tests/test_time_performance_order.py +++ b/tests/test_time_performance_order.py @@ -3,7 +3,7 @@ from rtichoke import prepare_performance_data_times -def test_prepare_performance_data_times_returns_deterministic_order(): +def test_prepare_performance_data_times_preserves_r_style_input_order(): probs = { "population_b": np.array([0.8, 0.2, 0.6, 0.4]), "population_a": np.array([0.7, 0.1, 0.9, 0.3]), @@ -16,32 +16,59 @@ def test_prepare_performance_data_times_returns_deterministic_order(): "population_b": np.array([2.0, 7.0, 4.0, 9.0]), "population_a": np.array([8.0, 3.0, 5.0, 10.0]), } + horizons = [8.0, 5.0] + heuristics_sets = [ + { + "censoring_heuristic": "excluded", + "competing_heuristic": "excluded", + }, + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + }, + ] result = prepare_performance_data_times( probs, reals, times, - fixed_time_horizons=[8.0, 5.0], - heuristics_sets=[ - { - "censoring_heuristic": "excluded", - "competing_heuristic": "excluded", - }, - { - "censoring_heuristic": "adjusted", - "competing_heuristic": "adjusted_as_negative", - }, - ], + fixed_time_horizons=horizons, + heuristics_sets=heuristics_sets, by=0.2, ) - sort_columns = [ - "reference_group", - "fixed_time_horizon", - "censoring_heuristic", - "competing_heuristic", - "stratified_by", - "chosen_cutoff", + block_keys = [] + for row in result.select( + [ + "reference_group", + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + ] + ).unique(maintain_order=True).iter_rows(): + block_keys.append(row) + + expected_block_keys = [ + ( + population, + horizon, + heuristics["censoring_heuristic"], + heuristics["competing_heuristic"], + ) + for population in probs + for horizon in horizons + for heuristics in heuristics_sets ] - assert result.equals(result.sort(sort_columns)) + assert block_keys == expected_block_keys + + for block in result.partition_by( + [ + "reference_group", + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + ], + maintain_order=True, + ): + assert block["chosen_cutoff"].to_list() == sorted(block["chosen_cutoff"].to_list()) From 4aea322d9a3ecd121b8248351cf0a4831587f79b Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:17:59 +0300 Subject: [PATCH 09/12] Order time performance data for cross-group comparison --- .../performance_data_times.py | 67 +++++++++++++------ 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/src/rtichoke/performance_data/performance_data_times.py b/src/rtichoke/performance_data/performance_data_times.py index 0b6e52e2..33558faf 100644 --- a/src/rtichoke/performance_data/performance_data_times.py +++ b/src/rtichoke/performance_data/performance_data_times.py @@ -97,26 +97,51 @@ def prepare_performance_data_times( # 3. Turn AJ output into performance metrics performance_data = _turn_cumulative_aj_to_performance_data(cumulative_aj_data) - ordered_blocks = [] - for reference_group in probs: - for fixed_time_horizon in fixed_time_horizons: - for heuristics in heuristics_sets: - block = performance_data.filter( - (pl.col("reference_group") == reference_group) - & (pl.col("fixed_time_horizon") == float(fixed_time_horizon)) - & ( - pl.col("censoring_heuristic") - == heuristics["censoring_heuristic"] - ) - & ( - pl.col("competing_heuristic") - == heuristics["competing_heuristic"] - ) - ).sort(["stratified_by", "chosen_cutoff"]) - if block.height: - ordered_blocks.append(block) - - return pl.concat(ordered_blocks, how="vertical") + group_order = {group: index for index, group in enumerate(probs)} + horizon_order = { + float(horizon): index for index, horizon in enumerate(fixed_time_horizons) + } + heuristic_order = { + ( + heuristics["censoring_heuristic"], + heuristics["competing_heuristic"], + ): index + for index, heuristics in enumerate(heuristics_sets) + } + + return ( + performance_data.with_columns( + pl.col("reference_group") + .replace_strict(group_order, default=len(group_order)) + .alias("_reference_group_order"), + pl.col("fixed_time_horizon") + .replace_strict(horizon_order, default=len(horizon_order)) + .alias("_fixed_time_horizon_order"), + pl.struct(["censoring_heuristic", "competing_heuristic"]) + .map_elements( + lambda row: heuristic_order.get( + (row["censoring_heuristic"], row["competing_heuristic"]), + len(heuristic_order), + ), + return_dtype=pl.Int64, + ) + .alias("_heuristic_order"), + ) + .sort( + [ + "_fixed_time_horizon_order", + "_heuristic_order", + "stratified_by", + "chosen_cutoff", + "_reference_group_order", + ] + ) + .drop( + "_reference_group_order", + "_fixed_time_horizon_order", + "_heuristic_order", + ) + ) def prepare_binned_classification_data_times( @@ -150,7 +175,7 @@ def prepare_binned_classification_data_times( A dictionary mapping model or dataset names (str) to their predicted probabilities. reals : Union[np.ndarray, Dict[str, np.ndarray]] - The true event statuses (e.g., 0=censored, 1=event, 2=competing). + The true event statuses (e.g., 0=censored, 1=event, 2=competing event). times : Union[np.ndarray, Dict[str, np.ndarray]] The event or censoring times. fixed_time_horizons : list[float] From 40a80cef4dd51978f9ca9c0227d2fb067111422f Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:18:35 +0300 Subject: [PATCH 10/12] Use native Polars ordering keys --- .../performance_data_times.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/rtichoke/performance_data/performance_data_times.py b/src/rtichoke/performance_data/performance_data_times.py index 33558faf..d3177406 100644 --- a/src/rtichoke/performance_data/performance_data_times.py +++ b/src/rtichoke/performance_data/performance_data_times.py @@ -79,7 +79,6 @@ def prepare_performance_data_times( thresholds and time horizons. It includes columns for cutoffs, time points, heuristics, and performance measures. """ - # 1. Get the underlying binned time-dependent classification data final_adjusted_data = prepare_binned_classification_data_times( probs=probs, reals=reals, @@ -91,10 +90,7 @@ def prepare_performance_data_times( risk_set_scope=["pooled_by_cutoff"], ) - # 2. Apply AJ cumulative machinery cumulative_aj_data = _calculate_cumulative_aj_data(final_adjusted_data) - - # 3. Turn AJ output into performance metrics performance_data = _turn_cumulative_aj_to_performance_data(cumulative_aj_data) group_order = {group: index for index, group in enumerate(probs)} @@ -102,10 +98,7 @@ def prepare_performance_data_times( float(horizon): index for index, horizon in enumerate(fixed_time_horizons) } heuristic_order = { - ( - heuristics["censoring_heuristic"], - heuristics["competing_heuristic"], - ): index + f'{heuristics["censoring_heuristic"]}\x1f{heuristics["competing_heuristic"]}': index for index, heuristics in enumerate(heuristics_sets) } @@ -117,14 +110,10 @@ def prepare_performance_data_times( pl.col("fixed_time_horizon") .replace_strict(horizon_order, default=len(horizon_order)) .alias("_fixed_time_horizon_order"), - pl.struct(["censoring_heuristic", "competing_heuristic"]) - .map_elements( - lambda row: heuristic_order.get( - (row["censoring_heuristic"], row["competing_heuristic"]), - len(heuristic_order), - ), - return_dtype=pl.Int64, + pl.concat_str( + ["censoring_heuristic", "competing_heuristic"], separator="\x1f" ) + .replace_strict(heuristic_order, default=len(heuristic_order)) .alias("_heuristic_order"), ) .sort( From 31fce8ec0317be1b1c1b3ddf785b3bd8af903b26 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:18:44 +0300 Subject: [PATCH 11/12] Test comparison-oriented time performance order --- tests/test_time_performance_order.py | 55 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/tests/test_time_performance_order.py b/tests/test_time_performance_order.py index 38fead78..bc51beee 100644 --- a/tests/test_time_performance_order.py +++ b/tests/test_time_performance_order.py @@ -3,7 +3,7 @@ from rtichoke import prepare_performance_data_times -def test_prepare_performance_data_times_preserves_r_style_input_order(): +def test_prepare_performance_data_times_groups_reference_groups_for_comparison(): probs = { "population_b": np.array([0.8, 0.2, 0.6, 0.4]), "population_a": np.array([0.7, 0.1, 0.9, 0.3]), @@ -37,38 +37,37 @@ def test_prepare_performance_data_times_preserves_r_style_input_order(): by=0.2, ) - block_keys = [] - for row in result.select( + comparison_keys = result.select( [ - "reference_group", "fixed_time_horizon", "censoring_heuristic", "competing_heuristic", + "stratified_by", + "chosen_cutoff", + "reference_group", ] - ).unique(maintain_order=True).iter_rows(): - block_keys.append(row) + ).iter_rows() - expected_block_keys = [ - ( - population, - horizon, - heuristics["censoring_heuristic"], - heuristics["competing_heuristic"], - ) - for population in probs - for horizon in horizons - for heuristics in heuristics_sets - ] + expected_group_order = list(probs) + previous_comparison_key = None + groups_for_key = [] - assert block_keys == expected_block_keys + for row in comparison_keys: + comparison_key = row[:-1] + reference_group = row[-1] - for block in result.partition_by( - [ - "reference_group", - "fixed_time_horizon", - "censoring_heuristic", - "competing_heuristic", - ], - maintain_order=True, - ): - assert block["chosen_cutoff"].to_list() == sorted(block["chosen_cutoff"].to_list()) + if previous_comparison_key is not None and comparison_key != previous_comparison_key: + assert groups_for_key == expected_group_order + groups_for_key = [] + + groups_for_key.append(reference_group) + previous_comparison_key = comparison_key + + assert groups_for_key == expected_group_order + + observed_horizons = ( + result.select("fixed_time_horizon") + .unique(maintain_order=True)["fixed_time_horizon"] + .to_list() + ) + assert observed_horizons == horizons From 9c7736050ff28bcba065f6283b6b763034e544b3 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 12:18:53 +0300 Subject: [PATCH 12/12] Simplify ordering regression assertion --- tests/test_time_performance_order.py | 34 ++++++++-------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/tests/test_time_performance_order.py b/tests/test_time_performance_order.py index bc51beee..1e877731 100644 --- a/tests/test_time_performance_order.py +++ b/tests/test_time_performance_order.py @@ -37,33 +37,17 @@ def test_prepare_performance_data_times_groups_reference_groups_for_comparison() by=0.2, ) - comparison_keys = result.select( - [ - "fixed_time_horizon", - "censoring_heuristic", - "competing_heuristic", - "stratified_by", - "chosen_cutoff", - "reference_group", - ] - ).iter_rows() - + comparison_columns = [ + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + "stratified_by", + "chosen_cutoff", + ] expected_group_order = list(probs) - previous_comparison_key = None - groups_for_key = [] - - for row in comparison_keys: - comparison_key = row[:-1] - reference_group = row[-1] - - if previous_comparison_key is not None and comparison_key != previous_comparison_key: - assert groups_for_key == expected_group_order - groups_for_key = [] - - groups_for_key.append(reference_group) - previous_comparison_key = comparison_key - assert groups_for_key == expected_group_order + for block in result.partition_by(comparison_columns, maintain_order=True): + assert block["reference_group"].to_list() == expected_group_order observed_horizons = ( result.select("fixed_time_horizon")