From 4f433605b78f4229593a2ef2d5466678c37c75d0 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 17:42:55 +0300 Subject: [PATCH 1/9] test: audit binary and time reference-line isolation --- tests/test_reference_line_audit.py | 101 +++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/test_reference_line_audit.py diff --git a/tests/test_reference_line_audit.py b/tests/test_reference_line_audit.py new file mode 100644 index 00000000..8b61b918 --- /dev/null +++ b/tests/test_reference_line_audit.py @@ -0,0 +1,101 @@ +import numpy as np +import pytest + +from rtichoke import ( + create_decision_curve, + create_gains_curve, + create_lift_curve, + create_precision_recall_curve, + create_gains_curve_times, +) + + +def _binary_inputs(): + probs = { + "population_a": np.array([0.05, 0.15, 0.35, 0.55, 0.75, 0.95]), + "population_b": np.array([0.10, 0.30, 0.50, 0.70]), + } + reals = { + "population_a": np.array([1, 0, 1, 0, 1, 0]), + "population_b": np.array([1, 1, 0, 1]), + } + return probs, reals + + +def _visible_trace(fig, name): + matches = [trace for trace in fig.data if trace.name == name and trace.visible is True] + assert len(matches) == 1 + return matches[0] + + +def test_binary_precision_recall_reference_is_population_specific(): + probs, reals = _binary_inputs() + fig = create_precision_recall_curve(probs, reals, by=0.1) + + a = _visible_trace(fig, "random_guess_population_a") + b = _visible_trace(fig, "random_guess_population_b") + + assert float(a.y[0]) == pytest.approx(3 / 6) + assert float(b.y[0]) == pytest.approx(3 / 4) + + +def test_binary_lift_reference_is_population_specific(): + probs, reals = _binary_inputs() + fig = create_lift_curve(probs, reals, by=0.1) + + a = _visible_trace(fig, "perfect_model_population_a") + b = _visible_trace(fig, "perfect_model_population_b") + + assert float(a.y[0]) == pytest.approx(2.0) + assert float(b.y[0]) == pytest.approx(4 / 3) + + +def test_binary_decision_reference_is_population_specific(): + probs, reals = _binary_inputs() + fig = create_decision_curve( + probs, reals, by=0.1, min_p_threshold=0.1, max_p_threshold=0.9 + ) + + a = _visible_trace(fig, "treat_all_population_a") + b = _visible_trace(fig, "treat_all_population_b") + + x = float(a.x[0]) + assert x == pytest.approx(0.1) + assert float(b.x[0]) == pytest.approx(x) + assert float(a.y[0]) == pytest.approx(0.5 - 0.5 * x / (1 - x)) + assert float(b.y[0]) == pytest.approx(0.75 - 0.25 * x / (1 - x)) + + +def test_binary_gains_reference_is_population_specific(): + probs, reals = _binary_inputs() + fig = create_gains_curve(probs, reals, by=0.1) + + a = _visible_trace(fig, "perfect_model_population_a") + b = _visible_trace(fig, "perfect_model_population_b") + + assert float(a.y[0]) == pytest.approx(0.01 / 0.5) + assert float(b.y[0]) == pytest.approx(0.01 / 0.75) + + +def test_time_gains_reference_is_population_and_horizon_specific(): + probs = { + "population_a": np.array([0.05, 0.15, 0.35, 0.55, 0.75, 0.95]), + "population_b": np.array([0.10, 0.30, 0.50, 0.70]), + } + reals = { + "population_a": np.array([1, 0, 1, 0, 1, 0]), + "population_b": np.array([1, 1, 0, 1]), + } + times = { + "population_a": np.array([3.0, 11.0, 7.0, 12.0, 13.0, 14.0]), + "population_b": np.array([4.0, 8.0, 11.0, 12.0]), + } + fig = create_gains_curve_times( + probs, reals, times, fixed_time_horizons=[5.0, 10.0], by=0.1 + ) + + traces = {(t.name, t.visible): t for t in fig.data} + assert float(traces[("perfect_model_population_a", True)].y[0]) == pytest.approx(0.06) + assert float(traces[("perfect_model_population_b", True)].y[0]) == pytest.approx(0.04) + assert float(traces[("perfect_model_population_a", False)].y[0]) == pytest.approx(0.03) + assert float(traces[("perfect_model_population_b", False)].y[0]) == pytest.approx(0.02) From c550257701a5af9d5f327d688fb5c9f6aa54fe4a Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 17:47:27 +0300 Subject: [PATCH 2/9] test: correct Plotly visibility and gains grid assumptions --- tests/test_reference_line_audit.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_reference_line_audit.py b/tests/test_reference_line_audit.py index 8b61b918..fb892ae1 100644 --- a/tests/test_reference_line_audit.py +++ b/tests/test_reference_line_audit.py @@ -23,7 +23,13 @@ def _binary_inputs(): def _visible_trace(fig, name): - matches = [trace for trace in fig.data if trace.name == name and trace.visible is True] + matches = [trace for trace in fig.data if trace.name == name and trace.visible is not False] + assert len(matches) == 1 + return matches[0] + + +def _y_at_x(trace, x): + matches = [float(y) for tx, y in zip(trace.x, trace.y) if float(tx) == pytest.approx(x)] assert len(matches) == 1 return matches[0] @@ -73,8 +79,8 @@ def test_binary_gains_reference_is_population_specific(): a = _visible_trace(fig, "perfect_model_population_a") b = _visible_trace(fig, "perfect_model_population_b") - assert float(a.y[0]) == pytest.approx(0.01 / 0.5) - assert float(b.y[0]) == pytest.approx(0.01 / 0.75) + assert _y_at_x(a, 0.01) == pytest.approx(0.01 / 0.5) + assert _y_at_x(b, 0.01) == pytest.approx(0.01 / 0.75) def test_time_gains_reference_is_population_and_horizon_specific(): @@ -95,7 +101,7 @@ def test_time_gains_reference_is_population_and_horizon_specific(): ) traces = {(t.name, t.visible): t for t in fig.data} - assert float(traces[("perfect_model_population_a", True)].y[0]) == pytest.approx(0.06) - assert float(traces[("perfect_model_population_b", True)].y[0]) == pytest.approx(0.04) - assert float(traces[("perfect_model_population_a", False)].y[0]) == pytest.approx(0.03) - assert float(traces[("perfect_model_population_b", False)].y[0]) == pytest.approx(0.02) + assert _y_at_x(traces[("perfect_model_population_a", True)], 0.01) == pytest.approx(0.06) + assert _y_at_x(traces[("perfect_model_population_b", True)], 0.01) == pytest.approx(0.04) + assert _y_at_x(traces[("perfect_model_population_a", False)], 0.01) == pytest.approx(0.03) + assert _y_at_x(traces[("perfect_model_population_b", False)], 0.01) == pytest.approx(0.02) From 42d1c74c1ec3c25ce87769e851376322e18ce66e Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 17:53:13 +0300 Subject: [PATCH 3/9] fix: honor binary decision threshold range --- src/rtichoke/utility/decision.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/rtichoke/utility/decision.py b/src/rtichoke/utility/decision.py index 7c3c0615..ea2cff62 100644 --- a/src/rtichoke/utility/decision.py +++ b/src/rtichoke/utility/decision.py @@ -5,13 +5,11 @@ from typing import Dict, List, Sequence, Union from plotly.graph_objs._figure import Figure from rtichoke.processing.binary_color_values import _apply_color_values_binary -from rtichoke.processing.plotly_helper_functions import ( - _create_rtichoke_plotly_curve_binary, - _plot_rtichoke_curve_binary, -) +from rtichoke.processing.plotly_helper_functions import _plot_rtichoke_curve_binary from rtichoke.processing.time_reference_lines import ( _create_rtichoke_plotly_curve_times_reference_safe, ) +from rtichoke.performance_data.performance_data import prepare_performance_data import numpy as np import polars as pl @@ -90,14 +88,17 @@ def create_decision_curve( else: curve = "interventions avoided" - fig = _create_rtichoke_plotly_curve_binary( - probs, - reals, - by=by, + performance_data = prepare_performance_data( + probs=probs, + reals=reals, stratified_by=stratified_by, - size=size, - color_values=color_values, + by=by, + ) + fig = _plot_rtichoke_curve_binary( + performance_data=performance_data, + stratified_by=stratified_by[0], curve=curve, + size=size, min_p_threshold=min_p_threshold, max_p_threshold=max_p_threshold, ) @@ -218,7 +219,7 @@ def create_decision_curve_times( max_p_threshold : float, optional The maximum probability threshold to plot. Defaults to 1. by : float, optional - The step size for the probability thresholds. Defaults to 0.01. + The step size for probability thresholds. Defaults to 0.01. stratified_by : Sequence[str], optional Variables for stratification. Defaults to ``["probability_threshold"]``. size : int, optional @@ -252,4 +253,4 @@ def create_decision_curve_times( max_p_threshold=max_p_threshold, ) - return fig + return fig \ No newline at end of file From 8ccf3bc6bee4a0fc1b011a545ac2b6f2703b9576 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 17:53:43 +0300 Subject: [PATCH 4/9] test: cover decision threshold range variants --- tests/test_reference_line_audit.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_reference_line_audit.py b/tests/test_reference_line_audit.py index fb892ae1..b036c9c3 100644 --- a/tests/test_reference_line_audit.py +++ b/tests/test_reference_line_audit.py @@ -67,11 +67,32 @@ def test_binary_decision_reference_is_population_specific(): x = float(a.x[0]) assert x == pytest.approx(0.1) + assert float(a.x[-1]) == pytest.approx(0.9) assert float(b.x[0]) == pytest.approx(x) assert float(a.y[0]) == pytest.approx(0.5 - 0.5 * x / (1 - x)) assert float(b.y[0]) == pytest.approx(0.75 - 0.25 * x / (1 - x)) +def test_binary_interventions_avoided_honors_threshold_range(): + probs, reals = _binary_inputs() + fig = create_decision_curve( + probs, + reals, + decision_type="interventions avoided", + by=0.1, + min_p_threshold=0.1, + max_p_threshold=0.9, + ) + + reference = next( + trace + for trace in fig.data + if trace.name.startswith("treat_none") and trace.visible is not False + ) + assert float(reference.x[0]) == pytest.approx(0.1) + assert float(reference.x[-1]) == pytest.approx(0.9) + + def test_binary_gains_reference_is_population_specific(): probs, reals = _binary_inputs() fig = create_gains_curve(probs, reals, by=0.1) From 5ebe2b3cff92d74b6de1bce6b487ee60605bb6f0 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 17:54:29 +0300 Subject: [PATCH 5/9] chore: keep decision diff scoped --- src/rtichoke/utility/decision.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rtichoke/utility/decision.py b/src/rtichoke/utility/decision.py index ea2cff62..11feb847 100644 --- a/src/rtichoke/utility/decision.py +++ b/src/rtichoke/utility/decision.py @@ -219,7 +219,7 @@ def create_decision_curve_times( max_p_threshold : float, optional The maximum probability threshold to plot. Defaults to 1. by : float, optional - The step size for probability thresholds. Defaults to 0.01. + The step size for the probability thresholds. Defaults to 0.01. stratified_by : Sequence[str], optional Variables for stratification. Defaults to ``["probability_threshold"]``. size : int, optional @@ -253,4 +253,4 @@ def create_decision_curve_times( max_p_threshold=max_p_threshold, ) - return fig \ No newline at end of file + return fig From f06259615a1db1a4254e467ab815a9163c7f498b Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 18:12:58 +0300 Subject: [PATCH 6/9] test: check non-empty interventions traces --- tests/test_reference_line_audit.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_reference_line_audit.py b/tests/test_reference_line_audit.py index b036c9c3..f75616a8 100644 --- a/tests/test_reference_line_audit.py +++ b/tests/test_reference_line_audit.py @@ -84,13 +84,13 @@ def test_binary_interventions_avoided_honors_threshold_range(): max_p_threshold=0.9, ) - reference = next( - trace - for trace in fig.data - if trace.name.startswith("treat_none") and trace.visible is not False - ) - assert float(reference.x[0]) == pytest.approx(0.1) - assert float(reference.x[-1]) == pytest.approx(0.9) + nonempty_visible = [ + trace for trace in fig.data if trace.visible is not False and len(trace.x) > 0 + ] + assert nonempty_visible + for trace in nonempty_visible: + assert min(float(x) for x in trace.x) >= pytest.approx(0.1) + assert max(float(x) for x in trace.x) <= pytest.approx(0.9) def test_binary_gains_reference_is_population_specific(): From 58d58bad6063724e53551b77664e5c74a55bb595 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 18:13:25 +0300 Subject: [PATCH 7/9] test: use numeric tolerance for range --- tests/test_reference_line_audit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_reference_line_audit.py b/tests/test_reference_line_audit.py index f75616a8..ce28c162 100644 --- a/tests/test_reference_line_audit.py +++ b/tests/test_reference_line_audit.py @@ -89,8 +89,8 @@ def test_binary_interventions_avoided_honors_threshold_range(): ] assert nonempty_visible for trace in nonempty_visible: - assert min(float(x) for x in trace.x) >= pytest.approx(0.1) - assert max(float(x) for x in trace.x) <= pytest.approx(0.9) + assert min(float(x) for x in trace.x) >= 0.1 - 1e-12 + assert max(float(x) for x in trace.x) <= 0.9 + 1e-12 def test_binary_gains_reference_is_population_specific(): From a7319fd22437c6457f71358d4e656f634f76d1a5 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 18:43:24 +0300 Subject: [PATCH 8/9] test: assert displayed interventions-avoided threshold range --- tests/test_reference_line_audit.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_reference_line_audit.py b/tests/test_reference_line_audit.py index ce28c162..012eec28 100644 --- a/tests/test_reference_line_audit.py +++ b/tests/test_reference_line_audit.py @@ -73,7 +73,7 @@ def test_binary_decision_reference_is_population_specific(): assert float(b.y[0]) == pytest.approx(0.75 - 0.25 * x / (1 - x)) -def test_binary_interventions_avoided_honors_threshold_range(): +def test_binary_interventions_avoided_honors_displayed_threshold_range(): probs, reals = _binary_inputs() fig = create_decision_curve( probs, @@ -84,13 +84,7 @@ def test_binary_interventions_avoided_honors_threshold_range(): max_p_threshold=0.9, ) - nonempty_visible = [ - trace for trace in fig.data if trace.visible is not False and len(trace.x) > 0 - ] - assert nonempty_visible - for trace in nonempty_visible: - assert min(float(x) for x in trace.x) >= 0.1 - 1e-12 - assert max(float(x) for x in trace.x) <= 0.9 + 1e-12 + assert tuple(float(x) for x in fig.layout.xaxis.range) == pytest.approx((0.1, 0.9)) def test_binary_gains_reference_is_population_specific(): From 88156f08dbc9bd6bcc76528076cf95042c78386a Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Thu, 20 Aug 2026 18:50:27 +0300 Subject: [PATCH 9/9] fix: enforce binary decision display range --- src/rtichoke/utility/decision.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rtichoke/utility/decision.py b/src/rtichoke/utility/decision.py index 11feb847..b47f1296 100644 --- a/src/rtichoke/utility/decision.py +++ b/src/rtichoke/utility/decision.py @@ -102,6 +102,7 @@ def create_decision_curve( min_p_threshold=min_p_threshold, max_p_threshold=max_p_threshold, ) + fig.update_xaxes(range=[min_p_threshold, max_p_threshold]) return _apply_color_values_binary(fig, color_values) @@ -152,6 +153,7 @@ def plot_decision_curve( min_p_threshold=min_p_threshold, max_p_threshold=max_p_threshold, ) + fig.update_xaxes(range=[min_p_threshold, max_p_threshold]) return fig