diff --git a/src/rtichoke/_decision_curve_viz_spec_v2.py b/src/rtichoke/_decision_curve_viz_spec_v2.py index ee836fe4..1139083a 100644 --- a/src/rtichoke/_decision_curve_viz_spec_v2.py +++ b/src/rtichoke/_decision_curve_viz_spec_v2.py @@ -34,6 +34,7 @@ def _decision_curve_v2_spec_from_performance_data( *, min_p_threshold: float = 0.0, max_p_threshold: float = 1.0, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical static Decision Curve v2 from production quantities.""" missing = _REQUIRED_COLUMNS.difference(performance_data.columns) @@ -163,7 +164,7 @@ def _decision_curve_v2_spec_from_performance_data( } ) - return { + spec = { "schemaVersion": "2.0", "type": "decision_curve", "evaluations": evaluations, @@ -178,6 +179,13 @@ def _decision_curve_v2_spec_from_performance_data( "yAxis": {"label": "Net benefit"}, "references": references, } + if operating_point_dimension is not None: + if operating_point_dimension not in {"probability_threshold", "ppcr"}: + raise ValueError( + f"Invalid operating_point_dimension: {operating_point_dimension!r}." + ) + spec["operatingPoint"] = {"dimension": operating_point_dimension} + return spec def _decision_curve_times_v2_spec_from_performance_data( @@ -186,6 +194,7 @@ def _decision_curve_times_v2_spec_from_performance_data( *, min_p_threshold: float = 0.0, max_p_threshold: float = 1.0, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical time-dependent Decision Curve v2 from production quantities.""" missing = _REQUIRED_TIMES_COLUMNS.difference(performance_data.columns) @@ -372,7 +381,7 @@ def _decision_curve_times_v2_spec_from_performance_data( } ) - return { + spec = { "schemaVersion": "2.0", "type": "decision_curve", "evaluations": evaluations, @@ -387,3 +396,10 @@ def _decision_curve_times_v2_spec_from_performance_data( "yAxis": {"label": "Net benefit"}, "references": references, } + if operating_point_dimension is not None: + if operating_point_dimension not in {"probability_threshold", "ppcr"}: + raise ValueError( + f"Invalid operating_point_dimension: {operating_point_dimension!r}." + ) + spec["operatingPoint"] = {"dimension": operating_point_dimension} + return spec diff --git a/src/rtichoke/_interventions_avoided_viz_spec_v2.py b/src/rtichoke/_interventions_avoided_viz_spec_v2.py index c7273757..d7a1a54f 100644 --- a/src/rtichoke/_interventions_avoided_viz_spec_v2.py +++ b/src/rtichoke/_interventions_avoided_viz_spec_v2.py @@ -34,6 +34,7 @@ def _interventions_avoided_v2_spec_from_performance_data( *, min_p_threshold: float = 0.0, max_p_threshold: float = 1.0, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical static Interventions Avoided from production quantities.""" missing = _REQUIRED_COLUMNS.difference(performance_data.columns) @@ -168,7 +169,7 @@ def _interventions_avoided_v2_spec_from_performance_data( } ) - return { + spec = { "schemaVersion": "2.0", "type": "interventions_avoided", "evaluations": evaluations, @@ -183,6 +184,13 @@ def _interventions_avoided_v2_spec_from_performance_data( "yAxis": {"label": "Interventions Avoided (per 100)"}, "references": references, } + if operating_point_dimension is not None: + if operating_point_dimension not in {"probability_threshold", "ppcr"}: + raise ValueError( + f"Invalid operating_point_dimension: {operating_point_dimension!r}." + ) + spec["operatingPoint"] = {"dimension": operating_point_dimension} + return spec def _interventions_avoided_times_v2_spec_from_performance_data( @@ -191,6 +199,7 @@ def _interventions_avoided_times_v2_spec_from_performance_data( *, min_p_threshold: float = 0.0, max_p_threshold: float = 1.0, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical time-dependent Interventions Avoided v2. @@ -363,7 +372,7 @@ def _interventions_avoided_times_v2_spec_from_performance_data( } ) - return { + spec = { "schemaVersion": "2.0", "type": "interventions_avoided", "evaluations": evaluations, @@ -378,3 +387,10 @@ def _interventions_avoided_times_v2_spec_from_performance_data( "yAxis": {"label": "Interventions Avoided (per 100)"}, "references": references, } + if operating_point_dimension is not None: + if operating_point_dimension not in {"probability_threshold", "ppcr"}: + raise ValueError( + f"Invalid operating_point_dimension: {operating_point_dimension!r}." + ) + spec["operatingPoint"] = {"dimension": operating_point_dimension} + return spec diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index b3aae624..b6b209d6 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -45,21 +45,38 @@ } +def _add_operating_point_to_spec( + spec: dict[str, object], operating_point_dimension: str | None +) -> None: + if operating_point_dimension is not None: + if operating_point_dimension not in {"probability_threshold", "ppcr"}: + raise ValueError( + f"Invalid operating_point_dimension: {operating_point_dimension!r}. " + "Must be 'probability_threshold' or 'ppcr'." + ) + spec["operatingPoint"] = {"dimension": operating_point_dimension} + + def _roc_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build a canonical ROC-v2 spec without recalculating statistics.""" return _curve_v2_spec_from_performance_data( performance_data, evaluation_metadata, chart_type="roc", + operating_point_dimension=operating_point_dimension, ) def _precision_recall_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical static Precision-Recall from production quantities.""" finite_rows = performance_data.filter( @@ -71,6 +88,7 @@ def _precision_recall_v2_spec_from_performance_data( finite_rows, evaluation_metadata, chart_type="precision_recall", + operating_point_dimension=operating_point_dimension, ) prevalence = _gains_population_prevalence(performance_data, evaluation_metadata) populations = list( @@ -92,6 +110,8 @@ def _precision_recall_v2_spec_from_performance_data( def _precision_recall_times_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical time-dependent precision-recall from calculated production data.""" required = _REQUIRED_PRECISION_RECALL_COLUMNS | { @@ -112,7 +132,7 @@ def _precision_recall_times_v2_spec_from_performance_data( & pl.col("ppv").is_finite() ) - rows = finite_rows.select( + selected_cols = [ "reference_group", "fixed_time_horizon", "censoring_heuristic", @@ -120,7 +140,11 @@ def _precision_recall_times_v2_spec_from_performance_data( "chosen_cutoff", "sensitivity", "ppv", - ).to_dicts() + ] + if "ppcr" in finite_rows.columns: + selected_cols.append("ppcr") + + rows = finite_rows.select(*selected_cols).to_dicts() row_groups = {str(row["reference_group"]) for row in rows} missing_metadata = row_groups.difference(evaluation_metadata) if missing_metadata: @@ -185,14 +209,15 @@ def _precision_recall_times_v2_spec_from_performance_data( str(row["censoring_heuristic"]), str(row["competing_heuristic"]), ) - data.append( - { - "seriesId": series_ids[key], - "cutoff": row["chosen_cutoff"], - "sensitivity": row["sensitivity"], - "ppv": row["ppv"], - } - ) + datum = { + "seriesId": series_ids[key], + "cutoff": row["chosen_cutoff"], + "sensitivity": row["sensitivity"], + "ppv": row["ppv"], + } + if "ppcr" in row: + datum["ppcr"] = row["ppcr"] + data.append(datum) risks = _gains_population_horizon_risk(performance_data, evaluation_metadata) references = [] @@ -208,7 +233,7 @@ def _precision_recall_times_v2_spec_from_performance_data( } ) - return { + spec = { "schemaVersion": "2.0", "type": "precision_recall", "evaluations": evaluations, @@ -220,17 +245,22 @@ def _precision_recall_times_v2_spec_from_performance_data( "yAxis": {"label": "Positive Predictive Value", "domain": [0, 1]}, "references": references, } + _add_operating_point_to_spec(spec, operating_point_dimension) + return spec def _gains_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build a canonical gains-v2 spec from production performance quantities.""" spec = _curve_v2_spec_from_performance_data( performance_data, evaluation_metadata, chart_type="gains", + operating_point_dimension=operating_point_dimension, ) prevalence = _gains_population_prevalence(performance_data, evaluation_metadata) @@ -260,6 +290,8 @@ def _gains_v2_spec_from_performance_data( def _lift_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build a canonical lift-v2 spec from production performance quantities.""" finite_rows = performance_data.filter( @@ -271,6 +303,7 @@ def _lift_v2_spec_from_performance_data( finite_rows, evaluation_metadata, chart_type="lift", + operating_point_dimension=operating_point_dimension, ) prevalence = _gains_population_prevalence(performance_data, evaluation_metadata) @@ -310,6 +343,8 @@ def _lift_v2_spec_from_performance_data( def _gains_times_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical time-dependent gains from calculated production data.""" required = _REQUIRED_GAINS_COLUMNS | { @@ -429,7 +464,7 @@ def _gains_times_v2_spec_from_performance_data( } ) - return { + spec = { "schemaVersion": "2.0", "type": "gains", "evaluations": evaluations, @@ -441,11 +476,15 @@ def _gains_times_v2_spec_from_performance_data( "yAxis": {"label": "Sensitivity", "domain": [0, 1]}, "references": references, } + _add_operating_point_to_spec(spec, operating_point_dimension) + return spec def _lift_times_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build canonical time-dependent lift from calculated production data.""" required = _REQUIRED_LIFT_COLUMNS | { @@ -571,7 +610,7 @@ def _lift_times_v2_spec_from_performance_data( } ) - return { + spec = { "schemaVersion": "2.0", "type": "lift", "evaluations": evaluations, @@ -586,6 +625,8 @@ def _lift_times_v2_spec_from_performance_data( }, "references": references, } + _add_operating_point_to_spec(spec, operating_point_dimension) + return spec def _gains_population_horizon_risk( @@ -693,6 +734,7 @@ def _curve_v2_spec_from_performance_data( evaluation_metadata: Mapping[str, _EvaluationMetadata], *, chart_type: str, + operating_point_dimension: str | None = "probability_threshold", ) -> dict[str, object]: """Build common canonical curve semantics without recalculating statistics. @@ -723,6 +765,9 @@ def _curve_v2_spec_from_performance_data( f"{chart_type.upper()} performance data is missing columns: {missing_columns}" ) + if "ppcr" in performance_data.columns and "ppcr" not in selected: + selected.append("ppcr") + rows = performance_data.select(*selected).to_dicts() row_groups = {str(row["reference_group"]) for row in rows} metadata_groups = set(evaluation_metadata) @@ -779,6 +824,8 @@ def _curve_v2_spec_from_performance_data( "seriesId": series_ids[str(row["reference_group"])], "cutoff": row["chosen_cutoff"], } + if "ppcr" in row: + datum["ppcr"] = row["ppcr"] if chart_type == "roc": datum["sensitivity"] = row["sensitivity"] datum["specificity"] = row["specificity"] @@ -787,14 +834,12 @@ def _curve_v2_spec_from_performance_data( datum["ppv"] = row["ppv"] elif chart_type == "gains": datum["sensitivity"] = row["sensitivity"] - datum["ppcr"] = row["ppcr"] elif chart_type == "lift": - datum["ppcr"] = row["ppcr"] datum["lift"] = row["lift"] data.append(datum) if chart_type == "roc": - return { + spec = { "schemaVersion": "2.0", "type": "roc", "evaluations": evaluations, @@ -806,9 +851,8 @@ def _curve_v2_spec_from_performance_data( "yAxis": {"label": "Sensitivity", "domain": [0, 1]}, "references": [{"type": "identity", "scope": "global"}], } - - if chart_type == "precision_recall": - return { + elif chart_type == "precision_recall": + spec = { "schemaVersion": "2.0", "type": "precision_recall", "evaluations": evaluations, @@ -820,9 +864,8 @@ def _curve_v2_spec_from_performance_data( "yAxis": {"label": "Positive Predictive Value", "domain": [0, 1]}, "references": [], } - - if chart_type == "gains": - return { + elif chart_type == "gains": + spec = { "schemaVersion": "2.0", "type": "gains", "evaluations": evaluations, @@ -834,19 +877,22 @@ def _curve_v2_spec_from_performance_data( "yAxis": {"label": "Sensitivity", "domain": [0, 1]}, "references": [], } + else: + spec = { + "schemaVersion": "2.0", + "type": "lift", + "evaluations": evaluations, + "series": series, + "data": data, + "x": "ppcr", + "y": "lift", + "xAxis": {"label": "Predicted Positives (Rate)", "domain": [0, 1]}, + "yAxis": { + "label": "Lift", + "domain": [0, _lift_y_axis_upper_bound(performance_data, [])], + }, + "references": [], + } - return { - "schemaVersion": "2.0", - "type": "lift", - "evaluations": evaluations, - "series": series, - "data": data, - "x": "ppcr", - "y": "lift", - "xAxis": {"label": "Predicted Positives (Rate)", "domain": [0, 1]}, - "yAxis": { - "label": "Lift", - "domain": [0, _lift_y_axis_upper_bound(performance_data, [])], - }, - "references": [], - } + _add_operating_point_to_spec(spec, operating_point_dimension) + return spec diff --git a/src/rtichoke/discrimination/gains.py b/src/rtichoke/discrimination/gains.py index a30b4d93..93e8c67d 100644 --- a/src/rtichoke/discrimination/gains.py +++ b/src/rtichoke/discrimination/gains.py @@ -21,6 +21,9 @@ _gains_times_v2_spec_from_performance_data, _gains_v2_spec_from_performance_data, ) +from rtichoke.discrimination.precision_recall import ( + _derive_op_dim_from_stratified_by, +) from rtichoke.performance_data.performance_data import prepare_performance_data from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata @@ -142,8 +145,11 @@ def create_gains_curve( by=by, ) evaluation_metadata = _build_evaluation_metadata(probs, reals, np.array([])) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) spec = _gains_v2_spec_from_performance_data( - performance_data, evaluation_metadata + performance_data, + evaluation_metadata, + operating_point_dimension=op_dim, ) return _render_gains_v2( spec, @@ -287,8 +293,11 @@ def create_gains_curve_times( stratified_by=stratified_by, ) evaluation_metadata = _build_evaluation_metadata(probs, reals, times) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) spec = _gains_times_v2_spec_from_performance_data( - performance_data, evaluation_metadata + performance_data, + evaluation_metadata, + operating_point_dimension=op_dim, ) return _render_gains_v2( spec, diff --git a/src/rtichoke/discrimination/lift.py b/src/rtichoke/discrimination/lift.py index f95dcc7f..5aec92a7 100644 --- a/src/rtichoke/discrimination/lift.py +++ b/src/rtichoke/discrimination/lift.py @@ -19,6 +19,9 @@ _lift_times_v2_spec_from_performance_data, _lift_v2_spec_from_performance_data, ) +from rtichoke.discrimination.precision_recall import ( + _derive_op_dim_from_stratified_by, +) from rtichoke.performance_data.performance_data import prepare_performance_data from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata @@ -97,8 +100,11 @@ def create_lift_curve( by=by, ) evaluation_metadata = _build_evaluation_metadata(probs, reals, np.array([])) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) spec = _lift_v2_spec_from_performance_data( - performance_data, evaluation_metadata + performance_data, + evaluation_metadata, + operating_point_dimension=op_dim, ) return _render_lift_v2( spec, @@ -243,8 +249,11 @@ def create_lift_curve_times( stratified_by=stratified_by, ) evaluation_metadata = _build_evaluation_metadata(probs, reals, times) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) spec = _lift_times_v2_spec_from_performance_data( - performance_data, evaluation_metadata + performance_data, + evaluation_metadata, + operating_point_dimension=op_dim, ) return _render_lift_v2( spec, diff --git a/src/rtichoke/discrimination/precision_recall.py b/src/rtichoke/discrimination/precision_recall.py index 1183e308..3d451d72 100644 --- a/src/rtichoke/discrimination/precision_recall.py +++ b/src/rtichoke/discrimination/precision_recall.py @@ -29,15 +29,23 @@ ) +def _derive_op_dim_from_stratified_by(stratified_by: Sequence[str]) -> str: + if "ppcr" in stratified_by: + return "ppcr" + return "probability_threshold" + + def _precision_recall_browser_chart( performance_data: pl.DataFrame, evaluation_metadata: dict[str, _EvaluationMetadata], *, size: int, + operating_point_dimension: str = "probability_threshold", ) -> RtichokeBrowserChart: spec = _precision_recall_v2_spec_from_performance_data( performance_data, evaluation_metadata, + operating_point_dimension=operating_point_dimension, ) return RtichokeBrowserChart(spec=spec, size=size) @@ -138,10 +146,12 @@ def create_precision_recall_curve( stratified_by=stratified_by, ) evaluation_metadata = _build_evaluation_metadata(probs, reals, np.array([])) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) return _precision_recall_browser_chart( performance_data, evaluation_metadata, size=size, + operating_point_dimension=op_dim, ) fig = _create_rtichoke_plotly_curve_binary( @@ -196,10 +206,12 @@ def plot_precision_recall_curve( "renderers." ) evaluation_metadata = _performance_data_evaluation_metadata(performance_data) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) return _precision_recall_browser_chart( performance_data, evaluation_metadata, size=size, + operating_point_dimension=op_dim, ) fig = _plot_rtichoke_curve_binary( @@ -300,9 +312,11 @@ def create_precision_recall_curve_times( stratified_by=stratified_by, ) evaluation_metadata = _build_evaluation_metadata(probs, reals, times) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) spec = _precision_recall_times_v2_spec_from_performance_data( performance_data, evaluation_metadata, + operating_point_dimension=op_dim, ) return RtichokeBrowserChart(spec=spec, size=size) diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index d0775a11..36b1509e 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -144,29 +144,33 @@ def create_summary_report_times( ) pr_thresh_spec = _precision_recall_times_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) gains_thresh_spec = _gains_times_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) lift_thresh_spec = _lift_times_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) pr_ppcr_spec = _precision_recall_times_v2_spec_from_performance_data( - perf_data_ppcr, metadata + perf_data_ppcr, metadata, operating_point_dimension="ppcr" ) gains_ppcr_spec = _gains_times_v2_spec_from_performance_data( - perf_data_ppcr, metadata + perf_data_ppcr, metadata, operating_point_dimension="ppcr" + ) + lift_ppcr_spec = _lift_times_v2_spec_from_performance_data( + perf_data_ppcr, metadata, operating_point_dimension="ppcr" ) - lift_ppcr_spec = _lift_times_v2_spec_from_performance_data(perf_data_ppcr, metadata) decision_curve_spec = _decision_curve_times_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) interventions_avoided_spec = ( _interventions_avoided_times_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, + metadata, + operating_point_dimension="probability_threshold", ) ) @@ -366,25 +370,37 @@ def _create_browser_summary_report( calibration_curve_list, metadata, calibration_type="discrete" ) - roc_thresh_spec = _roc_v2_spec_from_performance_data(perf_data_thresh, metadata) + roc_thresh_spec = _roc_v2_spec_from_performance_data( + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" + ) pr_thresh_spec = _precision_recall_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" + ) + gains_thresh_spec = _gains_v2_spec_from_performance_data( + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" + ) + lift_thresh_spec = _lift_v2_spec_from_performance_data( + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) - gains_thresh_spec = _gains_v2_spec_from_performance_data(perf_data_thresh, metadata) - lift_thresh_spec = _lift_v2_spec_from_performance_data(perf_data_thresh, metadata) - roc_ppcr_spec = _roc_v2_spec_from_performance_data(perf_data_ppcr, metadata) + roc_ppcr_spec = _roc_v2_spec_from_performance_data( + perf_data_ppcr, metadata, operating_point_dimension="ppcr" + ) pr_ppcr_spec = _precision_recall_v2_spec_from_performance_data( - perf_data_ppcr, metadata + perf_data_ppcr, metadata, operating_point_dimension="ppcr" + ) + gains_ppcr_spec = _gains_v2_spec_from_performance_data( + perf_data_ppcr, metadata, operating_point_dimension="ppcr" + ) + lift_ppcr_spec = _lift_v2_spec_from_performance_data( + perf_data_ppcr, metadata, operating_point_dimension="ppcr" ) - gains_ppcr_spec = _gains_v2_spec_from_performance_data(perf_data_ppcr, metadata) - lift_ppcr_spec = _lift_v2_spec_from_performance_data(perf_data_ppcr, metadata) decision_curve_spec = _decision_curve_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) interventions_avoided_spec = _interventions_avoided_v2_spec_from_performance_data( - perf_data_thresh, metadata + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) table_thresh_spec = _performance_table_spec_from_performance_data( diff --git a/src/rtichoke/utility/decision.py b/src/rtichoke/utility/decision.py index 05a300cd..75d568e7 100644 --- a/src/rtichoke/utility/decision.py +++ b/src/rtichoke/utility/decision.py @@ -38,12 +38,14 @@ def _decision_curve_browser_chart( size: int, min_p_threshold: float, max_p_threshold: float, + operating_point_dimension: str = "probability_threshold", ) -> RtichokeBrowserChart: spec = _decision_curve_v2_spec_from_performance_data( performance_data, evaluation_metadata, min_p_threshold=min_p_threshold, max_p_threshold=max_p_threshold, + operating_point_dimension=operating_point_dimension, ) return RtichokeBrowserChart(spec=spec, size=size) @@ -55,12 +57,14 @@ def _interventions_avoided_browser_chart( size: int, min_p_threshold: float, max_p_threshold: float, + operating_point_dimension: str = "probability_threshold", ) -> RtichokeBrowserChart: spec = _interventions_avoided_v2_spec_from_performance_data( performance_data, evaluation_metadata, min_p_threshold=min_p_threshold, max_p_threshold=max_p_threshold, + operating_point_dimension=operating_point_dimension, ) return RtichokeBrowserChart(spec=spec, size=size) @@ -320,6 +324,7 @@ def create_decision_curve_times( evaluation_metadata, min_p_threshold=min_p_threshold, max_p_threshold=max_p_threshold, + operating_point_dimension="probability_threshold", ) return RtichokeBrowserChart(spec=spec, size=size) diff --git a/tests/test_operating_point_metadata.py b/tests/test_operating_point_metadata.py new file mode 100644 index 00000000..b0a7d60b --- /dev/null +++ b/tests/test_operating_point_metadata.py @@ -0,0 +1,129 @@ +import numpy as np +import pytest + +from rtichoke._viz_spec_v2 import ( + _roc_v2_spec_from_performance_data, + _precision_recall_v2_spec_from_performance_data, + _gains_v2_spec_from_performance_data, + _lift_v2_spec_from_performance_data, + _precision_recall_times_v2_spec_from_performance_data, + _gains_times_v2_spec_from_performance_data, + _lift_times_v2_spec_from_performance_data, +) +from rtichoke._decision_curve_viz_spec_v2 import ( + _decision_curve_v2_spec_from_performance_data, + _decision_curve_times_v2_spec_from_performance_data, +) +from rtichoke._interventions_avoided_viz_spec_v2 import ( + _interventions_avoided_v2_spec_from_performance_data, + _interventions_avoided_times_v2_spec_from_performance_data, +) +from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) +from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata + + +@pytest.fixture +def sample_static_data(): + probs = {"Model A": np.array([0.1, 0.4, 0.7, 0.9])} + reals = {"Model A": np.array([0, 0, 1, 1], dtype=np.float64)} + perf_data = prepare_performance_data(probs, reals) + metadata = _build_evaluation_metadata(probs, reals, np.array([])) + return perf_data, metadata + + +@pytest.fixture +def sample_times_data(): + probs = {"Model A": np.array([0.1, 0.4, 0.7, 0.9, 0.2, 0.3, 0.8, 0.95])} + reals = {"Model A": np.array([0, 0, 1, 1, 0, 1, 0, 1], dtype=np.float64)} + times = {"Model A": np.array([1.0, 5.0, 3.0, 4.0, 2.0, 6.0, 7.0, 8.0])} + perf_data = prepare_performance_data_times( + probs, reals, times, fixed_time_horizons=[5.0] + ) + metadata = _build_evaluation_metadata(probs, reals, times) + return perf_data, metadata + + +def test_static_canonical_builders_operating_point_dimension(sample_static_data): + perf_data, metadata = sample_static_data + + # ROC + spec_roc_thresh = _roc_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_roc_thresh["operatingPoint"]["dimension"] == "probability_threshold" + + spec_roc_ppcr = _roc_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="ppcr" + ) + assert spec_roc_ppcr["operatingPoint"]["dimension"] == "ppcr" + + # PR + spec_pr_thresh = _precision_recall_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_pr_thresh["operatingPoint"]["dimension"] == "probability_threshold" + + spec_pr_ppcr = _precision_recall_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="ppcr" + ) + assert spec_pr_ppcr["operatingPoint"]["dimension"] == "ppcr" + + # Gains + spec_gains_thresh = _gains_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_gains_thresh["operatingPoint"]["dimension"] == "probability_threshold" + + # Lift + spec_lift_ppcr = _lift_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="ppcr" + ) + assert spec_lift_ppcr["operatingPoint"]["dimension"] == "ppcr" + + # Utility + spec_dc = _decision_curve_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_dc["operatingPoint"]["dimension"] == "probability_threshold" + + spec_ia = _interventions_avoided_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_ia["operatingPoint"]["dimension"] == "probability_threshold" + + +def test_time_canonical_builders_operating_point_dimension(sample_times_data): + perf_data, metadata = sample_times_data + + spec_pr_t = _precision_recall_times_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_pr_t["operatingPoint"]["dimension"] == "probability_threshold" + assert "horizon" in spec_pr_t["series"][0] + + spec_gains_t = _gains_times_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="ppcr" + ) + assert spec_gains_t["operatingPoint"]["dimension"] == "ppcr" + assert "horizon" in spec_gains_t["series"][0] + + spec_lift_t = _lift_times_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_lift_t["operatingPoint"]["dimension"] == "probability_threshold" + assert "horizon" in spec_lift_t["series"][0] + + spec_dc_t = _decision_curve_times_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_dc_t["operatingPoint"]["dimension"] == "probability_threshold" + assert "horizon" in spec_dc_t["series"][0] + + spec_ia_t = _interventions_avoided_times_v2_spec_from_performance_data( + perf_data, metadata, operating_point_dimension="probability_threshold" + ) + assert spec_ia_t["operatingPoint"]["dimension"] == "probability_threshold" + assert "horizon" in spec_ia_t["series"][0] diff --git a/tests/test_precision_recall_v2.py b/tests/test_precision_recall_v2.py index 9ec8cb75..73e1cb8f 100644 --- a/tests/test_precision_recall_v2.py +++ b/tests/test_precision_recall_v2.py @@ -67,7 +67,7 @@ def test_one_model_spec_is_pure_pass_through_with_deterministic_ids(): ] source_rows = performance_data.select( - "chosen_cutoff", "sensitivity", "ppv" + "chosen_cutoff", "ppcr", "sensitivity", "ppv" ).to_dicts() assert any(np.isnan(row["ppv"]) for row in source_rows) expected_rows = [ @@ -81,6 +81,7 @@ def test_one_model_spec_is_pure_pass_through_with_deterministic_ids(): { "seriesId": "series-1", "cutoff": row["chosen_cutoff"], + "ppcr": row["ppcr"], "sensitivity": row["sensitivity"], "ppv": row["ppv"], } diff --git a/tests/test_summary_report_times_browser.py b/tests/test_summary_report_times_browser.py index 1d1d1a41..bd66ad91 100644 --- a/tests/test_summary_report_times_browser.py +++ b/tests/test_summary_report_times_browser.py @@ -277,7 +277,7 @@ def test_summary_report_times_preserves_standalone_canonical_producers(tmp_path) perf_thresh, metadata ) expected_gains_ppcr = _gains_times_v2_spec_from_performance_data( - perf_ppcr, metadata + perf_ppcr, metadata, operating_point_dimension="ppcr" ) expected_lift_thresh = _lift_times_v2_spec_from_performance_data( perf_thresh, metadata