Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 138 additions & 0 deletions src/rtichoke/_viz_spec_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@
"sensitivity",
"specificity",
}
_REQUIRED_ROC_TIMES_COLUMNS = {
"reference_group",
"chosen_cutoff",
"sensitivity",
"specificity",
"false_positive_rate",
}
_REQUIRED_PRECISION_RECALL_COLUMNS = {
"reference_group",
"chosen_cutoff",
Expand Down Expand Up @@ -72,6 +79,137 @@ def _roc_v2_spec_from_performance_data(
)


def _roc_times_v2_spec_from_performance_data(
performance_data: pl.DataFrame,
evaluation_metadata: Mapping[str, _EvaluationMetadata],
*,
operating_point_dimension: str | None = "probability_threshold",
) -> dict[str, object]:
"""Build canonical time-dependent ROC from calculated production data."""
required = _REQUIRED_ROC_TIMES_COLUMNS | {
"fixed_time_horizon",
"censoring_heuristic",
"competing_heuristic",
}
missing = required.difference(performance_data.columns)
if missing:
raise ValueError(
"Time-dependent ROC performance data is missing columns: "
+ ", ".join(sorted(missing))
)

finite_rows = performance_data.filter(
pl.col("chosen_cutoff").is_finite()
& pl.col("sensitivity").is_finite()
& pl.col("specificity").is_finite()
& pl.col("false_positive_rate").is_finite()
)

selected_cols = [
"reference_group",
"fixed_time_horizon",
"censoring_heuristic",
"competing_heuristic",
"chosen_cutoff",
"sensitivity",
"specificity",
"false_positive_rate",
]
if "ppcr" in finite_rows.columns:
selected_cols.append("ppcr")

rows = finite_rows.select(*selected_cols).to_dicts()
row_groups = {str(row["reference_group"]) for row in rows}
missing_metadata = row_groups.difference(evaluation_metadata)
if missing_metadata:
raise ValueError(
"Time-dependent ROC rows are missing evaluation metadata: "
+ ", ".join(sorted(missing_metadata))
)

ordered_groups = [group for group in evaluation_metadata if group in row_groups]
evaluation_ids = {
group: f"evaluation-{index}"
for index, group in enumerate(ordered_groups, start=1)
}
evaluations = []
for group in ordered_groups:
metadata = evaluation_metadata[group]
evaluation: dict[str, object] = {
"id": evaluation_ids[group],
"population": metadata.population,
}
if metadata.model is not None:
evaluation["model"] = metadata.model
evaluations.append(evaluation)

series_keys = list(
dict.fromkeys(
(
str(row["reference_group"]),
float(row["fixed_time_horizon"]),
str(row["censoring_heuristic"]),
str(row["competing_heuristic"]),
)
for row in rows
)
)
series_ids = {
key: f"series-{index}" for index, key in enumerate(series_keys, start=1)
}
series = []
for key in series_keys:
group, horizon, _, _ = key
metadata = evaluation_metadata[group]
display_value = metadata.model or metadata.population
series.append(
{
"id": series_ids[key],
"evaluationId": evaluation_ids[group],
"horizon": horizon,
"display": {
"label": display_value,
"group": display_value,
"role": "model" if metadata.model is not None else "population",
},
}
)

data = []
for row in rows:
key = (
str(row["reference_group"]),
float(row["fixed_time_horizon"]),
str(row["censoring_heuristic"]),
str(row["competing_heuristic"]),
)
datum = {
"seriesId": series_ids[key],
"cutoff": row["chosen_cutoff"],
"sensitivity": row["sensitivity"],
"specificity": row["specificity"],
"false_positive_rate": row["false_positive_rate"],
}
if "ppcr" in row:
datum["ppcr"] = row["ppcr"]
data.append(datum)

spec = {
"schemaVersion": "2.0",
"type": "roc",
"evaluations": evaluations,
"series": series,
"data": data,
"x": "false_positive_rate",
"y": "sensitivity",
"xAxis": {"label": "False Positive Rate", "domain": [0, 1]},
"yAxis": {"label": "Sensitivity", "domain": [0, 1]},
"references": [{"type": "identity", "scope": "global"}],
}
_add_operating_point_to_spec(spec, operating_point_dimension)
return spec


def _precision_recall_v2_spec_from_performance_data(
performance_data: pl.DataFrame,
evaluation_metadata: Mapping[str, _EvaluationMetadata],
Expand Down
42 changes: 38 additions & 4 deletions src/rtichoke/discrimination/roc.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
A module for ROC Curves
"""

from typing import Dict, List, Union, Sequence
from typing import Any, Dict, List, Union, Sequence
from plotly.graph_objs._figure import Figure
from rtichoke.processing.binary_color_values import _apply_color_values_binary
from rtichoke.processing.plotly_helper_functions import (
Expand All @@ -12,6 +12,13 @@
from rtichoke.processing.time_reference_lines import (
_create_rtichoke_plotly_curve_times_reference_safe,
)
from rtichoke._renderers import RtichokeBrowserChart, _validate_renderer
from rtichoke._viz_spec_v2 import _roc_times_v2_spec_from_performance_data
from rtichoke.discrimination.precision_recall import _derive_op_dim_from_stratified_by
from rtichoke.performance_data.performance_data_times import (
prepare_performance_data_times,
)
from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata
import numpy as np
import polars as pl

Expand Down Expand Up @@ -164,7 +171,8 @@ def create_roc_curve_times(
"#D1603D",
"#585123",
],
) -> Figure:
renderer: str = "plotly",
) -> Any:
"""Creates a time-dependent Receiver Operating Characteristic (ROC) curve.

This function generates an ROC curve for time-to-event models. It evaluates
Expand All @@ -191,12 +199,38 @@ def create_roc_curve_times(
The width and height of the plot in pixels. Defaults to 600.
color_values : List[str], optional
A list of hex color strings for the plot lines.
renderer : {"plotly", "browser", "rtichoke_viz"}, optional
Rendering backend. ``"plotly"`` remains the default. ``"browser"`` and
its ``"rtichoke_viz"`` alias return a canonical offline browser chart.

Returns
-------
Figure
A Plotly ``Figure`` object representing the time-dependent ROC curve.
Figure or RtichokeBrowserChart
A Plotly ``Figure`` or canonical offline browser chart depending on ``renderer``.
"""
selected_renderer = _validate_renderer(renderer)
if selected_renderer != "plotly":
if selected_renderer == "matplotlib":
raise ValueError(
"ROC supports 'plotly', 'browser', and 'rtichoke_viz' renderers."
)
performance_data = prepare_performance_data_times(
probs,
reals,
times,
fixed_time_horizons=fixed_time_horizons,
heuristics_sets=heuristics_sets,
by=by,
stratified_by=stratified_by,
)
evaluation_metadata = _build_evaluation_metadata(probs, reals, times)
op_dim = _derive_op_dim_from_stratified_by(stratified_by)
spec = _roc_times_v2_spec_from_performance_data(
performance_data,
evaluation_metadata,
operating_point_dimension=op_dim,
)
return RtichokeBrowserChart(spec=spec, size=size)

fig = _create_rtichoke_plotly_curve_times_reference_safe(
probs,
Expand Down
9 changes: 9 additions & 0 deletions src/rtichoke/summary_report/summary_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
_lift_v2_spec_from_performance_data,
_precision_recall_times_v2_spec_from_performance_data,
_precision_recall_v2_spec_from_performance_data,
_roc_times_v2_spec_from_performance_data,
_roc_v2_spec_from_performance_data,
)
from rtichoke.calibration.calibration import (
Expand Down Expand Up @@ -143,6 +144,9 @@ def create_summary_report_times(
calibration_curve_list_discrete, metadata, calibration_type="discrete"
)

roc_thresh_spec = _roc_times_v2_spec_from_performance_data(
perf_data_thresh, metadata, operating_point_dimension="probability_threshold"
)
pr_thresh_spec = _precision_recall_times_v2_spec_from_performance_data(
perf_data_thresh, metadata, operating_point_dimension="probability_threshold"
)
Expand All @@ -153,6 +157,9 @@ def create_summary_report_times(
perf_data_thresh, metadata, operating_point_dimension="probability_threshold"
)

roc_ppcr_spec = _roc_times_v2_spec_from_performance_data(
perf_data_ppcr, metadata, operating_point_dimension="ppcr"
)
pr_ppcr_spec = _precision_recall_times_v2_spec_from_performance_data(
perf_data_ppcr, metadata, operating_point_dimension="ppcr"
)
Expand Down Expand Up @@ -206,6 +213,7 @@ def create_summary_report_times(
"id": "discrimination-probability-threshold",
"title": "By Probability Threshold",
"components": [
{"id": "roc", "title": "ROC", "spec": roc_thresh_spec},
{
"id": "precision-recall",
"title": "Precision-Recall",
Expand All @@ -223,6 +231,7 @@ def create_summary_report_times(
"id": "discrimination-ppcr",
"title": "By Predicted Positives Condition Rate (PPCR)",
"components": [
{"id": "roc-2", "title": "ROC", "spec": roc_ppcr_spec},
{
"id": "precision-recall-2",
"title": "Precision-Recall",
Expand Down
21 changes: 17 additions & 4 deletions tests/test_summary_report_times_browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,14 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path):

assert g1["id"] == "discrimination-probability-threshold"
assert g1["title"] == "By Probability Threshold"
assert [c["id"] for c in g1["components"]] == ["precision-recall", "gains", "lift"]
assert [c["id"] for c in g1["components"]] == [
"roc",
"precision-recall",
"gains",
"lift",
]
assert [c["title"] for c in g1["components"]] == [
"ROC",
"Precision-Recall",
"Gains",
"Lift",
Expand All @@ -184,11 +190,13 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path):
assert g2["id"] == "discrimination-ppcr"
assert g2["title"] == "By Predicted Positives Condition Rate (PPCR)"
assert [c["id"] for c in g2["components"]] == [
"roc-2",
"precision-recall-2",
"gains-2",
"lift-2",
]
assert [c["title"] for c in g2["components"]] == [
"ROC",
"Precision-Recall",
"Gains",
"Lift",
Expand Down Expand Up @@ -293,13 +301,18 @@ def test_summary_report_times_preserves_standalone_canonical_producers(tmp_path)
)

report_calib_smooth = report["sections"][0]["items"][0]["spec"]
report_pr_thresh = report["sections"][1]["items"][0]["components"][0]["spec"]
report_gains_ppcr = report["sections"][1]["items"][1]["components"][1]["spec"]
report_lift_thresh = report["sections"][1]["items"][0]["components"][2]["spec"]
report_roc_thresh = report["sections"][1]["items"][0]["components"][0]["spec"]
report_pr_thresh = report["sections"][1]["items"][0]["components"][1]["spec"]
report_roc_ppcr = report["sections"][1]["items"][1]["components"][0]["spec"]
report_gains_ppcr = report["sections"][1]["items"][1]["components"][2]["spec"]
report_lift_thresh = report["sections"][1]["items"][0]["components"][3]["spec"]
report_dc = report["sections"][2]["items"][0]["spec"]
report_ia = report["sections"][2]["items"][1]["spec"]
report_table_thresh = report["sections"][3]["items"][0]["components"][0]["spec"]

assert report_roc_thresh["operatingPoint"]["dimension"] == "probability_threshold"
assert report_roc_ppcr["operatingPoint"]["dimension"] == "ppcr"

assert report_calib_smooth == expected_calib_smooth
assert report_pr_thresh == expected_pr_thresh
assert report_gains_ppcr == expected_gains_ppcr
Expand Down
Loading
Loading