From 506074e7f59649b8bcb8821cf25e989825702da5 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:17:54 +0000 Subject: [PATCH] Add canonical browser support for time ROC Implement canonical adapter for time ROC and integrate standalone browser rendering and summary report insertion. Co-authored-by: uriahf <11351434+uriahf@users.noreply.github.com> --- src/rtichoke/_viz_spec_v2.py | 138 ++++++++++ src/rtichoke/discrimination/roc.py | 42 ++- src/rtichoke/summary_report/summary_report.py | 9 + tests/test_summary_report_times_browser.py | 21 +- tests/test_time_roc_v2.py | 250 ++++++++++++++++++ 5 files changed, 452 insertions(+), 8 deletions(-) create mode 100644 tests/test_time_roc_v2.py diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index b6b209d6..a861af66 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -19,6 +19,13 @@ "sensitivity", "specificity", } +_REQUIRED_ROC_TIMES_COLUMNS = { + "reference_group", + "chosen_cutoff", + "sensitivity", + "specificity", + "false_positive_rate", +} _REQUIRED_PRECISION_RECALL_COLUMNS = { "reference_group", "chosen_cutoff", @@ -72,6 +79,137 @@ def _roc_v2_spec_from_performance_data( ) +def _roc_times_v2_spec_from_performance_data( + performance_data: pl.DataFrame, + evaluation_metadata: Mapping[str, _EvaluationMetadata], + *, + operating_point_dimension: str | None = "probability_threshold", +) -> dict[str, object]: + """Build canonical time-dependent ROC from calculated production data.""" + required = _REQUIRED_ROC_TIMES_COLUMNS | { + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + } + missing = required.difference(performance_data.columns) + if missing: + raise ValueError( + "Time-dependent ROC performance data is missing columns: " + + ", ".join(sorted(missing)) + ) + + finite_rows = performance_data.filter( + pl.col("chosen_cutoff").is_finite() + & pl.col("sensitivity").is_finite() + & pl.col("specificity").is_finite() + & pl.col("false_positive_rate").is_finite() + ) + + selected_cols = [ + "reference_group", + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + "chosen_cutoff", + "sensitivity", + "specificity", + "false_positive_rate", + ] + if "ppcr" in finite_rows.columns: + selected_cols.append("ppcr") + + rows = finite_rows.select(*selected_cols).to_dicts() + row_groups = {str(row["reference_group"]) for row in rows} + missing_metadata = row_groups.difference(evaluation_metadata) + if missing_metadata: + raise ValueError( + "Time-dependent ROC rows are missing evaluation metadata: " + + ", ".join(sorted(missing_metadata)) + ) + + ordered_groups = [group for group in evaluation_metadata if group in row_groups] + evaluation_ids = { + group: f"evaluation-{index}" + for index, group in enumerate(ordered_groups, start=1) + } + evaluations = [] + for group in ordered_groups: + metadata = evaluation_metadata[group] + evaluation: dict[str, object] = { + "id": evaluation_ids[group], + "population": metadata.population, + } + if metadata.model is not None: + evaluation["model"] = metadata.model + evaluations.append(evaluation) + + series_keys = list( + dict.fromkeys( + ( + str(row["reference_group"]), + float(row["fixed_time_horizon"]), + str(row["censoring_heuristic"]), + str(row["competing_heuristic"]), + ) + for row in rows + ) + ) + series_ids = { + key: f"series-{index}" for index, key in enumerate(series_keys, start=1) + } + series = [] + for key in series_keys: + group, horizon, _, _ = key + metadata = evaluation_metadata[group] + display_value = metadata.model or metadata.population + series.append( + { + "id": series_ids[key], + "evaluationId": evaluation_ids[group], + "horizon": horizon, + "display": { + "label": display_value, + "group": display_value, + "role": "model" if metadata.model is not None else "population", + }, + } + ) + + data = [] + for row in rows: + key = ( + str(row["reference_group"]), + float(row["fixed_time_horizon"]), + str(row["censoring_heuristic"]), + str(row["competing_heuristic"]), + ) + datum = { + "seriesId": series_ids[key], + "cutoff": row["chosen_cutoff"], + "sensitivity": row["sensitivity"], + "specificity": row["specificity"], + "false_positive_rate": row["false_positive_rate"], + } + if "ppcr" in row: + datum["ppcr"] = row["ppcr"] + data.append(datum) + + spec = { + "schemaVersion": "2.0", + "type": "roc", + "evaluations": evaluations, + "series": series, + "data": data, + "x": "false_positive_rate", + "y": "sensitivity", + "xAxis": {"label": "False Positive Rate", "domain": [0, 1]}, + "yAxis": {"label": "Sensitivity", "domain": [0, 1]}, + "references": [{"type": "identity", "scope": "global"}], + } + _add_operating_point_to_spec(spec, operating_point_dimension) + return spec + + def _precision_recall_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], diff --git a/src/rtichoke/discrimination/roc.py b/src/rtichoke/discrimination/roc.py index 9884316f..f82f12d4 100644 --- a/src/rtichoke/discrimination/roc.py +++ b/src/rtichoke/discrimination/roc.py @@ -2,7 +2,7 @@ A module for ROC Curves """ -from typing import Dict, List, Union, Sequence +from typing import Any, Dict, List, Union, Sequence from plotly.graph_objs._figure import Figure from rtichoke.processing.binary_color_values import _apply_color_values_binary from rtichoke.processing.plotly_helper_functions import ( @@ -12,6 +12,13 @@ from rtichoke.processing.time_reference_lines import ( _create_rtichoke_plotly_curve_times_reference_safe, ) +from rtichoke._renderers import RtichokeBrowserChart, _validate_renderer +from rtichoke._viz_spec_v2 import _roc_times_v2_spec_from_performance_data +from rtichoke.discrimination.precision_recall import _derive_op_dim_from_stratified_by +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) +from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata import numpy as np import polars as pl @@ -164,7 +171,8 @@ def create_roc_curve_times( "#D1603D", "#585123", ], -) -> Figure: + renderer: str = "plotly", +) -> Any: """Creates a time-dependent Receiver Operating Characteristic (ROC) curve. This function generates an ROC curve for time-to-event models. It evaluates @@ -191,12 +199,38 @@ def create_roc_curve_times( The width and height of the plot in pixels. Defaults to 600. color_values : List[str], optional A list of hex color strings for the plot lines. + renderer : {"plotly", "browser", "rtichoke_viz"}, optional + Rendering backend. ``"plotly"`` remains the default. ``"browser"`` and + its ``"rtichoke_viz"`` alias return a canonical offline browser chart. Returns ------- - Figure - A Plotly ``Figure`` object representing the time-dependent ROC curve. + Figure or RtichokeBrowserChart + A Plotly ``Figure`` or canonical offline browser chart depending on ``renderer``. """ + selected_renderer = _validate_renderer(renderer) + if selected_renderer != "plotly": + if selected_renderer == "matplotlib": + raise ValueError( + "ROC supports 'plotly', 'browser', and 'rtichoke_viz' renderers." + ) + performance_data = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=fixed_time_horizons, + heuristics_sets=heuristics_sets, + by=by, + stratified_by=stratified_by, + ) + evaluation_metadata = _build_evaluation_metadata(probs, reals, times) + op_dim = _derive_op_dim_from_stratified_by(stratified_by) + spec = _roc_times_v2_spec_from_performance_data( + performance_data, + evaluation_metadata, + operating_point_dimension=op_dim, + ) + return RtichokeBrowserChart(spec=spec, size=size) fig = _create_rtichoke_plotly_curve_times_reference_safe( probs, diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index 36b1509e..a91e0225 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -34,6 +34,7 @@ _lift_v2_spec_from_performance_data, _precision_recall_times_v2_spec_from_performance_data, _precision_recall_v2_spec_from_performance_data, + _roc_times_v2_spec_from_performance_data, _roc_v2_spec_from_performance_data, ) from rtichoke.calibration.calibration import ( @@ -143,6 +144,9 @@ def create_summary_report_times( calibration_curve_list_discrete, metadata, calibration_type="discrete" ) + roc_thresh_spec = _roc_times_v2_spec_from_performance_data( + perf_data_thresh, metadata, operating_point_dimension="probability_threshold" + ) pr_thresh_spec = _precision_recall_times_v2_spec_from_performance_data( perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) @@ -153,6 +157,9 @@ def create_summary_report_times( perf_data_thresh, metadata, operating_point_dimension="probability_threshold" ) + roc_ppcr_spec = _roc_times_v2_spec_from_performance_data( + perf_data_ppcr, metadata, operating_point_dimension="ppcr" + ) pr_ppcr_spec = _precision_recall_times_v2_spec_from_performance_data( perf_data_ppcr, metadata, operating_point_dimension="ppcr" ) @@ -206,6 +213,7 @@ def create_summary_report_times( "id": "discrimination-probability-threshold", "title": "By Probability Threshold", "components": [ + {"id": "roc", "title": "ROC", "spec": roc_thresh_spec}, { "id": "precision-recall", "title": "Precision-Recall", @@ -223,6 +231,7 @@ def create_summary_report_times( "id": "discrimination-ppcr", "title": "By Predicted Positives Condition Rate (PPCR)", "components": [ + {"id": "roc-2", "title": "ROC", "spec": roc_ppcr_spec}, { "id": "precision-recall-2", "title": "Precision-Recall", diff --git a/tests/test_summary_report_times_browser.py b/tests/test_summary_report_times_browser.py index bd66ad91..49bedfdb 100644 --- a/tests/test_summary_report_times_browser.py +++ b/tests/test_summary_report_times_browser.py @@ -174,8 +174,14 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path): assert g1["id"] == "discrimination-probability-threshold" assert g1["title"] == "By Probability Threshold" - assert [c["id"] for c in g1["components"]] == ["precision-recall", "gains", "lift"] + assert [c["id"] for c in g1["components"]] == [ + "roc", + "precision-recall", + "gains", + "lift", + ] assert [c["title"] for c in g1["components"]] == [ + "ROC", "Precision-Recall", "Gains", "Lift", @@ -184,11 +190,13 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path): assert g2["id"] == "discrimination-ppcr" assert g2["title"] == "By Predicted Positives Condition Rate (PPCR)" assert [c["id"] for c in g2["components"]] == [ + "roc-2", "precision-recall-2", "gains-2", "lift-2", ] assert [c["title"] for c in g2["components"]] == [ + "ROC", "Precision-Recall", "Gains", "Lift", @@ -293,13 +301,18 @@ def test_summary_report_times_preserves_standalone_canonical_producers(tmp_path) ) report_calib_smooth = report["sections"][0]["items"][0]["spec"] - report_pr_thresh = report["sections"][1]["items"][0]["components"][0]["spec"] - report_gains_ppcr = report["sections"][1]["items"][1]["components"][1]["spec"] - report_lift_thresh = report["sections"][1]["items"][0]["components"][2]["spec"] + report_roc_thresh = report["sections"][1]["items"][0]["components"][0]["spec"] + report_pr_thresh = report["sections"][1]["items"][0]["components"][1]["spec"] + report_roc_ppcr = report["sections"][1]["items"][1]["components"][0]["spec"] + report_gains_ppcr = report["sections"][1]["items"][1]["components"][2]["spec"] + report_lift_thresh = report["sections"][1]["items"][0]["components"][3]["spec"] report_dc = report["sections"][2]["items"][0]["spec"] report_ia = report["sections"][2]["items"][1]["spec"] report_table_thresh = report["sections"][3]["items"][0]["components"][0]["spec"] + assert report_roc_thresh["operatingPoint"]["dimension"] == "probability_threshold" + assert report_roc_ppcr["operatingPoint"]["dimension"] == "ppcr" + assert report_calib_smooth == expected_calib_smooth assert report_pr_thresh == expected_pr_thresh assert report_gains_ppcr == expected_gains_ppcr diff --git a/tests/test_time_roc_v2.py b/tests/test_time_roc_v2.py new file mode 100644 index 00000000..0eb8f9ac --- /dev/null +++ b/tests/test_time_roc_v2.py @@ -0,0 +1,250 @@ +import shutil +import subprocess +from contextlib import contextmanager +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from typing import Iterator + +import numpy as np +import plotly.graph_objects as go +import pytest + +from rtichoke import create_roc_curve_times +from rtichoke._renderers import RtichokeBrowserChart +from rtichoke._viz_spec_v2 import ( + _roc_times_v2_spec_from_performance_data, +) +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) +from rtichoke.processing.evaluation_semantics import ( + _SHARED_POPULATION, + _build_evaluation_metadata, +) + +HORIZONS = [5.0, 10.0] +HEURISTICS = [ + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + } +] + + +def _shared_inputs(): + return ( + { + "Model A": np.array([0.05, 0.15, 0.35, 0.55, 0.75, 0.95]), + "Model B": np.array([0.10, 0.25, 0.45, 0.65, 0.80, 0.90]), + }, + np.array([1, 0, 1, 0, 1, 0]), + np.array([3.0, 12.0, 8.0, 13.0, 14.0, 15.0]), + ) + + +def _spec(probs, reals, times, operating_point_dimension="probability_threshold"): + performance = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + ) + return _roc_times_v2_spec_from_performance_data( + performance, + _build_evaluation_metadata(probs, reals, times), + operating_point_dimension=operating_point_dimension, + ) + + +def test_time_roc_uses_one_evaluation_and_series_per_model_horizon(): + probs, reals, times = _shared_inputs() + spec = _spec(probs, reals, times) + + assert spec["schemaVersion"] == "2.0" + assert spec["type"] == "roc" + assert spec["x"] == "false_positive_rate" + assert spec["y"] == "sensitivity" + assert spec["xAxis"] == {"label": "False Positive Rate", "domain": [0, 1]} + assert spec["yAxis"] == {"label": "Sensitivity", "domain": [0, 1]} + assert len(spec["evaluations"]) == 2 + assert {evaluation["population"] for evaluation in spec["evaluations"]} == { + _SHARED_POPULATION + } + assert len(spec["series"]) == 4 + assert { + (series["display"]["group"], series["horizon"]) for series in spec["series"] + } == { + ("Model A", 5.0), + ("Model A", 10.0), + ("Model B", 5.0), + ("Model B", 10.0), + } + + # Reference line + assert spec["references"] == [{"type": "identity", "scope": "global"}] + + # Check data payload fields + first_datum = spec["data"][0] + assert "cutoff" in first_datum + assert "sensitivity" in first_datum + assert "specificity" in first_datum + assert "false_positive_rate" in first_datum + assert "ppcr" in first_datum + assert first_datum["false_positive_rate"] == 1.0 - first_datum["specificity"] + + +def test_time_roc_operating_point_metadata(): + probs, reals, times = _shared_inputs() + + spec_thresh = _spec( + probs, reals, times, operating_point_dimension="probability_threshold" + ) + assert spec_thresh["operatingPoint"] == {"dimension": "probability_threshold"} + + spec_ppcr = _spec(probs, reals, times, operating_point_dimension="ppcr") + assert spec_ppcr["operatingPoint"] == {"dimension": "ppcr"} + + +def test_time_roc_renderers_preserve_plotly_default_and_browser(tmp_path: Path): + probs, reals, times = _shared_inputs() + default = create_roc_curve_times( + probs, reals, times, HORIZONS, heuristics_sets=HEURISTICS, by=0.25 + ) + assert isinstance(default, go.Figure) + + browser = create_roc_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="browser", + ) + assert isinstance(browser, RtichokeBrowserChart) + html = browser.write_html(tmp_path / "roc-times.html").read_text(encoding="utf-8") + assert "renderRocV2" in html + assert {series["horizon"] for series in browser.spec["series"]} == set(HORIZONS) + + rtichoke_viz_browser = create_roc_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="rtichoke_viz", + ) + assert isinstance(rtichoke_viz_browser, RtichokeBrowserChart) + + with pytest.raises(ValueError, match="ROC supports"): + create_roc_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="matplotlib", + ) + + +def _chrome_executable() -> str: + for candidate in ( + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + ): + executable = shutil.which(candidate) + if executable is not None: + return executable + pytest.skip("headless Chrome/Chromium is not available") + raise RuntimeError("unreachable") + + +@contextmanager +def _serve(directory: Path) -> Iterator[str]: + handler = partial(SimpleHTTPRequestHandler, directory=str(directory)) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join() + server.server_close() + + +def test_time_roc_browser_chart_executes_and_switches_horizons_in_chrome( + tmp_path: Path, +): + probs, reals, times = _shared_inputs() + chart = create_roc_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="browser", + ) + chart.write_html(tmp_path / "roc-times.html") + + script_path = tmp_path / "test_switch.js" + script_path.write_text( + """ + import { renderRocV2 } from "./rtichoke-viz.js"; + const spec = JSON.parse(document.querySelector("#rtichoke-spec").textContent); + const chart = renderRocV2(spec, { width: 600, height: 600 }); + document.querySelector("#rtichoke-chart").replaceChildren(chart); + + const select = chart.querySelector("select"); + if (!select) { + console.error("Horizon selector missing!"); + } else { + console.log("INITIAL_HORIZON:" + select.value); + select.value = "10"; + select.dispatchEvent(new Event("change", { bubbles: true })); + console.log("SWITCHED_HORIZON:" + select.value); + } + """, + encoding="utf-8", + ) + + html_file = tmp_path / "roc-times.html" + content = html_file.read_text(encoding="utf-8") + content = content.replace( + "", + '', + ) + html_file.write_text(content, encoding="utf-8") + + with _serve(tmp_path) as base_url: + result = subprocess.run( + [ + _chrome_executable(), + "--headless=new", + "--no-sandbox", + "--disable-gpu", + "--enable-logging=stderr", + "--log-level=0", + "--dump-dom", + f"{base_url}/roc-times.html", + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert "INITIAL_HORIZON:5" in result.stderr, result.stderr + assert "SWITCHED_HORIZON:10" in result.stderr, result.stderr + assert "