From c3a8a9cec547589b9365e994725e7cce76872a97 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:52:19 +0000 Subject: [PATCH] Adopt time-dependent Precision-Recall browser rendering Add browser opt-in (`renderer="browser"` and `renderer="rtichoke_viz"`) to `create_precision_recall_curve_times()`. Construct canonical `PrecisionRecallV2Spec` specifications from time-dependent performance data, preserving existing Plotly defaults and statistical calculations. Co-authored-by: uriahf <11351434+uriahf@users.noreply.github.com> --- src/rtichoke/_viz_spec_v2.py | 133 ++++++++ .../discrimination/precision_recall.py | 41 ++- tests/test_precision_recall_v2.py | 5 +- tests/test_time_precision_recall_v2.py | 290 ++++++++++++++++++ 4 files changed, 462 insertions(+), 7 deletions(-) create mode 100644 tests/test_time_precision_recall_v2.py diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index fc32cec5..db8b2de5 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -89,6 +89,139 @@ def _precision_recall_v2_spec_from_performance_data( return spec +def _precision_recall_times_v2_spec_from_performance_data( + performance_data: pl.DataFrame, + evaluation_metadata: Mapping[str, _EvaluationMetadata], +) -> dict[str, object]: + """Build canonical time-dependent precision-recall from calculated production data.""" + required = _REQUIRED_PRECISION_RECALL_COLUMNS | { + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + } + missing = required.difference(performance_data.columns) + if missing: + raise ValueError( + "Time-dependent precision-recall performance data is missing columns: " + + ", ".join(sorted(missing)) + ) + + finite_rows = performance_data.filter( + pl.col("chosen_cutoff").is_finite() + & pl.col("sensitivity").is_finite() + & pl.col("ppv").is_finite() + ) + + rows = finite_rows.select( + "reference_group", + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + "chosen_cutoff", + "sensitivity", + "ppv", + ).to_dicts() + row_groups = {str(row["reference_group"]) for row in rows} + missing_metadata = row_groups.difference(evaluation_metadata) + if missing_metadata: + raise ValueError( + "Time-dependent precision-recall rows are missing evaluation metadata: " + + ", ".join(sorted(missing_metadata)) + ) + + ordered_groups = [group for group in evaluation_metadata if group in row_groups] + evaluation_ids = { + group: f"evaluation-{index}" + for index, group in enumerate(ordered_groups, start=1) + } + evaluations = [] + for group in ordered_groups: + metadata = evaluation_metadata[group] + evaluation: dict[str, object] = { + "id": evaluation_ids[group], + "population": metadata.population, + } + if metadata.model is not None: + evaluation["model"] = metadata.model + evaluations.append(evaluation) + + series_keys = list( + dict.fromkeys( + ( + str(row["reference_group"]), + float(row["fixed_time_horizon"]), + str(row["censoring_heuristic"]), + str(row["competing_heuristic"]), + ) + for row in rows + ) + ) + series_ids = { + key: f"series-{index}" for index, key in enumerate(series_keys, start=1) + } + series = [] + for key in series_keys: + group, horizon, _, _ = key + metadata = evaluation_metadata[group] + display_value = metadata.model or metadata.population + series.append( + { + "id": series_ids[key], + "evaluationId": evaluation_ids[group], + "horizon": horizon, + "display": { + "label": display_value, + "group": display_value, + "role": "model" if metadata.model is not None else "population", + }, + } + ) + + data = [] + for row in rows: + key = ( + str(row["reference_group"]), + float(row["fixed_time_horizon"]), + str(row["censoring_heuristic"]), + str(row["competing_heuristic"]), + ) + data.append( + { + "seriesId": series_ids[key], + "cutoff": row["chosen_cutoff"], + "sensitivity": row["sensitivity"], + "ppv": row["ppv"], + } + ) + + risks = _gains_population_horizon_risk(performance_data, evaluation_metadata) + references = [] + for (population, horizon), risk in risks.items(): + references.append( + { + "type": "horizontal", + "scope": "population_horizon", + "population": population, + "horizon": horizon, + "value": risk, + "label": "Prevalence", + } + ) + + return { + "schemaVersion": "2.0", + "type": "precision_recall", + "evaluations": evaluations, + "series": series, + "data": data, + "x": "sensitivity", + "y": "ppv", + "xAxis": {"label": "Sensitivity", "domain": [0, 1]}, + "yAxis": {"label": "Positive Predictive Value", "domain": [0, 1]}, + "references": references, + } + + def _gains_v2_spec_from_performance_data( performance_data: pl.DataFrame, evaluation_metadata: Mapping[str, _EvaluationMetadata], diff --git a/src/rtichoke/discrimination/precision_recall.py b/src/rtichoke/discrimination/precision_recall.py index fa533828..1183e308 100644 --- a/src/rtichoke/discrimination/precision_recall.py +++ b/src/rtichoke/discrimination/precision_recall.py @@ -3,7 +3,6 @@ """ from typing import Any, Dict, List, Sequence, Union -from plotly.graph_objs._figure import Figure from rtichoke.processing.binary_color_values import _apply_color_values_binary from rtichoke.processing.plotly_helper_functions import ( _create_rtichoke_plotly_curve_binary, @@ -16,8 +15,14 @@ import polars as pl from rtichoke._renderers import RtichokeBrowserChart, _validate_renderer -from rtichoke._viz_spec_v2 import _precision_recall_v2_spec_from_performance_data +from rtichoke._viz_spec_v2 import ( + _precision_recall_times_v2_spec_from_performance_data, + _precision_recall_v2_spec_from_performance_data, +) from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) from rtichoke.processing.evaluation_semantics import ( _EvaluationMetadata, _build_evaluation_metadata, @@ -241,7 +246,8 @@ def create_precision_recall_curve_times( "#D1603D", "#585123", ], -) -> Figure: + renderer: str = "plotly", +) -> Any: """Creates a time-dependent Precision-Recall curve. Generates a Precision-Recall curve for time-to-event models, evaluating @@ -268,12 +274,37 @@ def create_precision_recall_curve_times( The width and height of the plot in pixels. Defaults to 600. color_values : List[str], optional A list of hex color strings for the plot lines. + renderer : {"plotly", "browser", "rtichoke_viz"}, optional + Rendering backend. ``"plotly"`` remains the default. ``"browser"`` and + its ``"rtichoke_viz"`` alias return a canonical offline browser chart. Returns ------- - Figure - A Plotly ``Figure`` object for the time-dependent Precision-Recall curve. + Figure or RtichokeBrowserChart + A Plotly ``Figure`` or canonical offline browser chart depending on ``renderer``. """ + selected_renderer = _validate_renderer(renderer) + if selected_renderer != "plotly": + if selected_renderer == "matplotlib": + raise ValueError( + "Precision-Recall supports 'plotly', 'browser', and 'rtichoke_viz' " + "renderers." + ) + performance_data = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=fixed_time_horizons, + heuristics_sets=heuristics_sets, + by=by, + stratified_by=stratified_by, + ) + evaluation_metadata = _build_evaluation_metadata(probs, reals, times) + spec = _precision_recall_times_v2_spec_from_performance_data( + performance_data, + evaluation_metadata, + ) + return RtichokeBrowserChart(spec=spec, size=size) fig = _create_rtichoke_plotly_curve_times_reference_safe( probs, diff --git a/tests/test_precision_recall_v2.py b/tests/test_precision_recall_v2.py index 90fb0c66..9ec8cb75 100644 --- a/tests/test_precision_recall_v2.py +++ b/tests/test_precision_recall_v2.py @@ -269,9 +269,10 @@ def test_default_and_explicit_plotly_behavior_are_unchanged(): assert pio.to_json(default_plot) == pio.to_json(explicit_plot) -def test_time_dependent_precision_recall_api_does_not_gain_renderer_selection(): +def test_time_dependent_precision_recall_api_has_renderer_selection(): parameters = inspect.signature(create_precision_recall_curve_times).parameters - assert "renderer" not in parameters + assert "renderer" in parameters + assert parameters["renderer"].default == "plotly" def test_vendored_v050_contains_static_precision_recall_contract_and_export(): diff --git a/tests/test_time_precision_recall_v2.py b/tests/test_time_precision_recall_v2.py new file mode 100644 index 00000000..73e485f3 --- /dev/null +++ b/tests/test_time_precision_recall_v2.py @@ -0,0 +1,290 @@ +import shutil +import subprocess +from contextlib import contextmanager +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from typing import Any, Iterator, cast + +import numpy as np +import plotly.graph_objects as go +import pytest + +from rtichoke import create_precision_recall_curve_times +from rtichoke._renderers import RtichokeBrowserChart +from rtichoke._viz_spec_v2 import ( + _precision_recall_times_v2_spec_from_performance_data, +) +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) +from rtichoke.processing.evaluation_semantics import ( + _SHARED_POPULATION, + _build_evaluation_metadata, +) + +HORIZONS = [5.0, 10.0] +HEURISTICS = [ + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + } +] + + +def _shared_inputs(): + return ( + { + "Model A": np.array([0.05, 0.15, 0.35, 0.55, 0.75, 0.95]), + "Model B": np.array([0.10, 0.25, 0.45, 0.65, 0.80, 0.90]), + }, + np.array([1, 0, 1, 0, 1, 0]), + np.array([3.0, 12.0, 8.0, 13.0, 14.0, 15.0]), + ) + + +def _spec(probs, reals, times): + performance = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + ) + return _precision_recall_times_v2_spec_from_performance_data( + performance, _build_evaluation_metadata(probs, reals, times) + ) + + +def test_time_precision_recall_uses_one_evaluation_and_series_per_model_horizon(): + probs, reals, times = _shared_inputs() + spec = _spec(probs, reals, times) + + assert spec["schemaVersion"] == "2.0" + assert spec["type"] == "precision_recall" + assert spec["x"] == "sensitivity" + assert spec["y"] == "ppv" + assert len(spec["evaluations"]) == 2 + assert {evaluation["population"] for evaluation in spec["evaluations"]} == { + _SHARED_POPULATION + } + assert len(spec["series"]) == 4 + assert { + (series["display"]["group"], series["horizon"]) for series in spec["series"] + } == { + ("Model A", 5.0), + ("Model A", 10.0), + ("Model B", 5.0), + ("Model B", 10.0), + } + + # Prevalence references (scope = population_horizon) + references = spec["references"] + assert len(references) == 2 + assert { + (reference["population"], reference["horizon"]) for reference in references + } == { + (_SHARED_POPULATION, 5.0), + (_SHARED_POPULATION, 10.0), + } + + +def test_equal_risk_population_horizons_remain_distinct_pr_reference_owners(): + probs = { + "Population A": np.array([0.05, 0.2, 0.7, 0.95]), + "Population B": np.array([0.1, 0.4, 0.6, 0.9]), + } + reals = { + "Population A": np.array([1, 0, 0, 0]), + "Population B": np.array([1, 0, 0, 0]), + } + times = { + "Population A": np.array([3.0, 12.0, 13.0, 14.0]), + "Population B": np.array([3.0, 12.0, 13.0, 14.0]), + } + references = _spec(probs, reals, times)["references"] + + assert len(references) == 4 + by_owner = { + (reference["population"], reference["horizon"]): reference["value"] + for reference in references + } + assert len(by_owner) == 4 + assert by_owner[("Population A", 5.0)] == by_owner[("Population B", 5.0)] + + +def test_time_precision_recall_censoring_and_competing_risk_reference_comes_from_performance_layer(): + probs = {"Model A": np.array([0.05, 0.2, 0.4, 0.6, 0.8, 0.95])} + reals = np.array([1, 0, 2, 1, 0, 2]) + times = np.array([2.0, 3.0, 4.0, 8.0, 12.0, 14.0]) + performance = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + ) + spec = _precision_recall_times_v2_spec_from_performance_data( + performance, _build_evaluation_metadata(probs, reals, times) + ) + references = cast(list[dict[str, Any]], spec["references"]) + calculated_risks = { + float(row["fixed_time_horizon"]): float(row["real_positives"] / row["n"]) + for row in performance.filter(performance["chosen_cutoff"] == 0) + .select("fixed_time_horizon", "real_positives", "n") + .unique() + .to_dicts() + } + + for reference in references: + horizon = reference["horizon"] + risk = calculated_risks[horizon] + assert reference["value"] == risk + assert reference["scope"] == "population_horizon" + assert reference["type"] == "horizontal" + + +def test_time_precision_recall_renderers_preserve_plotly_default_and_browser( + tmp_path: Path, +): + probs, reals, times = _shared_inputs() + default = create_precision_recall_curve_times( + probs, reals, times, HORIZONS, heuristics_sets=HEURISTICS, by=0.25 + ) + assert isinstance(default, go.Figure) + + browser = create_precision_recall_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="browser", + ) + assert isinstance(browser, RtichokeBrowserChart) + html = browser.write_html(tmp_path / "pr-times.html").read_text(encoding="utf-8") + assert "renderPrecisionRecallV2" in html + assert {series["horizon"] for series in browser.spec["series"]} == set(HORIZONS) + + rtichoke_viz_browser = create_precision_recall_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="rtichoke_viz", + ) + assert isinstance(rtichoke_viz_browser, RtichokeBrowserChart) + + with pytest.raises(ValueError, match="Precision-Recall supports"): + create_precision_recall_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="matplotlib", + ) + + +def _chrome_executable() -> str: + for candidate in ( + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + ): + executable = shutil.which(candidate) + if executable is not None: + return executable + pytest.skip("headless Chrome/Chromium is not available") + raise RuntimeError("unreachable") + + +@contextmanager +def _serve(directory: Path) -> Iterator[str]: + handler = partial(SimpleHTTPRequestHandler, directory=str(directory)) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join() + server.server_close() + + +def test_time_precision_recall_browser_chart_executes_and_switches_horizons_in_chrome( + tmp_path: Path, +): + probs, reals, times = _shared_inputs() + chart = create_precision_recall_curve_times( + probs, + reals, + times, + HORIZONS, + heuristics_sets=HEURISTICS, + by=0.25, + renderer="browser", + ) + chart.write_html(tmp_path / "precision-recall-times.html") + + script_path = tmp_path / "test_switch.js" + script_path.write_text( + """ + import { renderPrecisionRecallV2 } from "./rtichoke-viz.js"; + const spec = JSON.parse(document.querySelector("#rtichoke-spec").textContent); + const chart = renderPrecisionRecallV2(spec, { width: 600, height: 600 }); + document.querySelector("#rtichoke-chart").replaceChildren(chart); + + const select = chart.querySelector("select"); + if (!select) { + console.error("Horizon selector missing!"); + } else { + console.log("INITIAL_HORIZON:" + select.value); + select.value = "10"; + select.dispatchEvent(new Event("change", { bubbles: true })); + console.log("SWITCHED_HORIZON:" + select.value); + } + """, + encoding="utf-8", + ) + + html_file = tmp_path / "precision-recall-times.html" + content = html_file.read_text(encoding="utf-8") + content = content.replace( + "", + '', + ) + html_file.write_text(content, encoding="utf-8") + + with _serve(tmp_path) as base_url: + result = subprocess.run( + [ + _chrome_executable(), + "--headless=new", + "--no-sandbox", + "--disable-gpu", + "--enable-logging=stderr", + "--log-level=0", + "--dump-dom", + f"{base_url}/precision-recall-times.html", + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert "INITIAL_HORIZON:5" in result.stderr, result.stderr + assert "SWITCHED_HORIZON:10" in result.stderr, result.stderr + assert "