From 602c6ccd9979d6821b2f71c98d52bd8fdc7f104f Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 02:20:11 +0000 Subject: [PATCH 1/2] Add canonical browser time-dependent summary report Introduce public create_summary_report_times() as a canonical browser summary report composed entirely from existing time-dependent statistical infrastructure. Co-authored-by: uriahf <11351434+uriahf@users.noreply.github.com> --- src/rtichoke/__init__.py | 2 + src/rtichoke/_report_browser.py | 18 +- src/rtichoke/_viz_spec_v2.py | 14 +- src/rtichoke/summary_report/summary_report.py | 245 +++++++++++- tests/test_public_exports.py | 1 + tests/test_summary_report_times_browser.py | 364 ++++++++++++++++++ 6 files changed, 640 insertions(+), 4 deletions(-) create mode 100644 tests/test_summary_report_times_browser.py diff --git a/src/rtichoke/__init__.py b/src/rtichoke/__init__.py index d5df5ce7..896d7eb1 100644 --- a/src/rtichoke/__init__.py +++ b/src/rtichoke/__init__.py @@ -59,6 +59,7 @@ from rtichoke.summary_report.summary_report import ( create_summary_report as create_summary_report, + create_summary_report_times as create_summary_report_times, ) __all__ = [ @@ -87,4 +88,5 @@ "create_performance_table_times", "render_performance_table", "create_summary_report", + "create_summary_report_times", ] diff --git a/src/rtichoke/_report_browser.py b/src/rtichoke/_report_browser.py index 385fa71f..d3d628db 100644 --- a/src/rtichoke/_report_browser.py +++ b/src/rtichoke/_report_browser.py @@ -8,6 +8,19 @@ from typing import Any +def _sanitize_nan_values(obj: Any) -> Any: + """Recursively replace NaN and Inf float values with None for valid JSON serialization.""" + if isinstance(obj, dict): + return {k: _sanitize_nan_values(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_sanitize_nan_values(v) for v in obj] + if isinstance(obj, float) and not ( + obj == obj and obj != float("inf") and obj != float("-inf") + ): + return None + return obj + + class RtichokeBrowserReport: """A complete canonical ReportSpec rendered by shared ``rtichoke_viz``.""" @@ -23,7 +36,10 @@ def write_html(self, path: str | Path) -> Path: for asset in ("rtichoke-viz.js", "rtichoke-viz.css"): (output.parent / asset).write_bytes(vendor.joinpath(asset).read_bytes()) - spec_json = json.dumps(self.spec, separators=(",", ":")).replace(" diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index 48ff236d..b3aae624 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -324,7 +324,12 @@ def _gains_times_v2_spec_from_performance_data( + ", ".join(sorted(missing)) ) - rows = performance_data.select( + finite_performance_data = performance_data.filter( + pl.col("chosen_cutoff").is_finite() + & pl.col("ppcr").is_finite() + & pl.col("lift").is_finite() + ) + rows = finite_performance_data.select( "reference_group", "fixed_time_horizon", "censoring_heuristic", @@ -455,7 +460,12 @@ def _lift_times_v2_spec_from_performance_data( + ", ".join(sorted(missing)) ) - rows = performance_data.select( + finite_performance_data = performance_data.filter( + pl.col("chosen_cutoff").is_finite() + & pl.col("ppcr").is_finite() + & pl.col("lift").is_finite() + ) + rows = finite_performance_data.select( "reference_group", "fixed_time_horizon", "censoring_heuristic", diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index 9d58af54..d0775a11 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -10,13 +10,16 @@ from rtichoke._calibration_viz_spec_v2 import _calibration_v2_spec_from_curve_list from rtichoke._decision_curve_viz_spec_v2 import ( + _decision_curve_times_v2_spec_from_performance_data, _decision_curve_v2_spec_from_performance_data, ) from rtichoke._interventions_avoided_viz_spec_v2 import ( + _interventions_avoided_times_v2_spec_from_performance_data, _interventions_avoided_v2_spec_from_performance_data, ) from rtichoke._performance_table_spec import ( _performance_table_spec_from_performance_data, + _performance_table_times_spec_from_performance_data, ) from rtichoke._report_browser import RtichokeBrowserReport from rtichoke._report_spec import _build_report_spec_v11 @@ -25,13 +28,22 @@ _prevalence_summary_metrics_spec, ) from rtichoke._viz_spec_v2 import ( + _gains_times_v2_spec_from_performance_data, _gains_v2_spec_from_performance_data, + _lift_times_v2_spec_from_performance_data, _lift_v2_spec_from_performance_data, + _precision_recall_times_v2_spec_from_performance_data, _precision_recall_v2_spec_from_performance_data, _roc_v2_spec_from_performance_data, ) -from rtichoke.calibration.calibration import _create_calibration_curve_list +from rtichoke.calibration.calibration import ( + _create_calibration_curve_list, + _create_calibration_curve_list_times, +) from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata from rtichoke.processing.send_post_request_to_r_rtichoke import ( send_requests_to_rtichoke_r, @@ -40,6 +52,237 @@ SummaryReportRenderer = Literal["r", "browser"] +_DEFAULT_TIME_HEURISTICS = [ + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + } +] + + +def create_summary_report_times( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + times: Union[np.ndarray, Dict[str, np.ndarray]], + fixed_time_horizons: list[float], + heuristics_sets: list[dict] | None = None, + by: float = 0.01, + *, + output_file: str | Path = "summary_report_times.html", +) -> Path: + """Create a canonical browser time-dependent model-performance summary report. + + Parameters + ---------- + probs : Dict[str, np.ndarray] + A dictionary mapping model or population names to predicted probabilities. + reals : Union[np.ndarray, Dict[str, np.ndarray]] + The true outcome labels (0, 1, 2). + times : Union[np.ndarray, Dict[str, np.ndarray]] + Follow-up times. + fixed_time_horizons : list[float] + Fixed time horizons for evaluation. + heuristics_sets : list[dict], optional + List of heuristic configurations for censoring and competing events. + Defaults to ``[{"censoring_heuristic": "adjusted", "competing_heuristic": "adjusted_as_negative"}]``. + by : float, optional + Step size for probability thresholds / PPCR. Defaults to 0.01. + output_file : str or pathlib.Path, optional + HTML destination file path. Defaults to ``"summary_report_times.html"``. + + Returns + ------- + pathlib.Path + The generated HTML file path. + """ + if heuristics_sets is None: + heuristics_sets = [dict(_DEFAULT_TIME_HEURISTICS[0])] + + metadata = _build_evaluation_metadata(probs, reals, times) + + perf_data_thresh = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=fixed_time_horizons, + heuristics_sets=heuristics_sets, + stratified_by=("probability_threshold",), + by=by, + ) + perf_data_ppcr = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=fixed_time_horizons, + heuristics_sets=heuristics_sets, + stratified_by=("ppcr",), + by=by, + ) + + calibration_curve_list_smooth = _create_calibration_curve_list_times( + probs, + reals, + times, + fixed_time_horizons=fixed_time_horizons, + heuristics_sets=heuristics_sets, + calibration_type="smooth", + ) + calibration_curve_list_discrete = _create_calibration_curve_list_times( + probs, + reals, + times, + fixed_time_horizons=fixed_time_horizons, + heuristics_sets=heuristics_sets, + calibration_type="discrete", + ) + + calib_smooth_spec = _calibration_v2_spec_from_curve_list( + calibration_curve_list_smooth, metadata, calibration_type="smooth" + ) + calib_discrete_spec = _calibration_v2_spec_from_curve_list( + calibration_curve_list_discrete, metadata, calibration_type="discrete" + ) + + pr_thresh_spec = _precision_recall_times_v2_spec_from_performance_data( + perf_data_thresh, metadata + ) + gains_thresh_spec = _gains_times_v2_spec_from_performance_data( + perf_data_thresh, metadata + ) + lift_thresh_spec = _lift_times_v2_spec_from_performance_data( + perf_data_thresh, metadata + ) + + pr_ppcr_spec = _precision_recall_times_v2_spec_from_performance_data( + perf_data_ppcr, metadata + ) + gains_ppcr_spec = _gains_times_v2_spec_from_performance_data( + perf_data_ppcr, metadata + ) + lift_ppcr_spec = _lift_times_v2_spec_from_performance_data(perf_data_ppcr, metadata) + + decision_curve_spec = _decision_curve_times_v2_spec_from_performance_data( + perf_data_thresh, metadata + ) + interventions_avoided_spec = ( + _interventions_avoided_times_v2_spec_from_performance_data( + perf_data_thresh, metadata + ) + ) + + table_thresh_spec = _performance_table_times_spec_from_performance_data( + perf_data_thresh, metadata + ) + table_ppcr_spec = _performance_table_times_spec_from_performance_data( + perf_data_ppcr, metadata + ) + + sections = [ + { + "id": "calibration", + "title": "Calibration", + "components": [ + { + "id": "calibration-smooth", + "title": "Smooth", + "spec": calib_smooth_spec, + }, + { + "id": "calibration", + "title": "Discrete", + "spec": calib_discrete_spec, + }, + ], + }, + { + "id": "discrimination", + "title": "Discrimination", + "groups": [ + { + "id": "discrimination-probability-threshold", + "title": "By Probability Threshold", + "components": [ + { + "id": "precision-recall", + "title": "Precision-Recall", + "spec": pr_thresh_spec, + }, + { + "id": "gains", + "title": "Gains", + "spec": gains_thresh_spec, + }, + {"id": "lift", "title": "Lift", "spec": lift_thresh_spec}, + ], + }, + { + "id": "discrimination-ppcr", + "title": "By Predicted Positives Condition Rate (PPCR)", + "components": [ + { + "id": "precision-recall-2", + "title": "Precision-Recall", + "spec": pr_ppcr_spec, + }, + { + "id": "gains-2", + "title": "Gains", + "spec": gains_ppcr_spec, + }, + {"id": "lift-2", "title": "Lift", "spec": lift_ppcr_spec}, + ], + }, + ], + }, + { + "id": "utility", + "title": "Utility", + "components": [ + { + "id": "decision-curve", + "title": "Decision Curve", + "spec": decision_curve_spec, + }, + { + "id": "interventions-avoided", + "title": "Interventions Avoided", + "spec": interventions_avoided_spec, + }, + ], + }, + { + "id": "performance-table", + "title": "Performance Table", + "groups": [ + { + "id": "performance-table-probability-threshold", + "title": "By Probability Threshold", + "components": [ + { + "id": "performance-table", + "title": "Performance Table", + "spec": table_thresh_spec, + } + ], + }, + { + "id": "performance-table-ppcr", + "title": "By Predicted Positives Condition Rate (PPCR)", + "components": [ + { + "id": "performance-table-2", + "title": "Performance Table", + "spec": table_ppcr_spec, + } + ], + }, + ], + }, + ] + + report = _build_report_spec_v11(sections, title="rtichoke summary report") + return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html(output_file) + def create_summary_report( probs: Dict[str, np.ndarray], diff --git a/tests/test_public_exports.py b/tests/test_public_exports.py index 422da87e..72398d16 100644 --- a/tests/test_public_exports.py +++ b/tests/test_public_exports.py @@ -27,6 +27,7 @@ "create_performance_table_times", "render_performance_table", "create_summary_report", + "create_summary_report_times", } diff --git a/tests/test_summary_report_times_browser.py b/tests/test_summary_report_times_browser.py new file mode 100644 index 00000000..b8a3eed6 --- /dev/null +++ b/tests/test_summary_report_times_browser.py @@ -0,0 +1,364 @@ +import json +import shutil +import subprocess +from contextlib import contextmanager +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from typing import Any, Iterator, cast + +import numpy as np +import pytest + +import rtichoke +from rtichoke._calibration_viz_spec_v2 import _calibration_v2_spec_from_curve_list +from rtichoke._decision_curve_viz_spec_v2 import ( + _decision_curve_times_v2_spec_from_performance_data, +) +from rtichoke._interventions_avoided_viz_spec_v2 import ( + _interventions_avoided_times_v2_spec_from_performance_data, +) +from rtichoke._performance_table_spec import ( + _performance_table_times_spec_from_performance_data, +) +from rtichoke._viz_spec_v2 import ( + _gains_times_v2_spec_from_performance_data, + _lift_times_v2_spec_from_performance_data, + _precision_recall_times_v2_spec_from_performance_data, +) +from rtichoke.calibration.calibration import _create_calibration_curve_list_times +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) +from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata +from rtichoke.summary_report.summary_report import create_summary_report_times + + +def _inputs(): + probs = { + "Model A": np.array( + [0.05, 0.12, 0.20, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95] + ) + } + reals = np.array([0, 0, 1, 0, 1, 0, 1, 1, 1, 1]) + times = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) + fixed_time_horizons = [4.0, 8.0] + return probs, reals, times, fixed_time_horizons + + +def _embedded_report(html: str) -> dict[str, Any]: + start = html.index('", start) + return cast(dict[str, Any], json.loads(html[start:end])) + + +def _chrome_executable() -> str: + for candidate in ( + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + ): + executable = shutil.which(candidate) + if executable is not None: + return executable + pytest.skip("headless Chrome/Chromium is not available") + + +def _dump_dom(url: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + _chrome_executable(), + "--headless=new", + "--no-sandbox", + "--disable-gpu", + "--enable-logging=stderr", + "--log-level=0", + "--dump-dom", + url, + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + +def _rendered_report_html(dom: str) -> str: + marker = '
' + start = dom.index(marker) + len(marker) + end = dom.index('", start) + json_block = html_text[start:end] + + assert ":NaN" not in json_block + assert ":Infinity" not in json_block + assert ":-Infinity" not in json_block + + +def test_time_lift_excludes_only_non_finite_values_retains_boundary_points(tmp_path): + probs, reals, times, horizons = _inputs() + heuristics_sets = [ + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + } + ] + metadata = _build_evaluation_metadata(probs, reals, times) + + perf_thresh = prepare_performance_data_times( + probs, + reals, + times, + fixed_time_horizons=horizons, + heuristics_sets=heuristics_sets, + stratified_by=("probability_threshold",), + by=0.01, + ) + + lift_spec = _lift_times_v2_spec_from_performance_data(perf_thresh, metadata) + + # 1. Non-finite values (lift is null or non-existent in data points) + data_points = lift_spec["data"] + for pt in data_points: + assert pt["lift"] is not None + assert np.isfinite(pt["lift"]) + + # 2. Valid finite boundary points are retained (e.g. cutoff = 0.0 / low cutoffs) + cutoffs = [pt["cutoff"] for pt in data_points] + assert 0.0 in cutoffs or min(cutoffs) <= 0.05 + + # 3. Reference geometry remains present and intact + refs = lift_spec["references"] + ref_types = [r["type"] for r in refs] + assert "horizontal" in ref_types + assert "path" in ref_types + horizontal_ref = next(r for r in refs if r["type"] == "horizontal") + assert horizontal_ref["value"] == 1.0