diff --git a/great-docs.yml b/great-docs.yml index a8293201..cbe232c9 100644 --- a/great-docs.yml +++ b/great-docs.yml @@ -101,10 +101,14 @@ reference: - create_calibration_curve - create_calibration_curve_times + - title: Summary Reports + desc: Create historical R-backed reports or explicitly opt into canonical browser ReportSpec rendering. + contents: + - create_summary_report + - title: Utility desc: Decision-curve analysis for classification and time-to-event models. contents: - create_decision_curve - create_decision_curve_times - plot_decision_curve - diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index 0db544f6..156587e7 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -1,59 +1,126 @@ -""" -A module for Summary Report -""" +"""Public summary-report entry points.""" +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any, Dict, Literal, Union, cast + +import numpy as np + +from rtichoke._calibration_viz_spec_v2 import _calibration_v2_spec_from_curve_list +from rtichoke._performance_table_spec import ( + _performance_table_spec_from_performance_data, +) +from rtichoke._report_browser import RtichokeBrowserReport +from rtichoke._report_spec import _report_spec_from_components +from rtichoke._viz_spec_v2 import _roc_v2_spec_from_performance_data +from rtichoke.calibration.calibration import _create_calibration_curve_list +from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata from rtichoke.processing.send_post_request_to_r_rtichoke import ( send_requests_to_rtichoke_r, ) -from rtichoke.processing.transforms import ( - _create_list_data_to_adjust, -) -import subprocess +from rtichoke.processing.transforms import _create_list_data_to_adjust + +SummaryReportRenderer = Literal["r", "browser"] + + +def create_summary_report( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + url_api: str = "http://localhost:4242/", + *, + renderer: SummaryReportRenderer = "r", + output_file: str | Path = "summary_report.html", +) -> Path | None: + """Create an rtichoke model-performance summary report. + The default ``renderer="r"`` preserves the historical public behavior and + delegates to the R rtichoke backend at ``url_api``. ``renderer="browser"`` + is an explicit opt-in path that uses Python's existing production + calculations, canonical standalone component builders, canonical ReportSpec + assembly, and the vendored ``rtichoke_viz`` ``renderReport()`` composer. -def create_summary_report(probs, reals, url_api="http://localhost:4242/"): - """Creates a summary report for rtichoke model performance. + The first browser report contains a canonical PerformanceTable, ROC-v2, and + calibration-v2 component, in that order. The browser renderer writes an HTML + file plus the vendored ``rtichoke-viz.js`` and ``rtichoke-viz.css`` assets + beside it, and returns the written HTML path. The historical R path retains + its existing return behavior (``None``). Parameters ---------- probs : Dict[str, np.ndarray] - A dictionary mapping model names to predicted probabilities. + A dictionary mapping model or population names to predicted probabilities. reals : Union[np.ndarray, Dict[str, np.ndarray]] - The true outcome labels (0 or 1). + The true binary outcome labels. url_api : str, optional - The API endpoint URL of the R rtichoke backend. - Defaults to ``"http://localhost:4242/"``. + The API endpoint URL of the historical R rtichoke backend. Used only by + ``renderer="r"``. Defaults to ``"http://localhost:4242/"``. + renderer : {"r", "browser"}, optional + Summary-report backend. Defaults to ``"r"`` for backward compatibility. + output_file : str or pathlib.Path, optional + HTML destination for ``renderer="browser"``. Defaults to + ``"summary_report.html"``. + + Returns + ------- + pathlib.Path or None + The generated HTML path for ``renderer="browser"``; ``None`` for the + historical R backend. """ + if renderer == "browser": + return _create_browser_summary_report(probs, reals, output_file=output_file) + if renderer != "r": + raise ValueError("renderer must be either 'r' or 'browser'") + rtichoke_response = send_requests_to_rtichoke_r( dictionary_to_send={"probs": probs, "reals": reals}, url_api=url_api, endpoint="create_summary_report", ) print(rtichoke_response.json()[0].keys()) + return None -def render_summary_report(): - """ - Render the rtichoke Summary Report using Quarto. +def _create_browser_summary_report( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + *, + output_file: str | Path, +) -> Path: + """Build the first public canonical browser report from production outputs.""" + performance_data = prepare_performance_data(probs, reals) + metadata = _build_evaluation_metadata(probs, reals, np.array([])) + calibration_curve_list = _create_calibration_curve_list(probs, reals) + + performance_table = _performance_table_spec_from_performance_data( + performance_data, metadata + ) + roc = _roc_v2_spec_from_performance_data(performance_data, metadata) + calibration = _calibration_v2_spec_from_curve_list(calibration_curve_list, metadata) - Args: - probs (list): A list of probabilities. - reals (list): A list of real values. - times (list): A list of absolute numbers representing timestamps. + report = _report_spec_from_components( + [ + {"title": "Performance", "spec": performance_table}, + {"title": "ROC", "spec": roc}, + {"title": "Calibration", "spec": calibration}, + ], + title="rtichoke summary report", + ) + return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html(output_file) - Example: - probs = [0.1, 0.4, 0.8] - reals = [0, 1, 1] - times = [1, 3, 5] - render_summary_report(probs, reals, times) - This will generate a `summary_report.html` file based on the `summary_report_template.qmd`. +def render_summary_report(): + """Render the historical rtichoke Summary Report using Quarto. + + This function is unchanged by the canonical browser-report opt-in path. It + renders ``aj_estimate_summary_report.qmd`` to ``summary_report.html`` using + the local Quarto executable. """ - # Define the path to the template and output file template_path = "aj_estimate_summary_report.qmd" output_path = "summary_report.html" - # Prepare the command to render the Quarto document command = [ "quarto", "render", @@ -61,12 +128,8 @@ def render_summary_report(): "--to", "html", "--output", - output_path, # , - # "--execute-params", - # f"probs={probs},reals={reals},times={times}", + output_path, ] - - # Execute the command subprocess.run(command, check=True) diff --git a/tests/test_summary_report_browser.py b/tests/test_summary_report_browser.py new file mode 100644 index 00000000..7c32b8ed --- /dev/null +++ b/tests/test_summary_report_browser.py @@ -0,0 +1,164 @@ +import json +from pathlib import Path +from typing import Any, cast + +import numpy as np + +from rtichoke.summary_report import summary_report as summary_report_module +from rtichoke.summary_report.summary_report import create_summary_report + + +def _inputs(): + probs = { + "Model A": np.array( + [ + 0.03, + 0.08, + 0.12, + 0.18, + 0.25, + 0.32, + 0.40, + 0.50, + 0.62, + 0.75, + 0.88, + 0.96, + ] + ) + } + reals = np.array([0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1]) + return probs, reals + + +def _embedded_report(html: str) -> dict[str, Any]: + start = html.index('", start) + return cast(dict[str, Any], json.loads(html[start:end])) + + +def test_default_summary_report_keeps_historical_r_backend(monkeypatch, capsys): + probs, reals = _inputs() + calls = [] + + class Response: + def json(self): + return [{"historical": True}] + + def fake_send_requests_to_rtichoke_r(**kwargs): + calls.append(kwargs) + return Response() + + def fail_browser(*args, **kwargs): + raise AssertionError("default path must not invoke RtichokeBrowserReport") + + monkeypatch.setattr( + summary_report_module, + "send_requests_to_rtichoke_r", + fake_send_requests_to_rtichoke_r, + ) + monkeypatch.setattr( + summary_report_module.RtichokeBrowserReport, + "write_html", + fail_browser, + ) + + result = create_summary_report(probs, reals) + + assert result is None + assert len(calls) == 1 + assert calls[0]["dictionary_to_send"]["probs"] is probs + assert calls[0]["dictionary_to_send"]["reals"] is reals + assert calls[0]["url_api"] == "http://localhost:4242/" + assert calls[0]["endpoint"] == "create_summary_report" + assert "dict_keys(['historical'])" in capsys.readouterr().out + + +def test_browser_summary_report_is_opt_in_and_uses_real_canonical_components( + tmp_path, +): + probs, reals = _inputs() + output = tmp_path / "canonical-summary.html" + + result = create_summary_report( + probs, + reals, + renderer="browser", + output_file=output, + ) + + assert result == output + assert output.exists() + assert (tmp_path / "rtichoke-viz.js").exists() + assert (tmp_path / "rtichoke-viz.css").exists() + + html = output.read_text(encoding="utf-8") + report = _embedded_report(html) + assert [component["id"] for component in report["components"]] == [ + "performance-table", + "roc", + "calibration", + ] + assert [component["spec"]["type"] for component in report["components"]] == [ + "performance_table", + "roc", + "calibration", + ] + assert report["components"][1]["spec"]["schemaVersion"] == "2.0" + assert report["components"][2]["spec"]["schemaVersion"] == "2.0" + assert 'import { renderReport } from "./rtichoke-viz.js";' in html + assert "append(renderReport(spec))" in html + assert "renderPerformanceTable" not in html + assert "renderRocV2" not in html + assert "renderCalibrationV2" not in html + + +def test_browser_summary_report_preserves_component_local_identity(tmp_path): + probs = { + "Population A": np.array([0.05, 0.2, 0.7, 0.95]), + "Population B": np.array([0.1, 0.4, 0.6, 0.9]), + } + reals = { + "Population A": np.array([0, 0, 1, 1]), + "Population B": np.array([0, 1, 0, 1]), + } + + output = create_summary_report( + probs, + reals, + renderer="browser", + output_file=tmp_path / "populations.html", + ) + assert isinstance(output, Path) + report = _embedded_report(output.read_text(encoding="utf-8")) + + assert "evaluations" not in report + assert "models" not in report + assert "populations" not in report + assert "horizon" not in report + + table, roc, calibration = [component["spec"] for component in report["components"]] + assert table["evaluations"][0]["id"] == "evaluation-1" + assert roc["evaluations"][0]["id"] == "evaluation-1" + assert calibration["evaluations"][0]["id"] == "evaluation-1" + assert [item["population"] for item in calibration["evaluations"]] == [ + "Population A", + "Population B", + ] + assert all("model" not in item for item in calibration["evaluations"]) + assert [item["display"]["role"] for item in calibration["series"]] == [ + "population", + "population", + ] + + +def test_browser_summary_report_rejects_unknown_renderer(): + probs, reals = _inputs() + + try: + create_summary_report(probs, reals, renderer="unknown") # type: ignore[arg-type] + except ValueError as exc: + assert str(exc) == "renderer must be either 'r' or 'browser'" + else: + raise AssertionError("unknown renderer should fail") diff --git a/user_guide/06-summary-reports.qmd b/user_guide/06-summary-reports.qmd new file mode 100644 index 00000000..0b99cc1e --- /dev/null +++ b/user_guide/06-summary-reports.qmd @@ -0,0 +1,36 @@ +--- +title: "Summary reports" +--- + +`create_summary_report()` keeps the historical R-backed report path as its default. The canonical browser report is available only when explicitly requested with `renderer="browser"`. + +```python +import numpy as np +from rtichoke import create_summary_report + +probs = { + "Model A": np.array( + [0.03, 0.08, 0.12, 0.18, 0.25, 0.32, 0.40, 0.50, 0.62, 0.75, 0.88, 0.96] + ) +} +reals = np.array([0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1]) + +create_summary_report( + probs, + reals, + renderer="browser", + output_file="summary_report.html", +) +``` + +The browser path uses the same production calculations as the standalone Python components, converts those results with the existing canonical component builders, assembles a canonical ReportSpec, and delegates report composition to the vendored immutable `rtichoke_viz v0.5.0` `renderReport()` implementation. + +The first public browser report contains, in deterministic order: + +1. PerformanceTable; +2. ROC-v2; +3. calibration-v2. + +The generated HTML is accompanied by `rtichoke-viz.js` and `rtichoke-viz.css` in the same directory, so those three files should be kept together when moving the report. The browser backend returns the generated HTML `pathlib.Path`; the default historical R backend retains its existing `None` return behavior. + +The browser backend does not replace Quarto or the historical R backend, and it is not the default. Existing Plotly, Matplotlib, table, standalone browser-chart, and time-dependent APIs are unchanged by this opt-in report path.