From be84026f4482d482dbe0664a6c8fac16f8dd55c1 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:39:11 +0300 Subject: [PATCH 01/10] feat: add opt-in browser summary report --- src/rtichoke/summary_report/summary_report.py | 129 +++++++++++++----- 1 file changed, 96 insertions(+), 33 deletions(-) diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index 0db544f6..25424017 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -1,59 +1,126 @@ -""" -A module for Summary Report -""" +"""Public summary-report entry points.""" +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any, Dict, Literal, Union, cast + +import numpy as np + +from rtichoke._calibration_viz_spec_v2 import _calibration_v2_spec_from_curve_list +from rtichoke._performance_table_spec import _performance_table_spec_from_performance_data +from rtichoke._report_browser import RtichokeBrowserReport +from rtichoke._report_spec import _report_spec_from_components +from rtichoke._viz_spec_v2 import _roc_v2_spec_from_performance_data +from rtichoke.calibration.calibration import _create_calibration_curve_list +from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata from rtichoke.processing.send_post_request_to_r_rtichoke import ( send_requests_to_rtichoke_r, ) -from rtichoke.processing.transforms import ( - _create_list_data_to_adjust, -) -import subprocess +from rtichoke.processing.transforms import _create_list_data_to_adjust + +SummaryReportRenderer = Literal["r", "browser"] + +def create_summary_report( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + url_api: str = "http://localhost:4242/", + *, + renderer: SummaryReportRenderer = "r", + output_file: str | Path = "summary_report.html", +) -> Path | None: + """Create an rtichoke model-performance summary report. -def create_summary_report(probs, reals, url_api="http://localhost:4242/"): - """Creates a summary report for rtichoke model performance. + The default ``renderer="r"`` preserves the historical public behavior and + delegates to the R rtichoke backend at ``url_api``. ``renderer="browser"`` + is an explicit opt-in path that uses Python's existing production + calculations, canonical standalone component builders, canonical ReportSpec + assembly, and the vendored ``rtichoke_viz`` ``renderReport()`` composer. + + The first browser report contains a canonical PerformanceTable, ROC-v2, and + calibration-v2 component, in that order. The browser renderer writes an HTML + file plus the vendored ``rtichoke-viz.js`` and ``rtichoke-viz.css`` assets + beside it, and returns the written HTML path. The historical R path retains + its existing return behavior (``None``). Parameters ---------- probs : Dict[str, np.ndarray] - A dictionary mapping model names to predicted probabilities. + A dictionary mapping model or population names to predicted probabilities. reals : Union[np.ndarray, Dict[str, np.ndarray]] - The true outcome labels (0 or 1). + The true binary outcome labels. url_api : str, optional - The API endpoint URL of the R rtichoke backend. - Defaults to ``"http://localhost:4242/"``. + The API endpoint URL of the historical R rtichoke backend. Used only by + ``renderer="r"``. Defaults to ``"http://localhost:4242/"``. + renderer : {"r", "browser"}, optional + Summary-report backend. Defaults to ``"r"`` for backward compatibility. + output_file : str or pathlib.Path, optional + HTML destination for ``renderer="browser"``. Defaults to + ``"summary_report.html"``. + + Returns + ------- + pathlib.Path or None + The generated HTML path for ``renderer="browser"``; ``None`` for the + historical R backend. """ + if renderer == "browser": + return _create_browser_summary_report(probs, reals, output_file=output_file) + if renderer != "r": + raise ValueError("renderer must be either 'r' or 'browser'") + rtichoke_response = send_requests_to_rtichoke_r( dictionary_to_send={"probs": probs, "reals": reals}, url_api=url_api, endpoint="create_summary_report", ) print(rtichoke_response.json()[0].keys()) + return None -def render_summary_report(): - """ - Render the rtichoke Summary Report using Quarto. +def _create_browser_summary_report( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + *, + output_file: str | Path, +) -> Path: + """Build the first public canonical browser report from production outputs.""" + performance_data = prepare_performance_data(probs, reals) + metadata = _build_evaluation_metadata(probs, reals, np.array([])) + calibration_curve_list = _create_calibration_curve_list(probs, reals) + + performance_table = _performance_table_spec_from_performance_data( + performance_data, metadata + ) + roc = _roc_v2_spec_from_performance_data(performance_data, metadata) + calibration = _calibration_v2_spec_from_curve_list( + calibration_curve_list, metadata + ) - Args: - probs (list): A list of probabilities. - reals (list): A list of real values. - times (list): A list of absolute numbers representing timestamps. + report = _report_spec_from_components( + [ + {"title": "Performance", "spec": performance_table}, + {"title": "ROC", "spec": roc}, + {"title": "Calibration", "spec": calibration}, + ], + title="rtichoke summary report", + ) + return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html(output_file) - Example: - probs = [0.1, 0.4, 0.8] - reals = [0, 1, 1] - times = [1, 3, 5] - render_summary_report(probs, reals, times) - This will generate a `summary_report.html` file based on the `summary_report_template.qmd`. +def render_summary_report(): + """Render the historical rtichoke Summary Report using Quarto. + + This function is unchanged by the canonical browser-report opt-in path. It + renders ``aj_estimate_summary_report.qmd`` to ``summary_report.html`` using + the local Quarto executable. """ - # Define the path to the template and output file template_path = "aj_estimate_summary_report.qmd" output_path = "summary_report.html" - # Prepare the command to render the Quarto document command = [ "quarto", "render", @@ -61,12 +128,8 @@ def render_summary_report(): "--to", "html", "--output", - output_path, # , - # "--execute-params", - # f"probs={probs},reals={reals},times={times}", + output_path, ] - - # Execute the command subprocess.run(command, check=True) From 12e48b129fe698674c48ebca96c9c927cb2fade9 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:39:32 +0300 Subject: [PATCH 02/10] test: cover public browser summary report --- tests/test_summary_report_browser.py | 153 +++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 tests/test_summary_report_browser.py diff --git a/tests/test_summary_report_browser.py b/tests/test_summary_report_browser.py new file mode 100644 index 00000000..5a50f864 --- /dev/null +++ b/tests/test_summary_report_browser.py @@ -0,0 +1,153 @@ +import json +from pathlib import Path +from typing import Any, cast + +import numpy as np + +from rtichoke.summary_report import summary_report as summary_report_module +from rtichoke.summary_report.summary_report import create_summary_report + + +def _inputs(): + probs = { + "Model A": np.array( + [0.03, 0.08, 0.12, 0.18, 0.25, 0.32, 0.40, 0.50, 0.62, 0.75, 0.88, 0.96] + ) + } + reals = np.array([0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1]) + return probs, reals + + +def _embedded_report(html: str) -> dict[str, Any]: + start = html.index('", start) + return cast(dict[str, Any], json.loads(html[start:end])) + + +def test_default_summary_report_keeps_historical_r_backend(monkeypatch, capsys): + probs, reals = _inputs() + calls = [] + + class Response: + def json(self): + return [{"historical": True}] + + def fake_send_requests_to_rtichoke_r(**kwargs): + calls.append(kwargs) + return Response() + + def fail_browser(*args, **kwargs): + raise AssertionError("default path must not invoke RtichokeBrowserReport") + + monkeypatch.setattr( + summary_report_module, + "send_requests_to_rtichoke_r", + fake_send_requests_to_rtichoke_r, + ) + monkeypatch.setattr( + summary_report_module.RtichokeBrowserReport, + "write_html", + fail_browser, + ) + + result = create_summary_report(probs, reals) + + assert result is None + assert calls == [ + { + "dictionary_to_send": {"probs": probs, "reals": reals}, + "url_api": "http://localhost:4242/", + "endpoint": "create_summary_report", + } + ] + assert "dict_keys(['historical'])" in capsys.readouterr().out + + +def test_browser_summary_report_is_opt_in_and_uses_real_canonical_components(tmp_path): + probs, reals = _inputs() + output = tmp_path / "canonical-summary.html" + + result = create_summary_report( + probs, + reals, + renderer="browser", + output_file=output, + ) + + assert result == output + assert output.exists() + assert (tmp_path / "rtichoke-viz.js").exists() + assert (tmp_path / "rtichoke-viz.css").exists() + + html = output.read_text(encoding="utf-8") + report = _embedded_report(html) + assert [component["id"] for component in report["components"]] == [ + "performance-table", + "roc", + "calibration", + ] + assert [component["spec"]["type"] for component in report["components"]] == [ + "performance_table", + "roc", + "calibration", + ] + assert report["components"][1]["spec"]["schemaVersion"] == "2.0" + assert report["components"][2]["spec"]["schemaVersion"] == "2.0" + assert 'import { renderReport } from "./rtichoke-viz.js";' in html + assert "append(renderReport(spec))" in html + assert "renderPerformanceTable" not in html + assert "renderRocV2" not in html + assert "renderCalibrationV2" not in html + + +def test_browser_summary_report_preserves_component_local_identity(tmp_path): + probs = { + "Population A": np.array([0.05, 0.2, 0.7, 0.95]), + "Population B": np.array([0.1, 0.4, 0.6, 0.9]), + } + reals = { + "Population A": np.array([0, 0, 1, 1]), + "Population B": np.array([0, 1, 0, 1]), + } + + output = create_summary_report( + probs, + reals, + renderer="browser", + output_file=tmp_path / "populations.html", + ) + assert isinstance(output, Path) + report = _embedded_report(output.read_text(encoding="utf-8")) + + assert "evaluations" not in report + assert "models" not in report + assert "populations" not in report + assert "horizon" not in report + + table, roc, calibration = [ + component["spec"] for component in report["components"] + ] + assert table["evaluations"][0]["id"] == "evaluation-1" + assert roc["evaluations"][0]["id"] == "evaluation-1" + assert calibration["evaluations"][0]["id"] == "evaluation-1" + assert [item["population"] for item in calibration["evaluations"]] == [ + "Population A", + "Population B", + ] + assert all("model" not in item for item in calibration["evaluations"]) + assert [item["display"]["role"] for item in calibration["series"]] == [ + "population", + "population", + ] + + +def test_browser_summary_report_rejects_unknown_renderer(): + probs, reals = _inputs() + + try: + create_summary_report(probs, reals, renderer="unknown") # type: ignore[arg-type] + except ValueError as exc: + assert str(exc) == "renderer must be either 'r' or 'browser'" + else: + raise AssertionError("unknown renderer should fail") From 27183491567af8e99cf48d54c5bae273b5a8dde2 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:39:43 +0300 Subject: [PATCH 03/10] docs: document browser summary report opt in --- user_guide/06-summary-reports.qmd | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 user_guide/06-summary-reports.qmd diff --git a/user_guide/06-summary-reports.qmd b/user_guide/06-summary-reports.qmd new file mode 100644 index 00000000..0b99cc1e --- /dev/null +++ b/user_guide/06-summary-reports.qmd @@ -0,0 +1,36 @@ +--- +title: "Summary reports" +--- + +`create_summary_report()` keeps the historical R-backed report path as its default. The canonical browser report is available only when explicitly requested with `renderer="browser"`. + +```python +import numpy as np +from rtichoke import create_summary_report + +probs = { + "Model A": np.array( + [0.03, 0.08, 0.12, 0.18, 0.25, 0.32, 0.40, 0.50, 0.62, 0.75, 0.88, 0.96] + ) +} +reals = np.array([0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1]) + +create_summary_report( + probs, + reals, + renderer="browser", + output_file="summary_report.html", +) +``` + +The browser path uses the same production calculations as the standalone Python components, converts those results with the existing canonical component builders, assembles a canonical ReportSpec, and delegates report composition to the vendored immutable `rtichoke_viz v0.5.0` `renderReport()` implementation. + +The first public browser report contains, in deterministic order: + +1. PerformanceTable; +2. ROC-v2; +3. calibration-v2. + +The generated HTML is accompanied by `rtichoke-viz.js` and `rtichoke-viz.css` in the same directory, so those three files should be kept together when moving the report. The browser backend returns the generated HTML `pathlib.Path`; the default historical R backend retains its existing `None` return behavior. + +The browser backend does not replace Quarto or the historical R backend, and it is not the default. Existing Plotly, Matplotlib, table, standalone browser-chart, and time-dependent APIs are unchanged by this opt-in report path. From d887ae340ae453c7da13078d0b0b974aba63c520 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:39:57 +0300 Subject: [PATCH 04/10] docs: expose summary report reference --- great-docs.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/great-docs.yml b/great-docs.yml index a8293201..cbe232c9 100644 --- a/great-docs.yml +++ b/great-docs.yml @@ -101,10 +101,14 @@ reference: - create_calibration_curve - create_calibration_curve_times + - title: Summary Reports + desc: Create historical R-backed reports or explicitly opt into canonical browser ReportSpec rendering. + contents: + - create_summary_report + - title: Utility desc: Decision-curve analysis for classification and time-to-event models. contents: - create_decision_curve - create_decision_curve_times - plot_decision_curve - From 2cd13305ff9380ff7dc3ec76fb592175f64cd91f Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:41:13 +0300 Subject: [PATCH 05/10] style: format public browser summary report --- src/rtichoke/summary_report/summary_report.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index 25424017..f74fc722 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -9,7 +9,9 @@ import numpy as np from rtichoke._calibration_viz_spec_v2 import _calibration_v2_spec_from_curve_list -from rtichoke._performance_table_spec import _performance_table_spec_from_performance_data +from rtichoke._performance_table_spec import ( + _performance_table_spec_from_performance_data, +) from rtichoke._report_browser import RtichokeBrowserReport from rtichoke._report_spec import _report_spec_from_components from rtichoke._viz_spec_v2 import _roc_v2_spec_from_performance_data @@ -108,7 +110,9 @@ def _create_browser_summary_report( ], title="rtichoke summary report", ) - return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html(output_file) + return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html( + output_file + ) def render_summary_report(): From 7cb751d924313685b35a49246265b26b3bb51e8e Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:41:37 +0300 Subject: [PATCH 06/10] style: format browser summary report tests --- tests/test_summary_report_browser.py | 31 ++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/test_summary_report_browser.py b/tests/test_summary_report_browser.py index 5a50f864..8a773893 100644 --- a/tests/test_summary_report_browser.py +++ b/tests/test_summary_report_browser.py @@ -11,7 +11,20 @@ def _inputs(): probs = { "Model A": np.array( - [0.03, 0.08, 0.12, 0.18, 0.25, 0.32, 0.40, 0.50, 0.62, 0.75, 0.88, 0.96] + [ + 0.03, + 0.08, + 0.12, + 0.18, + 0.25, + 0.32, + 0.40, + 0.50, + 0.62, + 0.75, + 0.88, + 0.96, + ] ) } reals = np.array([0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1]) @@ -54,17 +67,17 @@ def fail_browser(*args, **kwargs): result = create_summary_report(probs, reals) assert result is None - assert calls == [ - { - "dictionary_to_send": {"probs": probs, "reals": reals}, - "url_api": "http://localhost:4242/", - "endpoint": "create_summary_report", - } - ] + assert len(calls) == 1 + assert calls[0]["dictionary_to_send"]["probs"] is probs + assert calls[0]["dictionary_to_send"]["reals"] is reals + assert calls[0]["url_api"] == "http://localhost:4242/" + assert calls[0]["endpoint"] == "create_summary_report" assert "dict_keys(['historical'])" in capsys.readouterr().out -def test_browser_summary_report_is_opt_in_and_uses_real_canonical_components(tmp_path): +def test_browser_summary_report_is_opt_in_and_uses_real_canonical_components( + tmp_path, +): probs, reals = _inputs() output = tmp_path / "canonical-summary.html" From c187f0012c7812ee819ce1c25997519aad717cf3 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:43:03 +0300 Subject: [PATCH 07/10] ci: expose formatter diff for summary report --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index bc2f64ac..c8b31daa 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,8 +31,8 @@ jobs: - name: Check lint run: uv run ruff check . - - name: Check format - run: uv run ruff format --check . + - name: Show format diff + run: uv run ruff format --diff . - name: Check types run: uv run ty check src/rtichoke From 9a74bd105c947b5f7a86b1d7779c0a9a2763e8b9 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:44:32 +0300 Subject: [PATCH 08/10] style: apply ruff formatting --- src/rtichoke/summary_report/summary_report.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index f74fc722..156587e7 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -98,9 +98,7 @@ def _create_browser_summary_report( performance_data, metadata ) roc = _roc_v2_spec_from_performance_data(performance_data, metadata) - calibration = _calibration_v2_spec_from_curve_list( - calibration_curve_list, metadata - ) + calibration = _calibration_v2_spec_from_curve_list(calibration_curve_list, metadata) report = _report_spec_from_components( [ @@ -110,9 +108,7 @@ def _create_browser_summary_report( ], title="rtichoke summary report", ) - return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html( - output_file - ) + return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html(output_file) def render_summary_report(): From a29c3b841304b71197bca4b36a324d1ed8c82c4c Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:44:54 +0300 Subject: [PATCH 09/10] style: apply ruff formatting to tests --- tests/test_summary_report_browser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_summary_report_browser.py b/tests/test_summary_report_browser.py index 8a773893..7c32b8ed 100644 --- a/tests/test_summary_report_browser.py +++ b/tests/test_summary_report_browser.py @@ -138,9 +138,7 @@ def test_browser_summary_report_preserves_component_local_identity(tmp_path): assert "populations" not in report assert "horizon" not in report - table, roc, calibration = [ - component["spec"] for component in report["components"] - ] + table, roc, calibration = [component["spec"] for component in report["components"]] assert table["evaluations"][0]["id"] == "evaluation-1" assert roc["evaluations"][0]["id"] == "evaluation-1" assert calibration["evaluations"][0]["id"] == "evaluation-1" From fa1ced84e6cb29b64f0aea382f43661c337975aa Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Mon, 24 Aug 2026 17:45:16 +0300 Subject: [PATCH 10/10] ci: restore standard validation workflow --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c8b31daa..bc2f64ac 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,8 +31,8 @@ jobs: - name: Check lint run: uv run ruff check . - - name: Show format diff - run: uv run ruff format --diff . + - name: Check format + run: uv run ruff format --check . - name: Check types run: uv run ty check src/rtichoke