diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 052253c3..990d6de1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -71,6 +71,19 @@ jobs: if: github.event.action != 'closed' run: uv run great-docs build + - name: Export Great Tables performance table demo + if: github.event.action != 'closed' + run: uv run marimo export html --no-include-code examples/performance_table_demo.py -o great-docs/_site/performance-table-demo.html + + - name: Export Reactable performance table demo + if: github.event.action != 'closed' + run: | + uv run quarto render examples/performance_table_reactable.qmd --output performance-table-reactable.html + grep -q '.Reactable {' performance-table-reactable.html + grep -q 'Real Positive' performance-table-reactable.html + mv performance-table-reactable.html great-docs/_site/performance-table-reactable.html + cp -R examples/performance_table_reactable_files great-docs/_site/performance_table_reactable_files + - name: Deploy PR preview uses: rossjrw/pr-preview-action@v1 with: diff --git a/examples/performance_table_demo.py b/examples/performance_table_demo.py new file mode 100644 index 00000000..fe2b8900 --- /dev/null +++ b/examples/performance_table_demo.py @@ -0,0 +1,88 @@ +import marimo + +__generated_with = "0.18.4" +app = marimo.App(width="full") + + +@app.cell +def _(): + import marimo as mo + import numpy as np + from rtichoke import create_performance_table, create_performance_table_times + + return create_performance_table, create_performance_table_times, mo, np + + +@app.cell +def _(mo): + mo.md( + """ + # rtichoke performance table + + PR preview for the Python port of `rtichoke::create_performance_table()`. + This Marimo preview uses the **Great Tables** renderer. The same public + API also supports `renderer="reactable"` for Quarto/Jupyter contexts. + """ + ) + return + + +@app.cell +def _(np): + reals = np.array([0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1]) + model_a = np.array([0.04, 0.10, 0.20, 0.24, 0.33, 0.42, 0.48, 0.61, 0.70, 0.82, 0.86, 0.94]) + model_b = np.array([0.08, 0.18, 0.14, 0.39, 0.30, 0.50, 0.43, 0.57, 0.65, 0.74, 0.76, 0.88]) + return model_a, model_b, reals + + +@app.cell +def _(create_performance_table, mo, model_a, reals): + mo.md("## One model — probability threshold") + create_performance_table(probs={"Model A": model_a}, reals=reals, by=0.10) + return + + +@app.cell +def _(create_performance_table, mo, model_a, model_b, reals): + mo.md("## Multiple models — probability threshold") + create_performance_table(probs={"Model A": model_a, "Model B": model_b}, reals=reals, by=0.10) + return + + +@app.cell +def _(create_performance_table, mo, model_a, model_b, reals): + mo.md("## Multiple models — PPCR") + create_performance_table(probs={"Model A": model_a, "Model B": model_b}, reals=reals, by=0.10, stratified_by=("ppcr",)) + return + + +@app.cell +def _(np): + time_probs = {"Model A": np.array([0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 1.00])} + time_reals = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) + times = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + return time_probs, time_reals, times + + +@app.cell +def _(create_performance_table_times, mo, time_probs, time_reals, times): + mo.md("## Fixed time horizons — 5 and 10") + create_performance_table_times(probs=time_probs, reals=time_reals, times=times, fixed_time_horizons=[5, 10], by=0.10) + return + + +@app.cell +def _(mo): + mo.md( + """ + The Great Tables backend is the Marimo-safe renderer. Reactable is kept + as an optional richer backend because it supports sortable columns and + expandable confusion-matrix details in environments that support its + Jupyter widget bridge. + """ + ) + return + + +if __name__ == "__main__": + app.run() diff --git a/examples/performance_table_reactable.qmd b/examples/performance_table_reactable.qmd new file mode 100644 index 00000000..c2f49b27 --- /dev/null +++ b/examples/performance_table_reactable.qmd @@ -0,0 +1,55 @@ +--- +title: "rtichoke performance table — Reactable" +format: + html: + toc: false +jupyter: python3 +page-layout: full +execute: + echo: false + warning: false +--- + +This page shows the optional `renderer="reactable"` output from the same public API used by the Great Tables preview. + +```{python} +import numpy as np +from reactable import embed_css +from rtichoke import create_performance_table, create_performance_table_times + +embed_css() +``` + +## One model — probability threshold + +```{python} +reals = np.array([0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1]) +model_a = np.array([0.04, 0.10, 0.20, 0.24, 0.33, 0.42, 0.48, 0.61, 0.70, 0.82, 0.86, 0.94]) +model_b = np.array([0.08, 0.18, 0.14, 0.39, 0.30, 0.50, 0.43, 0.57, 0.65, 0.74, 0.76, 0.88]) + +create_performance_table(probs={"Model A": model_a}, reals=reals, by=0.10, renderer="reactable") +``` + +## Multiple models — probability threshold + +```{python} +create_performance_table(probs={"Model A": model_a, "Model B": model_b}, reals=reals, by=0.10, renderer="reactable") +``` + +## Multiple models — PPCR + +```{python} +create_performance_table(probs={"Model A": model_a, "Model B": model_b}, reals=reals, by=0.10, stratified_by=("ppcr",), renderer="reactable") +``` + +## Fixed time horizons — 5 and 10 + +```{python} +time_probs = {"Model A": np.array([0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 1.00])} +time_reals = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) +times = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + +create_performance_table_times(probs=time_probs, reals=time_reals, times=times, fixed_time_horizons=[5, 10], by=0.10, renderer="reactable") +``` + +The Reactable renderer is intended for widget-capable notebook/Quarto contexts. Click a row to inspect its expandable confusion matrix, and click column headers to sort. diff --git a/great-docs.yml b/great-docs.yml index dc8f6b7d..2c40814f 100644 --- a/great-docs.yml +++ b/great-docs.yml @@ -50,6 +50,13 @@ reference: - prepare_performance_data_times - prepare_binned_classification_data_times + - title: Performance Tables + desc: Summarize model performance across thresholds and time horizons. + contents: + - create_performance_table + - create_performance_table_times + - render_performance_table + - title: Discrimination desc: ROC, precision-recall, gains, and lift visualizations. contents: diff --git a/pyproject.toml b/pyproject.toml index 43fd6ff8..3b39df7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "pyarrow>=21.0.0", "statsmodels>=0.14.0", "polars>=1.31.0", + "reactable>=0.1.5", + "great-tables>=0.18.0", ] name = "rtichoke" version = "0.1.28" diff --git a/src/rtichoke/__init__.py b/src/rtichoke/__init__.py index f489bb15..5e8b0855 100644 --- a/src/rtichoke/__init__.py +++ b/src/rtichoke/__init__.py @@ -51,6 +51,12 @@ prepare_binned_classification_data_times as prepare_binned_classification_data_times, ) +from rtichoke.performance_table import ( + create_performance_table as create_performance_table, + create_performance_table_times as create_performance_table_times, + render_performance_table as render_performance_table, +) + from rtichoke.summary_report.summary_report import ( create_summary_report as create_summary_report, ) @@ -69,5 +75,8 @@ "plot_decision_curve", "prepare_performance_data", "prepare_performance_data_times", + "create_performance_table", + "create_performance_table_times", + "render_performance_table", "create_summary_report", ] diff --git a/src/rtichoke/performance_table.py b/src/rtichoke/performance_table.py new file mode 100644 index 00000000..63445b92 --- /dev/null +++ b/src/rtichoke/performance_table.py @@ -0,0 +1,91 @@ +"""Performance-table API with multiple rendering backends.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Dict, Literal, Union + +import numpy as np +import polars as pl + +from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.performance_data.performance_data_times import prepare_performance_data_times +from rtichoke.performance_table_great_tables import render_performance_table_great_tables +from rtichoke.performance_table_reactable import ( + DEFAULT_COLORS, + render_performance_table_reactable, +) + +PerformanceTableRenderer = Literal["great_tables", "reactable"] + +_DEFAULT_HEURISTICS = [ + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + } +] + + +def render_performance_table( + performance_data: pl.DataFrame, + color_values: Sequence[str] = DEFAULT_COLORS, + renderer: PerformanceTableRenderer = "great_tables", +): + """Render prepared performance data with a selected table backend.""" + if renderer == "great_tables": + return render_performance_table_great_tables(performance_data, color_values=color_values) + if renderer == "reactable": + return render_performance_table_reactable(performance_data, color_values=color_values) + raise ValueError("renderer must be either 'great_tables' or 'reactable'") + + +def create_performance_table( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + by: float = 0.01, + stratified_by: Sequence[str] = ("probability_threshold",), + color_values: Sequence[str] = DEFAULT_COLORS, + renderer: PerformanceTableRenderer = "great_tables", +): + """Create an R-style rtichoke performance table.""" + performance_data = prepare_performance_data(probs=probs, reals=reals, by=by, stratified_by=stratified_by) + return render_performance_table(performance_data, color_values=color_values, renderer=renderer) + + +def create_performance_table_times( + probs: Dict[str, np.ndarray], + reals: Union[np.ndarray, Dict[str, np.ndarray]], + times: Union[np.ndarray, Dict[str, np.ndarray]], + fixed_time_horizons: list[float], + heuristics_sets: list[Dict] = _DEFAULT_HEURISTICS, + by: float = 0.01, + stratified_by: Sequence[str] = ("probability_threshold",), + color_values: Sequence[str] = DEFAULT_COLORS, + renderer: PerformanceTableRenderer = "great_tables", +): + """Create a time-dependent rtichoke performance table. + + Numerical results come from ``prepare_performance_data_times()``. The table + keeps time horizon and censoring/competing-event heuristics visible so that + multiple requested evaluation scenarios are not collapsed in presentation. + Observed times are normalized to floating point at this public wrapper + boundary; fixed-horizon normalization is handled by the shared time-dependent + performance pipeline. + """ + if isinstance(times, dict): + normalized_times = { + key: np.asarray(value, dtype=float) for key, value in times.items() + } + else: + normalized_times = np.asarray(times, dtype=float) + + performance_data = prepare_performance_data_times( + probs=probs, + reals=reals, + times=normalized_times, + fixed_time_horizons=fixed_time_horizons, + heuristics_sets=heuristics_sets, + by=by, + stratified_by=stratified_by, + ) + return render_performance_table(performance_data, color_values=color_values, renderer=renderer) diff --git a/src/rtichoke/performance_table_great_tables.py b/src/rtichoke/performance_table_great_tables.py new file mode 100644 index 00000000..afaf55f4 --- /dev/null +++ b/src/rtichoke/performance_table_great_tables.py @@ -0,0 +1,203 @@ +"""Great Tables renderer for rtichoke performance tables.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np +import polars as pl +from great_tables import GT, loc, style + + +DEFAULT_COLORS = ( + "#1b9e77", "#d95f02", "#7570b3", "#e7298a", "#07004D", + "#E6AB02", "#FE5F55", "#54494B", "#006E90", "#BC96E6", + "#52050A", "#1F271B", "#BE7C4D", "#63768D", "#08A045", + "#320A28", "#82FF9E", "#2176FF", "#D1603D", "#585123", +) + + +def _bar_css(value: float | None, maximum: float = 1.0, color: str = "lightgreen") -> str: + """CSS matching rtichoke R's reactable metric bars.""" + if value is None or not np.isfinite(value) or maximum <= 0: + width = 0.0 + else: + width = min(abs(float(value)) / maximum, 1.0) * 100 + return ( + f"background:linear-gradient(90deg,{color} {width:.4f}%,transparent {width:.4f}%);" + "background-size:98% 88%;background-repeat:no-repeat;background-position:center;" + ) + + +def _net_benefit_css(value: float | None, maximum: float) -> str: + """Diverging bar CSS matching rtichoke R's net-benefit treatment.""" + if value is None or not np.isfinite(value) or maximum <= 0: + scaled = 0.0 + else: + scaled = max(-1.0, min(float(value) / maximum, 1.0)) + position = (0.5 + scaled / 2.0) * 100 + if scaled >= 0: + background = ( + "linear-gradient(90deg,transparent 50%,lightgreen 50%," + f"lightgreen {position:.4f}%,transparent {position:.4f}%)" + ) + else: + background = ( + f"linear-gradient(90deg,transparent {position:.4f}%,pink {position:.4f}%," + "pink 50%,transparent 50%)" + ) + return ( + f"background:{background};background-size:98% 88%;" + "background-repeat:no-repeat;background-position:center;" + ) + + +def render_performance_table_great_tables( + performance_data: pl.DataFrame, + color_values: Sequence[str] = DEFAULT_COLORS, +) -> GT: + """Render prepared binary or time-dependent performance data.""" + if performance_data.is_empty(): + raise ValueError("performance_data must contain at least one row") + + stratifications = performance_data.get_column("stratified_by").unique().to_list() + if len(stratifications) != 1: + raise ValueError("performance_data must contain exactly one stratification") + stratified_by = stratifications[0] + + context_columns = [ + "fixed_time_horizon", + "censoring_heuristic", + "competing_heuristic", + ] + source_columns = [ + "reference_group", *context_columns, "chosen_cutoff", "sensitivity", + "specificity", "ppv", "npv", "lift", "predicted_positives", + "net_benefit", "ppcr", + ] + data = performance_data.select([c for c in source_columns if c in performance_data.columns]) + if "reference_group" in data.columns: + data = data.rename({"reference_group": "Model"}) + if "fixed_time_horizon" in data.columns: + data = data.rename({"fixed_time_horizon": "Time"}) + if "censoring_heuristic" in data.columns: + data = data.rename({"censoring_heuristic": "Censoring"}) + if "competing_heuristic" in data.columns: + data = data.rename({"competing_heuristic": "Competing Event"}) + + context_sort = [c for c in ("Time", "Censoring", "Competing Event") if c in data.columns] + if stratified_by == "probability_threshold": + data = data.rename({"chosen_cutoff": "Threshold"}) + sort_columns = context_sort + [c for c in ("Threshold", "Model") if c in data.columns] + else: + if "chosen_cutoff" in data.columns: + data = data.drop("chosen_cutoff") + sort_columns = context_sort + [c for c in ("ppcr", "Model") if c in data.columns] + if sort_columns: + data = data.sort(sort_columns) + + data = data.with_columns( + pl.concat_str( + [ + pl.col("predicted_positives").cast(pl.String), + pl.lit(" ("), + (pl.col("ppcr") * 100).round(2).cast(pl.String), + pl.lit("%)"), + ] + ).alias("Predicted Positives") + ) + + if stratified_by != "probability_threshold" and "net_benefit" in data.columns: + data = data.drop("net_benefit") + + metric_columns = [ + c for c in ["sensitivity", "specificity", "ppv", "npv", "lift", "net_benefit"] + if c in data.columns + ] + display_columns = [c for c in [ + "Model", "Time", "Censoring", "Competing Event", "Threshold", + "Predicted Positives", "sensitivity", "specificity", "ppv", "npv", + "lift", "net_benefit", + ] if c in data.columns] + display = data.select(display_columns) + + labels = { + "Model": "Model", + "Time": "Time Horizon", + "Censoring": "Censoring", + "Competing Event": "Competing Event", + "Threshold": "Probability Threshold", + "Predicted Positives": "Predicted Positives", + "sensitivity": "Sens", + "specificity": "Spec", + "ppv": "PPV", + "npv": "NPV", + "lift": "Lift", + "net_benefit": "Net Benefit", + } + + table = ( + GT(display) + .cols_label(cases={c: labels[c] for c in display.columns}) + .tab_spanner(label="Performance Metrics", columns=metric_columns) + .fmt_number(columns=metric_columns, decimals=2) + .cols_width({c: "100px" for c in metric_columns + ["Predicted Positives"]}) + .opt_vertical_padding(scale=0.75) + .opt_horizontal_padding(scale=0.8) + .opt_css( + ".gt_table{font-size:14px;}" + ".gt_col_heading{font-weight:600;}" + ".gt_row{vertical-align:middle;}" + ) + ) + + table = table.tab_style( + style=style.css("text-align:left;"), + locations=loc.body(columns=display.columns), + ) + if "net_benefit" in display.columns: + table = table.tab_style( + style=style.css("text-align:center;"), + locations=loc.body(columns="net_benefit"), + ) + + if "Model" in data.columns: + models = data.get_column("Model").unique(maintain_order=True).to_list() + for index, model in enumerate(models): + color = color_values[index % len(color_values)] + rows = [i for i, value in enumerate(data.get_column("Model").to_list()) if value == model] + table = table.tab_style( + style=style.css(f"color:{color};font-weight:600;text-shadow:0 0 0 currentColor;"), + locations=loc.body(columns="Model", rows=rows), + ) + + lift_max = float(data.get_column("lift").drop_nulls().max() or 1.0) + nb_max = 1.0 + if "net_benefit" in data.columns: + nb = data.get_column("net_benefit").drop_nulls().abs().max() + nb_max = float(nb or 1.0) + + for row_index in range(data.height): + ppcr_value = data[row_index, "ppcr"] + table = table.tab_style( + style=style.css(_bar_css(ppcr_value, color="lightgrey")), + locations=loc.body(columns="Predicted Positives", rows=[row_index]), + ) + for column in ["sensitivity", "specificity", "ppv", "npv"]: + if column in data.columns: + table = table.tab_style( + style=style.css(_bar_css(data[row_index, column])), + locations=loc.body(columns=column, rows=[row_index]), + ) + if "lift" in data.columns: + table = table.tab_style( + style=style.css(_bar_css(data[row_index, "lift"], maximum=lift_max)), + locations=loc.body(columns="lift", rows=[row_index]), + ) + if "net_benefit" in data.columns: + table = table.tab_style( + style=style.css(_net_benefit_css(data[row_index, "net_benefit"], nb_max)), + locations=loc.body(columns="net_benefit", rows=[row_index]), + ) + + return table diff --git a/src/rtichoke/performance_table_reactable.py b/src/rtichoke/performance_table_reactable.py new file mode 100644 index 00000000..b60ae62a --- /dev/null +++ b/src/rtichoke/performance_table_reactable.py @@ -0,0 +1,229 @@ +"""Reactable renderer for rtichoke performance tables.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import htmltools as html +import numpy as np +import polars as pl +from reactable import Reactable, Column, ColFormat, ColGroup +from reactable.models import CellInfo, RowInfo + + +DEFAULT_COLORS = ( + "#1b9e77", "#d95f02", "#7570b3", "#e7298a", "#07004D", + "#E6AB02", "#FE5F55", "#54494B", "#006E90", "#BC96E6", + "#52050A", "#1F271B", "#BE7C4D", "#63768D", "#08A045", + "#320A28", "#82FF9E", "#2176FF", "#D1603D", "#585123", +) + + +def _bar_style(value: float | None, maximum: float = 1.0, color: str = "lightgreen") -> dict[str, str]: + if value is None or not np.isfinite(value) or maximum <= 0: + return {} + width = min(abs(float(value)) / maximum, 1.0) * 100 + return { + "background": f"linear-gradient(90deg, {color} {width}%, transparent {width}%)", + "backgroundSize": "98% 88%", + "backgroundRepeat": "no-repeat", + "backgroundPosition": "center", + } + + +def _net_benefit_style(value: float | None, maximum: float) -> dict[str, str]: + if value is None or not np.isfinite(value) or maximum <= 0: + return {} + width = max(-1.0, min(float(value) / maximum, 1.0)) + position = (0.5 + width / 2) * 100 + if width >= 0: + background = ( + "linear-gradient(90deg, transparent 50%, lightgreen 50%, " + f"lightgreen {position}%, transparent {position}%)" + ) + else: + background = ( + f"linear-gradient(90deg, transparent {position}%, pink {position}%, " + "pink 50%, transparent 50%)" + ) + return { + "background": background, + "backgroundSize": "98% 88%", + "backgroundRepeat": "no-repeat", + "backgroundPosition": "center", + } + + +def _metric_column(column_id: str, name: str, maximum: float = 1.0) -> Column: + return Column( + id=column_id, + name=name, + format=ColFormat(digits=2), + style=lambda info: _bar_style(info.value, maximum), + ) + + +def render_performance_table_reactable( + performance_data: pl.DataFrame, + color_values: Sequence[str] = DEFAULT_COLORS, +) -> Reactable: + """Render prepared binary or time-dependent performance data.""" + if performance_data.is_empty(): + raise ValueError("performance_data must contain at least one row") + + stratifications = performance_data.get_column("stratified_by").unique().to_list() + if len(stratifications) != 1: + raise ValueError("performance_data must contain exactly one stratification") + stratified_by = stratifications[0] + + display_columns = [ + "reference_group", "fixed_time_horizon", "censoring_heuristic", + "competing_heuristic", "chosen_cutoff", "sensitivity", "specificity", + "ppv", "npv", "lift", "predicted_positives", "net_benefit", "ppcr", + "true_positives", "true_negatives", "false_positives", "false_negatives", + ] + data = performance_data.select([c for c in display_columns if c in performance_data.columns]) + rename_map = { + "reference_group": "Model", + "fixed_time_horizon": "Time", + "censoring_heuristic": "Censoring", + "competing_heuristic": "Competing Event", + } + data = data.rename({k: v for k, v in rename_map.items() if k in data.columns}) + + context_sort = [c for c in ("Time", "Censoring", "Competing Event") if c in data.columns] + if stratified_by == "probability_threshold": + data = data.rename({"chosen_cutoff": "Threshold"}) + sort_columns = context_sort + [c for c in ("Threshold", "Model") if c in data.columns] + else: + if "chosen_cutoff" in data.columns: + data = data.drop("chosen_cutoff") + sort_columns = context_sort + [c for c in ("ppcr", "Model") if c in data.columns] + if sort_columns: + data = data.sort(sort_columns) + + lift_max = data.get_column("lift").drop_nulls().max() or 1.0 + nb_max = 1.0 + if "net_benefit" in data.columns: + nb_max = data.get_column("net_benefit").drop_nulls().abs().max() or 1.0 + + models = data.get_column("Model").unique(maintain_order=True).to_list() if "Model" in data.columns else [] + colors = {model: color_values[i % len(color_values)] for i, model in enumerate(models)} + + def model_cell(info: CellInfo): + value = info.value + color = colors.get(value, "#aaa") + return html.span( + html.span(style=( + "display:inline-block;margin-right:8px;width:9px;height:9px;" + f"background-color:{color};border-radius:50%;" + )), + str(value), + ) + + def ppcr_cell(info: CellInfo) -> str: + if info.value is None: + return "" + count = data[info.row_index, "predicted_positives"] + return f"{count} ({float(info.value) * 100:.2f}%)" + + def confusion_matrix(info: RowInfo): + i = info.row_index + tp = data[i, "true_positives"] + tn = data[i, "true_negatives"] + fp = data[i, "false_positives"] + fn = data[i, "false_negatives"] + predicted_positive = tp + fp + predicted_negative = fn + tn + real_positive = tp + fn + real_negative = fp + tn + total = tp + fp + fn + tn + + matrix = pl.DataFrame({ + "Outcome": ["Predicted Positive", "Predicted Negative", " "], + "Real Positive": [tp, fn, real_positive], + "Real Negative": [fp, tn, real_negative], + "Total": [predicted_positive, predicted_negative, total], + }) + + def matrix_cell(info: CellInfo) -> str: + if info.value is None or total == 0: + return "" + return f"{info.value} ({float(info.value) / total * 100:.2f}%)" + + def matrix_style(colors: tuple[str, str, str]): + return lambda info: _bar_style(info.value, float(total), colors[info.row_index]) + + nested = Reactable( + matrix, + columns=[ + Column(id="Outcome", style={"fontWeight": "bold"}), + Column( + id="Real Positive", + align="left", + cell=matrix_cell, + style=matrix_style(("lightgreen", "pink", "lightgrey")), + ), + Column( + id="Real Negative", + align="left", + cell=matrix_cell, + style=matrix_style(("pink", "lightgreen", "lightgrey")), + ), + Column( + id="Total", + name=" ", + align="left", + cell=matrix_cell, + style=matrix_style(("lightgrey", "lightgrey", "lightgrey")), + ), + ], + full_width=False, + sortable=False, + pagination=False, + ) + return html.div(nested.to_widget(), style="padding:16px;") + + columns = [Column(id="Model", cell=model_cell, min_width=120)] + if "Time" in data.columns: + columns.append(Column(id="Time", name="Time Horizon", format=ColFormat(digits=2), min_width=100)) + if "Censoring" in data.columns: + columns.append(Column(id="Censoring", name="Censoring", min_width=110)) + if "Competing Event" in data.columns: + columns.append(Column(id="Competing Event", name="Competing Event", min_width=140)) + if stratified_by == "probability_threshold": + columns.append(Column(id="Threshold", name="Probability Threshold", format=ColFormat(digits=2), min_width=130)) + columns.extend([ + Column(id="ppcr", name="Predicted Positives", cell=ppcr_cell, min_width=150, + style=lambda info: _bar_style(info.value, 1.0, "#d3d3d3")), + Column(id="predicted_positives", show=False), + _metric_column("sensitivity", "Sens"), + _metric_column("specificity", "Spec"), + _metric_column("ppv", "PPV"), + _metric_column("npv", "NPV"), + _metric_column("lift", "Lift", float(lift_max)), + Column(id="net_benefit", name="Net Benefit", format=ColFormat(digits=2), + style=lambda info: _net_benefit_style(info.value, float(nb_max)), + show=stratified_by == "probability_threshold"), + Column(id="true_positives", show=False), + Column(id="true_negatives", show=False), + Column(id="false_positives", show=False), + Column(id="false_negatives", show=False), + ]) + + metric_columns = ["sensitivity", "specificity", "ppv", "npv", "lift"] + if stratified_by == "probability_threshold": + metric_columns.append("net_benefit") + + return Reactable( + data, + columns=columns, + column_groups=[ColGroup(name="Performance Metrics", columns=metric_columns)], + default_col_def=Column(align="left"), + bordered=True, + compact=True, + striped=True, + highlight=True, + details=confusion_matrix, + show_sort_icon=False, + ) diff --git a/tests/test_performance_table.py b/tests/test_performance_table.py new file mode 100644 index 00000000..dd9d4479 --- /dev/null +++ b/tests/test_performance_table.py @@ -0,0 +1,108 @@ +import numpy as np +import pytest +from great_tables import GT +from reactable import Reactable + +from rtichoke.performance_table_reactable import _bar_style, _net_benefit_style + +from rtichoke import ( + create_performance_table, + create_performance_table_times, + prepare_performance_data, + prepare_performance_data_times, + render_performance_table, +) + + +def _example(): + probs = {"Model 1": np.array([0.05, 0.15, 0.35, 0.55, 0.75, 0.95])} + reals = np.array([0, 0, 1, 0, 1, 1]) + return probs, reals + + +def _time_example(): + probs = {"Model 1": np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])} + reals = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) + times = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + return probs, reals, times + + +def test_create_performance_table_defaults_to_great_tables(): + probs, reals = _example() + assert isinstance(create_performance_table(probs, reals, by=0.1), GT) + + +def test_create_performance_table_supports_reactable(): + probs, reals = _example() + assert isinstance(create_performance_table(probs, reals, by=0.1, renderer="reactable"), Reactable) + + +def test_render_performance_table_accepts_prepared_polars_data(): + probs, reals = _example() + data = prepare_performance_data(probs, reals, by=0.1) + assert isinstance(render_performance_table(data), GT) + assert isinstance(render_performance_table(data, renderer="reactable"), Reactable) + + +def test_render_performance_table_rejects_empty_data(): + probs, reals = _example() + data = prepare_performance_data(probs, reals, by=0.1).clear() + with pytest.raises(ValueError, match="at least one row"): + render_performance_table(data) + + +def test_create_performance_table_supports_ppcr_stratification(): + probs, reals = _example() + assert isinstance(create_performance_table(probs, reals, by=0.1, stratified_by=("ppcr",)), GT) + + +def test_create_performance_table_times_defaults_to_great_tables(): + probs, reals, times = _time_example() + assert isinstance(create_performance_table_times(probs, reals, times, fixed_time_horizons=[5, 10], by=0.1), GT) + + +def test_create_performance_table_times_supports_reactable(): + probs, reals, times = _time_example() + assert isinstance(create_performance_table_times(probs, reals, times, fixed_time_horizons=[5], by=0.1, renderer="reactable"), Reactable) + + +def test_render_performance_table_preserves_multiple_time_horizons(): + probs, reals, times = _time_example() + data = prepare_performance_data_times(probs, reals, times.astype(float), fixed_time_horizons=[5, 10], by=0.1) + assert sorted(data.get_column("fixed_time_horizon").unique().to_list()) == [5.0, 10.0] + assert isinstance(render_performance_table(data), GT) + + +def test_create_performance_table_times_supports_multiple_heuristic_sets(): + probs, reals, times = _time_example() + heuristics_sets = [ + {"censoring_heuristic": "adjusted", "competing_heuristic": "adjusted_as_negative"}, + {"censoring_heuristic": "excluded", "competing_heuristic": "excluded"}, + ] + data = prepare_performance_data_times(probs, reals, times.astype(float), fixed_time_horizons=[5], heuristics_sets=heuristics_sets, by=0.1) + assert data.get_column("censoring_heuristic").n_unique() == 2 + assert data.get_column("competing_heuristic").n_unique() == 2 + assert isinstance(render_performance_table(data), GT) + + +def test_invalid_renderer_is_rejected(): + probs, reals = _example() + data = prepare_performance_data(probs, reals, by=0.1) + with pytest.raises(ValueError, match="renderer"): + render_performance_table(data, renderer="unknown") + + +def test_reactable_metric_bar_matches_r_colors_and_geometry(): + style = _bar_style(0.25) + assert "lightgreen 25.0%" in style["background"] + assert style["backgroundSize"] == "98% 88%" + assert style["backgroundRepeat"] == "no-repeat" + assert style["backgroundPosition"] == "center" + + +@pytest.mark.parametrize(("value", "color", "extent"), [(0.5, "lightgreen", "75.0%"), (-0.5, "pink", "25.0%")]) +def test_reactable_net_benefit_bar_matches_r_diverging_scale(value, color, extent): + style = _net_benefit_style(value, 1.0) + assert color in style["background"] + assert extent in style["background"] + assert "50%" in style["background"] diff --git a/user_guide/01-naming-conventions.qmd b/user_guide/01-naming-conventions.qmd index 6276beea..e874f009 100644 --- a/user_guide/01-naming-conventions.qmd +++ b/user_guide/01-naming-conventions.qmd @@ -10,14 +10,15 @@ guide-section: "Getting Started" | Prefix | Purpose | Typical input | Typical output | |---|---|---|---| | `prepare_*` | Prepare reusable performance data | predictions and observed outcomes | performance data | -| `create_*` | Prepare data and create a visualization in one call | predictions and observed outcomes | interactive figure | +| `create_*` | Prepare data and create a visualization or table in one call | predictions and observed outcomes | figure or rendered table | | `plot_*` | Visualize data that has already been prepared | performance data | interactive figure | +| `render_*` | Render already-prepared data as a table | prepared performance data | rendered table | -For example, a direct ROC workflow uses `create_roc_curve()`, while a workflow that first prepares reusable performance data can pass those results to `plot_roc_curve()`. +For example, a direct ROC workflow uses `create_roc_curve()`, while a workflow that first prepares reusable performance data can pass those results to `plot_roc_curve()`. Performance tables follow the same direct-versus-prepared-data idea: `create_performance_table()` prepares and renders in one call, while `render_performance_table()` renders an already-prepared performance-data frame. ## Curve families -The same naming pattern repeats across the main performance views: +The same naming pattern repeats across the main performance curves: | Performance view | Direct visualization | Plot prepared data | |---|---|---| @@ -29,6 +30,18 @@ The same naming pattern repeats across the main performance views: Calibration currently uses the direct `create_calibration_curve()` interface. +## Performance tables + +Performance tables use a closely related naming pattern: + +| Workflow | Function | +|---|---| +| Prepare and render a binary-outcome table | `create_performance_table()` | +| Prepare and render a time-to-event table | `create_performance_table_times()` | +| Render already-prepared performance data | `render_performance_table()` | + +The table constructors use the same underlying `prepare_performance_data()` and `prepare_performance_data_times()` pipelines as the curve functions. The `render_*` prefix is used when the numerical performance data already exist and only the presentation layer is needed. + ## Time-to-event variants Functions ending in `_times` extend the corresponding workflow to time-to-event outcomes. For example: @@ -39,6 +52,8 @@ Functions ending in `_times` extend the corresponding workflow to time-to-event - `create_calibration_curve_times()` → time-to-event calibration curve - `create_decision_curve()` → binary-outcome decision curve - `create_decision_curve_times()` → time-to-event decision curve +- `create_performance_table()` → binary-outcome performance table +- `create_performance_table_times()` → time-to-event performance table The same convention is used for the performance-data preparation functions, such as `prepare_performance_data()` and `prepare_performance_data_times()`. @@ -48,8 +63,9 @@ Think of the API as a small grammar: ```text prepare + performance data -> reusable data -create + metric/curve -> data to figure -plot + metric/curve -> prepared data to figure +create + curve/table -> data to rendered output +plot + curve -> prepared data to figure +render + table -> prepared data to rendered table *_times -> time-to-event version ``` diff --git a/user_guide/04-performance-tables.qmd b/user_guide/04-performance-tables.qmd new file mode 100644 index 00000000..4ef1e38e --- /dev/null +++ b/user_guide/04-performance-tables.qmd @@ -0,0 +1,136 @@ +--- +title: "Performance Tables" +guide-section: "Model Performance" +--- + +Performance tables summarize several model-performance quantities at the same probability threshold. They are useful when you want a compact comparison across models rather than a separate ROC, precision-recall, calibration, or decision curve. + +`rtichoke` provides two public constructors: + +- `create_performance_table()` for binary outcomes. +- `create_performance_table_times()` for time-to-event outcomes at one or more fixed horizons. + +Both use the existing `prepare_performance_data()` / `prepare_performance_data_times()` pipelines as their numerical source of truth. The table layer is presentation only. + +## Basic performance table + +A minimal two-model example: + +```python +import numpy as np +import rtichoke as rk + +reals = np.array([0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1]) + +probs = { + "Model A": np.array([0.04, 0.10, 0.20, 0.24, 0.33, 0.42, 0.48, 0.61, 0.70, 0.82, 0.86, 0.94]), + "Model B": np.array([0.08, 0.18, 0.14, 0.39, 0.30, 0.50, 0.43, 0.57, 0.65, 0.74, 0.76, 0.88]), +} + +table = rk.create_performance_table( + probs=probs, + reals=reals, + by=0.10, +) + +table +``` + +The default stratification is by `probability_threshold`, so each row corresponds to a threshold for one model. The table collects the performance quantities produced by `prepare_performance_data()` into one view, including discrimination, classification, and decision-analytic quantities where available. + +For an alternative view based on the predicted-positive proportion, use: + +```python +rk.create_performance_table( + probs=probs, + reals=reals, + by=0.10, + stratified_by=("ppcr",), +) +``` + +## Time-dependent performance tables + +`create_performance_table_times()` applies the same idea to time-to-event prediction. You supply observed times and one or more fixed horizons: + +```python +import numpy as np +import rtichoke as rk + +probs = { + "Model A": np.array([0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 1.00]) +} + +# 0 = censored, 1 = event of interest +reals = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1]) +times = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + +rk.create_performance_table_times( + probs=probs, + reals=reals, + times=times, + fixed_time_horizons=[5, 10], + by=0.10, +) +``` + +The time horizon remains visible in the output, so results from different horizons are not collapsed together. + +By default, time-dependent performance tables use: + +```python +heuristics_sets = [ + { + "censoring_heuristic": "adjusted", + "competing_heuristic": "adjusted_as_negative", + } +] +``` + +You can pass multiple heuristic sets. The censoring and competing-event heuristic columns remain visible so distinct evaluation scenarios stay distinguishable. + +As with the other time-dependent rtichoke functions, a censoring heuristic affects estimates only when censored observations are present, and a competing-event heuristic affects estimates only when competing events are present. + +## Renderer choice + +The default renderer is **Great Tables**: + +```python +rk.create_performance_table(probs=probs, reals=reals) +``` + +Great Tables is the recommended renderer for Marimo and ordinary HTML output. It is styled to preserve the visual ideas of the original R performance table, including model labeling, grouped performance columns, compact metric bars, predicted-positive bars, and diverging net-benefit bars. + +For Quarto or Jupyter environments, Reactable remains available explicitly: + +```python +rk.create_performance_table( + probs=probs, + reals=reals, + renderer="reactable", +) +``` + +The Reactable backend adds richer interaction such as sortable columns and expandable confusion-matrix details. It is retained as an option for environments that support its Jupyter widget bridge; it is **not** the Marimo renderer. + +The same `renderer=` argument is available on `create_performance_table_times()`. + +## Render prepared performance data directly + +If you already called `prepare_performance_data()` or `prepare_performance_data_times()`, render the resulting Polars DataFrame without recomputing it: + +```python +performance_data = rk.prepare_performance_data( + probs=probs, + reals=reals, + by=0.10, +) + +rk.render_performance_table(performance_data) +``` + +Use `renderer="reactable"` here as well if you want the Reactable backend. + +## Related documentation + +For the underlying numerical data, see the `prepare_performance_data()` and `prepare_performance_data_times()` API reference. For time-dependent censoring and competing-event semantics, see [Curve API Compatibility](curve-api-compatibility.html).