From 5fc5f5f661f95f0ea997954ec7cacf84ed45d612 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 21:55:14 +0300 Subject: [PATCH 1/8] Add internal canonical ROC v2 spec builder --- src/rtichoke/_viz_spec_v2.py | 114 +++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 src/rtichoke/_viz_spec_v2.py diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py new file mode 100644 index 00000000..96b29667 --- /dev/null +++ b/src/rtichoke/_viz_spec_v2.py @@ -0,0 +1,114 @@ +"""Internal builders for canonical ``rtichoke_viz`` v2 specifications. + +These helpers are deliberately not wired into production rendering yet. They +translate already-computed performance data plus semantic evaluation metadata +into the canonical visualization contract. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +import polars as pl + +from rtichoke.processing.evaluation_semantics import _EvaluationMetadata + +_REQUIRED_ROC_COLUMNS = { + "reference_group", + "chosen_cutoff", + "sensitivity", + "specificity", +} + + +def _roc_v2_spec_from_performance_data( + performance_data: pl.DataFrame, + evaluation_metadata: Mapping[str, _EvaluationMetadata], +) -> dict[str, object]: + """Build a canonical ROC-v2 spec without recalculating statistics. + + ``reference_group`` is used only to join existing performance rows to the + semantic metadata established by the high-level production inputs. Stable + evaluation/series IDs are ordinal over that semantic metadata, so they do + not encode labels, compatibility grouping names, prevalence, coordinates, + colors, or other presentation/statistical values. + """ + missing = _REQUIRED_ROC_COLUMNS.difference(performance_data.columns) + if missing: + missing_columns = ", ".join(sorted(missing)) + raise ValueError(f"ROC performance data is missing columns: {missing_columns}") + + rows = performance_data.select( + "reference_group", + "chosen_cutoff", + "sensitivity", + "specificity", + ).to_dicts() + row_groups = {str(row["reference_group"]) for row in rows} + metadata_groups = set(evaluation_metadata) + + missing_metadata = row_groups.difference(metadata_groups) + if missing_metadata: + groups = ", ".join(sorted(missing_metadata)) + raise ValueError(f"ROC performance rows are missing evaluation metadata: {groups}") + + ordered_groups = [group for group in evaluation_metadata if group in row_groups] + evaluation_ids = { + group: f"evaluation-{index}" for index, group in enumerate(ordered_groups, start=1) + } + series_ids = { + group: f"series-{index}" for index, group in enumerate(ordered_groups, start=1) + } + + evaluations: list[dict[str, object]] = [] + series: list[dict[str, object]] = [] + + for group in ordered_groups: + metadata = evaluation_metadata[group] + evaluation: dict[str, object] = { + "id": evaluation_ids[group], + "population": metadata.population, + } + if metadata.model is not None: + evaluation["model"] = metadata.model + evaluations.append(evaluation) + + if metadata.model is not None: + display_value = metadata.model + display_role = "model" + else: + display_value = metadata.population + display_role = "population" + + series.append( + { + "id": series_ids[group], + "evaluationId": evaluation_ids[group], + "display": { + "label": display_value, + "group": display_value, + "role": display_role, + }, + } + ) + + return { + "schemaVersion": "2.0", + "type": "roc", + "evaluations": evaluations, + "series": series, + "data": [ + { + "seriesId": series_ids[str(row["reference_group"])], + "cutoff": row["chosen_cutoff"], + "sensitivity": row["sensitivity"], + "specificity": row["specificity"], + } + for row in rows + ], + "x": "false_positive_rate", + "y": "sensitivity", + "xAxis": {"label": "1 - Specificity", "domain": [0, 1]}, + "yAxis": {"label": "Sensitivity", "domain": [0, 1]}, + "references": [{"type": "identity", "scope": "global"}], + } From a3dcd2da5054b57a0d7d08e0b10d559b0aaf01e5 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 21:55:41 +0300 Subject: [PATCH 2/8] Characterize canonical ROC v2 builder semantics --- tests/test_viz_spec_v2.py | 131 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 tests/test_viz_spec_v2.py diff --git a/tests/test_viz_spec_v2.py b/tests/test_viz_spec_v2.py new file mode 100644 index 00000000..7ec46dea --- /dev/null +++ b/tests/test_viz_spec_v2.py @@ -0,0 +1,131 @@ +import numpy as np + +from rtichoke._viz_spec_v2 import _roc_v2_spec_from_performance_data +from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.processing.evaluation_semantics import ( + _SHARED_POPULATION, + _build_evaluation_metadata, +) + + +def _static_metadata(probs, reals): + return _build_evaluation_metadata(probs, reals, np.array([])) + + +def test_roc_v2_one_model_one_population(): + probs = {"Model A": np.array([0.05, 0.2, 0.7, 0.95])} + reals = np.array([0, 0, 1, 1]) + performance_data = prepare_performance_data(probs, reals, by=0.25) + + spec = _roc_v2_spec_from_performance_data( + performance_data, + _static_metadata(probs, reals), + ) + + assert spec["schemaVersion"] == "2.0" + assert spec["type"] == "roc" + assert spec["evaluations"] == [ + { + "id": "evaluation-1", + "model": "Model A", + "population": _SHARED_POPULATION, + } + ] + assert spec["series"] == [ + { + "id": "series-1", + "evaluationId": "evaluation-1", + "display": {"label": "Model A", "group": "Model A", "role": "model"}, + } + ] + assert {row["seriesId"] for row in spec["data"]} == {"series-1"} + assert spec["references"] == [{"type": "identity", "scope": "global"}] + + +def test_roc_v2_multiple_models_share_one_population(): + probs = { + "Model A": np.array([0.05, 0.2, 0.7, 0.95]), + "Model B": np.array([0.1, 0.4, 0.6, 0.9]), + } + reals = np.array([0, 0, 1, 1]) + performance_data = prepare_performance_data(probs, reals, by=0.25) + + spec = _roc_v2_spec_from_performance_data( + performance_data, + _static_metadata(probs, reals), + ) + + evaluations = spec["evaluations"] + assert [evaluation["id"] for evaluation in evaluations] == [ + "evaluation-1", + "evaluation-2", + ] + assert {evaluation["population"] for evaluation in evaluations} == { + _SHARED_POPULATION + } + assert [evaluation["model"] for evaluation in evaluations] == ["Model A", "Model B"] + assert [series["id"] for series in spec["series"]] == ["series-1", "series-2"] + assert {series["display"]["role"] for series in spec["series"]} == {"model"} + assert {row["seriesId"] for row in spec["data"]} == {"series-1", "series-2"} + + +def test_roc_v2_keyed_populations_keep_model_identity_unknown(): + probs = { + "Population A": np.array([0.05, 0.2, 0.7, 0.95]), + "Population B": np.array([0.1, 0.4, 0.6, 0.9]), + } + reals = { + "Population A": np.array([0, 0, 1, 1]), + "Population B": np.array([0, 1, 0, 1]), + } + performance_data = prepare_performance_data(probs, reals, by=0.25) + + spec = _roc_v2_spec_from_performance_data( + performance_data, + _static_metadata(probs, reals), + ) + + assert spec["evaluations"] == [ + {"id": "evaluation-1", "population": "Population A"}, + {"id": "evaluation-2", "population": "Population B"}, + ] + assert [series["display"] for series in spec["series"]] == [ + {"label": "Population A", "group": "Population A", "role": "population"}, + {"label": "Population B", "group": "Population B", "role": "population"}, + ] + assert {row["seriesId"] for row in spec["data"]} == {"series-1", "series-2"} + + +def test_roc_v2_ids_do_not_encode_compatibility_group_labels(): + reals = { + "Population A": np.array([0, 0, 1, 1]), + "Population B": np.array([0, 1, 0, 1]), + } + probs = { + "Population A": np.array([0.05, 0.2, 0.7, 0.95]), + "Population B": np.array([0.1, 0.4, 0.6, 0.9]), + } + spec = _roc_v2_spec_from_performance_data( + prepare_performance_data(probs, reals, by=0.25), + _static_metadata(probs, reals), + ) + + renamed_reals = { + "Cohort X": reals["Population A"], + "Cohort Y": reals["Population B"], + } + renamed_probs = { + "Cohort X": probs["Population A"], + "Cohort Y": probs["Population B"], + } + renamed_spec = _roc_v2_spec_from_performance_data( + prepare_performance_data(renamed_probs, renamed_reals, by=0.25), + _static_metadata(renamed_probs, renamed_reals), + ) + + assert [evaluation["id"] for evaluation in spec["evaluations"]] == [ + evaluation["id"] for evaluation in renamed_spec["evaluations"] + ] + assert [series["id"] for series in spec["series"]] == [ + series["id"] for series in renamed_spec["series"] + ] From 2c34aa2eda72d16b69e73c25caaaa1060adae211 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 21:57:19 +0300 Subject: [PATCH 3/8] Format ROC v2 builder --- src/rtichoke/_viz_spec_v2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index 96b29667..64d3bf07 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -54,10 +54,12 @@ def _roc_v2_spec_from_performance_data( ordered_groups = [group for group in evaluation_metadata if group in row_groups] evaluation_ids = { - group: f"evaluation-{index}" for index, group in enumerate(ordered_groups, start=1) + group: f"evaluation-{index}" + for index, group in enumerate(ordered_groups, start=1) } series_ids = { - group: f"series-{index}" for index, group in enumerate(ordered_groups, start=1) + group: f"series-{index}" + for index, group in enumerate(ordered_groups, start=1) } evaluations: list[dict[str, object]] = [] From 58647a0a35ee84be5bc595b6ba1d4aea7bf30e91 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 21:58:46 +0300 Subject: [PATCH 4/8] Match Ruff formatting for ROC v2 builder --- src/rtichoke/_viz_spec_v2.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index 64d3bf07..191635e2 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -36,7 +36,9 @@ def _roc_v2_spec_from_performance_data( missing = _REQUIRED_ROC_COLUMNS.difference(performance_data.columns) if missing: missing_columns = ", ".join(sorted(missing)) - raise ValueError(f"ROC performance data is missing columns: {missing_columns}") + raise ValueError( + f"ROC performance data is missing columns: {missing_columns}" + ) rows = performance_data.select( "reference_group", @@ -50,7 +52,9 @@ def _roc_v2_spec_from_performance_data( missing_metadata = row_groups.difference(metadata_groups) if missing_metadata: groups = ", ".join(sorted(missing_metadata)) - raise ValueError(f"ROC performance rows are missing evaluation metadata: {groups}") + raise ValueError( + f"ROC performance rows are missing evaluation metadata: {groups}" + ) ordered_groups = [group for group in evaluation_metadata if group in row_groups] evaluation_ids = { From 7cd55d8c9dcb842634ddb3554d6be68bf65ff56c Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 22:00:06 +0300 Subject: [PATCH 5/8] Match Ruff line-length formatting --- src/rtichoke/_viz_spec_v2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index 191635e2..0b6692c3 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -36,9 +36,7 @@ def _roc_v2_spec_from_performance_data( missing = _REQUIRED_ROC_COLUMNS.difference(performance_data.columns) if missing: missing_columns = ", ".join(sorted(missing)) - raise ValueError( - f"ROC performance data is missing columns: {missing_columns}" - ) + raise ValueError(f"ROC performance data is missing columns: {missing_columns}") rows = performance_data.select( "reference_group", From 4c5f2c1f13a2f855a4fcd8b77bd151d413c21895 Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 22:01:03 +0300 Subject: [PATCH 6/8] Temporarily show Ruff format diff --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1bc44d40..69e05b5d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,7 +32,7 @@ jobs: run: uv run ruff check . - name: Check format - run: uv run ruff format --check . + run: uv run ruff format --diff . - name: Check types run: uv run ty check src/rtichoke From 58037c4da885744be89e2003531bcedce35aeace Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 22:01:47 +0300 Subject: [PATCH 7/8] Apply exact Ruff formatting --- src/rtichoke/_viz_spec_v2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index 0b6692c3..1ec30b70 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -60,8 +60,7 @@ def _roc_v2_spec_from_performance_data( for index, group in enumerate(ordered_groups, start=1) } series_ids = { - group: f"series-{index}" - for index, group in enumerate(ordered_groups, start=1) + group: f"series-{index}" for index, group in enumerate(ordered_groups, start=1) } evaluations: list[dict[str, object]] = [] From d9bfa113268aed5a625716479159f55027ae475e Mon Sep 17 00:00:00 2001 From: Uriah Finkel Date: Fri, 21 Aug 2026 22:01:57 +0300 Subject: [PATCH 8/8] Restore standard format check --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 69e05b5d..1bc44d40 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,7 +32,7 @@ jobs: run: uv run ruff check . - name: Check format - run: uv run ruff format --diff . + run: uv run ruff format --check . - name: Check types run: uv run ty check src/rtichoke