diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 82ddad50..7249bd13 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -52,7 +52,7 @@ jobs: prefix = "rtichoke/_vendor/rtichoke_viz/" required = { f"{prefix}VENDORED_FROM", - f"{prefix}rtichoke-viz-0.18.0.tar.gz", + f"{prefix}rtichoke-viz-0.19.0.tar.gz", f"{prefix}rtichoke-viz.js", f"{prefix}rtichoke-viz.css", f"{prefix}rtichoke-viz.schema.json", diff --git a/src/rtichoke/_report_spec.py b/src/rtichoke/_report_spec.py index 2f65082e..58eefbb6 100644 --- a/src/rtichoke/_report_spec.py +++ b/src/rtichoke/_report_spec.py @@ -32,7 +32,7 @@ def _validate_spec_schema_version(spec: Mapping[str, object]) -> None: """Validate that spec schemaVersion strictly matches type requirements. - summary_metrics -> "1.0" + summary_metrics -> "1.0" or "1.1" all v2 component types -> "2.0" """ spec_type = spec.get("type") @@ -46,9 +46,9 @@ def _validate_spec_schema_version(spec: Mapping[str, object]) -> None: raise ValueError("Report component spec is missing a string schemaVersion") if spec_type in _V10_SCHEMA_TYPES: - if schema_version != "1.0": + if schema_version not in {"1.0", "1.1"}: raise ValueError( - f"Component type {spec_type!r} requires schemaVersion '1.0', got {schema_version!r}" + f"Component type {spec_type!r} requires schemaVersion '1.0' or '1.1', got {schema_version!r}" ) elif spec_type in _V20_SCHEMA_TYPES: if schema_version != "2.0": diff --git a/src/rtichoke/_summary_metrics_spec.py b/src/rtichoke/_summary_metrics_spec.py index 589c53f1..d130b04d 100644 --- a/src/rtichoke/_summary_metrics_spec.py +++ b/src/rtichoke/_summary_metrics_spec.py @@ -50,6 +50,13 @@ class _PrevalenceMetric(TypedDict): estimate: float | None +class _EventRiskMetric(TypedDict): + metric: str + owner: _PopulationOwner + horizon: float + estimate: float | None + + class _AurocMetric(TypedDict): metric: str owner: _EvaluationOwner @@ -178,6 +185,94 @@ def _prevalence_summary_metrics_spec( } +def _event_risk_summary_metrics_spec( + performance_data: pl.DataFrame, + evaluation_metadata: Mapping[str, _EvaluationMetadata], + fixed_time_horizons: list[float], +) -> _SummaryMetricsSpec: + """Build canonical population-owned Event Risk SummaryMetricsSpec v1.1.""" + populations_list: list[_PopulationSpec] = [] + metrics_list: list[dict[str, Any]] = [] + + seen_populations: dict[tuple[str, str], str] = {} + pop_counter = 1 + + for group, metadata in evaluation_metadata.items(): + pop_name = metadata.population + label = "Population" if pop_name == _SHARED_POPULATION else pop_name + pop_key = ( + (group, pop_name) + if pop_name != _SHARED_POPULATION + and list(evaluation_metadata.values())[0].model is None + else (pop_name, pop_name) + ) + if pop_key not in seen_populations: + pop_id = f"population-{pop_counter}" + pop_counter += 1 + seen_populations[pop_key] = pop_id + populations_list.append({"id": pop_id, "label": label}) + + pop_horizon_estimates: dict[tuple[tuple[str, str], float], set[float]] = {} + cutoff_zero_rows = ( + performance_data.filter(pl.col("chosen_cutoff") == 0) + .select( + "reference_group", + "fixed_time_horizon", + (pl.col("real_positives") / pl.col("n")).alias("event_risk"), + ) + .to_dicts() + ) + for row in cutoff_zero_rows: + group = str(row["reference_group"]) + if group not in evaluation_metadata: + continue + metadata = evaluation_metadata[group] + pop_name = metadata.population + pop_key = ( + (group, pop_name) + if pop_name != _SHARED_POPULATION and metadata.model is None + else (pop_name, pop_name) + ) + horizon = float(row["fixed_time_horizon"]) + risk_val = row["event_risk"] + if risk_val is not None and math.isfinite(float(risk_val)): + pop_horizon_estimates.setdefault((pop_key, horizon), set()).add( + float(risk_val) + ) + + for horizon in fixed_time_horizons: + norm_horizon = float(horizon) + for pop_spec in populations_list: + pop_id = pop_spec["id"] + pop_key = next( + key for key, pid in seen_populations.items() if pid == pop_id + ) + estimates = pop_horizon_estimates.get((pop_key, norm_horizon), set()) + estimate: float | None = ( + next(iter(estimates)) if len(estimates) == 1 else None + ) + + metric_item: _EventRiskMetric = { + "metric": "event_risk", + "owner": { + "type": "population", + "populationId": pop_id, + }, + "horizon": norm_horizon, + "estimate": estimate, + } + metrics_list.append(cast(dict[str, Any], metric_item)) + + return { + "schemaVersion": "1.1", + "type": "summary_metrics", + "title": "Event Risk", + "evaluations": [], + "populations": populations_list, + "metrics": metrics_list, + } + + def _auroc_summary_metrics_spec( probs: dict[str, np.ndarray], reals: np.ndarray | dict[str, np.ndarray], diff --git a/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM b/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM index 1a70db5d..e2e6d181 100644 --- a/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM +++ b/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM @@ -1,5 +1,5 @@ repository=https://github.com/uriahf/rtichoke_viz -release=v0.18.0 -source_commit=dbabedb495ab70062ee635cd9d59eefcafe55a43 -archive=rtichoke-viz-0.18.0.tar.gz -sha256=1cd3af962be8357d0fd4c2f2ecf5e5953774683de8854ead2facbac807b6bc84 +release=v0.19.0 +source_commit=26f22617bb34664ad65d6591ce7b20ecc080e739 +archive=rtichoke-viz-0.19.0.tar.gz +sha256=bdeb1dd3041f700341730904492c7ab271bcdb9fa3637c7a9fd0a21f183bec70 diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.18.0.tar.gz b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.18.0.tar.gz deleted file mode 100644 index 000c1cf1..00000000 Binary files a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.18.0.tar.gz and /dev/null differ diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.19.0.tar.gz b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.19.0.tar.gz new file mode 100644 index 00000000..8f13c770 Binary files /dev/null and b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.19.0.tar.gz differ diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-report.schema.json b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-report.schema.json index 2fc2592c..9b237629 100644 --- a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-report.schema.json +++ b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-report.schema.json @@ -4971,156 +4971,357 @@ { "$id": "https://rtichoke.dev/schema/viz/summary-metrics.json", "title": "rtichoke summary metrics specification", - "type": "object", - "required": [ - "schemaVersion", - "type", - "evaluations", - "populations", - "metrics" - ], - "properties": { - "schemaVersion": { - "const": "1.0", - "type": "string" - }, - "type": { - "const": "summary_metrics", - "type": "string" - }, - "title": { - "type": "string" - }, - "evaluations": { - "type": "array", - "items": { - "type": "object", - "required": [ - "id", - "population" - ], - "properties": { - "id": { - "type": "string" - }, - "model": { - "type": "string" - }, - "population": { - "type": "string" - }, - "label": { - "type": "string" - } - } - } - }, - "populations": { - "type": "array", - "items": { - "type": "object", - "required": [ - "id", - "label" - ], - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" + "anyOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.0", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } } - } - } - }, - "metrics": { - "type": "array", - "items": { - "anyOf": [ - { + }, + "populations": { + "type": "array", + "items": { "type": "object", "required": [ - "metric", - "owner", - "estimate" + "id", + "label" ], "properties": { - "metric": { - "const": "auroc", + "id": { "type": "string" }, - "owner": { + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { "type": "object", "required": [ - "type", - "evaluationId" + "metric", + "owner", + "estimate" ], "properties": { - "type": { - "const": "evaluation", + "metric": { + "const": "auroc", "type": "string" }, - "evaluationId": { - "type": "string" + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } } }, - "estimate": { - "anyOf": [ - { - "type": "number" + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", + "type": "string" }, - { - "type": "null" + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } - ] + } + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.1", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" } } - }, - { + } + }, + "populations": { + "type": "array", + "items": { "type": "object", "required": [ - "metric", - "owner", - "estimate" + "id", + "label" ], "properties": { - "metric": { - "const": "prevalence", + "id": { "type": "string" }, - "owner": { + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { "type": "object", "required": [ - "type", - "populationId" + "metric", + "owner", + "estimate" ], "properties": { - "type": { - "const": "population", + "metric": { + "const": "auroc", "type": "string" }, - "populationId": { + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } } }, - "estimate": { - "anyOf": [ - { + { + "type": "object", + "required": [ + "metric", + "owner", + "horizon", + "estimate" + ], + "properties": { + "metric": { + "const": "event_risk", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "horizon": { + "minimum": 0, "type": "number" }, - { - "type": "null" + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } - ] + } } - } + ] } - ] + } } } - } + ] } ] } @@ -10122,156 +10323,357 @@ { "$id": "https://rtichoke.dev/schema/viz/summary-metrics.json", "title": "rtichoke summary metrics specification", - "type": "object", - "required": [ - "schemaVersion", - "type", - "evaluations", - "populations", - "metrics" - ], - "properties": { - "schemaVersion": { - "const": "1.0", - "type": "string" - }, - "type": { - "const": "summary_metrics", - "type": "string" - }, - "title": { - "type": "string" - }, - "evaluations": { - "type": "array", - "items": { - "type": "object", - "required": [ - "id", - "population" - ], - "properties": { - "id": { - "type": "string" - }, - "model": { - "type": "string" - }, - "population": { - "type": "string" - }, - "label": { - "type": "string" - } - } - } - }, - "populations": { - "type": "array", - "items": { - "type": "object", - "required": [ - "id", - "label" - ], - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" + "anyOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.0", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } } - } - } - }, - "metrics": { - "type": "array", - "items": { - "anyOf": [ - { + }, + "populations": { + "type": "array", + "items": { "type": "object", "required": [ - "metric", - "owner", - "estimate" + "id", + "label" ], "properties": { - "metric": { - "const": "auroc", + "id": { "type": "string" }, - "owner": { + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { "type": "object", "required": [ - "type", - "evaluationId" + "metric", + "owner", + "estimate" ], "properties": { - "type": { - "const": "evaluation", + "metric": { + "const": "auroc", "type": "string" }, - "evaluationId": { - "type": "string" + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } } }, - "estimate": { - "anyOf": [ - { - "type": "number" + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", + "type": "string" }, - { - "type": "null" + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } - ] + } + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.1", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" } } - }, - { + } + }, + "populations": { + "type": "array", + "items": { "type": "object", "required": [ - "metric", - "owner", - "estimate" + "id", + "label" ], "properties": { - "metric": { - "const": "prevalence", + "id": { "type": "string" }, - "owner": { + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { "type": "object", "required": [ - "type", - "populationId" + "metric", + "owner", + "estimate" ], "properties": { - "type": { - "const": "population", + "metric": { + "const": "auroc", "type": "string" }, - "populationId": { + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } } }, - "estimate": { - "anyOf": [ - { + { + "type": "object", + "required": [ + "metric", + "owner", + "horizon", + "estimate" + ], + "properties": { + "metric": { + "const": "event_risk", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "horizon": { + "minimum": 0, "type": "number" }, - { - "type": "null" + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } - ] + } } - } + ] } - ] + } } } - } + ] } ] } @@ -15236,171 +15638,372 @@ "type": "number" }, { - "type": "null" - } - ] - } - } - } - } - } - } - } - } - }, - { - "$id": "https://rtichoke.dev/schema/viz/summary-metrics.json", - "title": "rtichoke summary metrics specification", - "type": "object", - "required": [ - "schemaVersion", - "type", - "evaluations", - "populations", - "metrics" - ], - "properties": { - "schemaVersion": { - "const": "1.0", - "type": "string" - }, - "type": { - "const": "summary_metrics", - "type": "string" - }, - "title": { - "type": "string" - }, - "evaluations": { - "type": "array", - "items": { - "type": "object", - "required": [ - "id", - "population" - ], - "properties": { - "id": { - "type": "string" - }, - "model": { - "type": "string" - }, - "population": { - "type": "string" - }, - "label": { - "type": "string" + "type": "null" + } + ] + } + } + } } } } - }, - "populations": { - "type": "array", - "items": { - "type": "object", - "required": [ - "id", - "label" - ], - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" + } + } + }, + { + "$id": "https://rtichoke.dev/schema/viz/summary-metrics.json", + "title": "rtichoke summary metrics specification", + "anyOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.0", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } } - } - } - }, - "metrics": { - "type": "array", - "items": { - "anyOf": [ - { + }, + "populations": { + "type": "array", + "items": { "type": "object", "required": [ - "metric", - "owner", - "estimate" + "id", + "label" ], "properties": { - "metric": { - "const": "auroc", + "id": { "type": "string" }, - "owner": { + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { "type": "object", "required": [ - "type", - "evaluationId" + "metric", + "owner", + "estimate" ], "properties": { - "type": { - "const": "evaluation", + "metric": { + "const": "auroc", "type": "string" }, - "evaluationId": { - "type": "string" + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } } }, - "estimate": { - "anyOf": [ - { - "type": "number" + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", + "type": "string" }, - { - "type": "null" + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } - ] + } + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.1", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" } } - }, - { + } + }, + "populations": { + "type": "array", + "items": { "type": "object", "required": [ - "metric", - "owner", - "estimate" + "id", + "label" ], "properties": { - "metric": { - "const": "prevalence", + "id": { "type": "string" }, - "owner": { + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { "type": "object", "required": [ - "type", - "populationId" + "metric", + "owner", + "estimate" ], "properties": { - "type": { - "const": "population", + "metric": { + "const": "auroc", "type": "string" }, - "populationId": { + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } } }, - "estimate": { - "anyOf": [ - { + { + "type": "object", + "required": [ + "metric", + "owner", + "horizon", + "estimate" + ], + "properties": { + "metric": { + "const": "event_risk", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "horizon": { + "minimum": 0, "type": "number" }, - { - "type": "null" + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] } - ] + } } - } + ] } - ] + } } } - } + ] } ] } diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js index d1bcd572..61ff47cc 100644 --- a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js +++ b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js @@ -3269,19 +3269,43 @@ var PrevalenceSummaryMetricSchema = Type.Object({ }), estimate: Type.Union([Type.Number(), Type.Null()]) }); -var SummaryMetricSchema = Type.Union([ +var EventRiskSummaryMetricSchema = Type.Object({ + metric: Type.Literal("event_risk"), + owner: Type.Object({ + type: Type.Literal("population"), + populationId: Type.String() + }), + horizon: Type.Number({ minimum: 0 }), + estimate: Type.Union([Type.Number(), Type.Null()]) +}); +var SummaryMetricV1_0Schema = Type.Union([ AUROCSummaryMetricSchema, PrevalenceSummaryMetricSchema ]); -var SummaryMetricsSpecSchema = Type.Object( - { - schemaVersion: Type.Literal("1.0"), - type: Type.Literal("summary_metrics"), - title: Type.Optional(Type.String()), - evaluations: Type.Array(EvaluationSpecSchema), - populations: Type.Array(PopulationSummaryOwnerSpecSchema), - metrics: Type.Array(SummaryMetricSchema) - }, +var SummaryMetricV1_1Schema = Type.Union([ + AUROCSummaryMetricSchema, + PrevalenceSummaryMetricSchema, + EventRiskSummaryMetricSchema +]); +var SummaryMetricSchema = SummaryMetricV1_1Schema; +var SummaryMetricsSpecV1_0Schema = Type.Object({ + schemaVersion: Type.Literal("1.0"), + type: Type.Literal("summary_metrics"), + title: Type.Optional(Type.String()), + evaluations: Type.Array(EvaluationSpecSchema), + populations: Type.Array(PopulationSummaryOwnerSpecSchema), + metrics: Type.Array(SummaryMetricV1_0Schema) +}); +var SummaryMetricsSpecV1_1Schema = Type.Object({ + schemaVersion: Type.Literal("1.1"), + type: Type.Literal("summary_metrics"), + title: Type.Optional(Type.String()), + evaluations: Type.Array(EvaluationSpecSchema), + populations: Type.Array(PopulationSummaryOwnerSpecSchema), + metrics: Type.Array(SummaryMetricV1_1Schema) +}); +var SummaryMetricsSpecSchema = Type.Union( + [SummaryMetricsSpecV1_0Schema, SummaryMetricsSpecV1_1Schema], { $id: "https://rtichoke.dev/schema/viz/summary-metrics.json", title: "rtichoke summary metrics specification" @@ -3424,7 +3448,15 @@ function assertSummaryMetricsReferentialIntegrity(spec) { if (item.estimate !== null && !Number.isFinite(item.estimate)) { throw new Error(`non-finite metric estimate: ${item.estimate}`); } + if ("horizon" in item && item.horizon !== void 0) { + if (!Number.isFinite(item.horizon) || item.horizon < 0) { + throw new Error(`invalid horizon: ${item.horizon}`); + } + } if (item.metric === "auroc") { + if ("horizon" in item && item.horizon !== void 0) { + throw new Error("auroc metric cannot specify horizon"); + } if (!evaluationIds.has(item.owner.evaluationId)) { throw new Error(`unknown evaluation id: ${item.owner.evaluationId}`); } @@ -3436,6 +3468,9 @@ function assertSummaryMetricsReferentialIntegrity(spec) { } seenMetrics.add(key); } else if (item.metric === "prevalence") { + if ("horizon" in item && item.horizon !== void 0) { + throw new Error("prevalence metric cannot specify horizon"); + } if (!populationIds.has(item.owner.populationId)) { throw new Error(`unknown population id: ${item.owner.populationId}`); } @@ -3446,6 +3481,25 @@ function assertSummaryMetricsReferentialIntegrity(spec) { ); } seenMetrics.add(key); + } else if (item.metric === "event_risk") { + if (spec.schemaVersion !== "1.1") { + throw new Error( + `event_risk metric requires schemaVersion 1.1, got ${spec.schemaVersion}` + ); + } + if (!("horizon" in item) || item.horizon === void 0) { + throw new Error("event_risk metric requires horizon"); + } + if (!populationIds.has(item.owner.populationId)) { + throw new Error(`unknown population id: ${item.owner.populationId}`); + } + const key = `event_risk:${item.owner.populationId}:${item.horizon}`; + if (seenMetrics.has(key)) { + throw new Error( + `duplicate metric ownership: event_risk for population ${item.owner.populationId} at horizon ${item.horizon}` + ); + } + seenMetrics.add(key); } } } @@ -20340,6 +20394,37 @@ function renderSummaryMetrics(spec, document2 = globalThis.document) { title.textContent = spec.title; root2.append(title); } + const distinctHorizons = []; + for (const item of spec.metrics) { + if ("horizon" in item && item.horizon !== void 0) { + if (!distinctHorizons.includes(item.horizon)) { + distinctHorizons.push(item.horizon); + } + } + } + let selectedHorizon = distinctHorizons.length > 0 ? distinctHorizons[0] : void 0; + const rowsWithHorizon = []; + if (distinctHorizons.length > 1) { + const control = document2.createElement("label"); + control.className = "rtichoke-horizon-control"; + control.textContent = "Fixed Time Horizon: "; + const select = document2.createElement("select"); + select.className = "rtichoke-horizon-select"; + select.setAttribute("aria-label", "Fixed Time Horizon"); + for (const horizon of distinctHorizons) { + const option = document2.createElement("option"); + option.value = String(horizon); + option.textContent = String(horizon); + select.append(option); + } + select.value = String(selectedHorizon); + select.addEventListener("change", () => { + selectedHorizon = Number(select.value); + updateRowVisibility(); + }); + control.append(select); + root2.append(control); + } const table = document2.createElement("table"); table.className = "rtichoke-summary-metrics__table"; const head = document2.createElement("thead"); @@ -20360,6 +20445,7 @@ function renderSummaryMetrics(spec, document2 = globalThis.document) { const tr = document2.createElement("tr"); let ownerLabel = ""; let metricLabel = ""; + let horizon = void 0; if (item.metric === "auroc") { const evaluation = evaluations.get(item.owner.evaluationId); ownerLabel = evaluation.label ?? evaluation.model ?? evaluation.population ?? evaluation.id; @@ -20372,6 +20458,14 @@ function renderSummaryMetrics(spec, document2 = globalThis.document) { metricLabel = "Prevalence"; tr.dataset.metric = "prevalence"; tr.dataset.populationId = item.owner.populationId; + } else if (item.metric === "event_risk") { + const population = populations.get(item.owner.populationId); + ownerLabel = population.label; + metricLabel = "Event Risk"; + horizon = item.horizon; + tr.dataset.metric = "event_risk"; + tr.dataset.populationId = item.owner.populationId; + tr.dataset.horizon = String(item.horizon); } tr.append( cell2(document2, ownerLabel, "rtichoke-summary-metrics__owner"), @@ -20388,10 +20482,21 @@ function renderSummaryMetrics(spec, document2 = globalThis.document) { estimateCell.dataset.estimate = String(item.estimate); } tr.append(estimateCell); + rowsWithHorizon.push({ element: tr, horizon }); body.append(tr); } table.append(body); root2.append(table); + function updateRowVisibility() { + for (const { element, horizon } of rowsWithHorizon) { + if (horizon === void 0 || horizon === selectedHorizon) { + element.style.display = ""; + } else { + element.style.display = "none"; + } + } + } + updateRowVisibility(); return root2; } diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index a91e0225..e22eaa74 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -25,6 +25,7 @@ from rtichoke._report_spec import _build_report_spec_v11 from rtichoke._summary_metrics_spec import ( _auroc_summary_metrics_spec, + _event_risk_summary_metrics_spec, _prevalence_summary_metrics_spec, ) from rtichoke._viz_spec_v2 import ( @@ -188,7 +189,22 @@ def create_summary_report_times( perf_data_ppcr, metadata ) + event_risk_spec = _event_risk_summary_metrics_spec( + perf_data_thresh, metadata, fixed_time_horizons + ) + sections = [ + { + "id": "event-risk", + "title": "Event Risk", + "components": [ + { + "id": "event-risk", + "title": "Event Risk", + "spec": event_risk_spec, + } + ], + }, { "id": "calibration", "title": "Calibration", diff --git a/tests/test_report_spec.py b/tests/test_report_spec.py index efb78e99..5b2fcd4c 100644 --- a/tests/test_report_spec.py +++ b/tests/test_report_spec.py @@ -54,20 +54,24 @@ def _curve_spec( } -def _summary_metrics_spec(metric_type: str) -> dict[str, Any]: +def _summary_metrics_spec( + metric_type: str, schema_version: str = "1.0" +) -> dict[str, Any]: + metric_item: dict[str, Any] = { + "metric": metric_type, + "owner": {"type": "population", "populationId": "population-1"}, + "estimate": 0.25, + } + if metric_type == "event_risk": + metric_item["horizon"] = 5.0 + return { - "schemaVersion": "1.0", + "schemaVersion": schema_version, "type": "summary_metrics", "title": "Summary", "evaluations": [], "populations": [{"id": "population-1", "label": "Population"}], - "metrics": [ - { - "metric": metric_type, - "owner": {"type": "population", "populationId": "population-1"}, - "estimate": 0.25, - } - ], + "metrics": [metric_item], } @@ -111,9 +115,27 @@ def test_build_report_spec_v11_structure() -> None: assert report["sections"][1]["items"][0]["components"][0]["id"] == "roc" +def test_summary_metrics_schema_versions_accepted() -> None: + v10_spec = _summary_metrics_spec("prevalence", "1.0") + v11_spec = _summary_metrics_spec("event_risk", "1.1") + + sections = [ + { + "id": "event-risk", + "components": [{"id": "event-risk", "spec": v11_spec}], + }, + { + "id": "prevalence", + "components": [{"id": "prevalence-summary", "spec": v10_spec}], + }, + ] + report = _build_report_spec_v11(sections) + assert report["schemaVersion"] == "1.1" + + def test_type_aware_schema_version_validation() -> None: bad_v1_spec = _summary_metrics_spec("prevalence") - bad_v1_spec["schemaVersion"] = "2.0" # Invalid: summary_metrics must be 1.0 + bad_v1_spec["schemaVersion"] = "2.0" # Invalid: summary_metrics must be 1.0 or 1.1 sections = [ { diff --git a/tests/test_rtichoke_viz_vendor.py b/tests/test_rtichoke_viz_vendor.py index a1a14042..2c327ae0 100644 --- a/tests/test_rtichoke_viz_vendor.py +++ b/tests/test_rtichoke_viz_vendor.py @@ -3,19 +3,19 @@ from pathlib import Path _VENDOR = Path(__file__).parents[1] / "src" / "rtichoke" / "_vendor" / "rtichoke_viz" -_RELEASE_DIR = "rtichoke-viz-0.18.0" -_SHA256 = "1cd3af962be8357d0fd4c2f2ecf5e5953774683de8854ead2facbac807b6bc84" -_SOURCE_COMMIT = "dbabedb495ab70062ee635cd9d59eefcafe55a43" +_RELEASE_DIR = "rtichoke-viz-0.19.0" +_SHA256 = "bdeb1dd3041f700341730904492c7ab271bcdb9fa3637c7a9fd0a21f183bec70" +_SOURCE_COMMIT = "26f22617bb34664ad65d6591ce7b20ecc080e739" -def test_vendored_rtichoke_viz_v0180_provenance_archive_and_schemas(): +def test_vendored_rtichoke_viz_v0190_provenance_archive_and_schemas(): provenance = (_VENDOR / "VENDORED_FROM").read_text() - assert "release=v0.18.0" in provenance + assert "release=v0.19.0" in provenance assert f"source_commit={_SOURCE_COMMIT}" in provenance - assert "archive=rtichoke-viz-0.18.0.tar.gz" in provenance + assert "archive=rtichoke-viz-0.19.0.tar.gz" in provenance assert f"sha256={_SHA256}" in provenance - archive = _VENDOR / "rtichoke-viz-0.18.0.tar.gz" + archive = _VENDOR / "rtichoke-viz-0.19.0.tar.gz" assert hashlib.sha256(archive.read_bytes()).hexdigest() == _SHA256 with tarfile.open(archive, "r:gz") as release: assert set(release.getnames()) == { @@ -30,7 +30,7 @@ def test_vendored_rtichoke_viz_v0180_provenance_archive_and_schemas(): manifest = release.extractfile(f"{_RELEASE_DIR}/MANIFEST") assert manifest is not None assert manifest.read().decode() == ( - f"version=0.18.0\ncommit={_SOURCE_COMMIT}\n" + f"version=0.19.0\ncommit={_SOURCE_COMMIT}\n" ) for filename in ( "rtichoke-viz.css", @@ -58,7 +58,7 @@ def test_vendored_rtichoke_viz_v0180_provenance_archive_and_schemas(): assert '"summary_metrics"' in report_schema -def test_v0180_bundle_keeps_existing_exports_and_time_dependent_surfaces(): +def test_v0190_bundle_keeps_existing_exports_and_time_dependent_surfaces(): bundle = (_VENDOR / "rtichoke-viz.js").read_text(encoding="utf-8") for export_name in ( "renderRoc", diff --git a/tests/test_summary_metrics_spec.py b/tests/test_summary_metrics_spec.py index 1de69cd2..6d36c51c 100644 --- a/tests/test_summary_metrics_spec.py +++ b/tests/test_summary_metrics_spec.py @@ -5,9 +5,13 @@ from rtichoke._summary_metrics_spec import ( _auroc_summary_metrics_spec, _compute_auroc_proc_compatible, + _event_risk_summary_metrics_spec, _prevalence_summary_metrics_spec, ) from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.performance_data.performance_data_times import ( + prepare_performance_data_times, +) from rtichoke.processing.evaluation_semantics import ( _EvaluationMetadata, _build_evaluation_metadata, @@ -121,3 +125,89 @@ def test_prevalence_summary_metrics_spec_distinct_populations_same_label(): assert prev_spec_shared["populations"][1]["id"] == "population-2" assert prev_spec_shared["populations"][0]["label"] == "Shared Label" assert prev_spec_shared["populations"][1]["label"] == "Shared Label" + + +def test_event_risk_summary_metrics_spec_multiple_models_shared_population(): + probs = { + "Model 1": np.array([0.1, 0.3, 0.5, 0.7, 0.9]), + "Model 2": np.array([0.2, 0.4, 0.6, 0.8, 0.95]), + } + reals = np.array([0, 1, 0, 1, 1]) + times = np.array([1.0, 3.0, 5.0, 7.0, 9.0]) + horizons = [6.0, 2.0] + + perf_data = prepare_performance_data_times( + probs, reals, times, fixed_time_horizons=horizons + ) + metadata = _build_evaluation_metadata(probs, reals, times) + + spec = _event_risk_summary_metrics_spec(perf_data, metadata, horizons) + assert spec["schemaVersion"] == "1.1" + assert spec["type"] == "summary_metrics" + assert spec["title"] == "Event Risk" + assert spec["evaluations"] == [] + assert len(spec["populations"]) == 1 + assert spec["populations"][0]["id"] == "population-1" + assert spec["populations"][0]["label"] == "Population" + + # Exactly 2 metrics for 1 population x 2 horizons (no duplicates from 2 models) + assert len(spec["metrics"]) == 2 + assert spec["metrics"][0]["owner"] == { + "type": "population", + "populationId": "population-1", + } + assert spec["metrics"][0]["horizon"] == 6.0 + assert spec["metrics"][1]["owner"] == { + "type": "population", + "populationId": "population-1", + } + assert spec["metrics"][1]["horizon"] == 2.0 + + # Values match cutoff-0 real_positives / n from performance data + for metric in spec["metrics"]: + h = metric["horizon"] + cutoff_0_row = perf_data.filter( + (perf_data["fixed_time_horizon"] == h) & (perf_data["chosen_cutoff"] == 0) + ).to_dicts()[0] + expected_risk = float(cutoff_0_row["real_positives"]) / float(cutoff_0_row["n"]) + assert metric["estimate"] == pytest.approx(expected_risk) + + +def test_event_risk_summary_metrics_spec_distinct_populations(): + probs = { + "Cohort A": np.array([0.1, 0.3, 0.5]), + "Cohort B": np.array([0.2, 0.4, 0.6]), + } + reals = { + "Cohort A": np.array([0, 1, 0]), + "Cohort B": np.array([1, 0, 1]), + } + times = { + "Cohort A": np.array([2.0, 4.0, 6.0]), + "Cohort B": np.array([1.0, 3.0, 5.0]), + } + horizons = [3.0, 5.0] + + perf_data = prepare_performance_data_times( + probs, reals, times, fixed_time_horizons=horizons + ) + metadata = _build_evaluation_metadata(probs, reals, times) + + spec = _event_risk_summary_metrics_spec(perf_data, metadata, horizons) + assert len(spec["populations"]) == 2 + assert spec["populations"][0]["id"] == "population-1" + assert spec["populations"][0]["label"] == "Cohort A" + assert spec["populations"][1]["id"] == "population-2" + assert spec["populations"][1]["label"] == "Cohort B" + + # 2 populations x 2 horizons = 4 metrics in requested horizon order + assert len(spec["metrics"]) == 4 + m0, m1, m2, m3 = spec["metrics"] + assert m0["horizon"] == 3.0 + assert m0["owner"]["populationId"] == "population-1" + assert m1["horizon"] == 3.0 + assert m1["owner"]["populationId"] == "population-2" + assert m2["horizon"] == 5.0 + assert m2["owner"]["populationId"] == "population-1" + assert m3["horizon"] == 5.0 + assert m3["owner"]["populationId"] == "population-2" diff --git a/tests/test_summary_report_times_browser.py b/tests/test_summary_report_times_browser.py index 49bedfdb..89bb33bf 100644 --- a/tests/test_summary_report_times_browser.py +++ b/tests/test_summary_report_times_browser.py @@ -144,6 +144,7 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path): sections = report["sections"] section_ids = [s["id"] for s in sections] assert section_ids == [ + "event-risk", "calibration", "discrimination", "utility", @@ -152,23 +153,31 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path): # Section titles assert [s["title"] for s in sections] == [ + "Event Risk", "Calibration", "Discrimination", "Utility", "Performance Table", ] + # Event Risk section checks + event_risk_sec = sections[0] + assert [c["id"] for c in event_risk_sec["items"]] == ["event-risk"] + assert event_risk_sec["items"][0]["spec"]["schemaVersion"] == "1.1" + assert event_risk_sec["items"][0]["spec"]["type"] == "summary_metrics" + assert event_risk_sec["items"][0]["spec"]["metrics"][0]["metric"] == "event_risk" + # Explicit omissions check assert "prevalence" not in section_ids assert "auroc" not in json.dumps(report) # Calibration section components - calib = sections[0] + calib = sections[1] assert [c["id"] for c in calib["items"]] == ["calibration-smooth", "calibration"] assert [c["title"] for c in calib["items"]] == ["Smooth", "Discrete"] # Discrimination section groups and components - disc = sections[1] + disc = sections[2] assert len(disc["items"]) == 2 g1, g2 = disc["items"] @@ -203,7 +212,7 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path): ] # Utility section components - util = sections[2] + util = sections[3] assert [c["id"] for c in util["items"]] == [ "decision-curve", "interventions-avoided", @@ -214,7 +223,7 @@ def test_summary_report_times_spec_structure_and_ordering(tmp_path): ] # Performance Table section groups and components - table_sec = sections[3] + table_sec = sections[4] assert len(table_sec["items"]) == 2 tg1, tg2 = table_sec["items"] @@ -300,15 +309,17 @@ def test_summary_report_times_preserves_standalone_canonical_producers(tmp_path) perf_thresh, metadata ) - report_calib_smooth = report["sections"][0]["items"][0]["spec"] - report_roc_thresh = report["sections"][1]["items"][0]["components"][0]["spec"] - report_pr_thresh = report["sections"][1]["items"][0]["components"][1]["spec"] - report_roc_ppcr = report["sections"][1]["items"][1]["components"][0]["spec"] - report_gains_ppcr = report["sections"][1]["items"][1]["components"][2]["spec"] - report_lift_thresh = report["sections"][1]["items"][0]["components"][3]["spec"] - report_dc = report["sections"][2]["items"][0]["spec"] - report_ia = report["sections"][2]["items"][1]["spec"] - report_table_thresh = report["sections"][3]["items"][0]["components"][0]["spec"] + report_event_risk = report["sections"][0]["items"][0]["spec"] + assert report_event_risk["schemaVersion"] == "1.1" + report_calib_smooth = report["sections"][1]["items"][0]["spec"] + report_roc_thresh = report["sections"][2]["items"][0]["components"][0]["spec"] + report_pr_thresh = report["sections"][2]["items"][0]["components"][1]["spec"] + report_roc_ppcr = report["sections"][2]["items"][1]["components"][0]["spec"] + report_gains_ppcr = report["sections"][2]["items"][1]["components"][2]["spec"] + report_lift_thresh = report["sections"][2]["items"][0]["components"][3]["spec"] + report_dc = report["sections"][3]["items"][0]["spec"] + report_ia = report["sections"][3]["items"][1]["spec"] + report_table_thresh = report["sections"][4]["items"][0]["components"][0]["spec"] assert report_roc_thresh["operatingPoint"]["dimension"] == "probability_threshold" assert report_roc_ppcr["operatingPoint"]["dimension"] == "ppcr" @@ -337,7 +348,7 @@ def test_summary_report_times_multiple_horizons_ordering_and_context(tmp_path): ) report = _embedded_report(output.read_text(encoding="utf-8")) - pr_spec = report["sections"][1]["items"][0]["components"][0]["spec"] + pr_spec = report["sections"][2]["items"][0]["components"][0]["spec"] # Verify horizon identities in series preserve fixed_time_horizons order series_horizons = [s["horizon"] for s in pr_spec["series"]]