|
| 1 | +"""WO10: RESULTS.md -> metadata metrics generator. |
| 2 | +
|
| 3 | +Every IMF zip's metrics block is generated from a RESULTS.md table — |
| 4 | +never hand-written — and CI refuses to release a model whose metadata |
| 5 | +disagrees with the documented protocol numbers. |
| 6 | +
|
| 7 | +Sources are pinned in models/metrics-sources.yaml (repo, ref, path, |
| 8 | +anchor). Extraction is by table position (row label + optional column |
| 9 | +header), not regexes over prose: the mapping says where a number lives, |
| 10 | +the parser reads exactly that cell. |
| 11 | +""" |
| 12 | + |
| 13 | +from __future__ import annotations |
| 14 | + |
| 15 | +import re |
| 16 | +import urllib.request |
| 17 | +from dataclasses import dataclass |
| 18 | +from pathlib import Path |
| 19 | +from typing import Any |
| 20 | + |
| 21 | +import yaml |
| 22 | + |
| 23 | + |
| 24 | +class MetricsError(ValueError): |
| 25 | + """The RESULTS.md source cannot yield the mapped metrics.""" |
| 26 | + |
| 27 | + |
| 28 | +@dataclass(frozen=True) |
| 29 | +class TableSpec: |
| 30 | + row: str |
| 31 | + column: str | None = None # None: first value cell in the row |
| 32 | + as_name: str = "" |
| 33 | + protocol: str | None = None # override the source-level protocol |
| 34 | + |
| 35 | + |
| 36 | +@dataclass(frozen=True) |
| 37 | +class SourceSpec: |
| 38 | + repo: str |
| 39 | + ref: str |
| 40 | + path: str |
| 41 | + anchor: str |
| 42 | + protocol: str |
| 43 | + tables: tuple[TableSpec, ...] |
| 44 | + display_anchor: str = "" |
| 45 | + |
| 46 | + |
| 47 | +def _slugify(heading: str) -> str: |
| 48 | + text = heading.strip().lower() |
| 49 | + text = re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE) |
| 50 | + return re.sub(r"\s+", "-", text).strip("-") |
| 51 | + |
| 52 | + |
| 53 | +def _cell_to_value(cell: str) -> float: |
| 54 | + text = cell.replace("**", "").replace("%", "").replace(",", "").strip() |
| 55 | + match = re.search(r"-?\d+(?:\.\d+)?", text) |
| 56 | + if not match: |
| 57 | + raise MetricsError(f"no numeric value in cell {cell!r}") |
| 58 | + return float(match.group()) |
| 59 | + |
| 60 | + |
| 61 | +def parse_tables(markdown: str, anchor: str) -> list[dict[str, Any]]: |
| 62 | + """All tables in the section whose heading slugifies to `anchor`.""" |
| 63 | + lines = markdown.splitlines() |
| 64 | + start = None |
| 65 | + for index, line in enumerate(lines): |
| 66 | + if line.startswith("## "): |
| 67 | + if start is not None: |
| 68 | + end = index |
| 69 | + break |
| 70 | + if _slugify(line[3:]) == anchor: |
| 71 | + start = index |
| 72 | + else: |
| 73 | + end = len(lines) if start is not None else None |
| 74 | + if start is None: |
| 75 | + raise MetricsError(f"section anchor {anchor!r} not found") |
| 76 | + |
| 77 | + tables: list[dict[str, Any]] = [] |
| 78 | + index = start |
| 79 | + while index < end: |
| 80 | + line = lines[index] |
| 81 | + if line.startswith("|") and index + 1 < end and set(lines[index + 1]) <= set("|-: "): |
| 82 | + header = [cell.strip() for cell in line.strip("|").split("|")] |
| 83 | + index += 2 |
| 84 | + rows: list[dict[str, Any]] = [] |
| 85 | + while index < end and lines[index].startswith("|"): |
| 86 | + cells = [cell.strip() for cell in lines[index].strip("|").split("|")] |
| 87 | + rows.append({"label": cells[0], "cells": cells, "header": header}) |
| 88 | + index += 1 |
| 89 | + tables.append({"header": header, "rows": rows}) |
| 90 | + else: |
| 91 | + index += 1 |
| 92 | + if not tables: |
| 93 | + raise MetricsError(f"no tables under anchor {anchor!r}") |
| 94 | + return tables |
| 95 | + |
| 96 | + |
| 97 | +def extract( |
| 98 | + markdown: str, anchor: str, specs: tuple[TableSpec, ...] |
| 99 | +) -> list[dict[str, Any]]: |
| 100 | + tables = parse_tables(markdown, anchor) |
| 101 | + out: list[dict[str, Any]] = [] |
| 102 | + for spec in specs: |
| 103 | + found = None |
| 104 | + for table in tables: |
| 105 | + for row in table["rows"]: |
| 106 | + if spec.row.lower() in row["label"].lower(): |
| 107 | + found = row |
| 108 | + break |
| 109 | + if found: |
| 110 | + break |
| 111 | + if found is None: |
| 112 | + raise MetricsError(f"row {spec.row!r} not found under {anchor!r}") |
| 113 | + if spec.column is None: |
| 114 | + values = [ |
| 115 | + _cell_to_value(cell) |
| 116 | + for cell in found["cells"][1:] |
| 117 | + if "%" in cell or re.search(r"\d", cell.replace("**", "")) |
| 118 | + ] |
| 119 | + if not values: |
| 120 | + raise MetricsError(f"no value cells in row {spec.row!r}") |
| 121 | + value = values[0] |
| 122 | + else: |
| 123 | + try: |
| 124 | + column_index = found["header"].index(spec.column) |
| 125 | + except ValueError as e: |
| 126 | + raise MetricsError( |
| 127 | + f"column {spec.column!r} not in table header {found['header']}" |
| 128 | + ) from e |
| 129 | + value = _cell_to_value(found["cells"][column_index]) |
| 130 | + out.append({"name": spec.as_name or spec.row, "value": value}) |
| 131 | + return out |
| 132 | + |
| 133 | + |
| 134 | +def load_source(source: SourceSpec, cache_dir: Path | None = None) -> str: |
| 135 | + url = ( |
| 136 | + f"https://raw.githubusercontent.com/{source.repo}/{source.ref}/{source.path}" |
| 137 | + ) |
| 138 | + if cache_dir is not None: |
| 139 | + cached = cache_dir / _cache_name(source) |
| 140 | + if cached.is_file(): |
| 141 | + return cached.read_text(encoding="utf-8") |
| 142 | + with urllib.request.urlopen(url) as response: |
| 143 | + text = response.read().decode("utf-8") |
| 144 | + if cache_dir is not None: |
| 145 | + cache_dir.mkdir(parents=True, exist_ok=True) |
| 146 | + (cache_dir / _cache_name(source)).write_text(text, encoding="utf-8") |
| 147 | + return text |
| 148 | + |
| 149 | + |
| 150 | +def _cache_name(source: SourceSpec) -> str: |
| 151 | + return f"{source.repo.replace('/', '_')}@{source.ref}_{source.path.replace('/', '_')}" |
| 152 | + |
| 153 | + |
| 154 | + |
| 155 | + |
| 156 | +def generate_metrics( |
| 157 | + model_id: str, |
| 158 | + mapping_path: Path | str, |
| 159 | + cache_dir: Path | None = None, |
| 160 | +) -> list[dict[str, Any]]: |
| 161 | + raw = yaml.safe_load(Path(mapping_path).read_text(encoding="utf-8")) |
| 162 | + entry = raw.get("models", raw).get(model_id) |
| 163 | + if entry is None: |
| 164 | + raise MetricsError(f"no metrics source mapped for {model_id!r}") |
| 165 | + source = SourceSpec( |
| 166 | + repo=entry["repo"], |
| 167 | + ref=entry["ref"], |
| 168 | + path=entry["path"], |
| 169 | + anchor=entry["anchor"], |
| 170 | + protocol=entry["protocol"], |
| 171 | + display_anchor=entry.get("display_anchor", ""), |
| 172 | + tables=tuple( |
| 173 | + TableSpec( |
| 174 | + row=t["row"], |
| 175 | + column=t.get("column"), |
| 176 | + as_name=t.get("as", ""), |
| 177 | + protocol=t.get("protocol"), |
| 178 | + ) |
| 179 | + for t in entry["tables"] |
| 180 | + ), |
| 181 | + ) |
| 182 | + markdown = load_source(source, cache_dir) |
| 183 | + extracted = extract(markdown, source.anchor, source.tables) |
| 184 | + source_ref = f"{source.path}#{source.display_anchor or source.anchor}" |
| 185 | + return [ |
| 186 | + { |
| 187 | + "name": m["name"], |
| 188 | + "value": m["value"], |
| 189 | + "protocol": spec.protocol or source.protocol, |
| 190 | + "source": source_ref, |
| 191 | + } |
| 192 | + for m, spec in zip(extracted, source.tables, strict=True) |
| 193 | + ] |
| 194 | + |
| 195 | + |
| 196 | +def check_against_metadata( |
| 197 | + model_id: str, |
| 198 | + metadata_path: Path | str, |
| 199 | + mapping_path: Path | str, |
| 200 | + cache_dir: Path | None = None, |
| 201 | +) -> list[str]: |
| 202 | + """Diff generated metrics vs a metadata source file. Returns problems.""" |
| 203 | + generated = generate_metrics(model_id, mapping_path, cache_dir) |
| 204 | + meta = yaml.safe_load(Path(metadata_path).read_text(encoding="utf-8")) |
| 205 | + recorded = meta.get("metrics", []) |
| 206 | + problems: list[str] = [] |
| 207 | + if [(m["name"], m["value"]) for m in generated] != [(m["name"], m["value"]) for m in recorded]: |
| 208 | + problems.append( |
| 209 | + f"{model_id}: metrics mismatch — generated " |
| 210 | + f"{[(m['name'], m['value']) for m in generated]} vs metadata " |
| 211 | + f"{[(m['name'], m['value']) for m in recorded]}" |
| 212 | + ) |
| 213 | + return problems |
0 commit comments