From c744f16fd979d7896bc5ce57628695c28052fe04 Mon Sep 17 00:00:00 2001 From: Dipak Date: Thu, 6 Aug 2026 10:38:24 +0530 Subject: [PATCH] fix(report): write the report card as UTF-8, not the platform locale encoding build_report crashed on Windows: the card carries non-ASCII (the delta column header in the side-effects table, the warning sign on degenerate-trap callouts), and Path.write_text with no encoding= falls back to the locale encoding, which is cp1252 on a stock install and cannot encode either character. tests/test_cli.py's three rendering tests fail there on a clean checkout with UnicodeEncodeError; they pass on CI only because Linux defaults to UTF-8. Name the encoding at every text I/O site in the package: the two card writes, both CSV readers, the side-effects stub, and the MMLU/GSM8K slice cache. The HTML already declares charset=utf-8, so writing it in anything else was self-contradictory. The regression test runs the whole CSV-in / card-out path in a fresh interpreter under -X warn_default_encoding and fails if any EncodingWarning is attributed to a file inside the package, so it catches a regression on UTF-8 platforms too rather than only where the crash happens. Reverting the fix makes it fail and name all five call sites. --- src/steerbench/cli.py | 2 +- src/steerbench/metrics.py | 12 ++++--- src/steerbench/report.py | 15 ++++++--- tests/test_report.py | 71 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/steerbench/cli.py b/src/steerbench/cli.py index caa7a0c..3c34f46 100644 --- a/src/steerbench/cli.py +++ b/src/steerbench/cli.py @@ -56,7 +56,7 @@ def _ensure_side_csv(side_csv: Path | None, out_dir: Path) -> Path: return side_csv out_dir.mkdir(parents=True, exist_ok=True) stub = out_dir / "side_effects.csv" - stub.write_text("benchmark,unsteered_acc,steered_acc\n") + stub.write_text("benchmark,unsteered_acc,steered_acc\n", encoding="utf-8") return stub diff --git a/src/steerbench/metrics.py b/src/steerbench/metrics.py index ca2ca09..523055e 100644 --- a/src/steerbench/metrics.py +++ b/src/steerbench/metrics.py @@ -577,7 +577,7 @@ def load_mmlu_slice( raise ValueError(f"n must be >= 1, got {n}") path = _cache_path(cache_dir, f"mmlu_{n}_{seed}.json") if path.exists(): - rows = json.loads(path.read_text())[:n] + rows = json.loads(path.read_text(encoding="utf-8"))[:n] return [ MMLUExample(question=r["question"], choices=list(r["choices"]), answer=int(r["answer"])) for r in rows @@ -598,7 +598,8 @@ def load_mmlu_slice( path.write_text( json.dumps( [{"question": e.question, "choices": e.choices, "answer": e.answer} for e in examples] - ) + ), + encoding="utf-8", ) return examples @@ -617,7 +618,7 @@ def load_gsm8k_slice( raise ValueError(f"n must be >= 1, got {n}") path = _cache_path(cache_dir, f"gsm8k_{n}_{seed}.json") if path.exists(): - rows = json.loads(path.read_text())[:n] + rows = json.loads(path.read_text(encoding="utf-8"))[:n] return [GSM8KExample(question=r["question"], answer=str(r["answer"])) for r in rows] from datasets import load_dataset # local import: gated optional dependency @@ -628,7 +629,10 @@ def load_gsm8k_slice( for row in ds: gold = extract_gsm8k_answer(str(row["answer"])) examples.append(GSM8KExample(question=str(row["question"]), answer=gold if gold else "")) - path.write_text(json.dumps([{"question": e.question, "answer": e.answer} for e in examples])) + path.write_text( + json.dumps([{"question": e.question, "answer": e.answer} for e in examples]), + encoding="utf-8", + ) return examples diff --git a/src/steerbench/report.py b/src/steerbench/report.py index 661f8e3..96fc509 100644 --- a/src/steerbench/report.py +++ b/src/steerbench/report.py @@ -208,7 +208,7 @@ class _RawRow: def _read_sweep_rows(path: Path, x_column: str) -> list[_RawRow]: """Parse a raw sweep CSV keyed on ``x_column`` (``coeff`` for dose).""" rows: list[_RawRow] = [] - with path.open(newline="") as fh: + with path.open(newline="", encoding="utf-8") as fh: reader = csv.DictReader(fh) missing = {x_column, *_SWEEP_COLUMNS} - set(reader.fieldnames or []) if missing: @@ -273,7 +273,7 @@ def load_layer_curve(path: Path, x_column: str = "layer") -> list[SweepPoint]: def load_side_effects(path: Path) -> list[SideEffect]: """Parse the side-effect CSV (benchmark, unsteered_acc, steered_acc).""" effects: list[SideEffect] = [] - with path.open(newline="") as fh: + with path.open(newline="", encoding="utf-8") as fh: reader = csv.DictReader(fh) required = {"benchmark", "unsteered_acc", "steered_acc"} missing = required - set(reader.fieldnames or []) @@ -796,8 +796,15 @@ def build_report( md_path = out_dir / f"{stem}.md" html_path = out_dir / f"{stem}.html" - md_path.write_text(render_markdown(data, dose_png_path.name, layer_png_path.name)) - html_path.write_text(render_html(data, dose_png, layer_png)) + # UTF-8 explicitly: the card carries non-ASCII (the Δ column header, the ⚠️ + # trap warnings, ± in prose), and Path.write_text otherwise uses the + # platform's locale encoding — cp1252 on a default Windows install, which + # cannot encode them. The HTML also declares charset=utf-8, so writing it + # in anything else would be self-contradictory. + md_path.write_text( + render_markdown(data, dose_png_path.name, layer_png_path.name), encoding="utf-8" + ) + html_path.write_text(render_html(data, dose_png, layer_png), encoding="utf-8") return { "markdown": md_path, diff --git a/tests/test_report.py b/tests/test_report.py index 9e9d079..3aecf39 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -259,6 +259,77 @@ def test_render_html_self_contained(data: ReportData) -> None: assert "Degenerate trap at layer 1" in html +def test_report_files_are_written_as_utf8(csvs: tuple[Path, Path, Path]) -> None: + # The card is not ASCII: the side-effects table header carries Δ and the + # trap callouts carry ⚠️. Every text read/write in the package therefore has + # to name its encoding — Path.write_text/Path.open otherwise fall back to + # the platform locale encoding, which is cp1252 on a stock Windows install + # and cannot encode either character (build_report raises UnicodeEncodeError + # there). + # + # Asserting on the written bytes alone would not catch a regression on a + # UTF-8 platform, so a fresh interpreter runs the whole path under + # -X warn_default_encoding, where io.text_encoding() reports every + # unencoded call site as an EncodingWarning attributed to its caller. Only + # warnings raised from inside steerbench count; optional third-party + # dependencies are not this repo's problem. + import subprocess + import sys + import textwrap + + pytest.importorskip("matplotlib") + dose_csv, layer_csv, side_csv = csvs + out_dir = dose_csv.parent / "card" + + code = textwrap.dedent( + """ + import sys, warnings + from pathlib import Path + from steerbench import report as r + + package_dir = Path(r.__file__).resolve().parent + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + written = r.build_report( + dose_csv=Path(sys.argv[1]), + layer_csv=Path(sys.argv[2]), + side_csv=Path(sys.argv[3]), + out_dir=Path(sys.argv[4]), + ) + + offenders = sorted( + f"{Path(w.filename).name}:{w.lineno}" + for w in caught + if issubclass(w.category, EncodingWarning) + and package_dir in Path(w.filename).resolve().parents + ) + assert not offenders, "text I/O without an explicit encoding: " + ", ".join(offenders) + + markdown = written["markdown"].read_text(encoding="utf-8") + assert "\\u0394" in markdown, "side-effects delta header missing" + assert "\\u26a0" in markdown, "degenerate-trap warning missing" + html = written["html"].read_text(encoding="utf-8") + assert 'charset="utf-8"' in html, "html no longer declares utf-8" + assert "\\u26a0" in html, "degenerate-trap callout missing from html" + """ + ) + subprocess.run( + [ + sys.executable, + "-X", + "warn_default_encoding", + "-c", + code, + str(dose_csv), + str(layer_csv), + str(side_csv), + str(out_dir), + ], + check=True, + ) + + def test_no_matplotlib_imported() -> None: # The parse/analysis/markdown+html path must stay matplotlib-free. Checked in # a fresh interpreter: a global sys.modules check is unreliable in a suite