diff --git a/src/steerbench/cli.py b/src/steerbench/cli.py index caa7a0c..d39ba7a 100644 --- a/src/steerbench/cli.py +++ b/src/steerbench/cli.py @@ -95,6 +95,15 @@ def _build_parser() -> argparse.ArgumentParser: default=None, help="side-effects CSV (benchmark,unsteered_acc,steered_acc); optional", ) + parser.add_argument( + "--effect-col", + default=report.DEFAULT_EFFECT_COLUMN, + metavar="NAME", + help=( + "behaviour column to read from both sweep CSVs " + f"(default: {report.DEFAULT_EFFECT_COLUMN}; e.g. sentiment, verbosity)" + ), + ) parser.add_argument("--out", type=Path, default=Path("report_out"), help="output directory") parser.add_argument("--stem", default="report", help="output filename stem") parser.add_argument( @@ -139,6 +148,7 @@ def main(argv: list[str] | None = None) -> int: side_csv=side_csv, out_dir=args.out, stem=args.stem, + effect_column=args.effect_col, ) if args.json: print(json.dumps({kind: str(path) for kind, path in outputs.items()})) diff --git a/src/steerbench/report.py b/src/steerbench/report.py index 661f8e3..f679c83 100644 --- a/src/steerbench/report.py +++ b/src/steerbench/report.py @@ -18,16 +18,21 @@ coeff,seed,alpha_norm,formality,repetition,ppl - ``effect`` = ``formality``; coherence = the pair ``ppl`` (lower = better) and - ``repetition`` (higher = worse). ``alpha_norm`` is ignored. Extra columns are - tolerated; the required set is ``{coeff, seed, formality, repetition, ppl}``. + ``effect`` = ``formality`` by default; coherence = the pair ``ppl`` (lower = + better) and ``repetition`` (higher = worse). ``alpha_norm`` is ignored. Extra + columns are tolerated; the required set is + ``{coeff, seed, , repetition, ppl}``. The effect column is + selectable (``effect_column=`` / ``steer-report --effect-col``) so a sweep of + another shipped concept — ``sentiment``, ``verbosity`` — reports without + editing this module. * layer-sweep (``artifacts/layer_sweep_coeff.csv``) — one row per (layer, seed):: layer,layer_pos,seed,dir_norm,formality,repetition,ppl - Same effect/coherence tail keyed on ``layer``. ``dir_norm`` (per-layer - direction L2, ~1.0 since repeng unit-normalises) and ``layer_pos`` are - ignored. :func:`analyze_layers` flags coherent-peak vs degenerate-trap layers. + Same effect/coherence tail keyed on ``layer``, with the same selectable + effect column. ``dir_norm`` (per-layer direction L2, ~1.0 since repeng + unit-normalises) and ``layer_pos`` are ignored. :func:`analyze_layers` flags + coherent-peak vs degenerate-trap layers. * side-effects — one row per benchmark:: benchmark,unsteered_acc,steered_acc @@ -187,11 +192,14 @@ class ReportData: # Column names in the M0 sweep CSV. The producer emits one RAW row per seed # (no pre-aggregated mean/std); this reader computes mean±std across seeds. -# ``effect`` = formality; coherence is the pair (ppl, repetition). -_COL_EFFECT = "formality" +# Coherence is the pair (ppl, repetition); the effect column is per-concept and +# is chosen by the caller, defaulting to the M0 concept (formality). +DEFAULT_EFFECT_COLUMN = "formality" _COL_PERPLEXITY = "ppl" _COL_REPETITION = "repetition" -_SWEEP_COLUMNS = ("seed", _COL_EFFECT, _COL_REPETITION, _COL_PERPLEXITY) +# Fixed columns every sweep CSV carries, whatever the concept. The x column +# (``coeff``/``layer``) and the effect column are added per call. +_SWEEP_COLUMNS = ("seed", _COL_REPETITION, _COL_PERPLEXITY) @dataclass(frozen=True) @@ -205,20 +213,31 @@ class _RawRow: repetition: float -def _read_sweep_rows(path: Path, x_column: str) -> list[_RawRow]: - """Parse a raw sweep CSV keyed on ``x_column`` (``coeff`` for dose).""" +def _read_sweep_rows( + path: Path, x_column: str, effect_column: str = DEFAULT_EFFECT_COLUMN +) -> list[_RawRow]: + """Parse a raw sweep CSV keyed on ``x_column`` (``coeff`` for dose). + + ``effect_column`` names the per-concept behaviour column (``formality`` by + default). It is checked by the same missing-columns guard as the fixed + columns, so naming a column the sweep does not carry fails with the + available columns listed rather than a bare ``KeyError`` mid-parse. + """ rows: list[_RawRow] = [] with path.open(newline="") as fh: reader = csv.DictReader(fh) - missing = {x_column, *_SWEEP_COLUMNS} - set(reader.fieldnames or []) + available = list(reader.fieldnames or []) + missing = {x_column, effect_column, *_SWEEP_COLUMNS} - set(available) if missing: - raise ValueError(f"{path} is missing columns: {sorted(missing)}") + raise ValueError( + f"{path} is missing columns: {sorted(missing)} (available: {sorted(available)})" + ) for row in reader: rows.append( _RawRow( x=float(row[x_column]), seed=int(row["seed"]), - effect=float(row[_COL_EFFECT]), + effect=float(row[effect_column]), perplexity=float(row[_COL_PERPLEXITY]), repetition=float(row[_COL_REPETITION]), ) @@ -255,19 +274,29 @@ def _aggregate(rows: list[_RawRow]) -> list[SweepPoint]: return sorted(points, key=lambda p: p.x) -def load_dose_curve(path: Path) -> list[SweepPoint]: - """Load and aggregate the dose-response CSV (keyed on ``coeff``).""" - return _aggregate(_read_sweep_rows(path, "coeff")) +def load_dose_curve(path: Path, effect_column: str = DEFAULT_EFFECT_COLUMN) -> list[SweepPoint]: + """Load and aggregate the dose-response CSV (keyed on ``coeff``). + + ``effect_column`` selects the concept's behaviour column (``formality`` by + default; ``sentiment`` and ``verbosity`` are the other shipped concepts). + """ + return _aggregate(_read_sweep_rows(path, "coeff", effect_column)) -def load_layer_curve(path: Path, x_column: str = "layer") -> list[SweepPoint]: +def load_layer_curve( + path: Path, + x_column: str = "layer", + effect_column: str = DEFAULT_EFFECT_COLUMN, +) -> list[SweepPoint]: """Load and aggregate the layer-sweep CSV (keyed on ``layer``). - Columns ``layer,layer_pos,seed,dir_norm,formality,repetition,ppl``; - ``layer_pos``/``dir_norm`` are ignored (extra columns tolerated). Pair with - :func:`analyze_layers` to locate coherent peaks and degenerate traps. + Columns ``layer,layer_pos,seed,,repetition,ppl``; + ``layer_pos``/``dir_norm`` are ignored (extra columns tolerated). + ``effect_column`` selects the concept's behaviour column, as for + :func:`load_dose_curve`. Pair with :func:`analyze_layers` to locate + coherent peaks and degenerate traps. """ - return _aggregate(_read_sweep_rows(path, x_column)) + return _aggregate(_read_sweep_rows(path, x_column, effect_column)) def load_side_effects(path: Path) -> list[SideEffect]: @@ -768,16 +797,20 @@ def build_report( side_csv: Path, out_dir: Path, stem: str = "report", + effect_column: str = DEFAULT_EFFECT_COLUMN, ) -> dict[str, Path]: """Load CSVs, plot, and write ``.md`` + ``.html`` (+ PNGs). + ``effect_column`` selects the behaviour column read from both sweep CSVs + (``formality`` by default); it must be present in both. + Requires ``matplotlib``. Returns the written paths keyed by artefact (``markdown``, ``html``, ``dose_png``, ``layer_png``). The markdown references the sidecar PNGs; the HTML embeds them so it travels as one file. """ out_dir.mkdir(parents=True, exist_ok=True) - dose = load_dose_curve(dose_csv) - layer = load_layer_curve(layer_csv) + dose = load_dose_curve(dose_csv, effect_column=effect_column) + layer = load_layer_curve(layer_csv, effect_column=effect_column) side_effects = load_side_effects(side_csv) data = ReportData( dose=dose, diff --git a/tests/test_cli.py b/tests/test_cli.py index 686ae11..3a4bb3b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -64,6 +64,55 @@ def test_cli_json_emits_artifact_paths(tmp_path: Path, capsys: pytest.CaptureFix assert Path(payload[key]).exists(), f"artifact path does not exist: {payload[key]}" +def test_cli_effect_col_reports_a_non_formality_sweep(tmp_path: Path) -> None: + pytest.importorskip("matplotlib") + # The real M0 artifacts with the effect column renamed: same numbers, so a + # `sentiment` sweep must render exactly like the `formality` one. + dose = tmp_path / "dose_sentiment.csv" + layer = tmp_path / "layer_sentiment.csv" + dose.write_text(_DOSE.read_text().replace("formality", "sentiment", 1)) + layer.write_text(_LAYER.read_text().replace("formality", "sentiment", 1)) + + out = tmp_path / "card" + rc = cli.main( + [ + "--dose-csv", + str(dose), + "--layer-csv", + str(layer), + "--out", + str(out), + "--effect-col", + "sentiment", + ] + ) + assert rc == 0 + assert (out / "report.md").exists() + + reference = tmp_path / "reference" + assert ( + cli.main(["--dose-csv", str(_DOSE), "--layer-csv", str(_LAYER), "--out", str(reference)]) + == 0 + ) + assert (out / "report.md").read_text() == (reference / "report.md").read_text() + + +def test_cli_effect_col_unknown_column_is_reported(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="missing columns"): + cli.main( + [ + "--dose-csv", + str(_DOSE), + "--layer-csv", + str(_LAYER), + "--out", + str(tmp_path / "o"), + "--effect-col", + "no_such_column", + ] + ) + + def test_cli_errors_on_missing_csv(tmp_path: Path) -> None: with pytest.raises(SystemExit): cli.main(["--dose-csv", str(tmp_path / "nope.csv"), "--out", str(tmp_path / "o")]) diff --git a/tests/test_report.py b/tests/test_report.py index 9e9d079..c4e3ef2 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -126,6 +126,41 @@ def test_missing_column_raises(tmp_path: Path) -> None: load_dose_curve(bad) +def test_effect_column_is_selectable(tmp_path: Path) -> None: + # Same schema as the M0 sweep, but for another shipped concept: the effect + # column is `sentiment`, not `formality`. + csv_path = tmp_path / "sentiment_dose.csv" + csv_path.write_text(DOSE_CSV.replace("formality", "sentiment")) + + dose = load_dose_curve(csv_path, effect_column="sentiment") + assert [p.x for p in dose] == [-20.0, 0.0, 20.0, 40.0] + assert dose[1].effect.mean == pytest.approx(4.5) # mean of 4.4, 4.5, 4.6 + assert dose[1].perplexity.mean == pytest.approx(10.0) # coherence unchanged + + layer_path = tmp_path / "sentiment_layer.csv" + layer_path.write_text(LAYER_CSV.replace("formality", "sentiment")) + layer = load_layer_curve(layer_path, effect_column="sentiment") + assert [p.x for p in layer] == [1.0, 2.0, 13.0, 14.0, 20.0] + + +def test_effect_column_defaults_to_formality(tmp_path: Path) -> None: + # Default is unchanged, so a sweep without a `formality` column still fails + # the missing-columns guard rather than silently reading something else. + csv_path = tmp_path / "sentiment_dose.csv" + csv_path.write_text(DOSE_CSV.replace("formality", "sentiment")) + with pytest.raises(ValueError, match="missing columns"): + load_dose_curve(csv_path) + + +def test_unknown_effect_column_names_the_available_columns(csvs: tuple[Path, Path, Path]) -> None: + with pytest.raises(ValueError) as excinfo: + load_dose_curve(csvs[0], effect_column="verbosity") + message = str(excinfo.value) + assert "'verbosity'" in message # what was asked for + assert "available" in message + assert "'formality'" in message # what the CSV actually carries + + # --------------------------------------------------------------------------- # # Analysis — sweet spot + cliff # --------------------------------------------------------------------------- #