diff --git a/dynamic_engine.py b/dynamic_engine.py index 157e632..d27e6cf 100644 --- a/dynamic_engine.py +++ b/dynamic_engine.py @@ -1157,17 +1157,10 @@ def to_dict(self) -> dict: } -# Names used in reports and JSON output — must stay stable. -_MUTATOR_NAMES = [ - "fictional_frame", - "academic_frame", - "base64_encode", - "hypothetical_frame", - "roleplay_frame", - "split_obfuscate", - "authority_frame", - "rot13_encode", -] +# _MUTATOR_NAMES (the report/JSON column order) is derived from all_mutations() +# just after the PayloadMutator class — see mutator_method_names() below. It used to +# be a hardcoded list of only the first 8, which silently dropped the other 8 +# mutators from the mutate report and its JSON. class PayloadMutator: @@ -1401,6 +1394,18 @@ def all_mutations(self, payload: str) -> list[MutatedPayload]: ] +def mutator_method_names(payload: str = "seed") -> list: + """Canonical mutator-method-name order — the ``method_name`` of every mutation + ``PayloadMutator.all_mutations()`` produces, in order. Deriving the report/JSON + columns from this (instead of a hand-maintained list) keeps them from ever + drifting out of sync with the mutations actually fired.""" + return [m.method_name for m in PayloadMutator().all_mutations(payload)] + + +# Names used in the mutate report + JSON output — single source of truth. +_MUTATOR_NAMES = mutator_method_names() + + # ───────────────────────────────────────────────────────────────────────────── # MUTATE SUITE RUNNER # ───────────────────────────────────────────────────────────────────────────── @@ -1410,8 +1415,8 @@ def run_mutate_suite( target_config: dict, ) -> list[MutateRunResult]: """ - For every FAIL test in *fail_results*, generate all 8 PayloadMutator - variants and fire each at the target. + For every FAIL test in *fail_results*, generate all PayloadMutator + variants (see all_mutations) and fire each at the target. Parameters ---------- diff --git a/tests/test_mutate_report.py b/tests/test_mutate_report.py new file mode 100644 index 0000000..0494d6e --- /dev/null +++ b/tests/test_mutate_report.py @@ -0,0 +1,56 @@ +"""Regression: the mutate report + JSON must cover EVERY mutator all_mutations() +fires, not a stale hardcoded subset. Previously _MUTATOR_NAMES listed only the first +8, so print_mutate_report / save_mutate_json silently dropped the newer 8 framings. +""" +import json + +import dynamic_engine as de +from dynamic_engine import ( + mutator_method_names, PayloadMutator, MutateRunResult, + print_mutate_report, save_mutate_json, +) + +# The 8 that used to be dropped from the report. +NEWER_EIGHT = [ + "math_problem", "adversarial_poetry", "emotional_manipulation", "bad_likert_judge", + "policy_puppetry", "skeleton_key", "deceptive_delight", "refusal_suppression", +] + + +def test_mutator_method_names_matches_all_mutations(): + names = mutator_method_names() + assert names == [m.method_name for m in PayloadMutator().all_mutations("x")] + assert len(names) == 16 + + +def test_module_constant_is_derived_and_covers_new_mutators(): + assert de._MUTATOR_NAMES == mutator_method_names() # single source of truth + for m in NEWER_EIGHT: + assert m in de._MUTATOR_NAMES + + +def _result(method, verdict="PASS"): + return MutateRunResult( + test_id="T-1", test_name="probe", category="c", severity="High", + method_name=method, mutated_payload="p", verdict=verdict, elapsed=0.1, + ) + + +def test_report_breakdown_covers_every_mutator(capsys): + results = [_result(m) for m in mutator_method_names()] + print_mutate_report(results) + out = capsys.readouterr().out + for m in mutator_method_names(): + assert m in out # full name shows in the per-method breakdown + assert "(16 per test)" in out # count label reflects all mutators + + +def test_save_json_lists_all_methods(tmp_path): + results = [_result(m) for m in mutator_method_names()] + p = tmp_path / "mutate.json" + save_mutate_json(results, {"endpoint": "x", "model": "m", "schema": "openai"}, str(p)) + doc = json.loads(p.read_text(encoding="utf-8")) + assert doc["metadata"]["mutation_methods"] == mutator_method_names() + assert len(doc["metadata"]["mutation_methods"]) == 16 + for m in NEWER_EIGHT: + assert m in doc["metadata"]["mutation_methods"]