fix: mutate report/JSON cover all 16 mutators (not a stale 8)

dynamic_engine.py

@@ -1157,17 +1157,10 @@ def to_dict(self) -> dict:
         }
 
 
-# Names used in reports and JSON output — must stay stable.
-_MUTATOR_NAMES = [
-    "fictional_frame",
-    "academic_frame",
-    "base64_encode",
-    "hypothetical_frame",
-    "roleplay_frame",
-    "split_obfuscate",
-    "authority_frame",
-    "rot13_encode",
-]
+# _MUTATOR_NAMES (the report/JSON column order) is derived from all_mutations()
+# just after the PayloadMutator class — see mutator_method_names() below. It used to
+# be a hardcoded list of only the first 8, which silently dropped the other 8
+# mutators from the mutate report and its JSON.
 
 
 class PayloadMutator:
@@ -1401,6 +1394,18 @@ def all_mutations(self, payload: str) -> list[MutatedPayload]:
         ]
 
 
+def mutator_method_names(payload: str = "seed") -> list:
+    """Canonical mutator-method-name order — the ``method_name`` of every mutation
+    ``PayloadMutator.all_mutations()`` produces, in order. Deriving the report/JSON
+    columns from this (instead of a hand-maintained list) keeps them from ever
+    drifting out of sync with the mutations actually fired."""
+    return [m.method_name for m in PayloadMutator().all_mutations(payload)]
+
+
+# Names used in the mutate report + JSON output — single source of truth.
+_MUTATOR_NAMES = mutator_method_names()
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # MUTATE SUITE RUNNER
 # ─────────────────────────────────────────────────────────────────────────────
@@ -1410,8 +1415,8 @@ def run_mutate_suite(
     target_config: dict,
 ) -> list[MutateRunResult]:
     """
-    For every FAIL test in *fail_results*, generate all 8 PayloadMutator
-    variants and fire each at the target.
+    For every FAIL test in *fail_results*, generate all PayloadMutator
+    variants (see all_mutations) and fire each at the target.
 
     Parameters
     ----------

tests/test_mutate_report.py

@@ -0,0 +1,56 @@
+"""Regression: the mutate report + JSON must cover EVERY mutator all_mutations()
+fires, not a stale hardcoded subset. Previously _MUTATOR_NAMES listed only the first
+8, so print_mutate_report / save_mutate_json silently dropped the newer 8 framings.
+"""
+import json
+
+import dynamic_engine as de
+from dynamic_engine import (
+    mutator_method_names, PayloadMutator, MutateRunResult,
+    print_mutate_report, save_mutate_json,
+)
+
+# The 8 that used to be dropped from the report.
+NEWER_EIGHT = [
+    "math_problem", "adversarial_poetry", "emotional_manipulation", "bad_likert_judge",
+    "policy_puppetry", "skeleton_key", "deceptive_delight", "refusal_suppression",
+]
+
+
+def test_mutator_method_names_matches_all_mutations():
+    names = mutator_method_names()
+    assert names == [m.method_name for m in PayloadMutator().all_mutations("x")]
+    assert len(names) == 16
+
+
+def test_module_constant_is_derived_and_covers_new_mutators():
+    assert de._MUTATOR_NAMES == mutator_method_names()      # single source of truth
+    for m in NEWER_EIGHT:
+        assert m in de._MUTATOR_NAMES
+
+
+def _result(method, verdict="PASS"):
+    return MutateRunResult(
+        test_id="T-1", test_name="probe", category="c", severity="High",
+        method_name=method, mutated_payload="p", verdict=verdict, elapsed=0.1,
+    )
+
+
+def test_report_breakdown_covers_every_mutator(capsys):
+    results = [_result(m) for m in mutator_method_names()]
+    print_mutate_report(results)
+    out = capsys.readouterr().out
+    for m in mutator_method_names():
+        assert m in out                    # full name shows in the per-method breakdown
+    assert "(16 per test)" in out          # count label reflects all mutators
+
+
+def test_save_json_lists_all_methods(tmp_path):
+    results = [_result(m) for m in mutator_method_names()]
+    p = tmp_path / "mutate.json"
+    save_mutate_json(results, {"endpoint": "x", "model": "m", "schema": "openai"}, str(p))
+    doc = json.loads(p.read_text(encoding="utf-8"))
+    assert doc["metadata"]["mutation_methods"] == mutator_method_names()
+    assert len(doc["metadata"]["mutation_methods"]) == 16
+    for m in NEWER_EIGHT:
+        assert m in doc["metadata"]["mutation_methods"]