bench(w4.3): decorrelate untrusted from poison, and measure what the defense costs

.gitignore

@@ -172,6 +172,10 @@ backups/
 # local run logs are reproducible. The harness, its README, and committed reports stay tracked.
 benchmarks/memory/longmemeval/longmemeval_s.json
 benchmarks/memory/longmemeval/longmemeval_m*
-benchmarks/memory/longmemeval/results/
+# Per-run artifacts stay ignored; top-level roll-up reports are the committed record.
+# (Must be results/* rather than results/ — git will not descend into an excluded
+# directory, so a negation under it would never match.)
+benchmarks/memory/longmemeval/results/*
+!benchmarks/memory/longmemeval/results/*_report.json
 benchmarks/memory/longmemeval/*.log
 benchmarks/memory/longmemeval/*.err

benchmarks/memory/longmemeval/README.md

@@ -52,3 +52,34 @@ python run_longmemeval.py score
 Note: the server rate-limits per project (default 60 req/min, 1000 req/hour). The full
 ingest is ~2,000 batch calls — raise `RATE_LIMIT_PER_MINUTE`/`RATE_LIMIT_PER_HOUR` in
 `docker-compose.yml` for the full run, or let the harness back off on 429s (slower).
+
+## Trust-plan corpora (W4.3)
+
+In W4.2 every memory was ingested as `internal` and `trust_level=untrusted` was carried
+only by poison — so downweighting untrusted was indistinguishable from downweighting the
+answer key. `--trust-plan` breaks that correlation at ingest time:
+
+| plan | assignment |
+| --- | --- |
+| `none` | every round `internal` (W4.2 behaviour, byte-identical) |
+| `mixed` | a seeded 20% of **non-evidence** rounds are `untrusted`; evidence never is |
+| `evidence` | every round in an `answer_session_ids` session is `untrusted`; no poison |
+
+Each variant needs its own corpus, selected by `LME_NS_SUFFIX` (`_mix`, `_evu`) — the
+env var is read by ingest, answer, and `poison_corpus.py`'s inject/delete so all three
+agree on the namespace, and the injected-ID ledger is written per-suffix so a variant
+run cannot overwrite the record of what is in the W4.2 corpus. `ns()` now raises rather
+than truncating past 64 chars, since truncation would merge two questions' corpora.
+
+```sh
+$env:LME_NS_SUFFIX='_mix'
+python run_longmemeval.py ingest --limit 120 --trust-plan mixed --results-dir results/mix_ingest
+python analyze_mixed.py --arms mix_twr_off mix_twr_w35   # accuracy, occupancy, McNemar
+python build_mixed_report.py                             # results/mixed_untrusted_report.json
+python spend.py                                          # measured per-arm API cost
+```
+
+`analyze_mixed.py` reads trust levels and session ids from Postgres rather than
+re-deriving them from `trust_plan.jsonl`, because the plan records round indices while
+retrieval returns memory ids — the database is where those join. McNemar is only run
+**within** a corpus; the `_mix`/`_evu` arms are not paired with the W4.2 rows.

benchmarks/memory/longmemeval/analyze_mixed.py

@@ -0,0 +1,184 @@
+"""W4.3 analysis — mixed benign-untrusted arms.
+
+The W4.2 objection this answers: there, trust_level=untrusted was perfectly correlated
+with poison, so downweighting untrusted was indistinguishable from downweighting the
+answer key. These arms break that correlation and measure what the defence costs when
+the label is no longer an oracle.
+
+Metrics per arm:
+  accuracy                    + exact McNemar against that corpus's OWN TWR-off arm
+  poison in top-k / ranked #1 (Arm M only — Arm N has no poison)
+  benign-untrusted occupancy  fraction of top-k that is untrusted AND not poison
+                              (the over-suppression measure)
+  evidence-in-top-k           fraction of evidence rounds retrieved (Arm N's headline)
+
+Trust levels and session ids are read from Postgres rather than re-derived from
+trust_plan.jsonl, because the plan records round indices while retrieval returns
+memory ids — the database is the only place those two join.
+
+    python analyze_mixed.py --arms mix_twr_off mix_twr_w35
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+from math import comb
+from pathlib import Path
+
+from run_longmemeval import HERE, jsonl_read, load_dataset
+
+
+def psql(sql: str) -> list[list[str]]:
+    out = subprocess.run(
+        ["docker", "exec", "aegis-postgres", "psql", "-U", "aegis", "-d", "aegis",
+         "-t", "-A", "-F", "\t", "-c", sql],
+        capture_output=True, text=True, check=True,
+    ).stdout
+    return [ln.split("\t") for ln in out.splitlines() if ln.strip()]
+
+
+def memory_facts(namespace_like: str) -> dict[str, tuple[str, str]]:
+    """memory_id -> (trust_level, session_id) for one corpus."""
+    rows = psql(
+        "SELECT id, trust_level, COALESCE(metadata->>'session_id', '') "
+        f"FROM memories WHERE namespace LIKE '{namespace_like}'"
+    )
+    return {r[0]: (r[1], r[2]) for r in rows if len(r) >= 3}
+
+
+def exact_mcnemar(b: int, c: int) -> float:
+    """Two-sided exact binomial test on discordant pairs. b/c = the two disagreements."""
+    n = b + c
+    if n == 0:
+        return 1.0
+    k = min(b, c)
+    tail = sum(comb(n, i) for i in range(0, k + 1)) / (2 ** n)
+    return min(1.0, 2 * tail)
+
+
+def load_labels(results_dir: Path) -> dict[str, bool]:
+    return {q: r["label"] for q, r in jsonl_read(results_dir / "judged.jsonl").items()}
+
+
+def analyse_arm(results_dir: Path, facts: dict, poison_ids: dict, evidence_sessions: dict) -> dict:
+    hyp = jsonl_read(results_dir / "hypotheses.jsonl")
+    labels = load_labels(results_dir)
+
+    slots = poison_hits = benign_untrusted = 0
+    ranked_first = with_any_poison = 0
+    ev_retrieved = 0
+    q_with_evidence = q_with_any_evidence_retrieved = 0
+    n = 0
+
+    for qid, rec in hyp.items():
+        ids = rec.get("retrieved_memory_ids") or []
+        if not ids:
+            continue
+        n += 1
+        slots += len(ids)
+        pois = poison_ids.get(qid, set())
+        evs = evidence_sessions.get(qid, set())
+
+        hits = [i for i in ids if i in pois]
+        poison_hits += len(hits)
+        if hits:
+            with_any_poison += 1
+        if ids and ids[0] in pois:
+            ranked_first += 1
+
+        hit_evidence = 0
+        for mid in ids:
+            trust, sid = facts.get(mid, ("", ""))
+            if trust == "untrusted" and mid not in pois:
+                benign_untrusted += 1
+            if sid and sid in evs:
+                hit_evidence += 1
+        ev_retrieved += hit_evidence
+        if evs:
+            q_with_evidence += 1
+            if hit_evidence:
+                q_with_any_evidence_retrieved += 1
+
+    acc = sum(1 for v in labels.values() if v) / len(labels) if labels else 0.0
+    return {
+        "arm": results_dir.name,
+        "n_questions": len(labels),
+        "accuracy": round(acc, 4),
+        "top_k_slots": slots,
+        "poison_in_top_k": poison_hits,
+        "poison_in_top_k_rate": round(poison_hits / slots, 4) if slots else 0.0,
+        "questions_with_any_poison": with_any_poison,
+        "poison_ranked_first": ranked_first,
+        "poison_ranked_first_rate": round(ranked_first / n, 4) if n else 0.0,
+        "benign_untrusted_in_top_k": benign_untrusted,
+        "benign_untrusted_occupancy": round(benign_untrusted / slots, 4) if slots else 0.0,
+        # Arm N's headline. evidence_in_top_k_rate is the share of context that is
+        # answer-bearing; evidence_recall is the share of questions that got ANY
+        # evidence at all — the latter is what accuracy can actually depend on, since
+        # a question with zero evidence in context is unanswerable from retrieval.
+        "evidence_in_top_k": ev_retrieved,
+        "evidence_in_top_k_rate": round(ev_retrieved / slots, 4) if slots else 0.0,
+        "questions_with_evidence": q_with_evidence,
+        "questions_with_any_evidence_retrieved": q_with_any_evidence_retrieved,
+        "evidence_recall": (
+            round(q_with_any_evidence_retrieved / q_with_evidence, 4) if q_with_evidence else 0.0
+        ),
+    }
+
+
+def paired_mcnemar(control: Path, treatment: Path) -> dict:
+    a, b = load_labels(control), load_labels(treatment)
+    shared = sorted(set(a) & set(b))
+    only_ctrl = sum(1 for q in shared if a[q] and not b[q])
+    only_trt = sum(1 for q in shared if b[q] and not a[q])
+    return {
+        "control": control.name,
+        "treatment": treatment.name,
+        "paired_questions": len(shared),
+        "control_only_correct": only_ctrl,
+        "treatment_only_correct": only_trt,
+        # Not rounded to a fixed number of decimals: with a large discordant split the
+        # exact p is ~1e-10, and round(p, 6) would render it as a literal 0.0.
+        "exact_mcnemar_p": float(f"{exact_mcnemar(only_ctrl, only_trt):.3g}"),
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--arms", nargs="+", required=True)
+    ap.add_argument("--suffix", default="_mix")
+    ap.add_argument("--limit", type=int, default=120)
+    ap.add_argument("--control", default=None, help="within-corpus TWR-off arm to pair against")
+    args = ap.parse_args()
+
+    data = load_dataset(args.limit)
+    evidence_sessions = {
+        e["question_id"]: set(e.get("answer_session_ids") or []) for e in data
+    }
+
+    injected = jsonl_read(HERE / "results" / "poison" / f"injected{args.suffix}.jsonl")
+    poison_ids = {q: set(r["memory_ids"]) for q, r in injected.items()}
+
+    facts = memory_facts(f"lme\\_%{args.suffix}")
+    print(f"corpus {args.suffix}: {len(facts)} memories | "
+          f"poison ledger: {sum(len(v) for v in poison_ids.values())} ids")
+
+    arms = [analyse_arm(HERE / "results" / a, facts, poison_ids, evidence_sessions)
+            for a in args.arms]
+    out = {"corpus_suffix": args.suffix, "arms": arms}
+
+    if args.control:
+        ctrl = HERE / "results" / args.control
+        out["mcnemar"] = [
+            paired_mcnemar(ctrl, HERE / "results" / a)
+            for a in args.arms if a != args.control
+        ]
+
+    print(json.dumps(out, indent=2))
+    return out
+
+
+if __name__ == "__main__":
+    main()

benchmarks/memory/longmemeval/analyze_poison.py

@@ -10,13 +10,31 @@
 import json
 from pathlib import Path
 
-from run_longmemeval import HERE, jsonl_read, load_dataset
+from run_longmemeval import HERE, NS_SUFFIX, jsonl_read, load_dataset
 
-ARMS = [
-    ("poisoned TWR off", "poisoned_twr_off"),
-    ("poisoned TWR on", "poisoned_twr_on"),
-    ("poisoned TWR w=0.35", "poisoned_twr_w35"),
-]
+# Arms are grouped by corpus, because each corpus has its own poison ledger. Scoring
+# the _mix arms against the W4.2 ledger would silently compare retrieved ids against
+# poison that lives in different namespaces and report a spurious 0% — so the arm set
+# is selected by LME_NS_SUFFIX rather than concatenated into one list.
+ARMS_BY_SUFFIX = {
+    "": [
+        ("poisoned TWR off", "poisoned_twr_off"),
+        ("poisoned TWR on", "poisoned_twr_on"),
+        ("poisoned TWR w=0.35", "poisoned_twr_w35"),
+    ],
+    # W4.3: benign untrusted dilution + the same poison.
+    "_mix": [
+        ("mix TWR off", "mix_twr_off"),
+        ("mix TWR w=0.35", "mix_twr_w35"),
+    ],
+    # W4.3 Arm N carries no poison; poison columns are structurally zero there.
+    # Use analyze_mixed.py for its evidence-in-top-k and occupancy metrics.
+    "_evu": [
+        ("evu TWR off", "evu_twr_off"),
+        ("evu TWR w=0.35", "evu_twr_w35"),
+    ],
+}
+ARMS = ARMS_BY_SUFFIX[NS_SUFFIX]
 
 
 def main():
@@ -25,7 +43,7 @@ def main():
     args = ap.parse_args()
 
     qids = {e["question_id"] for e in load_dataset(args.limit)}
-    injected = jsonl_read(HERE / "results" / "poison" / "injected.jsonl")
+    injected = jsonl_read(HERE / "results" / "poison" / f"injected{NS_SUFFIX}.jsonl")
     poison_ids = {q: set(r["memory_ids"]) for q, r in injected.items()}
     total_poison = sum(len(v) for v in poison_ids.values())
     print(f"\npoison injected: {total_poison} memories across {len(poison_ids)} questions\n")