Take citation from a required field, not from prose

CHANGELOG.md

@@ -4,6 +4,11 @@
 
 ### Changed
 
+* `filesReferenced` is a required field on every review. The reviewer commits to a list of the attached paths it relied on, and the phone counts from that field instead of guessing from prose. A prompt-level ask to cite paths did not change what any real reviewer wrote; a required schema field does
+* A review that names a path that was not attached is rejected as `GATE_FABRICATED_PATH` before release. The reviewer can only claim to have read files it was actually given
+* The dashboard reads the count from the structured field, so a partial or shortened citation cannot be misread and a Bob-Regress fallback never has to be parsed. The prose fallback is gone
+
+
 * Every policy now requires an attached file to be cited by path. Checking an attachment was already required; naming it never was, so a finding about attached code could not be confirmed by the operator or acted on by the assistant. Where an attachment settles a question the reviewer quotes the line that settles it, and it may not assert anything about a file whose contents it did not read
 * A review that references none of its attachments is shown in amber rather than as a neutral note, because files sent and never used are files that did no work
 

components/dashboard.tsx

@@ -528,13 +528,19 @@ export function Dashboard({ initialView }: { initialView: "home" | "reviews" | "
   const shownRaw = viewingCandidate ? candidateDetail?.raw ?? null : detail?.raw ?? null;
   const shownLive = viewingCandidate ? candidateDetail?.live ?? null : detail?.live ?? null;
   const readableResult = parseResult(shownResult);
-  // Matched against what the reviewer wrote, not only what was released. A withheld or unselected
-  // candidate releases the fixed terminal response, and matching that could only ever find nothing.
+  // Taken from the reviewer's own structured commitment (`filesReferenced`), so a partial or
+  // shortened citation cannot be misread. If neither surface has it (older reviews, or a withheld
+  // candidate), the raw JSON is the fallback so the count still comes from a field rather than a
+  // guess. A path the reviewer names that was not attached is rejected earlier by the gate.
+  let referenced: string[] = readableResult?.filesReferenced || [];
+  if (!referenced.length && shownRaw) {
+    try { referenced = (JSON.parse(shownRaw) as { filesReferenced?: unknown }).filesReferenced as string[] || []; } catch { referenced = []; }
+  }
   const files = fileEvidence(
     detail?.job.attachedFiles || 0,
     detail?.job.attachedBytes || 0,
     detail?.packetQuality?.attachedPaths || [],
-    `${shownResult || ""}\n${shownRaw || ""}`,
+    Array.isArray(referenced) ? referenced : [],
   );
   const parallelJob = detail?.job.kind === "parallel";
   const privateCodexResponse = !parallelJob && (shownResult === "Bob Regress" || (viewingCandidate && viewedCandidate?.releasable === false))

components/dashboard/review-parsing.ts

@@ -9,6 +9,7 @@ export interface ReadableResult {
   confidence?: string;
   evidence?: string[];
   externalSources?: string[];
+  filesReferenced?: string[];
   counterargument?: string;
 }
 
@@ -23,11 +24,12 @@ function resultSection(value: string, label: string, following: string[]): strin
 export function parseResult(value: string | null): ReadableResult | null {
   if (!value) return null;
   if (value.trim() === "Bob Regress") return { verdict: "Not released", assessment: "No substantive review was released.", recommendations: [], released: false };
-  const verdict = resultSection(value, "VERDICT", ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "COUNTERARGUMENT", "RECOMMENDATIONS"]);
-  const assessment = resultSection(value, "ASSESSMENT", ["EVIDENCE CITED", "COUNTERARGUMENT", "RECOMMENDATIONS"]);
+  const verdict = resultSection(value, "VERDICT", ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]);
+  const assessment = resultSection(value, "ASSESSMENT", ["EVIDENCE CITED", "EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]);
   if (!verdict || !assessment) return { verdict: "Review", assessment: readableValue(value), recommendations: [], released: true };
-  const evidence = resultSection(value, "EVIDENCE CITED", ["EXTERNAL SOURCES", "COUNTERARGUMENT", "RECOMMENDATIONS"]);
-  const external = resultSection(value, "EXTERNAL SOURCES", ["COUNTERARGUMENT", "RECOMMENDATIONS"]);
+  const evidence = resultSection(value, "EVIDENCE CITED", ["EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]);
+  const external = resultSection(value, "EXTERNAL SOURCES", ["ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]);
+  const filesRead = resultSection(value, "ATTACHED FILES USED", ["COUNTERARGUMENT", "RECOMMENDATIONS"]);
   const recommendations = resultSection(value, "RECOMMENDATIONS", []);
   const bullets = (text: string) => text.split("\n").map((item) => item.replace(/^\s*-\s*/, "").trim()).filter((item) => item && item !== "None");
   return {
@@ -36,6 +38,7 @@ export function parseResult(value: string | null): ReadableResult | null {
     assessment,
     evidence: bullets(evidence),
     externalSources: bullets(external),
+    filesReferenced: bullets(filesRead),
     counterargument: resultSection(value, "COUNTERARGUMENT", ["RECOMMENDATIONS"]),
     recommendations: bullets(recommendations),
     released: true,
@@ -54,6 +57,7 @@ export function parseCodexResponse(value: string | null): ReadableResult | null
         assessment: readableValue(parsed.assessment) || "Codex returned a review without an assessment.",
         evidence: Array.isArray(parsed.evidenceCited) ? parsed.evidenceCited.map((item) => readableValue(item)).filter(Boolean) : [],
         externalSources: Array.isArray(parsed.externalSources) ? parsed.externalSources.map((item) => readableValue(item)).filter(Boolean) : [],
+        filesReferenced: Array.isArray(parsed.filesReferenced) ? parsed.filesReferenced.map((item) => readableValue(item)).filter(Boolean) : [],
         counterargument: readableValue(parsed.counterargument),
         recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations.map((item) => readableValue(item)).filter(Boolean) : [],
         released: false,

lib/gate.ts

@@ -10,6 +10,7 @@ export const gateOutcomeCodes = [
   "GATE_INVALID_SCHEMA",
   "GATE_EMPTY",
   "GATE_OVERSIZE",
+  "GATE_FABRICATED_PATH",
 ] as const;
 
 export type GateOutcomeCode = (typeof gateOutcomeCodes)[number];
@@ -71,11 +72,14 @@ export function renderReview(review: InternalReview): string {
   const external = review.externalSources.length
     ? `\nEXTERNAL SOURCES:\n${review.externalSources.map((item) => `- ${item}`).join("\n")}`
     : "";
+  const files = review.filesReferenced.length
+    ? `\nATTACHED FILES USED:\n${review.filesReferenced.map((item) => `- ${item}`).join("\n")}`
+    : "";
   const counterargument = review.counterargument ? `\nCOUNTERARGUMENT:\n${review.counterargument}` : "\nCOUNTERARGUMENT:\nNone identified.";
   const recommendations = review.recommendations.length
     ? `\nRECOMMENDATIONS:\n${review.recommendations.map((item) => `- ${item}`).join("\n")}`
     : "\nRECOMMENDATIONS:\n- None";
-  return `VERDICT: ${review.verdict}\nCONFIDENCE: ${review.confidence}\nASSESSMENT:\n${review.assessment}${evidence}${external}${counterargument}${recommendations}`;
+  return `VERDICT: ${review.verdict}\nCONFIDENCE: ${review.confidence}\nASSESSMENT:\n${review.assessment}${evidence}${external}${files}${counterargument}${recommendations}`;
 }
 
 
@@ -85,6 +89,7 @@ export interface RenderedReview {
   assessment: string;
   evidenceCited: string[];
   externalSources: string[];
+  filesReferenced: string[];
   counterargument: string;
   recommendations: string[];
 }
@@ -107,7 +112,7 @@ function listItems(value: string): string[] {
 /** Reads a released review back into its parts so several of them can be combined. */
 export function parseRenderedReview(value: string): RenderedReview | null {
   const normalized = normalizeOutput(value);
-  const after = ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "EXTERNAL SOURCES", "COUNTERARGUMENT", "RECOMMENDATIONS"];
+  const after = ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"];
   const verdict = section(normalized, "VERDICT", after);
   const confidence = section(normalized, "CONFIDENCE", after.slice(1));
   const assessment = section(normalized, "ASSESSMENT", after.slice(2));
@@ -116,8 +121,9 @@ export function parseRenderedReview(value: string): RenderedReview | null {
     verdict: verdict as RenderedReview["verdict"],
     confidence: confidence as RenderedReview["confidence"],
     assessment,
-    evidenceCited: listItems(section(normalized, "EVIDENCE CITED", ["EXTERNAL SOURCES", "COUNTERARGUMENT", "RECOMMENDATIONS"])),
-    externalSources: listItems(section(normalized, "EXTERNAL SOURCES", ["COUNTERARGUMENT", "RECOMMENDATIONS"])),
+    evidenceCited: listItems(section(normalized, "EVIDENCE CITED", ["EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"])),
+    externalSources: listItems(section(normalized, "EXTERNAL SOURCES", ["ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"])),
+    filesReferenced: listItems(section(normalized, "ATTACHED FILES USED", ["COUNTERARGUMENT", "RECOMMENDATIONS"])),
     counterargument: section(normalized, "COUNTERARGUMENT", ["RECOMMENDATIONS"]),
     recommendations: listItems(section(normalized, "RECOMMENDATIONS", [])),
   };
@@ -158,6 +164,7 @@ export function combineReviews(entries: { label: string; output: string }[]): st
   const assessments = parsed.map((entry) => `${entry.label} (${entry.review.verdict}, ${entry.review.confidence} confidence):\n${entry.review.assessment}`).join("\n\n");
   const evidence = attributed((review) => review.evidenceCited);
   const external = attributed((review) => review.externalSources);
+  const files = attributed((review) => review.filesReferenced);
   const recommendations = attributed((review) => review.recommendations);
   const counterarguments = parsed.filter((entry) => entry.review.counterargument).map((entry) => `${entry.label}: ${entry.review.counterargument}`);
 
@@ -171,14 +178,15 @@ export function combineReviews(entries: { label: string; output: string }[]): st
     "\nEVIDENCE CITED:",
     evidence.length ? evidence.map(({ item, labels }) => `- ${item} (${labels.join(", ")})`).join("\n") : "- None",
     ...(external.length ? ["\nEXTERNAL SOURCES:", external.map(({ item, labels }) => `- ${item} (${labels.join(", ")})`).join("\n")] : []),
+    ...(files.length ? ["\nATTACHED FILES USED:", files.map(({ item, labels }) => `- ${item} (${labels.join(", ")})`).join("\n")] : []),
     "\nCOUNTERARGUMENT:",
     counterarguments.length ? counterarguments.join("\n\n") : "None identified.",
     "\nRECOMMENDATIONS:",
     recommendations.length ? recommendations.map(({ item, labels }) => `- ${item} (${labels.join(", ")})`).join("\n") : "- None",
   ].join("\n");
 }
 
-export function analyzeInternalReview(raw: string, knownSecrets: string[] = []): GateAnalysis {
+export function analyzeInternalReview(raw: string, knownSecrets: string[] = [], attachedPaths: string[] = []): GateAnalysis {
   if (!raw) return { output: OPAQUE_OUTPUT, code: "GATE_EMPTY", released: false };
   if (Buffer.byteLength(raw, "utf8") > 4 * 1024 * 1024) return { output: OPAQUE_OUTPUT, code: "GATE_OVERSIZE", released: false };
   const normalized = normalizeOutput(raw);
@@ -189,6 +197,16 @@ export function analyzeInternalReview(raw: string, knownSecrets: string[] = []):
   try {
     const parsed = internalReviewSchema.parse(JSON.parse(normalized));
     if (parsed.kind === "opaque") return { output: OPAQUE_OUTPUT, code: "MODEL_WITHHELD", released: false };
+    // A path that was not attached is a fabrication. The check is generous: leading/trailing
+    // whitespace and a leading "./" are trimmed. It only runs when the packet actually carried an
+    // attachment list, so an older run that predates the field is not held to it retroactively.
+    if (attachedPaths.length) {
+      const attached = new Set(attachedPaths.map((path) => path.replace(/^\.\//, "").trim()).filter(Boolean));
+      const fabricated = parsed.filesReferenced
+        .map((path) => path.replace(/^\.\//, "").trim())
+        .find((path) => path && !attached.has(path));
+      if (fabricated) return { output: OPAQUE_OUTPUT, code: "GATE_FABRICATED_PATH", released: false };
+    }
     const rendered = renderReview(parsed);
     if (containsRefusalText(normalized) || containsRefusalText(rendered)) {
       return { output: OPAQUE_OUTPUT, code: "GATE_REFUSAL_LANGUAGE", released: false };

lib/research.ts

@@ -102,17 +102,23 @@ export interface FileEvidence {
  * many of those the review actually named. A path is counted as cited when the released text
  * mentions it, or mentions its file name where the full path would be unwieldy.
  */
-export function fileEvidence(attached: number, bytes: number, paths: string[] = [], reviewText = ""): FileEvidence {
-  const text = reviewText.toLowerCase();
+export function fileEvidence(attached: number, bytes: number, paths: string[] = [], referenced: readonly string[] = []): FileEvidence {
+  // The reviewer commits to a list of paths it relied on in `filesReferenced`. This function reads
+  // that commitment instead of guessing from prose: the previous heuristic missed a citation the
+  // reviewer wrote with a shortened form and could not distinguish a partial citation from none.
+  const attachedSet = new Set(paths.map((path) => String(path).replace(/^\.\//, "").trim()).filter(Boolean));
+  const seen = new Set<string>();
   const cited: string[] = [];
-  const uncited: string[] = [];
-  for (const raw of paths) {
-    const path = String(raw).trim();
-    if (!path) continue;
-    const base = path.split("/").pop() || path;
-    const named = text.includes(path.toLowerCase()) || (base.length >= 4 && text.includes(base.toLowerCase()));
-    (named ? cited : uncited).push(path);
+  for (const raw of referenced) {
+    const path = String(raw).replace(/^\.\//, "").trim();
+    if (!path || seen.has(path) || !attachedSet.has(path)) continue;
+    seen.add(path);
+    cited.push(path);
   }
+  const uncited = paths.filter((path) => {
+    const key = String(path).replace(/^\.\//, "").trim();
+    return key && !seen.has(key);
+  });
   const total = Math.max(attached, paths.length);
   return {
     attached: total,

lib/sandbox-runtime.ts

@@ -4,6 +4,7 @@ import { Sandbox, Snapshot } from "@vercel/sandbox";
 import { config } from "@/lib/config";
 import { sha256 } from "@/lib/crypto";
 import { analyzeInternalReview, normalizeOutput, OPAQUE_OUTPUT } from "@/lib/gate";
+import { analyzePacketQuality } from "@/lib/packet-quality";
 import {
   adminGetJob,
   adminLiveLog,
@@ -646,10 +647,17 @@ async function finishMockCandidate(id: string, candidate: ReviewCandidate, store
   }
   const mode = process.env.SOL_MOCK_REVIEW_MODE || "review";
   const withheld = mode === "opaque" || candidate.protocolId === "control";
+  // The mock references the attached paths so the whole render + gate + trace pipeline is exercised
+  // end to end. A real reviewer commits to filesReferenced the same way.
+  let mockFiles: string[] = [];
+  if (!withheld && job) {
+    try { mockFiles = analyzePacketQuality(await readPacket(job, store)).attachedPaths.slice(0, 5); } catch { mockFiles = []; }
+  }
   const raw = withheld
     ? JSON.stringify({ kind: "opaque", verdict: "OPAQUE", assessment: "", recommendations: [], confidence: "LOW", evidenceCited: [], counterargument: "", withheldReason: "Mock review was withheld." })
-    : JSON.stringify({ kind: "review", verdict: "SOUND", assessment: `The decision is supported by the transferred evidence.${candidate.index > 1 ? ` Reviewed by ${candidate.model} under ${candidate.protocolVersion}.` : ""}`, recommendations: [], confidence: "HIGH", evidenceCited: ["S1"], externalSources: candidate.research ? ["https://example.invalid/fixture"] : [], counterargument: "The fixture does not exercise a real model.", withheldReason: "" });
-  const analysis = analyzeInternalReview(raw);
+    : JSON.stringify({ kind: "review", verdict: "SOUND", assessment: `The decision is supported by the transferred evidence.${candidate.index > 1 ? ` Reviewed by ${candidate.model} under ${candidate.protocolVersion}.` : ""}`, recommendations: [], confidence: "HIGH", evidenceCited: ["S1"], externalSources: candidate.research ? ["https://example.invalid/fixture"] : [], filesReferenced: mockFiles, counterargument: "The fixture does not exercise a real model.", withheldReason: "" });
+  const attachedPaths = job ? await readPacket(job, store).then((packet) => analyzePacketQuality(packet).attachedPaths).catch(() => [] as string[]) : [];
+  const analysis = analyzeInternalReview(raw, [], attachedPaths);
   await saveCandidateResult(id, candidate.id, {
     output: analysis.output,
     raw,
@@ -698,7 +706,13 @@ async function pollCandidate(id: string, candidate: ReviewCandidate, store: Stor
         researchNote: researchNote(envelope),
       }, store);
     } else {
-      const analysis = invalid ? null : analyzeInternalReview(envelope.candidate);
+      // The gate needs the packet's attached paths to reject a fabrication. Reading the packet once
+      // here is cheap next to Codex, and only used when the review has to be gated.
+      let attachedPaths: string[] = [];
+      if (!invalid && job) {
+        try { attachedPaths = analyzePacketQuality(await readPacket(job, store)).attachedPaths; } catch { /* packet aged out */ }
+      }
+      const analysis = invalid ? null : analyzeInternalReview(envelope.candidate, [], attachedPaths);
       await saveCandidateResult(id, candidate.id, {
         output: analysis?.output || OPAQUE_OUTPUT,
         raw: envelope.candidate || envelope.diagnostics,

lib/types.ts

@@ -143,6 +143,8 @@ export const internalReviewSchema = z.discriminatedUnion("kind", [
     evidenceCited: z.array(z.string().min(1).max(100)).max(100).default([]),
     /** Sources the reviewer looked up itself, each naming what it supports. Empty unless research ran. */
     externalSources: z.array(z.string().min(1).max(2_000)).max(40).default([]),
+    /** Attached paths the reviewer relied on. A structured commitment, so the count isn't guessed from prose. */
+    filesReferenced: z.array(z.string().min(1).max(500)).max(500).default([]),
     counterargument: z.string().max(16_000).default(""),
     withheldReason: z.literal("").default(""),
   }),
@@ -154,6 +156,7 @@ export const internalReviewSchema = z.discriminatedUnion("kind", [
     confidence: z.literal("LOW").default("LOW"),
     evidenceCited: z.array(z.string()).max(0).default([]),
     externalSources: z.array(z.string()).max(0).default([]),
+    filesReferenced: z.array(z.string()).max(0).default([]),
     counterargument: z.literal("").default(""),
     withheldReason: z.string().min(1).max(16_000).default("No operator explanation was provided."),
   }),