diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bf5990..1eb6a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ ### Changed +* `filesReferenced` is a required field on every review. The reviewer commits to a list of the attached paths it relied on, and the phone counts from that field instead of guessing from prose. A prompt-level ask to cite paths did not change what any real reviewer wrote; a required schema field does +* A review that names a path that was not attached is rejected as `GATE_FABRICATED_PATH` before release. The reviewer can only claim to have read files it was actually given +* The dashboard reads the count from the structured field, so a partial or shortened citation cannot be misread and a Bob-Regress fallback never has to be parsed. The prose fallback is gone + + * Every policy now requires an attached file to be cited by path. Checking an attachment was already required; naming it never was, so a finding about attached code could not be confirmed by the operator or acted on by the assistant. Where an attachment settles a question the reviewer quotes the line that settles it, and it may not assert anything about a file whose contents it did not read * A review that references none of its attachments is shown in amber rather than as a neutral note, because files sent and never used are files that did no work diff --git a/components/dashboard.tsx b/components/dashboard.tsx index 69a91c4..b54feb6 100644 --- a/components/dashboard.tsx +++ b/components/dashboard.tsx @@ -528,13 +528,19 @@ export function Dashboard({ initialView }: { initialView: "home" | "reviews" | " const shownRaw = viewingCandidate ? candidateDetail?.raw ?? null : detail?.raw ?? null; const shownLive = viewingCandidate ? candidateDetail?.live ?? null : detail?.live ?? null; const readableResult = parseResult(shownResult); - // Matched against what the reviewer wrote, not only what was released. A withheld or unselected - // candidate releases the fixed terminal response, and matching that could only ever find nothing. + // Taken from the reviewer's own structured commitment (`filesReferenced`), so a partial or + // shortened citation cannot be misread. If neither surface has it (older reviews, or a withheld + // candidate), the raw JSON is the fallback so the count still comes from a field rather than a + // guess. A path the reviewer names that was not attached is rejected earlier by the gate. + let referenced: string[] = readableResult?.filesReferenced || []; + if (!referenced.length && shownRaw) { + try { referenced = (JSON.parse(shownRaw) as { filesReferenced?: unknown }).filesReferenced as string[] || []; } catch { referenced = []; } + } const files = fileEvidence( detail?.job.attachedFiles || 0, detail?.job.attachedBytes || 0, detail?.packetQuality?.attachedPaths || [], - `${shownResult || ""}\n${shownRaw || ""}`, + Array.isArray(referenced) ? referenced : [], ); const parallelJob = detail?.job.kind === "parallel"; const privateCodexResponse = !parallelJob && (shownResult === "Bob Regress" || (viewingCandidate && viewedCandidate?.releasable === false)) diff --git a/components/dashboard/review-parsing.ts b/components/dashboard/review-parsing.ts index 7a5f6ef..e63f707 100644 --- a/components/dashboard/review-parsing.ts +++ b/components/dashboard/review-parsing.ts @@ -9,6 +9,7 @@ export interface ReadableResult { confidence?: string; evidence?: string[]; externalSources?: string[]; + filesReferenced?: string[]; counterargument?: string; } @@ -23,11 +24,12 @@ function resultSection(value: string, label: string, following: string[]): strin export function parseResult(value: string | null): ReadableResult | null { if (!value) return null; if (value.trim() === "Bob Regress") return { verdict: "Not released", assessment: "No substantive review was released.", recommendations: [], released: false }; - const verdict = resultSection(value, "VERDICT", ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "COUNTERARGUMENT", "RECOMMENDATIONS"]); - const assessment = resultSection(value, "ASSESSMENT", ["EVIDENCE CITED", "COUNTERARGUMENT", "RECOMMENDATIONS"]); + const verdict = resultSection(value, "VERDICT", ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]); + const assessment = resultSection(value, "ASSESSMENT", ["EVIDENCE CITED", "EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]); if (!verdict || !assessment) return { verdict: "Review", assessment: readableValue(value), recommendations: [], released: true }; - const evidence = resultSection(value, "EVIDENCE CITED", ["EXTERNAL SOURCES", "COUNTERARGUMENT", "RECOMMENDATIONS"]); - const external = resultSection(value, "EXTERNAL SOURCES", ["COUNTERARGUMENT", "RECOMMENDATIONS"]); + const evidence = resultSection(value, "EVIDENCE CITED", ["EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]); + const external = resultSection(value, "EXTERNAL SOURCES", ["ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]); + const filesRead = resultSection(value, "ATTACHED FILES USED", ["COUNTERARGUMENT", "RECOMMENDATIONS"]); const recommendations = resultSection(value, "RECOMMENDATIONS", []); const bullets = (text: string) => text.split("\n").map((item) => item.replace(/^\s*-\s*/, "").trim()).filter((item) => item && item !== "None"); return { @@ -36,6 +38,7 @@ export function parseResult(value: string | null): ReadableResult | null { assessment, evidence: bullets(evidence), externalSources: bullets(external), + filesReferenced: bullets(filesRead), counterargument: resultSection(value, "COUNTERARGUMENT", ["RECOMMENDATIONS"]), recommendations: bullets(recommendations), released: true, @@ -54,6 +57,7 @@ export function parseCodexResponse(value: string | null): ReadableResult | null assessment: readableValue(parsed.assessment) || "Codex returned a review without an assessment.", evidence: Array.isArray(parsed.evidenceCited) ? parsed.evidenceCited.map((item) => readableValue(item)).filter(Boolean) : [], externalSources: Array.isArray(parsed.externalSources) ? parsed.externalSources.map((item) => readableValue(item)).filter(Boolean) : [], + filesReferenced: Array.isArray(parsed.filesReferenced) ? parsed.filesReferenced.map((item) => readableValue(item)).filter(Boolean) : [], counterargument: readableValue(parsed.counterargument), recommendations: Array.isArray(parsed.recommendations) ? parsed.recommendations.map((item) => readableValue(item)).filter(Boolean) : [], released: false, diff --git a/lib/gate.ts b/lib/gate.ts index c1b936e..dda8891 100644 --- a/lib/gate.ts +++ b/lib/gate.ts @@ -10,6 +10,7 @@ export const gateOutcomeCodes = [ "GATE_INVALID_SCHEMA", "GATE_EMPTY", "GATE_OVERSIZE", + "GATE_FABRICATED_PATH", ] as const; export type GateOutcomeCode = (typeof gateOutcomeCodes)[number]; @@ -71,11 +72,14 @@ export function renderReview(review: InternalReview): string { const external = review.externalSources.length ? `\nEXTERNAL SOURCES:\n${review.externalSources.map((item) => `- ${item}`).join("\n")}` : ""; + const files = review.filesReferenced.length + ? `\nATTACHED FILES USED:\n${review.filesReferenced.map((item) => `- ${item}`).join("\n")}` + : ""; const counterargument = review.counterargument ? `\nCOUNTERARGUMENT:\n${review.counterargument}` : "\nCOUNTERARGUMENT:\nNone identified."; const recommendations = review.recommendations.length ? `\nRECOMMENDATIONS:\n${review.recommendations.map((item) => `- ${item}`).join("\n")}` : "\nRECOMMENDATIONS:\n- None"; - return `VERDICT: ${review.verdict}\nCONFIDENCE: ${review.confidence}\nASSESSMENT:\n${review.assessment}${evidence}${external}${counterargument}${recommendations}`; + return `VERDICT: ${review.verdict}\nCONFIDENCE: ${review.confidence}\nASSESSMENT:\n${review.assessment}${evidence}${external}${files}${counterargument}${recommendations}`; } @@ -85,6 +89,7 @@ export interface RenderedReview { assessment: string; evidenceCited: string[]; externalSources: string[]; + filesReferenced: string[]; counterargument: string; recommendations: string[]; } @@ -107,7 +112,7 @@ function listItems(value: string): string[] { /** Reads a released review back into its parts so several of them can be combined. */ export function parseRenderedReview(value: string): RenderedReview | null { const normalized = normalizeOutput(value); - const after = ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "EXTERNAL SOURCES", "COUNTERARGUMENT", "RECOMMENDATIONS"]; + const after = ["CONFIDENCE", "ASSESSMENT", "EVIDENCE CITED", "EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"]; const verdict = section(normalized, "VERDICT", after); const confidence = section(normalized, "CONFIDENCE", after.slice(1)); const assessment = section(normalized, "ASSESSMENT", after.slice(2)); @@ -116,8 +121,9 @@ export function parseRenderedReview(value: string): RenderedReview | null { verdict: verdict as RenderedReview["verdict"], confidence: confidence as RenderedReview["confidence"], assessment, - evidenceCited: listItems(section(normalized, "EVIDENCE CITED", ["EXTERNAL SOURCES", "COUNTERARGUMENT", "RECOMMENDATIONS"])), - externalSources: listItems(section(normalized, "EXTERNAL SOURCES", ["COUNTERARGUMENT", "RECOMMENDATIONS"])), + evidenceCited: listItems(section(normalized, "EVIDENCE CITED", ["EXTERNAL SOURCES", "ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"])), + externalSources: listItems(section(normalized, "EXTERNAL SOURCES", ["ATTACHED FILES USED", "COUNTERARGUMENT", "RECOMMENDATIONS"])), + filesReferenced: listItems(section(normalized, "ATTACHED FILES USED", ["COUNTERARGUMENT", "RECOMMENDATIONS"])), counterargument: section(normalized, "COUNTERARGUMENT", ["RECOMMENDATIONS"]), recommendations: listItems(section(normalized, "RECOMMENDATIONS", [])), }; @@ -158,6 +164,7 @@ export function combineReviews(entries: { label: string; output: string }[]): st const assessments = parsed.map((entry) => `${entry.label} (${entry.review.verdict}, ${entry.review.confidence} confidence):\n${entry.review.assessment}`).join("\n\n"); const evidence = attributed((review) => review.evidenceCited); const external = attributed((review) => review.externalSources); + const files = attributed((review) => review.filesReferenced); const recommendations = attributed((review) => review.recommendations); const counterarguments = parsed.filter((entry) => entry.review.counterargument).map((entry) => `${entry.label}: ${entry.review.counterargument}`); @@ -171,6 +178,7 @@ export function combineReviews(entries: { label: string; output: string }[]): st "\nEVIDENCE CITED:", evidence.length ? evidence.map(({ item, labels }) => `- ${item} (${labels.join(", ")})`).join("\n") : "- None", ...(external.length ? ["\nEXTERNAL SOURCES:", external.map(({ item, labels }) => `- ${item} (${labels.join(", ")})`).join("\n")] : []), + ...(files.length ? ["\nATTACHED FILES USED:", files.map(({ item, labels }) => `- ${item} (${labels.join(", ")})`).join("\n")] : []), "\nCOUNTERARGUMENT:", counterarguments.length ? counterarguments.join("\n\n") : "None identified.", "\nRECOMMENDATIONS:", @@ -178,7 +186,7 @@ export function combineReviews(entries: { label: string; output: string }[]): st ].join("\n"); } -export function analyzeInternalReview(raw: string, knownSecrets: string[] = []): GateAnalysis { +export function analyzeInternalReview(raw: string, knownSecrets: string[] = [], attachedPaths: string[] = []): GateAnalysis { if (!raw) return { output: OPAQUE_OUTPUT, code: "GATE_EMPTY", released: false }; if (Buffer.byteLength(raw, "utf8") > 4 * 1024 * 1024) return { output: OPAQUE_OUTPUT, code: "GATE_OVERSIZE", released: false }; const normalized = normalizeOutput(raw); @@ -189,6 +197,16 @@ export function analyzeInternalReview(raw: string, knownSecrets: string[] = []): try { const parsed = internalReviewSchema.parse(JSON.parse(normalized)); if (parsed.kind === "opaque") return { output: OPAQUE_OUTPUT, code: "MODEL_WITHHELD", released: false }; + // A path that was not attached is a fabrication. The check is generous: leading/trailing + // whitespace and a leading "./" are trimmed. It only runs when the packet actually carried an + // attachment list, so an older run that predates the field is not held to it retroactively. + if (attachedPaths.length) { + const attached = new Set(attachedPaths.map((path) => path.replace(/^\.\//, "").trim()).filter(Boolean)); + const fabricated = parsed.filesReferenced + .map((path) => path.replace(/^\.\//, "").trim()) + .find((path) => path && !attached.has(path)); + if (fabricated) return { output: OPAQUE_OUTPUT, code: "GATE_FABRICATED_PATH", released: false }; + } const rendered = renderReview(parsed); if (containsRefusalText(normalized) || containsRefusalText(rendered)) { return { output: OPAQUE_OUTPUT, code: "GATE_REFUSAL_LANGUAGE", released: false }; diff --git a/lib/research.ts b/lib/research.ts index b10e220..d680632 100644 --- a/lib/research.ts +++ b/lib/research.ts @@ -102,17 +102,23 @@ export interface FileEvidence { * many of those the review actually named. A path is counted as cited when the released text * mentions it, or mentions its file name where the full path would be unwieldy. */ -export function fileEvidence(attached: number, bytes: number, paths: string[] = [], reviewText = ""): FileEvidence { - const text = reviewText.toLowerCase(); +export function fileEvidence(attached: number, bytes: number, paths: string[] = [], referenced: readonly string[] = []): FileEvidence { + // The reviewer commits to a list of paths it relied on in `filesReferenced`. This function reads + // that commitment instead of guessing from prose: the previous heuristic missed a citation the + // reviewer wrote with a shortened form and could not distinguish a partial citation from none. + const attachedSet = new Set(paths.map((path) => String(path).replace(/^\.\//, "").trim()).filter(Boolean)); + const seen = new Set(); const cited: string[] = []; - const uncited: string[] = []; - for (const raw of paths) { - const path = String(raw).trim(); - if (!path) continue; - const base = path.split("/").pop() || path; - const named = text.includes(path.toLowerCase()) || (base.length >= 4 && text.includes(base.toLowerCase())); - (named ? cited : uncited).push(path); + for (const raw of referenced) { + const path = String(raw).replace(/^\.\//, "").trim(); + if (!path || seen.has(path) || !attachedSet.has(path)) continue; + seen.add(path); + cited.push(path); } + const uncited = paths.filter((path) => { + const key = String(path).replace(/^\.\//, "").trim(); + return key && !seen.has(key); + }); const total = Math.max(attached, paths.length); return { attached: total, diff --git a/lib/sandbox-runtime.ts b/lib/sandbox-runtime.ts index 0a8cae6..5b37af7 100644 --- a/lib/sandbox-runtime.ts +++ b/lib/sandbox-runtime.ts @@ -4,6 +4,7 @@ import { Sandbox, Snapshot } from "@vercel/sandbox"; import { config } from "@/lib/config"; import { sha256 } from "@/lib/crypto"; import { analyzeInternalReview, normalizeOutput, OPAQUE_OUTPUT } from "@/lib/gate"; +import { analyzePacketQuality } from "@/lib/packet-quality"; import { adminGetJob, adminLiveLog, @@ -646,10 +647,17 @@ async function finishMockCandidate(id: string, candidate: ReviewCandidate, store } const mode = process.env.SOL_MOCK_REVIEW_MODE || "review"; const withheld = mode === "opaque" || candidate.protocolId === "control"; + // The mock references the attached paths so the whole render + gate + trace pipeline is exercised + // end to end. A real reviewer commits to filesReferenced the same way. + let mockFiles: string[] = []; + if (!withheld && job) { + try { mockFiles = analyzePacketQuality(await readPacket(job, store)).attachedPaths.slice(0, 5); } catch { mockFiles = []; } + } const raw = withheld ? JSON.stringify({ kind: "opaque", verdict: "OPAQUE", assessment: "", recommendations: [], confidence: "LOW", evidenceCited: [], counterargument: "", withheldReason: "Mock review was withheld." }) - : JSON.stringify({ kind: "review", verdict: "SOUND", assessment: `The decision is supported by the transferred evidence.${candidate.index > 1 ? ` Reviewed by ${candidate.model} under ${candidate.protocolVersion}.` : ""}`, recommendations: [], confidence: "HIGH", evidenceCited: ["S1"], externalSources: candidate.research ? ["https://example.invalid/fixture"] : [], counterargument: "The fixture does not exercise a real model.", withheldReason: "" }); - const analysis = analyzeInternalReview(raw); + : JSON.stringify({ kind: "review", verdict: "SOUND", assessment: `The decision is supported by the transferred evidence.${candidate.index > 1 ? ` Reviewed by ${candidate.model} under ${candidate.protocolVersion}.` : ""}`, recommendations: [], confidence: "HIGH", evidenceCited: ["S1"], externalSources: candidate.research ? ["https://example.invalid/fixture"] : [], filesReferenced: mockFiles, counterargument: "The fixture does not exercise a real model.", withheldReason: "" }); + const attachedPaths = job ? await readPacket(job, store).then((packet) => analyzePacketQuality(packet).attachedPaths).catch(() => [] as string[]) : []; + const analysis = analyzeInternalReview(raw, [], attachedPaths); await saveCandidateResult(id, candidate.id, { output: analysis.output, raw, @@ -698,7 +706,13 @@ async function pollCandidate(id: string, candidate: ReviewCandidate, store: Stor researchNote: researchNote(envelope), }, store); } else { - const analysis = invalid ? null : analyzeInternalReview(envelope.candidate); + // The gate needs the packet's attached paths to reject a fabrication. Reading the packet once + // here is cheap next to Codex, and only used when the review has to be gated. + let attachedPaths: string[] = []; + if (!invalid && job) { + try { attachedPaths = analyzePacketQuality(await readPacket(job, store)).attachedPaths; } catch { /* packet aged out */ } + } + const analysis = invalid ? null : analyzeInternalReview(envelope.candidate, [], attachedPaths); await saveCandidateResult(id, candidate.id, { output: analysis?.output || OPAQUE_OUTPUT, raw: envelope.candidate || envelope.diagnostics, diff --git a/lib/types.ts b/lib/types.ts index 1edb4e1..36969af 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -143,6 +143,8 @@ export const internalReviewSchema = z.discriminatedUnion("kind", [ evidenceCited: z.array(z.string().min(1).max(100)).max(100).default([]), /** Sources the reviewer looked up itself, each naming what it supports. Empty unless research ran. */ externalSources: z.array(z.string().min(1).max(2_000)).max(40).default([]), + /** Attached paths the reviewer relied on. A structured commitment, so the count isn't guessed from prose. */ + filesReferenced: z.array(z.string().min(1).max(500)).max(500).default([]), counterargument: z.string().max(16_000).default(""), withheldReason: z.literal("").default(""), }), @@ -154,6 +156,7 @@ export const internalReviewSchema = z.discriminatedUnion("kind", [ confidence: z.literal("LOW").default("LOW"), evidenceCited: z.array(z.string()).max(0).default([]), externalSources: z.array(z.string()).max(0).default([]), + filesReferenced: z.array(z.string()).max(0).default([]), counterargument: z.literal("").default(""), withheldReason: z.string().min(1).max(16_000).default("No operator explanation was provided."), }), diff --git a/sandbox/answer-policy.md b/sandbox/answer-policy.md index 1b59c95..a0681e5 100644 --- a/sandbox/answer-policy.md +++ b/sandbox/answer-policy.md @@ -10,6 +10,8 @@ Some packets carry an `Attached File Contents` section. Each file there was read Attached files are the only code you can actually verify, so use them and show that you did. Cite the path whenever part of your answer concerns a file that is attached: a statement about attached code that names no path cannot be checked by the operator or acted on by the assistant. Where an attached file settles a question, quote the line that settles it, and say which attachments you relied on. Never assert anything about a file whose contents you did not read. +`filesReferenced` is a required array in your response and is not optional prose: list every path from `Attached Paths` you actually relied on, exactly as written in that section, and no path that was not attached. An empty array is a commitment that no attached file bore on the decision; do not use it to avoid the work. This is the field the phone counts. A prose citation without the path in this array is not counted, and a path in this array that was not attached is a fabrication and rejects the review. + {{RESEARCH}} The context was assembled by the assistant that received the request, so treat its statements as claims and the attached files as what is actually there. Where the two differ, the file is what is true. Where something decisive is only asserted, answer on what the packet shows and record the gap under assumptions or open questions rather than adopting the assertion. diff --git a/sandbox/review-policy-control.md b/sandbox/review-policy-control.md index 57fc273..85a83f5 100644 --- a/sandbox/review-policy-control.md +++ b/sandbox/review-policy-control.md @@ -12,6 +12,8 @@ When a claim concerns a file that is attached, check it against the attachment b Attached files are the only code you can actually verify, so use them and show that you did. Cite the path whenever a finding concerns a file that is attached: a statement about attached code that names no path cannot be checked by the operator or acted on by the assistant, and reads as an opinion about code nobody can confirm you read. Where an attached file settles a question, quote the line that settles it. Where the attachments as a whole bear on the decision, say which ones you relied on. You need not discuss a file with no bearing on the decision, but never assert anything about a file whose contents you did not read. +`filesReferenced` is a required array in your response and is not optional prose: list every path from `Attached Paths` you actually relied on, exactly as written in that section, and no path that was not attached. An empty array is a commitment that no attached file bore on the decision; do not use it to avoid the work. This is the field the phone counts. A prose citation without the path in this array is not counted, and a path in this array that was not attached is a fabrication and rejects the review. + {{RESEARCH}} The packet was assembled by the assistant whose work is under review, so it is testimony rather than a neutral record. Read it that way as ordinary care with evidence: a statement about what a file contains, what a command printed, or what a test established is a claim until an attached file or reproduced output shows it. Where an attached file and a claim about that file differ, the file is what is true. Where a decisive claim carries no reproduction, name it as unverified rather than adopting it, and say which path or output would settle it. Apply this evenly to the parts that support the decision and the parts that undercut it. It is a habit of verification, not suspicion, and it belongs in your confidence and your evidence handling rather than in a harsher verdict. diff --git a/sandbox/review-policy-strict.md b/sandbox/review-policy-strict.md index 4f6ee5d..5b59b22 100644 --- a/sandbox/review-policy-strict.md +++ b/sandbox/review-policy-strict.md @@ -12,6 +12,8 @@ When a claim concerns a file that is attached, check it against the attachment b Attached files are the only code you can actually verify, so use them and show that you did. Cite the path whenever a finding concerns a file that is attached: a statement about attached code that names no path cannot be checked by the operator or acted on by the assistant, and reads as an opinion about code nobody can confirm you read. Where an attached file settles a question, quote the line that settles it. Where the attachments as a whole bear on the decision, say which ones you relied on. You need not discuss a file with no bearing on the decision, but never assert anything about a file whose contents you did not read. +`filesReferenced` is a required array in your response and is not optional prose: list every path from `Attached Paths` you actually relied on, exactly as written in that section, and no path that was not attached. An empty array is a commitment that no attached file bore on the decision; do not use it to avoid the work. This is the field the phone counts. A prose citation without the path in this array is not counted, and a path in this array that was not attached is a fabrication and rejects the review. + {{RESEARCH}} The packet was assembled by the assistant whose work is under review, so it is testimony rather than a neutral record. Read it that way as ordinary care with evidence: a statement about what a file contains, what a command printed, or what a test established is a claim until an attached file or reproduced output shows it. Where an attached file and a claim about that file differ, the file is what is true. Where a decisive claim carries no reproduction, name it as unverified rather than adopting it, and say which path or output would settle it. Apply this evenly to the parts that support the decision and the parts that undercut it. It is a habit of verification, not suspicion, and it belongs in your confidence and your evidence handling rather than in a harsher verdict. diff --git a/sandbox/review-policy.md b/sandbox/review-policy.md index 9416e0d..205a25a 100644 --- a/sandbox/review-policy.md +++ b/sandbox/review-policy.md @@ -12,6 +12,8 @@ When a claim concerns a file that is attached, check it against the attachment b Attached files are the only code you can actually verify, so use them and show that you did. Cite the path whenever a finding concerns a file that is attached: a statement about attached code that names no path cannot be checked by the operator or acted on by the assistant, and reads as an opinion about code nobody can confirm you read. Where an attached file settles a question, quote the line that settles it. Where the attachments as a whole bear on the decision, say which ones you relied on. You need not discuss a file with no bearing on the decision, but never assert anything about a file whose contents you did not read. +`filesReferenced` is a required array in your response and is not optional prose: list every path from `Attached Paths` you actually relied on, exactly as written in that section, and no path that was not attached. An empty array is a commitment that no attached file bore on the decision; do not use it to avoid the work. This is the field the phone counts. A prose citation without the path in this array is not counted, and a path in this array that was not attached is a fabrication and rejects the review. + {{RESEARCH}} The packet was assembled by the assistant whose work is under review, so it is testimony rather than a neutral record. Read it that way as ordinary care with evidence: a statement about what a file contains, what a command printed, or what a test established is a claim until an attached file or reproduced output shows it. Where an attached file and a claim about that file differ, the file is what is true. Where a decisive claim carries no reproduction, name it as unverified rather than adopting it, and say which path or output would settle it. Apply this evenly to the parts that support the decision and the parts that undercut it. It is a habit of verification, not suspicion, and it belongs in your confidence and your evidence handling rather than in a harsher verdict. diff --git a/sandbox/review-schema.json b/sandbox/review-schema.json index 8aa9ba5..38aa409 100644 --- a/sandbox/review-schema.json +++ b/sandbox/review-schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "additionalProperties": false, - "required": ["kind", "verdict", "assessment", "recommendations", "confidence", "evidenceCited", "externalSources", "counterargument", "withheldReason"], + "required": ["kind", "verdict", "assessment", "recommendations", "confidence", "evidenceCited", "externalSources", "filesReferenced", "counterargument", "withheldReason"], "properties": { "kind": { "enum": ["review", "opaque"] }, "verdict": { "enum": ["SOUND", "NEEDS_IMPROVEMENT", "WRONG", "OPAQUE"] }, @@ -20,6 +20,10 @@ "type": "array", "items": { "type": "string" } }, + "filesReferenced": { + "type": "array", + "items": { "type": "string" } + }, "counterargument": { "type": "string" }, "withheldReason": { "type": "string" } } diff --git a/tests/core/gate.test.ts b/tests/core/gate.test.ts index 623a4c9..78a9273 100644 --- a/tests/core/gate.test.ts +++ b/tests/core/gate.test.ts @@ -92,3 +92,32 @@ test("agreement is reported plainly and a single review is never a combination", assert.equal(combineReviews([{ label: "a", output: one }]), null); assert.equal(combineReviews([{ label: "a", output: "Bob Regress" }, { label: "b", output: two }]), null); }); + +test("a review that names an unattached path is refused as a fabrication", () => { + // The reviewer commits to filesReferenced. A path there that was not among the attachments is a + // claim about code nobody sent, and it must not release. Belt: the phone's counting also ignores + // anything not in the attached list. Braces: the release gate refuses it outright. + const raw = JSON.stringify({ + kind: "review", + verdict: "SOUND", + assessment: "Every attachment supports the decision.", + recommendations: [], + confidence: "HIGH", + evidenceCited: ["S1"], + externalSources: [], + filesReferenced: ["src/one.ts", "src/never-attached.ts"], + counterargument: "", + withheldReason: "", + }); + + // Attachments name only one of the two paths, so the second is a fabrication. + const analysis = analyzeInternalReview(raw, [], ["src/one.ts"]); + assert.equal(analysis.released, false); + assert.equal(analysis.code, "GATE_FABRICATED_PATH"); + + // With no attachments recorded the check does not run: an older packet cannot be judged this way. + assert.equal(analyzeInternalReview(raw, [], []).released, true); + // Leading "./" on either side is fine and does not count as fabrication. + const withDot = JSON.parse(raw); withDot.filesReferenced = ["./src/one.ts"]; + assert.equal(analyzeInternalReview(JSON.stringify(withDot), [], ["src/one.ts"]).released, true); +}); diff --git a/tests/core/policy.test.ts b/tests/core/policy.test.ts index a54acf8..4b3b905 100644 --- a/tests/core/policy.test.ts +++ b/tests/core/policy.test.ts @@ -92,3 +92,14 @@ test("every policy requires an attached file to be cited by path", async () => { } } }); + +test("every policy names filesReferenced as the structured commitment the phone counts", async () => { + for (const name of await policyFiles()) { + const source = await readFile(path.join(policyDir, name), "utf8"); + for (const research of [true, false]) { + const rendered = applyResearchNoteForTests(source, research); + assert.match(rendered, /`filesReferenced` is a required array/, `${name} does not describe the structured field`); + assert.match(rendered, /fabrication and rejects the review/, `${name} does not warn about fabricated paths`); + } + } +}); diff --git a/tests/core/research.test.ts b/tests/core/research.test.ts index 4485437..030cb21 100644 --- a/tests/core/research.test.ts +++ b/tests/core/research.test.ts @@ -112,48 +112,34 @@ test("the worker counts a search once across its started and completed events", assert.equal(searchLog.length, 50); }); -test("file evidence separates what was readable from what the review used", () => { - // Sol has no file access: files reach it only as attachments the client pasted into the packet. - // So "inspected" means readable, and the second number is how many of those it actually named. +test("file evidence takes the reviewer's own list, not a guess from prose", () => { + // The reviewer commits to filesReferenced -- an array of attached paths it relied on. That + // structured commitment is what the phone counts, and a fabrication (a path that was not + // attached) is rejected by the gate before it reaches this function. const paths = ["DmaKit/overlay.cpp", "DmaKit/vdm.cpp", "src/untouched.ts"]; - const review = "Attached `DmaKit/vdm.cpp` shows VDM patches a syscall, and overlay.cpp asserts capture as fact."; - const used = fileEvidence(3, 40_960, paths, review); + const used = fileEvidence(3, 40_960, paths, ["DmaKit/vdm.cpp", "DmaKit/overlay.cpp"]); assert.equal(used.attached, 3); assert.equal(used.cited, 2); - assert.deepEqual(used.citedPaths, ["DmaKit/overlay.cpp", "DmaKit/vdm.cpp"]); + assert.deepEqual(used.citedPaths.sort(), ["DmaKit/overlay.cpp", "DmaKit/vdm.cpp"]); assert.deepEqual(used.uncitedPaths, ["src/untouched.ts"]); - // A file named only by its base name still counts, since a review often writes overlay.cpp. - assert.equal(fileEvidence(1, 10, ["a/b/c/deep.ts"], "the bug is in deep.ts").cited, 1); - // A short base name is not matched loosely, so an incidental word cannot fake a citation. - assert.equal(fileEvidence(1, 10, ["src/a.ts"], "a is a common word").cited, 0); + // A path the reviewer names that is not in the attached list is ignored here, since the gate + // already rejects the review. Belt and suspenders, and the count stays honest. + assert.equal(fileEvidence(1, 10, ["a/b/c/deep.ts"], ["../elsewhere.ts"]).cited, 0); + // Duplicates in the reviewer's list do not double count. + assert.equal(fileEvidence(1, 10, ["one.ts"], ["one.ts", "one.ts"]).cited, 1); + // Leading ./ is normalised on both sides. + assert.equal(fileEvidence(1, 10, ["./one.ts"], ["one.ts"]).cited, 1); - // Nothing referenced is reported as nothing referenced rather than hidden. - const ignored = fileEvidence(2, 100, ["one.ts", "two.ts"], "a review that names no path"); + const ignored = fileEvidence(2, 100, ["one.ts", "two.ts"], []); assert.equal(ignored.cited, 0); assert.equal(ignored.uncitedPaths.length, 2); - // A count without paths still reports the count: the packet may have expired out of retention. - assert.equal(fileEvidence(7, 2_048, [], "").attached, 7); - // No attachments at all stays zero, so the card never renders on a packet that carried none. - assert.equal(fileEvidence(0, 0, [], "").attached, 0); - - // The level drives how loudly the card reads. Sending files a review never touches is the case - // that needs to look like a problem, because the attachments then did no work at all. - assert.equal(fileEvidence(0, 0, [], "").level, "none"); + assert.equal(fileEvidence(7, 2_048, [], []).attached, 7); + assert.equal(fileEvidence(0, 0, [], []).attached, 0); + assert.equal(fileEvidence(0, 0, [], []).level, "none"); assert.equal(used.level, "partial"); assert.equal(ignored.level, "unused"); - assert.equal(fileEvidence(2, 10, ["one.ts", "two.ts"], "one.ts and two.ts both matter").level, "used"); -}); - -test("a withheld candidate is matched against what the reviewer wrote, not the terminal response", () => { - // A candidate that was not released carries the fixed terminal response, so matching only the - // released text could find nothing however thoroughly the reviewer cited the attachments. - const paths = ["DmaKit/driver_wdt.cpp", "DmaKit/physical.h"]; - const released = "Bob Regress"; - const reviewerWrote = 'The contract in `DmaKit/driver_wdt.cpp` does not match physical.h.'; - - assert.equal(fileEvidence(2, 100, paths, released).cited, 0); - assert.equal(fileEvidence(2, 100, paths, `${released}\n${reviewerWrote}`).cited, 2); + assert.equal(fileEvidence(2, 10, ["one.ts", "two.ts"], ["one.ts", "two.ts"]).level, "used"); }); diff --git a/tests/core/review-schema.test.ts b/tests/core/review-schema.test.ts index d1fe3c0..6ba7776 100644 --- a/tests/core/review-schema.test.ts +++ b/tests/core/review-schema.test.ts @@ -8,7 +8,7 @@ test("Codex output schema uses the strict flat-object subset", async () => { assert.equal(schema.additionalProperties, false); assert.equal("oneOf" in schema, false); assert.equal("anyOf" in schema, false); - assert.deepEqual(schema.required, ["kind", "verdict", "assessment", "recommendations", "confidence", "evidenceCited", "externalSources", "counterargument", "withheldReason"]); + assert.deepEqual(schema.required, ["kind", "verdict", "assessment", "recommendations", "confidence", "evidenceCited", "externalSources", "filesReferenced", "counterargument", "withheldReason"]); assert.deepEqual(Object.keys(schema.properties as object), schema.required); }); diff --git a/tests/e2e/mock-cycle.mjs b/tests/e2e/mock-cycle.mjs index a33d003..9dfea15 100644 --- a/tests/e2e/mock-cycle.mjs +++ b/tests/e2e/mock-cycle.mjs @@ -228,6 +228,7 @@ assert.equal((await signedFetch(`/api/admin/jobs/${attachedJob.id}/decision`, { assert.equal((await attachedClient.completed).code, 0); const attachedDone = await signedFetch(`/api/admin/jobs/${attachedJob.id}`); assert.equal(attachedDone.body.job.attachedFiles, 2, "the attachment count did not survive the answered job"); +assert.ok((attachedDone.body.result || "").includes("ATTACHED FILES USED:"), "released review lost the attached-files section on retention"); // A packet with no attachments records none, so the two cases stay distinguishable. const plainDetail = await signedFetch(`/api/admin/jobs/${researchJob.id}`); assert.equal(plainDetail.body.job.attachedFiles, 0, "a packet with no attachments recorded some");