diff --git a/scripts/ci-forensics.mjs b/scripts/ci-forensics.mjs index 67fc09f..50a779c 100644 --- a/scripts/ci-forensics.mjs +++ b/scripts/ci-forensics.mjs @@ -14,6 +14,7 @@ import { pathToFileURL } from "node:url"; import { spawnSync } from "node:child_process"; import { captureLiveSnapshot } from "./lib/live-snapshot.mjs"; +import { ownedHelperEffect } from "./lib/watchdog-evidence-registry.mjs"; const USAGE = "Usage: node scripts/ci-forensics.mjs OWNER/REPO PR_NUMBER [--log-lines N] [--annotations N] [--json]"; @@ -142,6 +143,10 @@ async function main() { const summary = { schemaVersion: 1, kind: "github-delivery/ci-forensics", + gdEffect: { + ...ownedHelperEffect("ci-forensics.mjs"), + key: `pr-ci:${args.repo}:${args.pr}`, + }, repo: args.repo, pr: args.pr, headOid: snapshot.headOid, diff --git a/scripts/lib/codex-watchdog-hook.mjs b/scripts/lib/codex-watchdog-hook.mjs index d208ebd..d302fae 100644 --- a/scripts/lib/codex-watchdog-hook.mjs +++ b/scripts/lib/codex-watchdog-hook.mjs @@ -1,7 +1,8 @@ +import { createProgressWatchdog } from "./agent-progress-watchdog.mjs"; import { - compactToolOutput, - createProgressWatchdog, -} from "./agent-progress-watchdog.mjs"; + createEvidenceRegistry, + deriveShellEvidenceDescriptor, +} from "./watchdog-evidence-registry.mjs"; import { classifyHookTool } from "./watchdog-progress-classifier.mjs"; export function classifyCodexTool(toolName, toolInput = {}) { @@ -17,21 +18,22 @@ export function classifyCodexTool(toolName, toolInput = {}) { function hydrate(state, options) { const snapshot = state?.watchdog || state || {}; return createProgressWatchdog({ - stateGeneration: snapshot.stateGeneration, - reads: snapshot.reads, - consecutiveEvidenceAttempts: snapshot.consecutiveEvidenceAttempts, - totalEvidenceAttempts: snapshot.totalEvidenceAttempts, - evidenceWarningIssued: snapshot.evidenceWarningIssued, - executionProgressCount: snapshot.executionProgressCount, - stateProgressCount: snapshot.stateProgressCount, + ...snapshot, volatileReadIntervalMs: options.volatileReadIntervalMs, evidenceSoftLimit: options.evidenceSoftLimit, evidenceHardLimit: options.evidenceHardLimit, }); } -function stateOf(watchdog) { - return { watchdog: watchdog.snapshot() }; +function hydrateEvidence(state) { + return createEvidenceRegistry(state?.evidenceRegistry || null); +} + +function stateOf(watchdog, evidenceRegistry) { + return { + watchdog: watchdog.snapshot(), + evidenceRegistry: evidenceRegistry.snapshot(), + }; } function duplicateReason(decision) { @@ -41,6 +43,10 @@ function duplicateReason(decision) { return "Duplicate read blocked on unchanged state. Reuse the valid evidence already captured; read again only after relevant state changes or the prior result becomes failed, ambiguous, or stale."; } +function coveredEvidenceReason(descriptor) { + return `Authoritative evidence for ${descriptor.key} already covers this request in the current state. Reuse the captured evidence instead of re-reading the same resource with another filter or command shape.`; +} + function evidenceBudgetReason(decision) { return `Evidence exploration budget exhausted after ${decision.consecutiveEvidenceAttempts} consecutive reads/searches without execution or state progress. Synthesise the evidence already gathered and take the next focused execution step, make the authorised change, or report the concrete blocker before reading more.`; } @@ -57,6 +63,21 @@ function inputChars(value) { } } +function shellEvidenceDescriptor(input) { + const name = String(input?.tool_name || input?.toolName || ""); + if (name !== "Bash" && !/(?:^|__)shell(?:_|$)/i.test(name)) return null; + const toolInput = input?.tool_input ?? input?.toolInput ?? {}; + return deriveShellEvidenceDescriptor(toolInput?.command); +} + +function responseExplicitlyFailed(response) { + if (!response || typeof response !== "object") return false; + if (response.error) return true; + if (response.success === false || response.ok === false) return true; + const status = String(response.status || response.conclusion || "").toLowerCase(); + return ["failed", "failure", "error", "cancelled", "canceled", "rejected"].includes(status); +} + function stopDecision(watchdog, input) { const decision = watchdog.observeAssistantDelta(input.last_assistant_message || ""); if (decision.action !== "interrupt") return null; @@ -77,43 +98,57 @@ export function evaluateCodexHook(input, state = {}, options = {}) { const config = { now: options.now ?? Date.now(), volatileReadIntervalMs: options.volatileReadIntervalMs ?? 30_000, - maxToolOutputChars: options.maxToolOutputChars ?? 4_000, maxSubagentInputChars: options.maxSubagentInputChars ?? 6_000, evidenceSoftLimit: options.evidenceSoftLimit ?? 8, evidenceHardLimit: options.evidenceHardLimit ?? 12, }; const watchdog = hydrate(state, config); + const evidenceRegistry = hydrateEvidence(state); const event = input?.hook_event_name; let output = null; if (event === "PreToolUse") { const classification = classifyHookTool(input); if (classification.kind === "evidence") { - const read = { - toolName: input.tool_name, - input: input.tool_input, - volatility: classification.volatility || "stable", - now: config.now, - }; - const readDecision = watchdog.decideRead({ ...read, record: false }); - if (readDecision.action === "block") { - output = { decision: "block", reason: duplicateReason(readDecision) }; + const descriptor = shellEvidenceDescriptor(input); + const generation = watchdog.snapshot().stateGeneration; + const coverageDecision = descriptor + ? evidenceRegistry.decide({ + stateGeneration: generation, + key: descriptor.key, + requires: descriptor.covers, + }) + : { action: "allow" }; + + if (coverageDecision.action === "block") { + output = { decision: "block", reason: coveredEvidenceReason(descriptor) }; } else { - const budgetDecision = watchdog.chargeEvidenceAttempt(); - if (budgetDecision.action === "block") { - output = { - decision: "block", - reason: evidenceBudgetReason(budgetDecision), - }; + const read = { + toolName: input.tool_name, + input: input.tool_input, + volatility: classification.volatility || "stable", + now: config.now, + }; + const readDecision = watchdog.decideRead({ ...read, record: false }); + if (readDecision.action === "block") { + output = { decision: "block", reason: duplicateReason(readDecision) }; } else { - watchdog.decideRead({ ...read, record: true }); - if (budgetDecision.action === "warn") { + const budgetDecision = watchdog.chargeEvidenceAttempt(); + if (budgetDecision.action === "block") { output = { - hookSpecificOutput: { - hookEventName: "PreToolUse", - additionalContext: evidenceWarning(budgetDecision), - }, + decision: "block", + reason: evidenceBudgetReason(budgetDecision), }; + } else { + watchdog.decideRead({ ...read, record: true }); + if (budgetDecision.action === "warn") { + output = { + hookSpecificOutput: { + hookEventName: "PreToolUse", + additionalContext: evidenceWarning(budgetDecision), + }, + }; + } } } } @@ -132,20 +167,23 @@ export function evaluateCodexHook(input, state = {}, options = {}) { watchdog.recordStateProgress("tool_state_change_completed"); } else if (classification.kind === "execution") { watchdog.recordExecutionProgress({ kind: "tool_execution_completed", toolName: input.tool_name }); + } else if (classification.kind === "evidence" && !responseExplicitlyFailed(input.tool_response)) { + const descriptor = shellEvidenceDescriptor(input); + if (descriptor) { + evidenceRegistry.record({ + stateGeneration: watchdog.snapshot().stateGeneration, + key: descriptor.key, + covers: descriptor.covers, + authoritative: descriptor.authoritative, + }); + } } - - const compacted = compactToolOutput(input.tool_response ?? "", { - maxChars: config.maxToolOutputChars, - }); - if (compacted.truncated) { - output = { - continue: false, - stopReason: `tool_output_compacted: ${compacted.originalChars} chars -> ${compacted.text.length} chars; omitted ${compacted.omittedChars}. Omitted content is not positive evidence.\n${compacted.text}`, - }; - } + // PostToolUse must never replace or truncate a successful tool result. Doing + // so destroys evidence after the tool ran and can cause the model to re-read + // the same source with another command. Compact at the source/helper instead. } else if (event === "Stop" || event === "SubagentStop") { output = stopDecision(watchdog, input); } - return { output, state: stateOf(watchdog) }; + return { output, state: stateOf(watchdog, evidenceRegistry) }; } diff --git a/scripts/lib/watchdog-evidence-registry.mjs b/scripts/lib/watchdog-evidence-registry.mjs new file mode 100644 index 0000000..87bb825 --- /dev/null +++ b/scripts/lib/watchdog-evidence-registry.mjs @@ -0,0 +1,201 @@ +const OWNED_HELPERS = Object.freeze({ + "ci-forensics.mjs": { + effect: "evidence", + keyKind: "pr-ci", + authoritative: true, + covers: ["checks", "failure-origin", "annotations", "failure-log-tail"], + }, + "runtime-capabilities.mjs": { + effect: "evidence", + keyKind: "runtime-capabilities", + authoritative: true, + covers: ["runtime-capabilities"], + }, + "review-brief.mjs": { + effect: "evidence", + keyKind: "pr-review-brief", + authoritative: true, + covers: ["scope", "diff", "review-lenses", "required-probes"], + }, + "ship-gate.mjs": { + effect: "evidence", + keyKind: "pr-ship-gate", + authoritative: true, + covers: ["ship-gate", "checks", "review-state", "mergeability"], + }, +}); + +function normalizeRepo(value) { + const repo = String(value || "").trim().replace(/^['"]|['"]$/g, ""); + return /^[^/\s]+\/[^/\s]+$/.test(repo) ? repo : null; +} + +function parsePositiveInteger(value) { + const parsed = Number(String(value || "").replace(/^['"]|['"]$/g, "")); + return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +} + +function helperName(command) { + const match = String(command || "").match( + /\bnode(?:\.exe)?\s+(?:['"]?[^\s'"]*[\\/])?([^\\/\s'"]+\.mjs)\b/i, + ); + return match?.[1]?.toLowerCase() || null; +} + +function repoFlag(command) { + const value = String(command || ""); + const match = value.match(/(?:^|\s)(?:-R|--repo)(?:=|\s+)(['"]?[^\s'"]+\/[^\s'"]+['"]?)/i); + return normalizeRepo(match?.[1]); +} + +function ghRunDescriptor(command) { + const value = String(command || ""); + const match = value.match(/\bgh(?:\.exe)?\b[\s\S]*?\brun\s+view\s+(\d+)\b/i); + if (!match) return null; + const runId = parsePositiveInteger(match[1]); + if (!runId) return null; + const repo = repoFlag(value) || "current"; + return { + effect: "evidence", + key: `github-actions-run:${repo}:${runId}`, + authoritative: true, + covers: ["run-state", "failure-log-tail"], + }; +} + +function helperDescriptor(command) { + const name = helperName(command); + const manifest = name ? OWNED_HELPERS[name] : null; + if (!manifest) return null; + + const value = String(command || ""); + const afterScript = value.slice(value.toLowerCase().indexOf(name) + name.length).trim(); + const positional = afterScript + .split(/\s+/) + .filter(Boolean) + .filter((part) => !part.startsWith("-")); + + let repo = null; + let subject = null; + if (name === "runtime-capabilities.mjs") { + repo = repoFlag(value) || normalizeRepo(positional[0]) || "current"; + } else { + repo = normalizeRepo(positional[0]) || repoFlag(value) || "current"; + subject = parsePositiveInteger(positional[1]); + } + + const key = subject + ? `${manifest.keyKind}:${repo}:${subject}` + : `${manifest.keyKind}:${repo}`; + return { + effect: manifest.effect, + key, + authoritative: manifest.authoritative, + covers: [...manifest.covers], + }; +} + +export function ownedHelperEffect(name) { + const base = String(name || "").replace(/\\/g, "/").split("/").pop().toLowerCase(); + const manifest = OWNED_HELPERS[base]; + return manifest + ? { + effect: manifest.effect, + authoritative: manifest.authoritative, + covers: [...manifest.covers], + } + : null; +} + +export function deriveShellEvidenceDescriptor(command) { + return helperDescriptor(command) || ghRunDescriptor(command); +} + +function entryKey(stateGeneration, key) { + return `${Number(stateGeneration) || 0}\0${String(key || "")}`; +} + +function sortedUnique(values) { + return [...new Set((values || []).map(String).filter(Boolean))].sort(); +} + +export function createEvidenceRegistry(snapshot = null) { + const entries = new Map(); + for (const raw of snapshot?.entries || []) { + if (!raw?.key) continue; + const generation = Number.isInteger(raw.stateGeneration) && raw.stateGeneration >= 0 + ? raw.stateGeneration + : 0; + entries.set(entryKey(generation, raw.key), { + stateGeneration: generation, + key: String(raw.key), + covers: new Set(sortedUnique(raw.covers)), + authoritative: Boolean(raw.authoritative), + }); + } + + function record({ stateGeneration = 0, key, covers = [], authoritative = false } = {}) { + if (!key) throw new Error("evidence key is required"); + const generation = Number.isInteger(stateGeneration) && stateGeneration >= 0 + ? stateGeneration + : 0; + const id = entryKey(generation, key); + const prior = entries.get(id) || { + stateGeneration: generation, + key: String(key), + covers: new Set(), + authoritative: false, + }; + for (const dimension of sortedUnique(covers)) prior.covers.add(dimension); + prior.authoritative = prior.authoritative || Boolean(authoritative); + entries.set(id, prior); + return { + stateGeneration: generation, + key: prior.key, + covers: [...prior.covers].sort(), + authoritative: prior.authoritative, + }; + } + + function decide({ stateGeneration = 0, key, requires = [] } = {}) { + if (!key) throw new Error("evidence key is required"); + const generation = Number.isInteger(stateGeneration) && stateGeneration >= 0 + ? stateGeneration + : 0; + const needed = sortedUnique(requires); + const prior = entries.get(entryKey(generation, key)); + const missing = prior + ? needed.filter((dimension) => !prior.covers.has(dimension)) + : needed; + if (prior?.authoritative && needed.length > 0 && missing.length === 0) { + return { + action: "block", + reason: "evidence_already_covered", + missing: [], + }; + } + return { action: "allow", missing }; + } + + function snapshotState() { + return { + schemaVersion: 1, + entries: [...entries.values()] + .map((entry) => ({ + stateGeneration: entry.stateGeneration, + key: entry.key, + covers: [...entry.covers].sort(), + authoritative: entry.authoritative, + })) + .sort((a, b) => + a.stateGeneration - b.stateGeneration || a.key.localeCompare(b.key), + ), + }; + } + + return { + record, + decide, + snapshot: snapshotState, + }; +} diff --git a/scripts/lib/watchdog-progress-classifier.mjs b/scripts/lib/watchdog-progress-classifier.mjs index bcc3e59..e08f037 100644 --- a/scripts/lib/watchdog-progress-classifier.mjs +++ b/scripts/lib/watchdog-progress-classifier.mjs @@ -1,8 +1,13 @@ +import { deriveShellEvidenceDescriptor } from "./watchdog-evidence-registry.mjs"; + const VOLATILE_NAME = /(checks?|workflow|runs?|status|mergeable|pull_request|pr_|queue|jobs?)/i; const EVIDENCE_NAME = /(?:^|__|_)(fetch|get|list|search|read|view|status|diff|compare|find|inspect|lookup|show|logs?|checks?)(?:_|$)/i; const STATE_CHANGE_NAME = /(?:^|__|_)(create|update|delete|remove|merge|reply|push|close|reopen|mark|set|add|apply|write|edit|patch|commit|move|rename|archive|restore|publish)(?:_|$)/i; const DELEGATE_NAME = /(?:^|__|_)(agent|spawn_agent|delegate|collab)(?:_|$)/i; const EXPLICIT_SHELL_WRITE = /\b(?:set-content|add-content|out-file|clear-content|new-item|remove-item|move-item|copy-item|rename-item)\b/i; +const GIT_WRITE = /\bgit(?:\.exe)?(?:\s+-C\s+(?:"[^"]+"|'[^']+'|\S+))?\s+(?:commit|push|merge|rebase|checkout|switch|reset|restore|clean|add|rm|mv)\b/i; +const GIT_READ = /(?:^|[;|&(]\s*|\s)git(?:\.exe)?(?:\s+-C\s+(?:"[^"]+"|'[^']+'|\S+))?\s+(?:status|diff|log|show|branch|rev-parse)\b/i; +const POWERSHELL_READ = /(?:^|[;|&(]\s*)(?:get-content|get-childitem|select-string|rg|grep|cat|head|tail|findstr|type|ls|dir|pwd)\b/i; function commandText(command) { if (Array.isArray(command)) return command.join(" "); @@ -14,11 +19,11 @@ function hasOutputRedirection(value) { } function classifyGhApi(value) { - if (!/\bgh\s+api\b/i.test(value)) return null; + if (!/\bgh(?:\.exe)?\s+api\b/i.test(value)) return null; const explicitGet = /(?:--method(?:=|\s+)get\b|-x\s*get\b)/i.test(value); const explicitMutationMethod = /(?:--method(?:=|\s+)(?:post|put|patch|delete)\b|-x\s*(?:post|put|patch|delete)\b)/i.test(value); - if (/\bgh\s+api\s+graphql\b/i.test(value)) { + if (/\bgh(?:\.exe)?\s+api\s+graphql\b/i.test(value)) { if (/\bmutation\b/i.test(value)) return { kind: "state-change" }; if (explicitMutationMethod && !/\bquery\s*=\s*['"]?\s*query\b/i.test(value)) { return { kind: "neutral" }; @@ -48,25 +53,31 @@ function classifyCommand(command) { return ghApi; } - if (EXPLICIT_SHELL_WRITE.test(value)) return { kind: "state-change" }; + if (EXPLICIT_SHELL_WRITE.test(value) || GIT_WRITE.test(value)) { + return { kind: "state-change" }; + } - if (/\bgh\s+(?:pr\s+(?:checks|view|diff)|run\s+(?:view|list))/i.test(value)) { + if ( + /\bgh(?:\.exe)?\b[\s\S]*?\b(?:pr\s+(?:checks|view|diff)|run\s+(?:view|list))\b/i.test(value) + ) { return hasOutputRedirection(value) ? { kind: "neutral" } : { kind: "evidence", volatility: "volatile" }; } - if ( - /^(?:get-content\b|select-string\b|rg\b|grep\b|cat\b|head\b|tail\b|findstr\b|type\b|ls\b|dir\b|pwd\b|git\s+(?:status|diff|log|show|branch|rev-parse)\b)/i.test( - value, - ) - ) { + const ownedEvidence = deriveShellEvidenceDescriptor(raw); + if (ownedEvidence?.effect === "evidence") { + const volatile = /^(?:pr-ci|pr-ship-gate|github-actions-run):/.test(ownedEvidence.key); + return { kind: "evidence", volatility: volatile ? "volatile" : "stable" }; + } + + if (POWERSHELL_READ.test(value) || GIT_READ.test(value)) { if (hasOutputRedirection(value)) return { kind: "neutral" }; return { kind: "evidence", volatility: "stable" }; } if ( - /\b(?:git\s+(?:commit|push|merge|rebase|checkout|switch|reset|restore|clean|add|rm|mv)|gh\s+(?:pr\s+(?:create|edit|merge|ready|close|reopen)|issue\s+(?:create|edit|close|reopen)|release\s+(?:create|edit|delete)))\b/i.test( + /\bgh(?:\.exe)?\s+(?:pr\s+(?:create|edit|merge|ready|close|reopen)|issue\s+(?:create|edit|close|reopen)|release\s+(?:create|edit|delete))\b/i.test( value, ) ) { diff --git a/scripts/review-brief.mjs b/scripts/review-brief.mjs index ed96891..8ede95d 100644 --- a/scripts/review-brief.mjs +++ b/scripts/review-brief.mjs @@ -16,6 +16,7 @@ import { collectPrReviewInput, planReviewScope } from "./lib/review-scope.mjs"; import { projectBugScope, projectSecurityScope } from "./lib/review-scope-compat.mjs"; import { planReviewDepthExecution } from "./lib/review-depth-execution.mjs"; import { extractRequiredProbeBlocks } from "./lib/probe-blocks.mjs"; +import { ownedHelperEffect } from "./lib/watchdog-evidence-registry.mjs"; const USAGE = "Usage: node scripts/review-brief.mjs OWNER/REPO PR_NUMBER [--max-hunk-lines N] [--no-reference-map] [--json]"; @@ -171,6 +172,10 @@ async function main() { const out = { schemaVersion: 1, kind: "github-delivery/review-brief", + gdEffect: { + ...ownedHelperEffect("review-brief.mjs"), + key: `pr-review-brief:${args.repo}:${args.pr}`, + }, repo: args.repo, pr: args.pr, headRefOid: plan.headRefOid, diff --git a/scripts/runtime-capabilities.mjs b/scripts/runtime-capabilities.mjs index 9df45f1..90036be 100644 --- a/scripts/runtime-capabilities.mjs +++ b/scripts/runtime-capabilities.mjs @@ -6,6 +6,7 @@ import { join, resolve } from "node:path"; import { buildRuntimeCapabilities } from "./lib/runtime-capabilities.mjs"; import { readActivationReceipt } from "./lib/watchdog-activation.mjs"; +import { ownedHelperEffect } from "./lib/watchdog-evidence-registry.mjs"; const usage = "Usage: node scripts/runtime-capabilities.mjs [--repo OWNER/REPO] [--input FILE]"; @@ -134,6 +135,10 @@ try { ? JSON.parse(readFileSync(args.input, "utf8")) : liveInput(args.repo); const output = buildRuntimeCapabilities({ ...input, repo: input.repo ?? args.repo }); + output.gdEffect = { + ...ownedHelperEffect("runtime-capabilities.mjs"), + key: `runtime-capabilities:${output.repo || "current"}`, + }; process.stdout.write(`${JSON.stringify(output, null, 2)}\n`); if (!output.readyForReadOnly) process.exitCode = 2; } catch (error) { diff --git a/scripts/ship-gate.mjs b/scripts/ship-gate.mjs index 012423f..7aafe73 100755 --- a/scripts/ship-gate.mjs +++ b/scripts/ship-gate.mjs @@ -24,6 +24,7 @@ import { } from "./lib/snapshot-input.mjs"; import { combineShipGateResults } from "./lib/ship-gate-policy.mjs"; import { validateWorkflowMutationMode } from "./lib/workflow-mode.mjs"; +import { ownedHelperEffect } from "./lib/watchdog-evidence-registry.mjs"; const usage = "Usage: node scripts/ship-gate.mjs OWNER/REPO PR_NUMBER [--snapshot FILE] [--expected-head SHA] [--max-age-seconds N] [--mutation-mode MODE] [--workflow WORKFLOW]"; @@ -82,6 +83,11 @@ try { output.workflow = args.workflow; output.evidenceMode = replay ? "snapshot_replay" : "live_capture"; output.authoritative = !replay; + output.gdEffect = { + ...ownedHelperEffect("ship-gate.mjs"), + key: `pr-ship-gate:${args.repo}:${args.pr}`, + authoritative: !replay, + }; if (replay && output.ready) { output.replayDecision = output.decision; output.decision = "unknown"; diff --git a/tests/unit/codex-watchdog-hook.test.mjs b/tests/unit/codex-watchdog-hook.test.mjs index 6638348..67a97fc 100644 --- a/tests/unit/codex-watchdog-hook.test.mjs +++ b/tests/unit/codex-watchdog-hook.test.mjs @@ -80,12 +80,11 @@ test("PreToolUse allows a focused subagent brief", () => { assert.equal(result.output, null); }); -test("PostToolUse replaces only oversized model-facing output with a bounded excerpt", () => { - const toolResponse = [ - ...Array.from({ length: 300 }, (_, index) => `ordinary output ${index}`), - "ERROR unsponsored_surface", - "exit code: 1", - ].join("\n"); +test("PostToolUse preserves oversized model-facing output", () => { + const toolResponse = Array.from( + { length: 300 }, + (_, index) => `ordinary output ${index}`, + ).join("\n"); const result = evaluateCodexHook( { hook_event_name: "PostToolUse", @@ -99,10 +98,7 @@ test("PostToolUse replaces only oversized model-facing output with a bounded exc { maxToolOutputChars: 900 }, ); - assert.equal(result.output.continue, false); - assert.match(result.output.stopReason, /tool_output_compacted/); - assert.match(result.output.stopReason, /ERROR unsponsored_surface/); - assert.ok(result.output.stopReason.length < 1_200); + assert.equal(result.output, null); }); test("Stop requests one corrective continuation for a narration stall", () => { diff --git a/tests/unit/watchdog-classifier-safety.test.mjs b/tests/unit/watchdog-classifier-safety.test.mjs index ccc5049..4d4d3f3 100644 --- a/tests/unit/watchdog-classifier-safety.test.mjs +++ b/tests/unit/watchdog-classifier-safety.test.mjs @@ -70,3 +70,28 @@ test("read-looking shell redirection stays neutral while explicit write cmdlets ); assert.equal(classify("Set-Content README.copy.md 'x'").kind, "state-change"); }); + +test("Windows PowerShell read forms from real incidents are evidence", () => { + const commands = [ + "Get-ChildItem -LiteralPath 'D:\\repo\\src' -Recurse", + "(Get-Content -LiteralPath 'D:\\repo\\src\\core.ts' | Select-Object -First 40)", + "Write-Output 'status'; git -C 'D:\\repo' status --short", + "git -C 'D:\\repo' diff -- src/core.ts", + "git -C 'D:\\repo' log -5 --oneline", + ]; + for (const command of commands) { + assert.equal(classify(command).kind, "evidence", command); + } +}); + +test("GitHub Delivery owned evidence helpers are classified explicitly", () => { + const evidenceHelpers = [ + "node scripts/ci-forensics.mjs o/r 42", + "node scripts/runtime-capabilities.mjs --repo o/r", + "node scripts/review-brief.mjs o/r 42 --json", + "node scripts/ship-gate.mjs o/r 42 --json", + ]; + for (const command of evidenceHelpers) { + assert.equal(classify(command).kind, "evidence", command); + } +}); diff --git a/tests/unit/watchdog-evidence-contract.test.mjs b/tests/unit/watchdog-evidence-contract.test.mjs new file mode 100644 index 0000000..ed5aa4a --- /dev/null +++ b/tests/unit/watchdog-evidence-contract.test.mjs @@ -0,0 +1,149 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { evaluateCodexHook } from "../../scripts/lib/codex-watchdog-hook.mjs"; +import { + createEvidenceRegistry, + deriveShellEvidenceDescriptor, +} from "../../scripts/lib/watchdog-evidence-registry.mjs"; + +test("PostToolUse never replaces a successful large tool response", () => { + const result = evaluateCodexHook( + { + hook_event_name: "PostToolUse", + session_id: "session-output", + turn_id: "turn-output", + tool_name: "Bash", + tool_input: { command: "Get-Content README.md" }, + tool_response: "x".repeat(12_000), + }, + {}, + { maxToolOutputChars: 1_000 }, + ); + + assert.equal(result.output, null); +}); + +test("different local filters over one Actions run share one semantic evidence key", () => { + const first = deriveShellEvidenceDescriptor( + "gh -R o/r run view 31542325111 --log-failed | Select-String timeout", + ); + const second = deriveShellEvidenceDescriptor( + "gh -R o/r run view 31542325111 --log-failed | Select-String SIGSEGV", + ); + + assert.equal(first.key, "github-actions-run:o/r:31542325111"); + assert.equal(second.key, first.key); + assert.equal(first.authoritative, true); +}); + +test("owned CI helper declares authoritative coverage instead of looking like opaque shell", () => { + const descriptor = deriveShellEvidenceDescriptor( + "node scripts/ci-forensics.mjs o/r 1499 --json", + ); + assert.deepEqual(descriptor, { + effect: "evidence", + key: "pr-ci:o/r:1499", + authoritative: true, + covers: ["checks", "failure-origin", "annotations", "failure-log-tail"], + }); +}); + +test("evidence registry blocks already covered dimensions but permits missing dimensions", () => { + const registry = createEvidenceRegistry(); + registry.record({ + stateGeneration: 7, + key: "pr-ci:o/r:1499", + covers: ["checks", "failure-origin", "annotations", "failure-log-tail"], + authoritative: true, + }); + + assert.deepEqual( + registry.decide({ + stateGeneration: 7, + key: "pr-ci:o/r:1499", + requires: ["checks", "failure-origin"], + }), + { + action: "block", + reason: "evidence_already_covered", + missing: [], + }, + ); + + assert.deepEqual( + registry.decide({ + stateGeneration: 7, + key: "pr-ci:o/r:1499", + requires: ["checks", "artifact-download"], + }), + { + action: "allow", + missing: ["artifact-download"], + }, + ); +}); + +test("semantic evidence coverage is invalidated by a relevant state generation change", () => { + const registry = createEvidenceRegistry(); + registry.record({ + stateGeneration: 3, + key: "github-actions-run:o/r:99", + covers: ["failure-log-tail"], + authoritative: true, + }); + + assert.deepEqual( + registry.decide({ + stateGeneration: 4, + key: "github-actions-run:o/r:99", + requires: ["failure-log-tail"], + }), + { + action: "allow", + missing: ["failure-log-tail"], + }, + ); +}); + +test("hook blocks a differently filtered second read after the first run evidence completed", () => { + const common = { + session_id: "session-semantic", + turn_id: "turn-semantic", + tool_name: "Bash", + }; + const firstInput = { + ...common, + hook_event_name: "PreToolUse", + tool_input: { + command: "gh -R o/r run view 31542325111 --log-failed | Select-String timeout", + }, + }; + const firstPre = evaluateCodexHook(firstInput, {}, { now: 1_000 }); + assert.equal(firstPre.output, null); + + const firstPost = evaluateCodexHook( + { + ...firstInput, + hook_event_name: "PostToolUse", + tool_response: "failure log evidence", + }, + firstPre.state, + { now: 1_100 }, + ); + assert.equal(firstPost.output, null); + + const secondPre = evaluateCodexHook( + { + ...common, + hook_event_name: "PreToolUse", + tool_input: { + command: "gh -R o/r run view 31542325111 --log-failed | Select-String SIGSEGV", + }, + }, + firstPost.state, + { now: 1_200 }, + ); + assert.equal(secondPre.output?.decision, "block"); + assert.match(secondPre.output?.reason || "", /already covered|authoritative evidence/i); +});