diff --git a/package.json b/package.json index 9839988..f82bb56 100644 --- a/package.json +++ b/package.json @@ -34,12 +34,13 @@ }, "scripts": { "test": "node --test \"tests/unit/*.test.mjs\"", + "reliability:gate": "node --test tests/unit/reliability-regression-gate.test.mjs", "evals:offline": "node scripts/validate-evals.mjs", "evals:behavioural:compare": "node scripts/compare-behavioural-evals.mjs", "build:dist": "node scripts/build-dist.mjs", "dist:check": "node scripts/build-dist.mjs --verify-reproducible", "package:check": "node scripts/validate-npm-package.mjs", - "check": "node scripts/check-syntax.mjs && node scripts/policy-bundle.mjs --validate && node scripts/pre-open-gate.mjs --self-test && npm run security:repo && npm run dist:check && npm run package:check && npm run evals:offline && npm test", + "check": "node scripts/check-syntax.mjs && node scripts/policy-bundle.mjs --validate && node scripts/pre-open-gate.mjs --self-test && npm run security:repo && npm run dist:check && npm run package:check && npm run evals:offline && npm run reliability:gate && npm test", "release:prepare": "node scripts/prepare-release.mjs", "security:repo": "node scripts/validate-repository-security.mjs" } diff --git a/scripts/lib/agent-progress-watchdog.mjs b/scripts/lib/agent-progress-watchdog.mjs index 5083016..0dac120 100644 --- a/scripts/lib/agent-progress-watchdog.mjs +++ b/scripts/lib/agent-progress-watchdog.mjs @@ -13,11 +13,11 @@ const DEFAULTS = Object.freeze({ noProgressTokenSoftLimit: 4_000, noProgressTokenHardLimit: 8_000, toolEmissionIntentThreshold: 6, - protocolArtifactThreshold: 3, + protocolArtifactThreshold: 2, }); const INTENT_PREFIX = /^\s*(?:(?:now|next|first|then|actually|meanwhile)[,:]?\s+)?(?:let me|i(?:'|’)ll|i will|i need to|i'm going to|i am going to)\s+/i; -const TOOL_EMISSION_INTENT = /^\s*(?:(?:now|next|then|actually|enough|finally|stop narrating)[,:.!]?\s+)?(?:(?:let me|i(?:'|’)ll|i will|i need to|i(?:'|’)m going to|i am going to)\s+)?(?:(?:just|actually)\s+)?(?:run|running|execute|executing|invoke|invoking|call|calling|issue|issuing|emit|emitting|grep|search|read|open|inspect|apply|patch|use)\b/i; +const TOOL_EMISSION_INTENT = /^\s*(?:(?:now|next|then|actually|enough|finally|stop narrating)[,:.!]?\s+)?(?:(?:let me|i(?:'|’)ll|i will|i need to|i(?:'|’)m going to|i am going to)\s+)?(?:(?:just|actually)\s+)?(?:run|running|execute|executing|invoke|invoking|call|calling|issue|issuing|emit|emitting|grep|search|read|open|inspect|apply|patch|use|add|adding|wire|wiring|edit|editing|write|writing|modify|modifying|update|updating|remove|removing|delete|deleting|fix|fixing|change|changing)\b/i; const TOOL_PROTOCOL_ARTIFACT = /<\/?(?:atool|invoke|tool_calls?|function_calls?)\b[^>]*>/gi; const FAILURE_SIGNAL = /\b(error|errors|fail|failed|failure|failing|blocked|blocker|exception|traceback|denied|timeout|timed out|exit(?: code)?|conclusion|status|unsponsored_surface)\b/i; @@ -281,7 +281,7 @@ export function createProgressWatchdog(options = {}) { const protocolArtifacts = delta.match(TOOL_PROTOCOL_ARTIFACT); if (protocolArtifacts?.length) { - protocolArtifactCount += protocolArtifacts.length; + protocolArtifactCount += 1; if (protocolArtifactCount >= config.protocolArtifactThreshold) { return { action: "interrupt", diff --git a/scripts/lib/codex-progress-watchdog.mjs b/scripts/lib/codex-progress-watchdog.mjs index c568178..d1647f2 100644 --- a/scripts/lib/codex-progress-watchdog.mjs +++ b/scripts/lib/codex-progress-watchdog.mjs @@ -1,3 +1,4 @@ +import { createProgressWatchdog } from "./agent-progress-watchdog.mjs"; import { classifyAppServerItem, isSuccessfulAppServerItem, @@ -20,6 +21,13 @@ const RUNTIME_WORK_ITEM_TYPES = new Set([ "imageView", ]); +const FINALIZATION_WATCHDOG_OPTIONS = Object.freeze({ + generatedCharSoftLimit: 40_000, + generatedCharHardLimit: 64_000, + noProgressTokenSoftLimit: 12_000, + noProgressTokenHardLimit: 16_000, +}); + export function isCodexGeneratedTextMethod(method) { return GENERATED_TEXT_METHODS.has(String(method || "")); } @@ -59,6 +67,25 @@ function maybeInterrupt(decision, params, context) { }; } +function planIsComplete(plan) { + return ( + Array.isArray(plan) && + plan.length > 0 && + plan.every((entry) => String(entry?.status || "").toLowerCase() === "completed") + ); +} + +function finalizationWatchdog(context) { + if (!context.finalizationWatchdog) { + context.finalizationWatchdog = createProgressWatchdog(FINALIZATION_WATCHDOG_OPTIONS); + } + return context.finalizationWatchdog; +} + +function activeTextWatchdog(watchdog, context) { + return context.finalizing ? finalizationWatchdog(context) : watchdog; +} + export function observeCodexAppServerMessage(watchdog, message, context = {}) { if (!watchdog || typeof watchdog.observeAssistantDelta !== "function") { throw new Error("watchdog is required"); @@ -68,12 +95,14 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) { const { method, params = {} } = message; if (isCodexGeneratedTextMethod(method)) { - const decision = watchdog.observeAssistantDelta(params.delta || ""); + const decision = activeTextWatchdog(watchdog, context).observeAssistantDelta(params.delta || ""); return maybeInterrupt(decision, params, context); } if (method === "thread/tokenUsage/updated") { - const decision = watchdog.observeTokenUsage(generatedOutputTokens(params)); + const decision = activeTextWatchdog(watchdog, context).observeTokenUsage( + generatedOutputTokens(params), + ); return maybeInterrupt(decision, params, context); } @@ -84,6 +113,14 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) { if (method === "turn/plan/updated") { watchdog.observePlanProgress(params.plan || []); + const complete = planIsComplete(params.plan); + if (complete && !context.finalizing) { + context.finalizing = true; + context.finalizationWatchdog = createProgressWatchdog(FINALIZATION_WATCHDOG_OPTIONS); + } else if (!complete && context.finalizing) { + context.finalizing = false; + context.finalizationWatchdog = null; + } return { decision: { action: "allow" } }; } @@ -91,6 +128,8 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) { const item = params.item; if (RUNTIME_WORK_ITEM_TYPES.has(String(item?.type || ""))) { watchdog.recordToolStart({ type: item.type, id: item.id || null }); + context.finalizing = false; + context.finalizationWatchdog = null; } const classification = classifyAppServerItem(item); if (classification.kind === "evidence") { @@ -122,6 +161,8 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) { if (method === "turn/completed" && params.turn?.id) { context.interruptedTurns.delete(params.turn.id); + context.finalizing = false; + context.finalizationWatchdog = null; } return { decision: { action: "allow" } }; diff --git a/tests/unit/codex-watchdog-progress-bounds.test.mjs b/tests/unit/codex-watchdog-progress-bounds.test.mjs index ed05894..c77c341 100644 --- a/tests/unit/codex-watchdog-progress-bounds.test.mjs +++ b/tests/unit/codex-watchdog-progress-bounds.test.mjs @@ -12,7 +12,7 @@ function router(options = {}) { generatedCharSoftLimit: 80, generatedCharHardLimit: 160, toolEmissionIntentThreshold: 3, - protocolArtifactThreshold: 3, + protocolArtifactThreshold: 2, ...options, }, }); diff --git a/tests/unit/reliability-regression-gate.test.mjs b/tests/unit/reliability-regression-gate.test.mjs new file mode 100644 index 0000000..8dfa480 --- /dev/null +++ b/tests/unit/reliability-regression-gate.test.mjs @@ -0,0 +1,236 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { evaluateCodexHook } from "../../scripts/lib/codex-watchdog-hook.mjs"; +import { createAppServerWatchdogRouter } from "../../scripts/lib/codex-app-server-watchdog-proxy.mjs"; +import { replayCodexWatchdogTrace } from "../../scripts/lib/codex-watchdog-replay.mjs"; + +function replay(messages, watchdogOptions = undefined) { + return replayCodexWatchdogTrace(messages, { + router: createAppServerWatchdogRouter({ + internalRequestIdPrefix: "gd-regression", + watchdogOptions, + }), + }); +} + +function delta(method, text, itemId = "reasoning") { + return { + method, + params: { + threadId: "thr-regression", + turnId: "turn-regression", + itemId, + delta: text, + }, + }; +} + +function usage(outputTokens, totalTokens = outputTokens + 10_000) { + return { + method: "thread/tokenUsage/updated", + params: { + threadId: "thr-regression", + turnId: "turn-regression", + tokenUsage: { + total: { inputTokens: totalTokens - outputTokens, outputTokens, totalTokens }, + last: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + }, + }, + }; +} + +test("incident: Baseline-is-green tool-emission stall interrupts within six generated clauses", () => { + const lines = [ + "Baseline is green. Let me wire attribution into core.ts.\n", + "Baseline is green. Let me add the import.\n", + "Let me read the main attempt creation region.\n", + "Green baseline. Let me run the exact inspection.\n", + "Now execute the read of the target region.\n", + "I'll invoke the command now.\n", + "Running the tool next.\n", + ]; + const result = replay(lines.map((text) => delta("item/reasoning/summaryTextDelta", text))); + assert.equal(result.interruptCount, 1); + assert.ok(result.firstInterruptEvent <= 6, `interrupted at event ${result.firstInterruptEvent}`); +}); + +test("incident: malformed tool-protocol emission interrupts on the second malformed emission", () => { + const result = replay([ + delta("item/reasoning/summaryTextDelta", "Let me grep. \n"), + delta("item/reasoning/summaryTextDelta", "Run grep. \n"), + delta("item/reasoning/summaryTextDelta", "exec. \n"), + ]); + assert.equal(result.interruptCount, 1); + assert.equal(result.firstInterruptEvent, 2); +}); + +test("incident: channel hopping cannot evade repeated tool intent", () => { + const methods = [ + "item/reasoning/summaryTextDelta", + "item/agentMessage/delta", + "item/plan/delta", + "item/reasoning/textDelta", + ]; + const result = replay( + Array.from({ length: 8 }, (_, index) => + delta(methods[index % methods.length], "Let me run the grep now.\n", `item-${index}`), + ), + ); + assert.equal(result.interruptCount, 1); + assert.ok(result.firstInterruptEvent <= 6); +}); + +test("incident: differently filtered reads of one Actions run are blocked after first evidence", () => { + const common = { + session_id: "session-ci-loop", + turn_id: "turn-ci-loop", + tool_name: "Bash", + }; + const first = evaluateCodexHook( + { + ...common, + hook_event_name: "PreToolUse", + tool_input: { + command: "gh -R lidge-jun/opencodex run view 31542325111 --log-failed | Select-String timeout", + }, + }, + {}, + { now: 1_000 }, + ); + assert.equal(first.output, null); + const captured = evaluateCodexHook( + { + ...common, + hook_event_name: "PostToolUse", + tool_input: { + command: "gh -R lidge-jun/opencodex run view 31542325111 --log-failed | Select-String timeout", + }, + tool_response: "captured failure evidence", + }, + first.state, + { now: 1_100 }, + ); + const repeated = evaluateCodexHook( + { + ...common, + hook_event_name: "PreToolUse", + tool_input: { + command: "gh -R lidge-jun/opencodex run view 31542325111 --log-failed | Select-String SIGSEGV", + }, + }, + captured.state, + { now: 1_200 }, + ); + assert.equal(repeated.output?.decision, "block"); +}); + +test("active workflow unique no-progress generation is hard bounded by production defaults", () => { + const messages = []; + for (let index = 0; index < 20; index += 1) { + messages.push( + delta( + "item/reasoning/summaryTextDelta", + `Distinct analysis paragraph ${index}: ${"x".repeat(2_000)}\n`, + `reason-${index}`, + ), + ); + } + const result = replay(messages); + assert.equal(result.interruptCount, 1); + assert.ok(result.firstInterruptEvent <= 16, `unbounded until event ${result.firstInterruptEvent}`); +}); + +test("active workflow cumulative output tokens are bounded while large input growth is ignored", () => { + const result = replay([ + usage(100, 50_000), + usage(4_000, 90_000), + usage(8_101, 150_000), + ]); + assert.equal(result.interruptCount, 1); + assert.equal(result.firstInterruptEvent, 3); +}); + +test("false-positive corpus: completed-plan final verdict may exceed ordinary in-workflow character budget", () => { + const result = replay([ + { + method: "turn/plan/updated", + params: { + threadId: "thr-regression", + turnId: "turn-regression", + plan: [ + { step: "inspect", status: "completed" }, + { step: "verify", status: "completed" }, + { step: "publish final verdict", status: "completed" }, + ], + }, + }, + usage(100), + delta("item/agentMessage/delta", `Final review verdict:\n${"v".repeat(20_000)}`, "final-answer"), + usage(9_000), + ]); + assert.equal(result.interruptCount, 0); +}); + +test("finalization allowance does not disable malformed tool-emission detection", () => { + const result = replay([ + { + method: "turn/plan/updated", + params: { + threadId: "thr-regression", + turnId: "turn-regression", + plan: [{ step: "all work", status: "completed" }], + }, + }, + delta("item/agentMessage/delta", "Let me run it. \n", "final-1"), + delta("item/agentMessage/delta", "Executing. \n", "final-2"), + ]); + assert.equal(result.interruptCount, 1); + assert.equal(result.firstInterruptEvent, 3); +}); + +test("false-positive corpus: legitimate tool-rich investigation with real progress is not interrupted", () => { + const messages = []; + messages.push(usage(100)); + for (let index = 0; index < 5; index += 1) { + messages.push(delta("item/reasoning/summaryTextDelta", `Inspecting distinct required area ${index}.\n`, `r-${index}`)); + messages.push({ + method: "item/started", + params: { + threadId: "thr-regression", + turnId: "turn-regression", + item: { + id: `cmd-${index}`, + type: "commandExecution", + command: `npm test -- area-${index}`, + status: "inProgress", + }, + }, + }); + messages.push({ + method: "item/completed", + params: { + threadId: "thr-regression", + turnId: "turn-regression", + item: { + id: `cmd-${index}`, + type: "commandExecution", + command: `npm test -- area-${index}`, + status: "completed", + exitCode: 0, + }, + }, + }); + messages.push({ + method: "turn/diff/updated", + params: { + threadId: "thr-regression", + turnId: "turn-regression", + diff: `diff --git a/f${index}.ts b/f${index}.ts\n+change-${index}\n`, + }, + }); + messages.push(usage(500 + index * 1_000)); + } + const result = replay(messages); + assert.equal(result.interruptCount, 0); +});