Reliability: gate real loop regressions and token efficiency

package.json

@@ -34,12 +34,13 @@
   },
   "scripts": {
     "test": "node --test \"tests/unit/*.test.mjs\"",
+    "reliability:gate": "node --test tests/unit/reliability-regression-gate.test.mjs",
     "evals:offline": "node scripts/validate-evals.mjs",
     "evals:behavioural:compare": "node scripts/compare-behavioural-evals.mjs",
     "build:dist": "node scripts/build-dist.mjs",
     "dist:check": "node scripts/build-dist.mjs --verify-reproducible",
     "package:check": "node scripts/validate-npm-package.mjs",
-    "check": "node scripts/check-syntax.mjs && node scripts/policy-bundle.mjs --validate && node scripts/pre-open-gate.mjs --self-test && npm run security:repo && npm run dist:check && npm run package:check && npm run evals:offline && npm test",
+    "check": "node scripts/check-syntax.mjs && node scripts/policy-bundle.mjs --validate && node scripts/pre-open-gate.mjs --self-test && npm run security:repo && npm run dist:check && npm run package:check && npm run evals:offline && npm run reliability:gate && npm test",
     "release:prepare": "node scripts/prepare-release.mjs",
     "security:repo": "node scripts/validate-repository-security.mjs"
   }

scripts/lib/agent-progress-watchdog.mjs

@@ -13,11 +13,11 @@ const DEFAULTS = Object.freeze({
   noProgressTokenSoftLimit: 4_000,
   noProgressTokenHardLimit: 8_000,
   toolEmissionIntentThreshold: 6,
-  protocolArtifactThreshold: 3,
+  protocolArtifactThreshold: 2,
 });
 
 const INTENT_PREFIX = /^\s*(?:(?:now|next|first|then|actually|meanwhile)[,:]?\s+)?(?:let me|i(?:'|’)ll|i will|i need to|i'm going to|i am going to)\s+/i;
-const TOOL_EMISSION_INTENT = /^\s*(?:(?:now|next|then|actually|enough|finally|stop narrating)[,:.!]?\s+)?(?:(?:let me|i(?:'|’)ll|i will|i need to|i(?:'|’)m going to|i am going to)\s+)?(?:(?:just|actually)\s+)?(?:run|running|execute|executing|invoke|invoking|call|calling|issue|issuing|emit|emitting|grep|search|read|open|inspect|apply|patch|use)\b/i;
+const TOOL_EMISSION_INTENT = /^\s*(?:(?:now|next|then|actually|enough|finally|stop narrating)[,:.!]?\s+)?(?:(?:let me|i(?:'|’)ll|i will|i need to|i(?:'|’)m going to|i am going to)\s+)?(?:(?:just|actually)\s+)?(?:run|running|execute|executing|invoke|invoking|call|calling|issue|issuing|emit|emitting|grep|search|read|open|inspect|apply|patch|use|add|adding|wire|wiring|edit|editing|write|writing|modify|modifying|update|updating|remove|removing|delete|deleting|fix|fixing|change|changing)\b/i;
 const TOOL_PROTOCOL_ARTIFACT = /<\/?(?:atool|invoke|tool_calls?|function_calls?)\b[^>]*>/gi;
 const FAILURE_SIGNAL = /\b(error|errors|fail|failed|failure|failing|blocked|blocker|exception|traceback|denied|timeout|timed out|exit(?: code)?|conclusion|status|unsponsored_surface)\b/i;
 
@@ -281,7 +281,7 @@ export function createProgressWatchdog(options = {}) {
 
     const protocolArtifacts = delta.match(TOOL_PROTOCOL_ARTIFACT);
     if (protocolArtifacts?.length) {
-      protocolArtifactCount += protocolArtifacts.length;
+      protocolArtifactCount += 1;
       if (protocolArtifactCount >= config.protocolArtifactThreshold) {
         return {
           action: "interrupt",

scripts/lib/codex-progress-watchdog.mjs

@@ -1,3 +1,4 @@
+import { createProgressWatchdog } from "./agent-progress-watchdog.mjs";
 import {
   classifyAppServerItem,
   isSuccessfulAppServerItem,
@@ -20,6 +21,13 @@ const RUNTIME_WORK_ITEM_TYPES = new Set([
   "imageView",
 ]);
 
+const FINALIZATION_WATCHDOG_OPTIONS = Object.freeze({
+  generatedCharSoftLimit: 40_000,
+  generatedCharHardLimit: 64_000,
+  noProgressTokenSoftLimit: 12_000,
+  noProgressTokenHardLimit: 16_000,
+});
+
 export function isCodexGeneratedTextMethod(method) {
   return GENERATED_TEXT_METHODS.has(String(method || ""));
 }
@@ -59,6 +67,25 @@ function maybeInterrupt(decision, params, context) {
   };
 }
 
+function planIsComplete(plan) {
+  return (
+    Array.isArray(plan) &&
+    plan.length > 0 &&
+    plan.every((entry) => String(entry?.status || "").toLowerCase() === "completed")
+  );
+}
+
+function finalizationWatchdog(context) {
+  if (!context.finalizationWatchdog) {
+    context.finalizationWatchdog = createProgressWatchdog(FINALIZATION_WATCHDOG_OPTIONS);
+  }
+  return context.finalizationWatchdog;
+}
+
+function activeTextWatchdog(watchdog, context) {
+  return context.finalizing ? finalizationWatchdog(context) : watchdog;
+}
+
 export function observeCodexAppServerMessage(watchdog, message, context = {}) {
   if (!watchdog || typeof watchdog.observeAssistantDelta !== "function") {
     throw new Error("watchdog is required");
@@ -68,12 +95,14 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) {
 
   const { method, params = {} } = message;
   if (isCodexGeneratedTextMethod(method)) {
-    const decision = watchdog.observeAssistantDelta(params.delta || "");
+    const decision = activeTextWatchdog(watchdog, context).observeAssistantDelta(params.delta || "");
     return maybeInterrupt(decision, params, context);
   }
 
   if (method === "thread/tokenUsage/updated") {
-    const decision = watchdog.observeTokenUsage(generatedOutputTokens(params));
+    const decision = activeTextWatchdog(watchdog, context).observeTokenUsage(
+      generatedOutputTokens(params),
+    );
     return maybeInterrupt(decision, params, context);
   }
 
@@ -84,13 +113,23 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) {
 
   if (method === "turn/plan/updated") {
     watchdog.observePlanProgress(params.plan || []);
+    const complete = planIsComplete(params.plan);
+    if (complete && !context.finalizing) {
+      context.finalizing = true;
+      context.finalizationWatchdog = createProgressWatchdog(FINALIZATION_WATCHDOG_OPTIONS);
+    } else if (!complete && context.finalizing) {
+      context.finalizing = false;
+      context.finalizationWatchdog = null;
+    }
     return { decision: { action: "allow" } };
   }
 
   if (method === "item/started") {
     const item = params.item;
     if (RUNTIME_WORK_ITEM_TYPES.has(String(item?.type || ""))) {
       watchdog.recordToolStart({ type: item.type, id: item.id || null });
+      context.finalizing = false;
+      context.finalizationWatchdog = null;
     }
     const classification = classifyAppServerItem(item);
     if (classification.kind === "evidence") {
@@ -122,6 +161,8 @@ export function observeCodexAppServerMessage(watchdog, message, context = {}) {
 
   if (method === "turn/completed" && params.turn?.id) {
     context.interruptedTurns.delete(params.turn.id);
+    context.finalizing = false;
+    context.finalizationWatchdog = null;
   }
 
   return { decision: { action: "allow" } };

tests/unit/codex-watchdog-progress-bounds.test.mjs

@@ -12,7 +12,7 @@ function router(options = {}) {
       generatedCharSoftLimit: 80,
       generatedCharHardLimit: 160,
       toolEmissionIntentThreshold: 3,
-      protocolArtifactThreshold: 3,
+      protocolArtifactThreshold: 2,
       ...options,
     },
   });

tests/unit/reliability-regression-gate.test.mjs

@@ -0,0 +1,236 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { evaluateCodexHook } from "../../scripts/lib/codex-watchdog-hook.mjs";
+import { createAppServerWatchdogRouter } from "../../scripts/lib/codex-app-server-watchdog-proxy.mjs";
+import { replayCodexWatchdogTrace } from "../../scripts/lib/codex-watchdog-replay.mjs";
+
+function replay(messages, watchdogOptions = undefined) {
+  return replayCodexWatchdogTrace(messages, {
+    router: createAppServerWatchdogRouter({
+      internalRequestIdPrefix: "gd-regression",
+      watchdogOptions,
+    }),
+  });
+}
+
+function delta(method, text, itemId = "reasoning") {
+  return {
+    method,
+    params: {
+      threadId: "thr-regression",
+      turnId: "turn-regression",
+      itemId,
+      delta: text,
+    },
+  };
+}
+
+function usage(outputTokens, totalTokens = outputTokens + 10_000) {
+  return {
+    method: "thread/tokenUsage/updated",
+    params: {
+      threadId: "thr-regression",
+      turnId: "turn-regression",
+      tokenUsage: {
+        total: { inputTokens: totalTokens - outputTokens, outputTokens, totalTokens },
+        last: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+      },
+    },
+  };
+}
+
+test("incident: Baseline-is-green tool-emission stall interrupts within six generated clauses", () => {
+  const lines = [
+    "Baseline is green. Let me wire attribution into core.ts.\n",
+    "Baseline is green. Let me add the import.\n",
+    "Let me read the main attempt creation region.\n",
+    "Green baseline. Let me run the exact inspection.\n",
+    "Now execute the read of the target region.\n",
+    "I'll invoke the command now.\n",
+    "Running the tool next.\n",
+  ];
+  const result = replay(lines.map((text) => delta("item/reasoning/summaryTextDelta", text)));
+  assert.equal(result.interruptCount, 1);
+  assert.ok(result.firstInterruptEvent <= 6, `interrupted at event ${result.firstInterruptEvent}`);
+});
+
+test("incident: malformed tool-protocol emission interrupts on the second malformed emission", () => {
+  const result = replay([
+    delta("item/reasoning/summaryTextDelta", "Let me grep. <atool></atool>\n"),
+    delta("item/reasoning/summaryTextDelta", "Run grep. <atool></atool>\n"),
+    delta("item/reasoning/summaryTextDelta", "exec. <invoke></invoke>\n"),
+  ]);
+  assert.equal(result.interruptCount, 1);
+  assert.equal(result.firstInterruptEvent, 2);
+});
+
+test("incident: channel hopping cannot evade repeated tool intent", () => {
+  const methods = [
+    "item/reasoning/summaryTextDelta",
+    "item/agentMessage/delta",
+    "item/plan/delta",
+    "item/reasoning/textDelta",
+  ];
+  const result = replay(
+    Array.from({ length: 8 }, (_, index) =>
+      delta(methods[index % methods.length], "Let me run the grep now.\n", `item-${index}`),
+    ),
+  );
+  assert.equal(result.interruptCount, 1);
+  assert.ok(result.firstInterruptEvent <= 6);
+});
+
+test("incident: differently filtered reads of one Actions run are blocked after first evidence", () => {
+  const common = {
+    session_id: "session-ci-loop",
+    turn_id: "turn-ci-loop",
+    tool_name: "Bash",
+  };
+  const first = evaluateCodexHook(
+    {
+      ...common,
+      hook_event_name: "PreToolUse",
+      tool_input: {
+        command: "gh -R lidge-jun/opencodex run view 31542325111 --log-failed | Select-String timeout",
+      },
+    },
+    {},
+    { now: 1_000 },
+  );
+  assert.equal(first.output, null);
+  const captured = evaluateCodexHook(
+    {
+      ...common,
+      hook_event_name: "PostToolUse",
+      tool_input: {
+        command: "gh -R lidge-jun/opencodex run view 31542325111 --log-failed | Select-String timeout",
+      },
+      tool_response: "captured failure evidence",
+    },
+    first.state,
+    { now: 1_100 },
+  );
+  const repeated = evaluateCodexHook(
+    {
+      ...common,
+      hook_event_name: "PreToolUse",
+      tool_input: {
+        command: "gh -R lidge-jun/opencodex run view 31542325111 --log-failed | Select-String SIGSEGV",
+      },
+    },
+    captured.state,
+    { now: 1_200 },
+  );
+  assert.equal(repeated.output?.decision, "block");
+});
+
+test("active workflow unique no-progress generation is hard bounded by production defaults", () => {
+  const messages = [];
+  for (let index = 0; index < 20; index += 1) {
+    messages.push(
+      delta(
+        "item/reasoning/summaryTextDelta",
+        `Distinct analysis paragraph ${index}: ${"x".repeat(2_000)}\n`,
+        `reason-${index}`,
+      ),
+    );
+  }
+  const result = replay(messages);
+  assert.equal(result.interruptCount, 1);
+  assert.ok(result.firstInterruptEvent <= 16, `unbounded until event ${result.firstInterruptEvent}`);
+});
+
+test("active workflow cumulative output tokens are bounded while large input growth is ignored", () => {
+  const result = replay([
+    usage(100, 50_000),
+    usage(4_000, 90_000),
+    usage(8_101, 150_000),
+  ]);
+  assert.equal(result.interruptCount, 1);
+  assert.equal(result.firstInterruptEvent, 3);
+});
+
+test("false-positive corpus: completed-plan final verdict may exceed ordinary in-workflow character budget", () => {
+  const result = replay([
+    {
+      method: "turn/plan/updated",
+      params: {
+        threadId: "thr-regression",
+        turnId: "turn-regression",
+        plan: [
+          { step: "inspect", status: "completed" },
+          { step: "verify", status: "completed" },
+          { step: "publish final verdict", status: "completed" },
+        ],
+      },
+    },
+    usage(100),
+    delta("item/agentMessage/delta", `Final review verdict:\n${"v".repeat(20_000)}`, "final-answer"),
+    usage(9_000),
+  ]);
+  assert.equal(result.interruptCount, 0);
+});
+
+test("finalization allowance does not disable malformed tool-emission detection", () => {
+  const result = replay([
+    {
+      method: "turn/plan/updated",
+      params: {
+        threadId: "thr-regression",
+        turnId: "turn-regression",
+        plan: [{ step: "all work", status: "completed" }],
+      },
+    },
+    delta("item/agentMessage/delta", "Let me run it. <atool></atool>\n", "final-1"),
+    delta("item/agentMessage/delta", "Executing. <atool></atool>\n", "final-2"),
+  ]);
+  assert.equal(result.interruptCount, 1);
+  assert.equal(result.firstInterruptEvent, 3);
+});
+
+test("false-positive corpus: legitimate tool-rich investigation with real progress is not interrupted", () => {
+  const messages = [];
+  messages.push(usage(100));
+  for (let index = 0; index < 5; index += 1) {
+    messages.push(delta("item/reasoning/summaryTextDelta", `Inspecting distinct required area ${index}.\n`, `r-${index}`));
+    messages.push({
+      method: "item/started",
+      params: {
+        threadId: "thr-regression",
+        turnId: "turn-regression",
+        item: {
+          id: `cmd-${index}`,
+          type: "commandExecution",
+          command: `npm test -- area-${index}`,
+          status: "inProgress",
+        },
+      },
+    });
+    messages.push({
+      method: "item/completed",
+      params: {
+        threadId: "thr-regression",
+        turnId: "turn-regression",
+        item: {
+          id: `cmd-${index}`,
+          type: "commandExecution",
+          command: `npm test -- area-${index}`,
+          status: "completed",
+          exitCode: 0,
+        },
+      },
+    });
+    messages.push({
+      method: "turn/diff/updated",
+      params: {
+        threadId: "thr-regression",
+        turnId: "turn-regression",
+        diff: `diff --git a/f${index}.ts b/f${index}.ts\n+change-${index}\n`,
+      },
+    });
+    messages.push(usage(500 + index * 1_000));
+  }
+  const result = replay(messages);
+  assert.equal(result.interruptCount, 0);
+});