From 9665489578ce8048a1f79d0d1d062662fb6d32ae Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:18:19 +0000 Subject: [PATCH 01/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20deliver=20terminal?= =?UTF-8?q?=20wakes=20for=20kernel-launched=20background=20workflow=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 14 +++ .../services/agentWorkflowRunReferences.ts | 5 +- src/node/services/taskService.ts | 5 + src/node/services/workspaceService.test.ts | 98 +++++++++++++++++++ src/node/services/workspaceService.ts | 47 ++++++--- 5 files changed, 153 insertions(+), 16 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 88a64361395..44568222c86 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -31,4 +31,18 @@ describe("agent workflow run references", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("keeps the newest createdAtMs across re-records", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_re_recorded"; + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 2_000 }); + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 1_000 }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toEqual([{ runId, createdAtMs: 2_000 }]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index b202f4a83db..909b69a9264 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -75,7 +75,10 @@ export async function recordAgentWorkflowRunReference(input: { const previous = byRunId.get(input.runId); byRunId.set(input.runId, { runId: input.runId, - createdAtMs: previous ? Math.min(previous.createdAtMs, createdAtMs) : createdAtMs, + // Latest record wins: workflow_resume re-records the reference, and a resume issued after + // a manual user message must re-establish provenance for supersession-timestamp + // comparisons (isWorkflowInvocationCurrent, listAgentReferencedWorkflowRunIds). + createdAtMs: previous ? Math.max(previous.createdAtMs, createdAtMs) : createdAtMs, }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 7e43d21fde6..15bb3c686f7 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8232,6 +8232,11 @@ export class TaskService { notification.sourceId ); if (workflowPrompt == null) { + // Dropping a notify_on_terminal wake strands the run's owner; keep the drop diagnosable. + log.warn("Dropping superseded workflow terminal attention", { + ownerWorkspaceId, + runId: notification.sourceId, + }); await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); continue; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ec69f044d2d..9bd9871048a 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -54,9 +54,11 @@ import type { SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/typ import { createMuxMessage } from "@/common/types/message"; import { buildStagedAttachmentNotice } from "@/browser/features/ChatInput/stagedAttachments"; import { + WORKFLOW_RESULT_METADATA_TYPE, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, } from "@/common/utils/workflowRunMessages"; +import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; import * as todoStorageModule from "@/node/services/todos/todoStorage"; import * as runtimeFactory from "@/node/runtime/runtimeFactory"; @@ -5924,6 +5926,102 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("counts a kernel-launched run recorded in the sidecar as the current invocation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-kernel"; + const runId = "wfr_currentness_kernel"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-kernel", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // mux.workflow_run inside code_execution leaves no workflow_run tool part in history; the + // agent-workflow-runs sidecar reference is the only durable invocation evidence. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-kernel-launch", "assistant", "", { timestamp: 1_100 }, [ + { + type: "dynamic-tool", + toolCallId: "code-exec-1", + toolName: "code_execution", + state: "output-available", + input: { code: "return xum.workflow_run({ script_path: './workflows/demo.js' })" }, + output: { success: true, result: { status: "running", runId } }, + }, + ]) + ); + + // The nested runId in the code_execution output alone is not invocation evidence. + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A newer manual user message supersedes the sidecar reference. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel workflow_resume re-records the reference after the supersession and + // re-establishes provenance (latest record wins). + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_250, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Once the terminal result was delivered, the sidecar must not resurrect the invocation. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("workflow-result", "user", "The workflow below has finished.", { + timestamp: 1_300, + synthetic: true, + muxMetadata: { + type: WORKFLOW_RESULT_METADATA_TYPE, + rawCommand: "workflow_run ./workflows/demo.js", + runId, + }, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6393454098e..48f0dbc3ada 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3,6 +3,7 @@ import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { EventEmitter } from "events"; import * as path from "path"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; +import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; @@ -10935,21 +10936,17 @@ export class WorkspaceService extends EventEmitter { assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); - let current = false; - let foundDecision = false; + let outcome: "invocation" | "consumed" | "superseded" | null = null; + let supersededAtMs: number | null = null; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { - if (isManualUserSupersessionMessage(message)) { - current = false; - foundDecision = true; - return false; - } - if (isResetBoundaryMessage(message)) { - current = false; - foundDecision = true; + if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { + outcome = "superseded"; + const timestamp = message.metadata?.timestamp; + supersededAtMs = typeof timestamp === "number" ? timestamp : null; return false; } if ( @@ -10957,13 +10954,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - current = false; - foundDecision = true; + outcome = "consumed"; return false; } if (isWorkflowInvocationMessage(message, runId)) { - current = true; - foundDecision = true; + outcome = "invocation"; return false; } } @@ -10979,7 +10974,29 @@ export class WorkspaceService extends EventEmitter { return false; } - return foundDecision && current; + if (outcome === "invocation") { + return true; + } + if (outcome === "consumed") { + return false; + } + + // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave + // no recognizable invocation part in history, so the backward walk above stops at the prior + // real user message and would wrongly treat the run as superseded, silently dropping its + // notify_on_terminal wake. Their durable provenance is the agent-workflow-runs sidecar: a + // reference recorded after the latest supersession boundary counts as the current + // invocation. A boundary without a durable timestamp fails safe to superseded, mirroring + // TaskService.listAgentReferencedWorkflowRunIds. + if (outcome === "superseded" && supersededAtMs === null) { + return false; + } + const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + const reference = references.find((candidate) => candidate.runId === runId); + if (reference == null) { + return false; + } + return supersededAtMs === null || reference.createdAtMs > supersededAtMs; } /** From f41cbfbb1113b47d72b83bc3346c7d72af797ab9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:39:44 +0000 Subject: [PATCH 02/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20persist=20workflow?= =?UTF-8?q?=5Fresume=20terminal=20consumption=20for=20kernel-nested=20call?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/tools/workflow_resume.test.ts | 100 ++++++++++++++++++ src/node/services/tools/workflow_resume.ts | 24 ++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 1ae69b94021..63df2868bc6 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -6,6 +6,7 @@ import { TestTempDir, createTestToolConfig } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import { WORKFLOW_CHECKPOINT_RETRY_ERROR_MESSAGE } from "@/common/utils/workflowRetryEligibility"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +import type { TaskService } from "@/node/services/taskService"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; const mockToolCallOptions: ToolExecutionOptions = { @@ -229,6 +230,105 @@ describe("workflow_resume tool", () => { }); }); + test("marks terminal attention consumed when returning an already-completed run's result", async () => { + using tempDir = new TestTempDir("test-workflow-resume-consumed"); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { + sequence: 2, + type: "result", + at: "2026-05-29T00:00:01.000Z", + result: { reportMarkdown: "already done" }, + }, + { sequence: 3, type: "status", at: "2026-05-29T00:00:01.000Z", status: "completed" }, + ], + }); + const workflowService = buildWorkflowService({ getRun: mock(async () => completedRun) }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + }); + + test("does not mark terminal attention consumed for background dispatches", async () => { + using tempDir = new TestTempDir("test-workflow-resume-background-no-consume"); + // The refresh after a background dispatch can still observe the stale pre-dispatch failed + // status; consuming it would tombstone the retried run's future terminal wake. + const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: true, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).not.toHaveBeenCalled(); + }); + + test("marks terminal attention consumed when a foreground retry finishes terminal", async () => { + using tempDir = new TestTempDir("test-workflow-resume-foreground-consume"); + const failedRun = buildFailedRun(); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { sequence: 2, type: "status", at: "2026-05-29T00:00:02.000Z", status: "completed" }, + ], + }); + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? failedRun : completedRun; + }), + retryRunFromCheckpoint: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "retried" }, + })), + }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + }); + test("rejects default resume of a failed run with checkpoint retry guidance", async () => { using tempDir = new TestTempDir("test-workflow-resume-failed"); const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index a94188569a6..649c1ca90bf 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -2,7 +2,7 @@ import { tool } from "ai"; import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; -import type { WorkflowRunRecord } from "@/common/types/workflow"; +import { isTerminalWorkflowRunStatus, type WorkflowRunRecord } from "@/common/types/workflow"; import { getWorkflowCheckpointRetryEligibility } from "@/common/utils/workflowRetryEligibility"; import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import { @@ -154,9 +154,25 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const mode: WorkflowResumeMode = args.mode ?? "resume"; const invocationStartedAtMs = Date.now(); + // A kernel-nested resume (mux.workflow_resume inside code_execution) leaves no top-level + // workflow_resume part in history, so the history-walk consumption predicates cannot see + // that this turn already received the terminal result. Persist consumption durably so the + // terminal-attention drain never re-delivers it. + const markTerminalAttentionConsumed = async (terminalRun: WorkflowRunRecord) => { + if (!isTerminalWorkflowRunStatus(terminalRun.status)) { + return; + } + await config.taskService?.markWorkflowRunTerminalAttentionConsumed?.({ + ownerWorkspaceId: workspaceId, + runId: terminalRun.id, + status: terminalRun.status, + }); + }; + // Idempotent success: the work is already done, so hand back the durable result instead // of failing the agent's recovery loop (e.g. resuming after a crash that actually finished). if (run.status === "completed" && mode === "resume") { + await markTerminalAttentionConsumed(run); return parseToolResult( WorkflowResumeToolResultSchema, { @@ -231,6 +247,12 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const refreshedRunIsStale = isBackgroundDispatch && refreshedRun != null && refreshedRun.status === run.status; + // Foreground only: a background dispatch can still observe the stale pre-dispatch terminal + // status, and consuming it would tombstone the retried run's future terminal wake. + if (!isBackgroundDispatch && refreshedRun != null) { + await markTerminalAttentionConsumed(refreshedRun); + } + return parseToolResult( WorkflowResumeToolResultSchema, { From a19c5a304c63c58a04f55c592c564c3fce1f36e9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:46:20 +0000 Subject: [PATCH 03/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20let=20newer=20sidec?= =?UTF-8?q?ar=20records=20outrank=20consumed=20results;=20clamp=20future?= =?UTF-8?q?=20timestamps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 18 ++++++++++++++ .../services/agentWorkflowRunReferences.ts | 9 +++++-- src/node/services/workspaceService.test.ts | 9 +++++++ src/node/services/workspaceService.ts | 24 +++++++++---------- 4 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 44568222c86..fde568bd491 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -32,6 +32,24 @@ describe("agent workflow run references", () => { } }); + test("clamps future-dated createdAtMs to the current time", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_future"; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs: Date.now() + 86_400_000, + }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]?.createdAtMs).toBeLessThanOrEqual(Date.now()); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("keeps the newest createdAtMs across re-records", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 909b69a9264..eab9be58d8b 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -30,6 +30,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { } const parsed: AgentWorkflowRunReference[] = []; + const now = Date.now(); for (const reference of references) { if (reference == null || typeof reference !== "object") { continue; @@ -41,7 +42,10 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); + // Self-heal implausible future timestamps (clock correction, corruption): a future-dated + // reference would otherwise outrank every later user/reset boundary in supersession + // comparisons until wall time catches up. + parsed.push({ runId: record.runId, createdAtMs: Math.min(record.createdAtMs, now) }); } return parsed; } @@ -71,7 +75,8 @@ export async function recordAgentWorkflowRunReference(input: { await referenceFileLocks.withLock(filePath, async () => { const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); - const createdAtMs = input.createdAtMs ?? Date.now(); + // Clamp like parseReferences: never persist a future-dated timestamp. + const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); byRunId.set(input.runId, { runId: input.runId, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 9bd9871048a..08ecf7a1c05 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6016,6 +6016,15 @@ describe("WorkspaceService workflow invocation events", () => { }) ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel background resume issued after the delivered result re-records the reference, + // so the retried run's next terminal wake must count as current again. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_350, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); workspaceService.disposeSession(workspaceId); } finally { await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 48f0dbc3ada..b04e43a550f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10937,7 +10937,7 @@ export class WorkspaceService extends EventEmitter { assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); let outcome: "invocation" | "consumed" | "superseded" | null = null; - let supersededAtMs: number | null = null; + let boundaryAtMs: number | null = null; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", @@ -10946,7 +10946,7 @@ export class WorkspaceService extends EventEmitter { if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { outcome = "superseded"; const timestamp = message.metadata?.timestamp; - supersededAtMs = typeof timestamp === "number" ? timestamp : null; + boundaryAtMs = typeof timestamp === "number" ? timestamp : null; return false; } if ( @@ -10955,6 +10955,8 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowToolResultMessage(message, runId) ) { outcome = "consumed"; + const timestamp = message.metadata?.timestamp; + boundaryAtMs = typeof timestamp === "number" ? timestamp : null; return false; } if (isWorkflowInvocationMessage(message, runId)) { @@ -10977,18 +10979,16 @@ export class WorkspaceService extends EventEmitter { if (outcome === "invocation") { return true; } - if (outcome === "consumed") { - return false; - } // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave // no recognizable invocation part in history, so the backward walk above stops at the prior - // real user message and would wrongly treat the run as superseded, silently dropping its - // notify_on_terminal wake. Their durable provenance is the agent-workflow-runs sidecar: a - // reference recorded after the latest supersession boundary counts as the current - // invocation. A boundary without a durable timestamp fails safe to superseded, mirroring - // TaskService.listAgentReferencedWorkflowRunIds. - if (outcome === "superseded" && supersededAtMs === null) { + // real user message (or, after a delivered result, at that consumed terminal message) and + // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the + // agent-workflow-runs sidecar: a reference recorded after that boundary counts as the + // current invocation. For a consumed boundary that means a background resume/retry issued + // after the prior result was delivered. A boundary without a durable timestamp fails safe + // to not-current, mirroring TaskService.listAgentReferencedWorkflowRunIds. + if (outcome !== null && boundaryAtMs === null) { return false; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); @@ -10996,7 +10996,7 @@ export class WorkspaceService extends EventEmitter { if (reference == null) { return false; } - return supersededAtMs === null || reference.createdAtMs > supersededAtMs; + return boundaryAtMs === null || reference.createdAtMs > boundaryAtMs; } /** From 30a3437d0b9bd6e890b44c73af0b24ead7a24f8b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:59:12 +0000 Subject: [PATCH 04/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20reject=20persisted?= =?UTF-8?q?=20future-dated=20sidecar=20references=20instead=20of=20clampin?= =?UTF-8?q?g=20per=20read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 37 +++++++++++++++++++ .../services/agentWorkflowRunReferences.ts | 12 ++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index fde568bd491..b89d1185215 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -32,6 +32,43 @@ describe("agent workflow run references", () => { } }); + test("drops persisted future-dated references and repairs them on the next record", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const futureMs = Date.now() + 86_400_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_corrupt_future", createdAtMs: futureMs }, + { runId: "wfr_sane", createdAtMs: 1_000 }, + ], + }) + ); + + // A per-read clamp would re-evaluate to "now" on every read and outrank every later + // user/reset boundary; the corrupted entry must be dropped instead. + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_sane", createdAtMs: 1_000 }, + ]); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_corrupt_future", + createdAtMs: 2_000, + }); + const repaired = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(repaired).toContainEqual({ runId: "wfr_corrupt_future", createdAtMs: 2_000 }); + const raw = await fs.readFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + "utf-8" + ); + expect(raw).not.toContain(String(futureMs)); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("clamps future-dated createdAtMs to the current time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index eab9be58d8b..90992b5a900 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -42,10 +42,14 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - // Self-heal implausible future timestamps (clock correction, corruption): a future-dated - // reference would otherwise outrank every later user/reset boundary in supersession - // comparisons until wall time catches up. - parsed.push({ runId: record.runId, createdAtMs: Math.min(record.createdAtMs, now) }); + // Reject future-dated references (clock correction, corruption) instead of clamping at + // read time: a per-read clamp re-evaluates to "now" on every read, so the entry would + // outrank every later user/reset boundary until wall time catches up. Rejected entries are + // replaced with a sane timestamp by the next legitimate record. + if (record.createdAtMs > now) { + continue; + } + parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); } return parsed; } From 89263dd6b0c85445c1ab4a9aa57af4b9860302ae Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:13:03 +0000 Subject: [PATCH 05/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20fail=20safe=20after?= =?UTF-8?q?=20full=20history=20clear;=20dedupe=20sidecar=20references=20at?= =?UTF-8?q?=20parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 26 +++++++++++ .../services/agentWorkflowRunReferences.ts | 12 +++-- src/node/services/workspaceService.test.ts | 44 +++++++++++++++++++ src/node/services/workspaceService.ts | 12 +++-- 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index b89d1185215..dde03f870ef 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -32,6 +32,32 @@ describe("agent workflow run references", () => { } }); + test("collapses persisted duplicate entries to the newest sane timestamp", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Corrupted files can carry duplicates in either order; order-sensitive consumers must + // never observe a stale duplicate ahead of a legitimate re-record. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_dup", createdAtMs: 1_000 }, + { runId: "wfr_dup", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 1_000 }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(2); + expect(references).toContainEqual({ runId: "wfr_dup", createdAtMs: 2_000 }); + expect(references).toContainEqual({ runId: "wfr_dup_reversed", createdAtMs: 2_000 }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("drops persisted future-dated references and repairs them on the next record", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 90992b5a900..5aec88b834a 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -29,7 +29,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { return []; } - const parsed: AgentWorkflowRunReference[] = []; + const parsedByRunId = new Map(); const now = Date.now(); for (const reference of references) { if (reference == null || typeof reference !== "object") { @@ -49,9 +49,15 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (record.createdAtMs > now) { continue; } - parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); + // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive + // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run + // superseded. + const existing = parsedByRunId.get(record.runId); + if (existing == null || record.createdAtMs > existing.createdAtMs) { + parsedByRunId.set(record.runId, { runId: record.runId, createdAtMs: record.createdAtMs }); + } } - return parsed; + return Array.from(parsedByRunId.values()); } export async function readAgentWorkflowRunReferences( diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 08ecf7a1c05..407ad8242c3 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6031,6 +6031,50 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("does not treat sidecar references as current after a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-cleared"; + const runId = "wfr_currentness_cleared"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-cleared", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A full clear (truncateHistory) removes every row without appending a reset boundary + // and leaves the sidecar intact; the surviving reference must not inject a workflow + // result into the freshly cleared conversation. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + }); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index b04e43a550f..41c99a20d80 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10986,9 +10986,13 @@ export class WorkspaceService extends EventEmitter { // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the // agent-workflow-runs sidecar: a reference recorded after that boundary counts as the // current invocation. For a consumed boundary that means a background resume/retry issued - // after the prior result was delivered. A boundary without a durable timestamp fails safe - // to not-current, mirroring TaskService.listAgentReferencedWorkflowRunIds. - if (outcome !== null && boundaryAtMs === null) { + // after the prior result was delivered. The fallback requires a datable boundary: an + // undatable boundary fails safe to not-current (mirroring + // TaskService.listAgentReferencedWorkflowRunIds), and so does a decision-free history, + // because a full clear (truncateHistory) removes every row WITHOUT appending a reset + // boundary while leaving the sidecar intact — a surviving reference must not inject a + // workflow result into the freshly cleared conversation. + if (boundaryAtMs === null) { return false; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); @@ -10996,7 +11000,7 @@ export class WorkspaceService extends EventEmitter { if (reference == null) { return false; } - return boundaryAtMs === null || reference.createdAtMs > boundaryAtMs; + return reference.createdAtMs > boundaryAtMs; } /** From 59ce349492310c223b00f63de909829d52573f0f Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:28:39 +0000 Subject: [PATCH 06/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20tolerate=20bounded?= =?UTF-8?q?=20backward-clock=20skew=20in=20sidecar=20reference=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 19 +++++++++++++++++++ .../services/agentWorkflowRunReferences.ts | 16 +++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index dde03f870ef..2747eb1fe65 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -95,6 +95,25 @@ describe("agent workflow run references", () => { } }); + test("keeps references within the backward-clock skew tolerance", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A backward clock correction makes a legitimately recorded reference look slightly + // future-dated; dropping it would strand the run's terminal wake. + const slightlyFutureMs = Date.now() + 5 * 60_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ references: [{ runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }] }) + ); + + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }, + ]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("clamps future-dated createdAtMs to the current time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 5aec88b834a..11d7bb6c870 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -13,6 +13,11 @@ export interface AgentWorkflowRunReference { const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; +// Backward clock corrections (e.g. an NTP step after booting with a fast clock) can make a +// legitimately recorded reference look future-dated. Tolerate that bounded skew so the run's +// terminal wake is not dropped; only implausibly future values are treated as corruption. +const MAX_FUTURE_SKEW_MS = 60 * 60_000; + const referenceFileLocks = new MutexMap(); function referencesPath(workspaceSessionDir: string): string { @@ -42,11 +47,12 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - // Reject future-dated references (clock correction, corruption) instead of clamping at - // read time: a per-read clamp re-evaluates to "now" on every read, so the entry would - // outrank every later user/reset boundary until wall time catches up. Rejected entries are - // replaced with a sane timestamp by the next legitimate record. - if (record.createdAtMs > now) { + // Reject implausibly future-dated references (corruption) instead of clamping at read + // time: a per-read clamp re-evaluates to "now" on every read, so the entry would outrank + // every later user/reset boundary until wall time catches up. Values within + // MAX_FUTURE_SKEW_MS are kept as-is (backward clock correction, not corruption). Rejected + // entries are replaced with a sane timestamp by the next legitimate record. + if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { continue; } // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive From 76e6e8070d8c0211afa522120a14987118b534df Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:07:54 +0000 Subject: [PATCH 07/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20decide=20kernel=20w?= =?UTF-8?q?orkflow=20currentness=20by=20boundary-row=20identity,=20not=20w?= =?UTF-8?q?all=20clock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/agentWorkflowRunReferences.ts | 31 ++++- src/node/services/taskService.ts | 11 ++ src/node/services/tools/toolUtils.ts | 17 ++- src/node/services/tools/workflow_run.test.ts | 15 ++- src/node/services/workspaceService.test.ts | 108 ++++++++++++++++++ src/node/services/workspaceService.ts | 107 +++++++++++------ 6 files changed, 249 insertions(+), 40 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 11d7bb6c870..58f23023f9e 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -9,6 +9,14 @@ import { MutexMap } from "@/node/utils/concurrency/mutexMap"; export interface AgentWorkflowRunReference { runId: string; createdAtMs: number; + /** + * Message ID of the newest invocation-decision row (manual user/reset supersession, consumed + * terminal result for this run, or direct invocation part) at record time; null when history + * had none. Row identity, not wall clock: currentness compares this against the row the + * history walk stops at, so a backward clock correction cannot reorder the comparison. + * Absent on legacy entries, which fail safe to not-current. + */ + afterBoundaryMessageId?: string | null; } const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; @@ -55,12 +63,23 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { continue; } + const boundaryRaw = record.afterBoundaryMessageId; + const afterBoundaryMessageId = + typeof boundaryRaw === "string" && boundaryRaw.length > 0 + ? boundaryRaw + : boundaryRaw === null + ? null + : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run - // superseded. + // superseded. The chosen record is kept wholesale, including its boundary snapshot. const existing = parsedByRunId.get(record.runId); if (existing == null || record.createdAtMs > existing.createdAtMs) { - parsedByRunId.set(record.runId, { runId: record.runId, createdAtMs: record.createdAtMs }); + parsedByRunId.set(record.runId, { + runId: record.runId, + createdAtMs: record.createdAtMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + }); } } return Array.from(parsedByRunId.values()); @@ -84,6 +103,7 @@ export async function recordAgentWorkflowRunReference(input: { workspaceSessionDir: string; runId: string; createdAtMs?: number; + afterBoundaryMessageId?: string | null; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -98,8 +118,13 @@ export async function recordAgentWorkflowRunReference(input: { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after // a manual user message must re-establish provenance for supersession-timestamp - // comparisons (isWorkflowInvocationCurrent, listAgentReferencedWorkflowRunIds). + // comparisons (listAgentReferencedWorkflowRunIds). createdAtMs: previous ? Math.max(previous.createdAtMs, createdAtMs) : createdAtMs, + // The new record event defines currentness provenance wholesale; a caller without + // boundary knowledge produces a legacy-style entry that fails safe to not-current. + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 15bb3c686f7..42ef463b37b 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7730,6 +7730,17 @@ export class TaskService { ); } + /** + * Tool-path access to the invocation-boundary snapshot recorded into the + * agent-workflow-runs sidecar (see recordBackgroundWorkflowRunReference). + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + return this.workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); + } + async markWorkflowRunTerminalAttentionConsumed(params: { ownerWorkspaceId: string; runId: string; diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index eb60511d88c..2fa64f7a332 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -98,7 +98,22 @@ export async function recordBackgroundWorkflowRunReference( } try { - await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs }); + // Snapshot which invocation-decision row is newest at launch so the terminal-wake + // currentness check compares row identity instead of wall-clock order, which clock + // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). + const afterBoundaryMessageId = + config.workspaceId != null + ? ((await config.taskService?.getWorkflowInvocationBoundaryMessageId?.( + config.workspaceId, + runId + )) ?? null) + : null; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs, + afterBoundaryMessageId, + }); } catch (error: unknown) { log.warn("Failed to record agent workflow run reference", { runId, diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 0636a54d8b0..d8779d47e1b 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -13,6 +13,7 @@ import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { resolveWorkflowScript } from "@/node/services/workflows/workflowScriptResolver"; import { TestTempDir, createTestToolConfig, writeProjectSkill } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { TaskService } from "@/node/services/taskService"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkflowRunRecord } from "@/common/types/workflow"; @@ -650,9 +651,11 @@ describe("workflow_run tool", () => { result: null, })); const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); const tool = createWorkflowRunTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, startWorkflowInBackground, @@ -665,8 +668,18 @@ describe("workflow_run tool", () => { mockToolCallOptions ); + // The reference must persist the invocation-boundary snapshot: currentness compares row + // identity, so a reference recorded without it fails safe and the wake is dropped. const references = await readAgentWorkflowRunReferences(tempDir.path); - expect(references.map((reference) => reference.runId)).toContain("wfr_background"); + expect(references).toHaveLength(1); + expect(references[0]).toMatchObject({ + runId: "wfr_background", + afterBoundaryMessageId: "boundary-row-1", + }); + expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( + "workspace-1", + "wfr_background" + ); expect(startWorkflowInBackground).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 407ad8242c3..7ba2643cf5e 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5981,6 +5981,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); @@ -5999,6 +6000,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_250, + afterBoundaryMessageId: "manual-user-2", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); @@ -6023,6 +6025,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_350, + afterBoundaryMessageId: "workflow-result", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); workspaceService.disposeSession(workspaceId); @@ -6066,6 +6069,7 @@ describe("WorkspaceService workflow invocation events", () => { workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); @@ -6075,6 +6079,110 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-clock"; + const runId = "wfr_currentness_clock"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-clock", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // A backward clock correction after recording makes the reference timestamp future-dated + // relative to every later history row; identity comparison must still deliver the wake. + const skewedCreatedAtMs = Date.now() + 30 * 60_000; + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: skewedCreatedAtMs, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A user message written after the correction has a smaller timestamp than the reference; + // wall-clock ordering would keep the stale reference current, identity must not. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("fails safe for legacy sidecar references without a boundary snapshot", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-legacy"; + const runId = "wfr_currentness_legacy"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-legacy", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // Entries written before boundary snapshots existed carry only a timestamp; without an + // orderable identity they must not count as the current invocation. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 41c99a20d80..73cc90e1a16 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10936,17 +10936,63 @@ export class WorkspaceService extends EventEmitter { assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); - let outcome: "invocation" | "consumed" | "superseded" | null = null; - let boundaryAtMs: number | null = null; + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + if (decision.status === "error") { + return false; + } + if (decision.status === "found" && decision.outcome === "invocation") { + return true; + } + + // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave + // no recognizable invocation part in history, so the backward walk above stops at the prior + // real user message (or, after a delivered result, at that consumed terminal message) and + // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the + // agent-workflow-runs sidecar, which snapshots the ID of the decision row that was newest + // at record time: the run is current exactly when that row is still the newest decision row. + // Row identity, not wall-clock ordering, so a backward clock correction can neither strand + // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a + // consumed boundary, equality means a background resume/retry was recorded after the prior + // result was delivered. A decision-free history fails safe to not-current, because a full + // clear (truncateHistory) removes every row WITHOUT appending a reset boundary while + // leaving the sidecar intact — a surviving reference must not inject a workflow result into + // the freshly cleared conversation. Legacy references without a boundary snapshot fail safe + // the same way. + if (decision.status === "none") { + return false; + } + const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + const reference = references.find((candidate) => candidate.runId === runId); + if (reference?.afterBoundaryMessageId == null) { + return false; + } + return reference.afterBoundaryMessageId === decision.messageId; + } + + /** + * The newest invocation-decision row for this run: a manual user/reset supersession, a + * consumed terminal result for the run, or a direct invocation part. Shared by + * isWorkflowInvocationCurrent and the sidecar record path so both sides of the identity + * comparison classify rows identically. + */ + private async findWorkflowInvocationDecisionRow( + workspaceId: string, + runId: string + ): Promise< + | { status: "found"; outcome: "invocation" | "consumed" | "superseded"; messageId: string } + | { status: "none" } + | { status: "error" } + > { + const state: { + found: { outcome: "invocation" | "consumed" | "superseded"; messageId: string } | null; + } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { - outcome = "superseded"; - const timestamp = message.metadata?.timestamp; - boundaryAtMs = typeof timestamp === "number" ? timestamp : null; + state.found = { outcome: "superseded", messageId: message.id }; return false; } if ( @@ -10954,13 +11000,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - outcome = "consumed"; - const timestamp = message.metadata?.timestamp; - boundaryAtMs = typeof timestamp === "number" ? timestamp : null; + state.found = { outcome: "consumed", messageId: message.id }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - outcome = "invocation"; + state.found = { outcome: "invocation", messageId: message.id }; return false; } } @@ -10973,34 +11017,27 @@ export class WorkspaceService extends EventEmitter { runId, error: historyResult.error, }); - return false; - } - - if (outcome === "invocation") { - return true; + return { status: "error" }; } + return state.found != null + ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } + : { status: "none" }; + } - // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave - // no recognizable invocation part in history, so the backward walk above stops at the prior - // real user message (or, after a delivered result, at that consumed terminal message) and - // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the - // agent-workflow-runs sidecar: a reference recorded after that boundary counts as the - // current invocation. For a consumed boundary that means a background resume/retry issued - // after the prior result was delivered. The fallback requires a datable boundary: an - // undatable boundary fails safe to not-current (mirroring - // TaskService.listAgentReferencedWorkflowRunIds), and so does a decision-free history, - // because a full clear (truncateHistory) removes every row WITHOUT appending a reset - // boundary while leaving the sidecar intact — a surviving reference must not inject a - // workflow result into the freshly cleared conversation. - if (boundaryAtMs === null) { - return false; - } - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - const reference = references.find((candidate) => candidate.runId === runId); - if (reference == null) { - return false; - } - return reference.createdAtMs > boundaryAtMs; + /** + * Boundary snapshot for the agent-workflow-runs sidecar: the message ID of the newest + * invocation-decision row for this run, or null when history has none. Recorded at + * background launch/resume so isWorkflowInvocationCurrent can compare row identity instead + * of wall-clock timestamps, which clock corrections can reorder. + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + assert(workspaceId.length > 0, "getWorkflowInvocationBoundaryMessageId requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationBoundaryMessageId requires runId"); + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + return decision.status === "found" ? decision.messageId : null; } /** From 046286a0d1c344f9184f8a5200a9a31bb8b60d98 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:40:55 +0000 Subject: [PATCH 08/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20never=20persist=20a?= =?UTF-8?q?=20boundary=20snapshot=20from=20an=20unreadable=20history;=20de?= =?UTF-8?q?fer=20drains=20on=20indeterminate=20currentness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 53 +++++++++++++++ src/node/services/taskService.ts | 58 +++++++++++----- src/node/services/tools/toolUtils.ts | 35 +++++++--- src/node/services/tools/workflow_run.test.ts | 38 +++++++++++ src/node/services/workspaceService.test.ts | 71 ++++++++++++++++++++ src/node/services/workspaceService.ts | 34 ++++++++-- 6 files changed, 256 insertions(+), 33 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b4d4131a4d6..d444e4402b3 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -665,6 +665,14 @@ function createWorkspaceServiceMocks( mock((_workspaceId: string, _message: WorkspaceChatMessage) => undefined); const isWorkflowInvocationCurrent = overrides?.isWorkflowInvocationCurrent ?? mock(() => Promise.resolve(true)); + // Derived from the boolean mock so tests that override isWorkflowInvocationCurrent keep + // steering the drain's three-state check. + const getWorkflowInvocationCurrentness = mock( + async (workspaceId: string, runId: string) => + ((await isWorkflowInvocationCurrent(workspaceId, runId)) === true + ? "current" + : "not_current") as "current" | "not_current" | "indeterminate" + ); const countQueuedAgentPeerMessages = overrides?.countQueuedAgentPeerMessages ?? mock(() => 0); // Granted by default (no live user activity): interrupt_active tests exercise the // interruption/archive flow; the hold's own refusal logic lives in workspaceService.test.ts. @@ -728,6 +736,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, + getWorkflowInvocationCurrentness, countQueuedAgentPeerMessages, } as unknown as WorkspaceService, create, @@ -6239,6 +6248,50 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("terminal workflow wake-up defers when history is unreadable", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // History unreadable at drain time: currentness is indeterminate, so the notification must + // stay pending for a later drain instead of being tombstoned as superseded. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 42ef463b37b..17cc65a065d 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7893,7 +7893,9 @@ export class TaskService { private async buildWorkflowTerminalPrompt( ownerWorkspaceId: string, runId: string - ): Promise { + ): Promise< + { outcome: "deliver"; prompt: string } | { outcome: "superseded" } | { outcome: "defer" } + > { assert(ownerWorkspaceId.length > 0, "buildWorkflowTerminalPrompt requires ownerWorkspaceId"); assert(runId.length > 0, "buildWorkflowTerminalPrompt requires runId"); const runStore = new WorkflowRunStore({ @@ -7908,25 +7910,39 @@ export class TaskService { runId, error: getErrorMessage(error), }); - return null; + return { outcome: "superseded" }; } if ( run.workspaceId !== ownerWorkspaceId || run.parentWorkflow != null || - !isTerminalWorkflowRunStatus(run.status) || - !(await this.workspaceService.isWorkflowInvocationCurrent(ownerWorkspaceId, run.id)) + !isTerminalWorkflowRunStatus(run.status) ) { - return null; + return { outcome: "superseded" }; + } + const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( + ownerWorkspaceId, + run.id + ); + // Indeterminate means history was unreadable, not that the run was superseded: tombstoning + // now would permanently drop the wake over a transient fault, so defer and retry instead. + if (currentness === "indeterminate") { + return { outcome: "defer" }; + } + if (currentness === "not_current") { + return { outcome: "superseded" }; } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; - return buildWorkflowResultContextMessage({ - rawCommand: `workflow_run ${scriptPath}`, - name: scriptPath, - runId: run.id, - status: run.status, - result: null, - run, - }); + return { + outcome: "deliver", + prompt: buildWorkflowResultContextMessage({ + rawCommand: `workflow_run ${scriptPath}`, + name: scriptPath, + runId: run.id, + status: run.status, + result: null, + run, + }), + }; } private async ensureAgentTerminalMessages( @@ -8242,7 +8258,19 @@ export class TaskService { ownerWorkspaceId, notification.sourceId ); - if (workflowPrompt == null) { + if (workflowPrompt.outcome === "defer") { + // Currentness was indeterminate (history unreadable): keep the notification pending so + // the next drain trigger (a later terminal event, idle scheduling, or startup recovery) + // retries it, rather than permanently dropping the wake. No active reschedule here: an + // idle-wait resolves immediately on an idle owner and would busy-loop while the fault + // persists. + log.warn("Deferring workflow terminal attention; history unavailable", { + ownerWorkspaceId, + runId: notification.sourceId, + }); + continue; + } + if (workflowPrompt.outcome === "superseded") { // Dropping a notify_on_terminal wake strands the run's owner; keep the drop diagnosable. log.warn("Dropping superseded workflow terminal attention", { ownerWorkspaceId, @@ -8252,7 +8280,7 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); - promptSections.push(workflowPrompt); + promptSections.push(workflowPrompt.prompt); } // Sub-agent reports and failures are already durable user-context messages. Resume from history diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 2fa64f7a332..a56f4574762 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -97,22 +97,35 @@ export async function recordBackgroundWorkflowRunReference( return; } + // Snapshot which invocation-decision row is newest at launch so the terminal-wake + // currentness check compares row identity instead of wall-clock order, which clock + // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). A history read + // failure must not be persisted as a verified-empty boundary (null): record without the + // field instead, so the run stays rediscoverable (listAgentReferencedWorkflowRunIds) and a + // later workflow_resume re-record can repair provenance, while the unverifiable boundary + // fails safe for wake delivery. + let afterBoundaryMessageId: string | null | undefined; + const taskService = config.taskService; + if (config.workspaceId != null && taskService?.getWorkflowInvocationBoundaryMessageId != null) { + try { + afterBoundaryMessageId = await taskService.getWorkflowInvocationBoundaryMessageId( + config.workspaceId, + runId + ); + } catch (error: unknown) { + log.error("Failed to snapshot workflow invocation boundary for run reference", { + runId, + error: getErrorMessage(error), + }); + } + } + try { - // Snapshot which invocation-decision row is newest at launch so the terminal-wake - // currentness check compares row identity instead of wall-clock order, which clock - // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). - const afterBoundaryMessageId = - config.workspaceId != null - ? ((await config.taskService?.getWorkflowInvocationBoundaryMessageId?.( - config.workspaceId, - runId - )) ?? null) - : null; await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs, - afterBoundaryMessageId, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), }); } catch (error: unknown) { log.warn("Failed to record agent workflow run reference", { diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index d8779d47e1b..05064896ffa 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -694,6 +694,44 @@ describe("workflow_run tool", () => { expect(result).toEqual({ status: "running", runId: "wfr_background", result: null }); }); + test("records a rediscovery-only reference when the boundary snapshot fails", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-boundary-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + const startWorkflowInBackground = mock(async () => ({ + runId: "wfr_boundary_error", + status: "running" as const, + result: null, + })); + const getWorkflowInvocationBoundaryMessageId = mock(async () => { + throw new Error("history read failed"); + }); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { + startWorkflow: mock(async () => { + throw new Error("foreground start should not be used"); + }), + startWorkflowInBackground, + getRun: mock(async () => null), + }, + }); + + const result = await tool.execute!( + { script_path: scriptPath, args: { topic: "workflow tools" }, run_in_background: true }, + mockToolCallOptions + ); + expect(result).toEqual({ status: "running", runId: "wfr_boundary_error", result: null }); + + // A read failure must not persist a verified-empty boundary (null): the entry keeps the run + // rediscoverable while a later resume re-record can repair provenance. + const references = await readAgentWorkflowRunReferences(tempDir.path); + expect(references).toHaveLength(1); + expect(references[0]?.runId).toBe("wfr_boundary_error"); + expect(references[0] != null && "afterBoundaryMessageId" in references[0]).toBe(false); + }); + test("requires the workflow service", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-missing"); const scriptPath = await writeWorkflowScript(tempDir.path); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7ba2643cf5e..d9eee2e3740 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6183,6 +6183,77 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("treats an unreadable history as indeterminate, not superseded", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-io-error"; + const runId = "wfr_currentness_io_error"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-io-error", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + const readSpy = spyOn(historyService, "iterateFullHistory").mockResolvedValue( + Err("disk read failed") + ); + try { + // The drain distinguishes a read failure (retain and retry) from supersession + // (tombstone); the boolean view stays fail-safe false for non-destructive callers. + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + // The record path must fail loudly instead of persisting a verified-empty boundary that + // would permanently strand the run's wake after storage recovers. + let boundaryError: unknown; + try { + await workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); + } catch (error: unknown) { + boundaryError = error; + } + expect(String(boundaryError)).toContain("boundary unavailable"); + } finally { + readSpy.mockRestore(); + } + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" + ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 73cc90e1a16..472fc748390 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10933,15 +10933,29 @@ export class WorkspaceService extends EventEmitter { } async isWorkflowInvocationCurrent(workspaceId: string, runId: string): Promise { - assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); - assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); + return (await this.getWorkflowInvocationCurrentness(workspaceId, runId)) === "current"; + } + + /** + * Three-state currentness: "indeterminate" means history could not be read, so the answer is + * unknown rather than no. Callers that would permanently drop a terminal wake on a negative + * answer (the terminal-attention drain tombstones notifications) must retain and retry on + * "indeterminate" instead; boolean callers treat it as not-current, the pre-existing + * fail-safe for non-destructive decisions. + */ + async getWorkflowInvocationCurrentness( + workspaceId: string, + runId: string + ): Promise<"current" | "not_current" | "indeterminate"> { + assert(workspaceId.length > 0, "getWorkflowInvocationCurrentness requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationCurrentness requires runId"); const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); if (decision.status === "error") { - return false; + return "indeterminate"; } if (decision.status === "found" && decision.outcome === "invocation") { - return true; + return "current"; } // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave @@ -10959,14 +10973,14 @@ export class WorkspaceService extends EventEmitter { // the freshly cleared conversation. Legacy references without a boundary snapshot fail safe // the same way. if (decision.status === "none") { - return false; + return "not_current"; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); const reference = references.find((candidate) => candidate.runId === runId); if (reference?.afterBoundaryMessageId == null) { - return false; + return "not_current"; } - return reference.afterBoundaryMessageId === decision.messageId; + return reference.afterBoundaryMessageId === decision.messageId ? "current" : "not_current"; } /** @@ -11037,6 +11051,12 @@ export class WorkspaceService extends EventEmitter { assert(workspaceId.length > 0, "getWorkflowInvocationBoundaryMessageId requires workspaceId"); assert(runId.length > 0, "getWorkflowInvocationBoundaryMessageId requires runId"); const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + // A read failure must not masquerade as a verified-empty history: persisting null would + // permanently fail the run's currentness check even after storage recovers. Throw so the + // record path can distinguish and record a rediscovery-only reference instead. + if (decision.status === "error") { + throw new Error("workflow invocation boundary unavailable: history read failed"); + } return decision.status === "found" ? decision.messageId : null; } From e5de1dd3d12a3d8cb05da80e2db5c308f933860c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:00:58 +0000 Subject: [PATCH 09/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20migrate=20pre-snaps?= =?UTF-8?q?hot=20sidecar=20references=20through=20a=20wall-clock=20fallbac?= =?UTF-8?q?k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/workspaceService.test.ts | 22 ++++++++-- src/node/services/workspaceService.ts | 49 ++++++++++++++++++---- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index d9eee2e3740..22cb999be36 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6137,7 +6137,7 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("fails safe for legacy sidecar references without a boundary snapshot", async () => { + test("migrates legacy sidecar references through the wall-clock fallback", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-legacy"; const runId = "wfr_currentness_legacy"; @@ -6169,13 +6169,29 @@ describe("WorkspaceService workflow invocation events", () => { workspaceId, createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); - // Entries written before boundary snapshots existed carry only a timestamp; without an - // orderable identity they must not count as the current invocation. + // Entries written before boundary snapshots existed carry only a timestamp. An in-flight + // run recorded after the newest boundary must keep its wake across the upgrade. await recordAgentWorkflowRunReference({ workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A newer boundary still supersedes a legacy entry. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // An undatable boundary cannot be ordered against a legacy timestamp: fail safe. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-undated", "user", "another instruction", {}) + ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); } finally { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 472fc748390..6dffcdb66b0 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10970,14 +10970,29 @@ export class WorkspaceService extends EventEmitter { // result was delivered. A decision-free history fails safe to not-current, because a full // clear (truncateHistory) removes every row WITHOUT appending a reset boundary while // leaving the sidecar intact — a surviving reference must not inject a workflow result into - // the freshly cleared conversation. Legacy references without a boundary snapshot fail safe - // the same way. + // the freshly cleared conversation. References without a boundary snapshot (pre-upgrade + // entries, record-time read failures) take a wall-clock migration fallback below. if (decision.status === "none") { return "not_current"; } const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); const reference = references.find((candidate) => candidate.runId === runId); - if (reference?.afterBoundaryMessageId == null) { + if (reference == null) { + return "not_current"; + } + if (reference.afterBoundaryMessageId === undefined) { + // Migration fallback: entries written before boundary snapshots existed (or after a + // record-time history read failure) carry only a timestamp, and an in-flight run must not + // lose its wake across the upgrade. Fall back to wall-clock ordering against a datable + // boundary; an undatable boundary fails safe. All new records take the identity path + // above, so clock-correction edge cases are confined to this shrinking population. + if (decision.timestampMs == null) { + return "not_current"; + } + return reference.createdAtMs > decision.timestampMs ? "current" : "not_current"; + } + if (reference.afterBoundaryMessageId === null) { + // Verified-empty snapshot: a decision row now exists, so it appeared after the record. return "not_current"; } return reference.afterBoundaryMessageId === decision.messageId ? "current" : "not_current"; @@ -10993,20 +11008,31 @@ export class WorkspaceService extends EventEmitter { workspaceId: string, runId: string ): Promise< - | { status: "found"; outcome: "invocation" | "consumed" | "superseded"; messageId: string } + | { + status: "found"; + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + timestampMs: number | null; + } | { status: "none" } | { status: "error" } > { const state: { - found: { outcome: "invocation" | "consumed" | "superseded"; messageId: string } | null; + found: { + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + timestampMs: number | null; + } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { + const timestamp = message.metadata?.timestamp; + const timestampMs = typeof timestamp === "number" ? timestamp : null; if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { - state.found = { outcome: "superseded", messageId: message.id }; + state.found = { outcome: "superseded", messageId: message.id, timestampMs }; return false; } if ( @@ -11014,11 +11040,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - state.found = { outcome: "consumed", messageId: message.id }; + state.found = { outcome: "consumed", messageId: message.id, timestampMs }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - state.found = { outcome: "invocation", messageId: message.id }; + state.found = { outcome: "invocation", messageId: message.id, timestampMs }; return false; } } @@ -11034,7 +11060,12 @@ export class WorkspaceService extends EventEmitter { return { status: "error" }; } return state.found != null - ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } + ? { + status: "found", + outcome: state.found.outcome, + messageId: state.found.messageId, + timestampMs: state.found.timestampMs, + } : { status: "none" }; } From 60bb82d89f7325043832fd833c14df998ae2e3f6 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:14:54 +0000 Subject: [PATCH 10/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20deliver=20kernel=20?= =?UTF-8?q?workflow=20wakes=20launched=20from=20a=20decision-free=20histor?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A kernel background workflow launched from a synthetic turn in a new or fully cleared workspace records a verified-empty boundary snapshot (afterBoundaryMessageId: null), but getWorkflowInvocationCurrentness declared every decision-free history not_current, permanently superseding the run's terminal wake. Honor the verified-empty snapshot: null matching a still decision-free history means the launch context is unchanged, so the run is current. References pointing at a cleared row or lacking a verified snapshot keep failing safe, preserving the full-clear protection. --- src/node/services/workspaceService.test.ts | 65 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 21 ++++--- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 22cb999be36..0ac43c1a3cd 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6079,6 +6079,71 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("delivers kernel launches recorded against a decision-free history", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-empty"; + const runId = "wfr_currentness_empty"; + const legacyRunId = "wfr_currentness_empty_legacy"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-empty", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A kernel launch from a synthetic turn in a new (or fully cleared) workspace records a + // verified-empty snapshot (null). History still having no decision row means the launch + // context is unchanged, so the wake must deliver. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A reference without a verified snapshot cannot claim the empty history as its launch + // context; it may merely have survived a full clear. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: legacyRunId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, legacyRunId)).toBe( + false + ); + + // A decision row appearing after the launch supersedes the verified-empty snapshot. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6dffcdb66b0..1c6d3c18389 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10967,16 +10967,21 @@ export class WorkspaceService extends EventEmitter { // Row identity, not wall-clock ordering, so a backward clock correction can neither strand // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a // consumed boundary, equality means a background resume/retry was recorded after the prior - // result was delivered. A decision-free history fails safe to not-current, because a full - // clear (truncateHistory) removes every row WITHOUT appending a reset boundary while - // leaving the sidecar intact — a surviving reference must not inject a workflow result into - // the freshly cleared conversation. References without a boundary snapshot (pre-upgrade - // entries, record-time read failures) take a wall-clock migration fallback below. - if (decision.status === "none") { - return "not_current"; - } + // result was delivered. References without a boundary snapshot (pre-upgrade entries, + // record-time read failures) take a wall-clock migration fallback below. const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); const reference = references.find((candidate) => candidate.runId === runId); + if (decision.status === "none") { + // A decision-free history is current only for a reference whose snapshot verified an + // empty history at record time (null): kernel launches from a new or fully cleared + // workspace (e.g. a heartbeat turn) have no decision row before or after, and their wake + // must still deliver. Every other surviving reference fails safe, because a full clear + // (truncateHistory) removes every row WITHOUT appending a reset boundary while leaving + // the sidecar intact, and a reference pointing at a cleared row, or one without a + // verified snapshot, must not inject a workflow result into the freshly cleared + // conversation. + return reference?.afterBoundaryMessageId === null ? "current" : "not_current"; + } if (reference == null) { return "not_current"; } From a73a35428447759bce3eefc3b18f19b7b09ac5da Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:36:23 +0000 Subject: [PATCH 11/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20kernel=20w?= =?UTF-8?q?orkflow=20wake=20delivery=20against=20sidecar=20faults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reject sidecar entries whose afterBoundaryMessageId is present but invalid (empty string or non-string) instead of migrating them into the wall-clock legacy fallback, where they could outrank a newer boundary during tolerated clock skew. - Propagate non-ENOENT sidecar read failures instead of flattening them to an empty list; workflow currentness reports indeterminate so the terminal drain defers rather than tombstoning the wake, while rediscovery listings skip the pass and record keeps its atomic rewrite. - Arm a bounded per-owner retry timer when a drain defers on indeterminate currentness: an already-idle owner otherwise produces no further drain trigger until restart. --- .../agentWorkflowRunReferences.test.ts | 55 +++++++++++++++ .../services/agentWorkflowRunReferences.ts | 44 +++++++++--- src/node/services/taskService.test.ts | 63 +++++++++++++++++ src/node/services/taskService.ts | 47 +++++++++++-- src/node/services/workspaceService.test.ts | 69 +++++++++++++++++++ src/node/services/workspaceService.ts | 20 +++++- 6 files changed, 282 insertions(+), 16 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 2747eb1fe65..d4e893ef177 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -114,6 +114,61 @@ describe("agent workflow run references", () => { } }); + test("rejects entries with a present-but-invalid boundary snapshot", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // "" and non-string values are corruption, not legacy records: migrating them into the + // wall-clock fallback could outrank a newer boundary during tolerated clock skew. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_empty_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "" }, + { runId: "wfr_numeric_boundary", createdAtMs: 1_000, afterBoundaryMessageId: 42 }, + { runId: "wfr_valid_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "row-1" }, + { runId: "wfr_null_boundary", createdAtMs: 1_000, afterBoundaryMessageId: null }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(new Set(references.map((reference) => reference.runId))).toEqual( + new Set(["wfr_valid_boundary", "wfr_null_boundary"]) + ); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("propagates non-ENOENT read failures instead of flattening them to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A directory at the file path fails reads with EISDIR. Callers deciding wake delivery + // must observe the failure rather than "no references". + await fs.mkdir(path.join(workspaceSessionDir, "agent-workflow-runs.json")); + let readError: unknown; + try { + await readAgentWorkflowRunReferences(workspaceSessionDir); + } catch (error: unknown) { + readError = error; + } + expect(String(readError)).toContain("EISDIR"); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("self-heals unparseable file contents to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Unlike a failed read, corrupted contents cannot be repaired by rereading. + await fs.writeFile(path.join(workspaceSessionDir, "agent-workflow-runs.json"), "{not json"); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("clamps future-dated createdAtMs to the current time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 58f23023f9e..04aab142e65 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -63,13 +63,24 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { continue; } + const hasBoundary = "afterBoundaryMessageId" in record; const boundaryRaw = record.afterBoundaryMessageId; - const afterBoundaryMessageId = - typeof boundaryRaw === "string" && boundaryRaw.length > 0 + // A present-but-invalid snapshot ("" or a non-string) is corruption, not a legacy record: + // migrating it into the wall-clock fallback could let a stale reference outrank a newer + // boundary within the tolerated clock skew. Reject the entry; absence stays reserved for + // records that genuinely predate the field. + if ( + hasBoundary && + boundaryRaw !== null && + (typeof boundaryRaw !== "string" || boundaryRaw.length === 0) + ) { + continue; + } + const afterBoundaryMessageId = hasBoundary + ? typeof boundaryRaw === "string" ? boundaryRaw - : boundaryRaw === null - ? null - : undefined; + : null + : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. @@ -88,13 +99,23 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { export async function readAgentWorkflowRunReferences( workspaceSessionDir: string ): Promise { + let raw: string; try { - const raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); - return parseReferences(JSON.parse(raw) as unknown); + raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); } catch (error: unknown) { if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { return []; } + // For kernel-launched runs this file is the only durable invocation evidence, and callers + // deciding wake delivery must distinguish "no reference" from "cannot know right now": + // flattening a transient read failure into [] would let the terminal drain tombstone the + // run's wake. Corrupted contents below stay self-healing because rereading cannot repair + // them, while a failed read can succeed later. + throw error; + } + try { + return parseReferences(JSON.parse(raw) as unknown); + } catch { return []; } } @@ -109,7 +130,14 @@ export async function recordAgentWorkflowRunReference(input: { const filePath = referencesPath(input.workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + let existing: AgentWorkflowRunReference[]; + try { + existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + } catch { + // Recording must survive an unreadable file: the atomic rewrite below replaces it, and + // failing here would leave the new run without any sidecar entry, stranding its wake. + existing = []; + } const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index d444e4402b3..2466f5292ea 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6292,6 +6292,69 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); }); + test("deferred terminal wake-up retries on the bounded timer", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer_retry"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // The first drain sees a transient storage fault that clears before the retry fires. An + // already-idle owner produces no other drain trigger, so only the bounded retry delivers. + let currentnessCalls = 0; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => { + currentnessCalls += 1; + return Promise.resolve(currentnessCalls === 1 ? "indeterminate" : "current"); + }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + ( + taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } + ).terminalAttentionDeferRetryDelayMs = 10; + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + + // Real timers: poll until the armed retry fires and the follow-up drain delivers. + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + await flushTerminalAttentionDrains(taskService); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 17cc65a065d..785df8ba671 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -862,6 +862,11 @@ function isWorkspaceBusyIdleOnlySend(error: unknown): boolean { const REMOVED_AGENT_TASKS_DIR = "removed-agent-tasks"; const COMPLETED_REPORT_CACHE_MAX_ENTRIES = 128; +// Retry cadence for terminal-attention drains deferred by indeterminate workflow currentness +// (unreadable history/sidecar). There is no deterministic "storage recovered" signal, so a +// bounded timer is the re-trigger; each retry that defers again arms the next one. +const TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS = 30_000; + /** Maximum consecutive auto-resumes before stopping. Prevents infinite loops when descendants are stuck. */ // Task-recovery paths must stay deterministic and editing-capable even when // workspace/default agent preferences evolve (e.g., auto router defaults). @@ -1538,6 +1543,13 @@ export class TaskService { // tests and shutdown can await them; drains are idempotent and re-triggered on owner idle events. private readonly pendingTerminalAttentionDrainsByOwner = new Map>(); private readonly pendingTerminalAttentionDrains = new Set>(); + // One armed defer-retry timer per owner (see scheduleTerminalAttentionDeferRetry). The delay + // is a field, not a constant, so tests can shrink it without waiting out the real backoff. + private readonly terminalAttentionDeferRetryTimers = new Map< + string, + ReturnType + >(); + private terminalAttentionDeferRetryDelayMs = TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS; private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -1690,7 +1702,13 @@ export class TaskService { } const runIds = new Set(); - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + let references: Awaited> = []; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. + log.warn("Failed to read agent workflow run references", { workspaceId, error }); + } for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -7872,6 +7890,24 @@ export class TaskService { this.pendingTerminalAttentionDrains.add(promise); } + /** + * A deferred (indeterminate) terminal wake has no deterministic "storage recovered" signal + * to re-trigger the drain, and an idle-wait would resolve immediately on an already-idle + * owner and busy-loop while the fault persists. Retry on a bounded timer instead, one armed + * timer per owner; each retry that defers again arms the next one. + */ + private scheduleTerminalAttentionDeferRetry(ownerWorkspaceId: string): void { + if (this.terminalAttentionDeferRetryTimers.has(ownerWorkspaceId)) { + return; + } + const timer = setTimeout(() => { + this.terminalAttentionDeferRetryTimers.delete(ownerWorkspaceId); + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + }, this.terminalAttentionDeferRetryDelayMs); + timer.unref?.(); + this.terminalAttentionDeferRetryTimers.set(ownerWorkspaceId, timer); + } + private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { const promise = this.workspaceService .waitForIdleAndNoQueuedMessages(ownerWorkspaceId) @@ -8259,15 +8295,14 @@ export class TaskService { notification.sourceId ); if (workflowPrompt.outcome === "defer") { - // Currentness was indeterminate (history unreadable): keep the notification pending so - // the next drain trigger (a later terminal event, idle scheduling, or startup recovery) - // retries it, rather than permanently dropping the wake. No active reschedule here: an - // idle-wait resolves immediately on an idle owner and would busy-loop while the fault - // persists. + // Currentness was indeterminate (history or sidecar unreadable): keep the notification + // pending rather than permanently dropping the wake, and arm a bounded retry, because an + // already-idle owner produces no further drain trigger on its own. log.warn("Deferring workflow terminal attention; history unavailable", { ownerWorkspaceId, runId: notification.sourceId, }); + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); continue; } if (workflowPrompt.outcome === "superseded") { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 0ac43c1a3cd..9fb652f6cf0 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6335,6 +6335,75 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("treats an unreadable sidecar as indeterminate, not superseded", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-sidecar-error"; + const runId = "wfr_currentness_sidecar_error"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-sidecar-error", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The sidecar is the only invocation evidence for kernel-launched runs: an unreadable + // file must read as "cannot know right now", not "no reference", or the drain would + // tombstone the wake on a transient storage fault. + const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); + await fsPromises.rm(sidecarPath); + await fsPromises.mkdir(sidecarPath); + try { + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + } finally { + await fsPromises.rmdir(sidecarPath); + } + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" + ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test.each(["workflow_run", "workflow_resume"] as const)( "treats terminal %s output as a consumed workflow result", async (toolName) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 1c6d3c18389..c9a3182dd62 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3,7 +3,10 @@ import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { EventEmitter } from "events"; import * as path from "path"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; -import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import { + readAgentWorkflowRunReferences, + type AgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; @@ -10969,7 +10972,20 @@ export class WorkspaceService extends EventEmitter { // consumed boundary, equality means a background resume/retry was recorded after the prior // result was delivered. References without a boundary snapshot (pre-upgrade entries, // record-time read failures) take a wall-clock migration fallback below. - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + let references: AgentWorkflowRunReference[]; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // The sidecar is the only invocation evidence a kernel-launched run has, so an + // unreadable file is "cannot know right now", not "no reference": defer wake decisions + // exactly like an unreadable history. + log.warn("Could not read workflow run references for currentness", { + workspaceId, + runId, + error, + }); + return "indeterminate"; + } const reference = references.find((candidate) => candidate.runId === runId); if (decision.status === "none") { // A decision-free history is current only for a reference whose snapshot verified an From 983d3c5eac4e32446262960e774cb4d0f0e16b8c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:46:45 +0000 Subject: [PATCH 12/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20retire=20kernel=20w?= =?UTF-8?q?orkflow=20run=20references=20on=20a=20full=20history=20clear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verified-empty (null) boundary snapshot recorded before a full clear is indistinguishable from one recorded after it, because the clear removes every row without appending a reset boundary. A pre-clear reference could therefore inject its workflow result into the freshly cleared conversation. Retire the sidecar with the transcript, durably like the post-compaction carryover discard; a post-clear resume re-records provenance. --- .../services/agentWorkflowRunReferences.ts | 14 +++++ src/node/services/workspaceService.test.ts | 57 +++++++++++++++++++ src/node/services/workspaceService.ts | 15 +++++ 3 files changed, 86 insertions(+) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 04aab142e65..d4ef37394be 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -120,6 +120,20 @@ export async function readAgentWorkflowRunReferences( } } +/** + * Retire every reference for this workspace. A full history clear removes all rows without + * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded + * before the clear indistinguishable from one recorded after it; retiring the references with + * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear + * workflow_resume re-records provenance. + */ +export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { + const filePath = referencesPath(workspaceSessionDir); + await referenceFileLocks.withLock(filePath, async () => { + await fs.rm(filePath, { force: true }); + }); +} + export async function recordAgentWorkflowRunReference(input: { workspaceSessionDir: string; runId: string; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 9fb652f6cf0..ed803ed0890 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6144,6 +6144,63 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("retires kernel workflow run references on a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire"; + const runId = "wfr_currentness_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // Launched from a decision-free history: the verified-empty snapshot delivers. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A full clear returns history to decision-free, making the pre-clear null snapshot + // indistinguishable from a fresh empty-history launch; the clear must retire the + // reference so the stale result cannot inject into the fresh conversation. + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(true); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c9a3182dd62..6dfde259062 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4,6 +4,7 @@ import { EventEmitter } from "events"; import * as path from "path"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; import { + clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, type AgentWorkflowRunReference, } from "@/node/services/agentWorkflowRunReferences"; @@ -12958,6 +12959,20 @@ export class WorkspaceService extends EventEmitter { `be re-injected after a restart; retry once the session storage is writable.` ); } + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them with the transcript (a post-clear resume + // re-records provenance), durably like the carryover discard above. + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `History was cleared, but stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + + `result into the cleared conversation; retry once the session storage is writable.` + ); + } // The persistent RLM sandbox holds context DERIVED from the cleared // transcript (vars populated by code execution), and its latest durable // snapshot would restore it after a restart — later turns could read From 56c789b44da7e8229cf393bc894a5b169c2985e9 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:13:15 +0000 Subject: [PATCH 13/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20close=20round-10=20?= =?UTF-8?q?wake-delivery=20gaps:=20retirement=20ordering,=20record=20clobb?= =?UTF-8?q?er,=20coalesced=20replay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Retire kernel workflow run references immediately after the truncation commits, before post-clear steps (goal acknowledgment, carryover discard) that can fail and return early while leaving the transcript deleted. - Propagate a sidecar read failure out of recordAgentWorkflowRunReference instead of treating the file as empty: the atomic rewrite would replace valid-but-unreadable contents with only the new run, destroying every other run's durable provenance. The failed record is retryable; parse corruption still self-heals. - Recognize coalesced terminal-attention prompts as consumption during currentness checks: the drain's synthetic user row carries no workflow-result metadata, so a crash between durable acceptance and the outbox delivery mark would otherwise replay the same terminal result after restart. Payload blocks are parsed back and matched on the exact workflow.runId the builder wrote. --- src/common/utils/workflowRunMessages.ts | 35 +++++ .../agentWorkflowRunReferences.test.ts | 32 ++++ .../services/agentWorkflowRunReferences.ts | 14 +- src/node/services/workspaceService.test.ts | 147 ++++++++++++++++++ src/node/services/workspaceService.ts | 49 ++++-- 5 files changed, 255 insertions(+), 22 deletions(-) diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 9fc393e9f7e..66a2b3b5fa9 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,6 +190,41 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } +/** + * Recognize this run's result payload inside a coalesced terminal-attention prompt from the + * builder's own output format. The drain's synthetic user row can coalesce several runs into + * one message and carries no workflow-result metadata, so currentness checks must read the + * consumption evidence out of the text: each payload block is parsed back and matched on the + * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted + * inside another run's report cannot count as consumption. + */ +export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { + assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); + if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { + return false; + } + const blockPattern = new RegExp( + `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, + "g" + ); + for (const match of text.matchAll(blockPattern)) { + let payload: unknown; + try { + payload = JSON.parse(match[1] ?? ""); + } catch { + continue; + } + if (!isRecordValue(payload)) { + continue; + } + const workflow = payload.workflow; + if (isRecordValue(workflow) && workflow.runId === runId) { + return true; + } + } + return false; +} + export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index d4e893ef177..69171c081cf 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -158,6 +158,38 @@ describe("agent workflow run references", () => { } }); + test("record propagates a sidecar read failure instead of clobbering existing references", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_existing", + createdAtMs: 1_000, + }); + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + // Unreadable file, writable directory: the atomic rewrite could replace contents it + // never saw, destroying every other run's only durable provenance. + await fs.chmod(filePath, 0o000); + let recordError: unknown; + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_new", + createdAtMs: 2_000, + }); + } catch (error: unknown) { + recordError = error; + } finally { + await fs.chmod(filePath, 0o600); + } + expect(String(recordError)).toContain("EACCES"); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references.map((reference) => reference.runId)).toEqual(["wfr_existing"]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("self-heals unparseable file contents to empty", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index d4ef37394be..43a9d23a778 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -144,14 +144,12 @@ export async function recordAgentWorkflowRunReference(input: { const filePath = referencesPath(input.workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - let existing: AgentWorkflowRunReference[]; - try { - existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); - } catch { - // Recording must survive an unreadable file: the atomic rewrite below replaces it, and - // failing here would leave the new run without any sidecar entry, stranding its wake. - existing = []; - } + // A read failure propagates instead of being treated as empty: the atomic rewrite below + // would otherwise replace valid-but-momentarily-unreadable contents with only this run, + // destroying every other active run's sole durable provenance. The failed record is + // retryable (workflow_resume re-records), while parse corruption still self-heals to + // empty inside readAgentWorkflowRunReferences because rereading cannot repair it. + const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ed803ed0890..daac7889e48 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -57,6 +57,7 @@ import { WORKFLOW_RESULT_METADATA_TYPE, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, + buildWorkflowResultContextMessage, } from "@/common/utils/workflowRunMessages"; import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; @@ -6201,6 +6202,152 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("retires kernel workflow run references even when a later post-clear step fails", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-early"; + const runId = "wfr_currentness_retire_early"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-early", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + + // The truncation commits, then a later post-clear step fails. Retirement must already + // have happened, or the stale null-snapshot reference survives the committed clear and + // reads current against the emptied history. + const sessionAccessor = workspaceService as unknown as { + getOrCreateSession(id: string): { clearPostCompactionState(): Promise }; + }; + const session = sessionAccessor.getOrCreateSession(workspaceId); + const carryoverSpy = spyOn(session, "clearPostCompactionState").mockImplementationOnce(() => + Promise.reject(new Error("carryover discard failed")) + ); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + } finally { + carryoverSpy.mockRestore(); + } + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-coalesced"; + const runId = "wfr_currentness_coalesced"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-coalesced", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Another run's payload quoting nothing about this run must not count as consumption. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-other", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run other.js", + name: "other.js", + runId: "wfr_currentness_other", + status: "completed", + result: { reportMarkdown: "other done" }, + run: null, + }), + { timestamp: 1_250, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a + // crash between durable acceptance and the outbox delivery mark, this row is the only + // evidence the result already reached history; it must read as consumption or restart + // recovery injects the same terminal result again. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-result", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run research.js", + name: "research.js", + runId, + status: "completed", + result: { reportMarkdown: "done" }, + run: null, + }), + { timestamp: 1_300, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6dfde259062..bb95cabd55c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -204,6 +204,7 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, + textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -475,6 +476,23 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) ); } +/** + * The terminal-attention drain delivers workflow results as one synthetic user prompt that may + * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands + * between the send's durable acceptance and the outbox delivery mark, restart recovery drains + * the notification again; recognizing the accepted row as consumption is what suppresses the + * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is + * classified before this check runs. + */ +function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { + if (message.role !== "user" || message.metadata?.synthetic !== true) { + return false; + } + return message.parts.some( + (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) + ); +} + function isResetBoundaryMessage(message: MuxMessage): boolean { return message.metadata?.contextBoundaryKind === CONTEXT_BOUNDARY_KINDS.RESET; } @@ -11059,6 +11077,7 @@ export class WorkspaceService extends EventEmitter { } if ( isWorkflowResultContinuationMessage(message, runId) || + isCoalescedWorkflowResultMessage(message, runId) || isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { @@ -12901,6 +12920,22 @@ export class WorkspaceService extends EventEmitter { // admitted afterwards (their content references the discarded context). if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them immediately after the truncation commits, + // before any later post-clear step that can fail and return early (goal acknowledgment, + // carryover discard), or the stale reference would survive the committed clear. A + // post-clear resume re-records provenance. + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `History was cleared, but stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + + `result into the cleared conversation; retry once the session storage is writable.` + ); + } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the @@ -12959,20 +12994,6 @@ export class WorkspaceService extends EventEmitter { `be re-injected after a restart; retry once the session storage is writable.` ); } - // Kernel workflow run references belong to the cleared conversation: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one - // recorded after it, so a surviving reference could inject a pre-clear workflow result - // into the fresh conversation. Retire them with the transcript (a post-clear resume - // re-records provenance), durably like the carryover discard above. - try { - await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error) { - return Err( - `History was cleared, but stale workflow run references could not be retired ` + - `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + - `result into the cleared conversation; retry once the session storage is writable.` - ); - } // The persistent RLM sandbox holds context DERIVED from the cleared // transcript (vars populated by code execution), and its latest durable // snapshot would restore it after a restart — later turns could read From 521fba2b778747b7a727fd963e9a2b463f87274c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:30:53 +0000 Subject: [PATCH 14/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20close=20round-11=20?= =?UTF-8?q?gaps:=20record=20retry,=20indeterminate=20recovery,=20clear-emi?= =?UTF-8?q?t=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A failed provenance record now schedules bounded background retries (1s/10s/60s, reusing the launch-time boundary snapshot): the only natural re-record sites are a new dispatch and workflow_resume, which an untouched active run never hits, so a single failed write would permanently supersede its wake once storage recovers. - Startup recovery keeps indeterminate runs: it now consults three-state currentness and skips only not_current, because no pending notification exists yet to arm the drain's defer retry; the drain re-evaluates and defers or supersedes with full context. - A failed sidecar retirement after a committed full clear now emits the DeleteMessage before returning the cleanup error, so the renderer does not keep showing a transcript that no longer exists on disk. --- src/node/services/taskService.test.ts | 45 ++++++++++++++++ src/node/services/taskService.ts | 10 +++- src/node/services/tools/toolUtils.test.ts | 52 ++++++++++++++++++ src/node/services/tools/toolUtils.ts | 56 +++++++++++++++++++- src/node/services/workspaceService.test.ts | 61 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 38 +++++++++----- 6 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 src/node/services/tools/toolUtils.test.ts diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 2466f5292ea..23b0f450909 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6752,6 +6752,51 @@ describe("TaskService", () => { }); }); + test("initialize recovery keeps indeterminate workflow runs enqueued for a later drain", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_recovery_indeterminate"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // History/sidecar unreadable at startup. Recovery is the only reconstruction point for a + // wake that never reached the outbox, and no pending notification exists yet to arm the + // drain's defer retry, so skipping here would strand the run until another restart. The + // boolean wrapper collapses indeterminate to false, which is what recovery must NOT use. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + (workspaceService as unknown as Record).isWorkflowInvocationCurrent = mock( + () => Promise.resolve(false) + ); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + await taskService.initialize(); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + test("initialize recovers terminal notify workspace turns without pending notification", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 785df8ba671..1bc7797e56b 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7596,7 +7596,15 @@ export class TaskService { ) { continue; } - if (!(await this.workspaceService.isWorkflowInvocationCurrent(workspace.id, run.id))) { + const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( + workspace.id, + run.id + ); + // Indeterminate (unreadable history/sidecar) must still enqueue: startup recovery is + // the only reconstruction point for wakes that never reached the outbox, and no + // pending notification exists yet to arm the drain's defer retry. The drain + // re-evaluates currentness and defers or supersedes with full context. + if (currentness === "not_current") { continue; } const created = await this.terminalAttentionStore.enqueueIfAbsent({ diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts new file mode 100644 index 00000000000..7552b08a7da --- /dev/null +++ b/src/node/services/tools/toolUtils.test.ts @@ -0,0 +1,52 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; + +import { describe, expect, test } from "bun:test"; + +import type { ToolConfiguration } from "@/common/utils/tools/tools"; +import { + readAgentWorkflowRunReferences, + recordAgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; +import { recordBackgroundWorkflowRunReference } from "@/node/services/tools/toolUtils"; + +describe("recordBackgroundWorkflowRunReference", () => { + test("retries a failed provenance record until storage recovers", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-record-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_existing", + createdAtMs: 1_000, + }); + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + // Unreadable at record time: the tool has already returned by the time storage recovers, + // and an untouched active run never hits a natural re-record site, so only the bounded + // background retry can persist provenance for the terminal wake. + await fs.chmod(filePath, 0o000); + await recordBackgroundWorkflowRunReference( + { workspaceSessionDir } as unknown as ToolConfiguration, + "wfr_retry", + 2_000, + [25, 25, 25] + ); + await fs.chmod(filePath, 0o600); + + const deadline = Date.now() + 5_000; + let runIds: string[] = []; + while (Date.now() < deadline) { + runIds = (await readAgentWorkflowRunReferences(workspaceSessionDir)).map( + (reference) => reference.runId + ); + if (runIds.includes("wfr_retry")) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(new Set(runIds)).toEqual(new Set(["wfr_existing", "wfr_retry"])); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index a56f4574762..bf90c448775 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -79,6 +79,51 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } +// Bounded background retries for a transiently unreadable sidecar. The only natural re-record +// sites are a new dispatch and workflow_resume, which an untouched active run never hits, so +// giving up after one failed write would permanently supersede the run's terminal wake once +// storage recovers. Retries reuse the launch-time boundary snapshot: provenance describes the +// launch, not the retry moment. +const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; + +function scheduleRecordReferenceRetry(input: { + workspaceSessionDir: string; + runId: string; + createdAtMs: number; + afterBoundaryMessageId: string | null | undefined; + retryDelaysMs: readonly number[]; + attempt: number; +}): void { + const delayMs = input.retryDelaysMs[input.attempt]; + if (delayMs == null) { + log.error("Giving up on agent workflow run reference record after retries", { + runId: input.runId, + attempts: input.attempt, + }); + return; + } + const timer = setTimeout(() => { + // Detached by design: the launching tool already returned, so only this chain can finish + // the write. Failures reschedule until the bounded delays are exhausted. + void recordAgentWorkflowRunReference({ + workspaceSessionDir: input.workspaceSessionDir, + runId: input.runId, + createdAtMs: input.createdAtMs, + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), + }).catch((error: unknown) => { + log.warn("Agent workflow run reference record retry failed", { + runId: input.runId, + attempt: input.attempt + 1, + error: getErrorMessage(error), + }); + scheduleRecordReferenceRetry({ ...input, attempt: input.attempt + 1 }); + }); + }, delayMs); + timer.unref?.(); +} + /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -89,7 +134,8 @@ export async function emitWorkflowRunAttachedEvent(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number + createdAtMs: number, + retryDelaysMs?: readonly number[] | null ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { @@ -132,5 +178,13 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); + scheduleRecordReferenceRetry({ + workspaceSessionDir, + runId, + createdAtMs, + afterBoundaryMessageId, + retryDelaysMs: retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS, + attempt: 0, + }); } } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index daac7889e48..45322ca77af 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6266,6 +6266,67 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a failed reference retirement still emits the committed clear to the renderer", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-emit"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-emit", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) + ); + const sessionAccessor = workspaceService as unknown as { + getOrCreateSession(id: string): { emitChatEvent(message: unknown): void }; + }; + const session = sessionAccessor.getOrCreateSession(workspaceId); + const emitSpy = spyOn(session, "emitChatEvent"); + // A directory at the sidecar path makes retirement fail after the truncation committed. + const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); + await fsPromises.mkdir(sidecarPath); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + if (!clearResult.success) { + expect(clearResult.error).toContain("workflow run references"); + } + // The transcript is already gone on disk and the deleted sequences cannot be recovered + // by a retry; the renderer must learn about the deletion even though the cleanup error + // aborts the remaining post-clear steps. + expect( + emitSpy.mock.calls.some((call) => (call[0] as { type?: string }).type === "delete") + ).toBe(true); + } finally { + emitSpy.mockRestore(); + await fsPromises.rmdir(sidecarPath); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index bb95cabd55c..2069613ee69 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12918,6 +12918,28 @@ export class WorkspaceService extends EventEmitter { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). + // The truncation is committed: any early error return below must first emit the deletion, + // or the renderer keeps showing a transcript that no longer exists on disk (the original + // deletedSequences cannot be recovered by a retry). + const deletedSequences = truncateResult.data; + let deletionsEmitted = false; + const emitDeletedSequences = () => { + if (deletionsEmitted || deletedSequences.length === 0) { + return; + } + deletionsEmitted = true; + const deleteMessage: DeleteMessage = { + type: "delete", + historySequences: deletedSequences, + }; + // Emit through the session so ORPC subscriptions receive the event + if (session) { + session.emitChatEvent(deleteMessage); + } else { + // Fallback to direct emit (legacy path) + this.emit("chat", { workspaceId, message: deleteMessage }); + } + }; if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); // Kernel workflow run references belong to the cleared conversation: a verified-empty @@ -12930,6 +12952,7 @@ export class WorkspaceService extends EventEmitter { try { await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); } catch (error) { + emitDeletedSequences(); return Err( `History was cleared, but stale workflow run references could not be retired ` + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + @@ -12952,20 +12975,7 @@ export class WorkspaceService extends EventEmitter { await clearPendingBranchSummary(workspaceId); } - const deletedSequences = truncateResult.data; - if (deletedSequences.length > 0) { - const deleteMessage: DeleteMessage = { - type: "delete", - historySequences: deletedSequences, - }; - // Emit through the session so ORPC subscriptions receive the event - if (session) { - session.emitChatEvent(deleteMessage); - } else { - // Fallback to direct emit (legacy path) - this.emit("chat", { workspaceId, message: deleteMessage }); - } - } + emitDeletedSequences(); // On full clear, also delete plan file and clear file change tracking if (isFullClear) { From 3eb19a5229be8df68efb4959bd19492eccf433db Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:21:42 +0000 Subject: [PATCH 15/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20close=20round-12=20?= =?UTF-8?q?lifecycle=20gaps=20for=20kernel=20workflow=20wake=20provenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detached record retries are lifecycle-governed: they only fill absence (onlyIfAbsent), so a later dispatch or workflow_resume record wins, and a full history clear cancels the sidecar path's pending retries (with a registry-identity guard against a raced timer), so a stale retry can neither overwrite newer provenance nor resurrect a retired reference. - Sidecar retirement now happens durably BEFORE the truncation, like the r41 retry discard: no crash window exists in which the transcript is gone but the sidecar survives, and a retirement failure aborts the clear with the transcript intact instead of returning a partial cleanup error after commit. - Terminal-attention drain sends carry an admissionStale probe bound to the context-mutation epoch captured before prompt validation, so a full clear between the currentness check and send admission refuses the stale workflow result instead of injecting it into the cleared conversation; the refused send leaves notifications pending. --- .../agentWorkflowRunReferences.test.ts | 52 +++++++++ .../services/agentWorkflowRunReferences.ts | 102 +++++++++++++++++- src/node/services/taskService.test.ts | 55 ++++++++++ src/node/services/taskService.ts | 16 ++- src/node/services/tools/toolUtils.ts | 57 ++-------- src/node/services/workspaceService.test.ts | 17 +-- src/node/services/workspaceService.ts | 79 +++++++------- 7 files changed, 280 insertions(+), 98 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 69171c081cf..7f5b66a0636 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -5,8 +5,10 @@ import * as path from "node:path"; import { describe, expect, test } from "bun:test"; import { + clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, + scheduleAgentWorkflowRunReferenceRecordRetry, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -190,6 +192,56 @@ describe("agent workflow run references", () => { } }); + test("a pending record retry never overwrites newer provenance", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // The retry carries the stale launch-time snapshot; a workflow_resume records newer + // provenance before the timer fires. Fill-absence semantics must let the newer record win. + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_lifecycle", + createdAtMs: 1_000, + afterBoundaryMessageId: "stale-row", + retryDelaysMs: [150], + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_lifecycle", + createdAtMs: 2_000, + afterBoundaryMessageId: "resume-row", + }); + await new Promise((resolve) => setTimeout(resolve, 400)); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]).toMatchObject({ + runId: "wfr_lifecycle", + afterBoundaryMessageId: "resume-row", + }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("a full history clear cancels pending record retries", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A stale detached retry must not resurrect a reference the clear retired; against the + // then decision-free history it would read current and inject the pre-clear result. + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_cleared", + createdAtMs: 1_000, + afterBoundaryMessageId: null, + retryDelaysMs: [100], + }); + await clearAgentWorkflowRunReferences(workspaceSessionDir); + await new Promise((resolve) => setTimeout(resolve, 350)); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("self-heals unparseable file contents to empty", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 43a9d23a778..b1f55971d6f 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -5,6 +5,7 @@ import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; +import { log } from "@/node/services/log"; export interface AgentWorkflowRunReference { runId: string; @@ -28,6 +29,26 @@ const MAX_FUTURE_SKEW_MS = 60 * 60_000; const referenceFileLocks = new MutexMap(); +// Detached record retries keyed by sidecar path and runId so lifecycle events can govern them: +// a full clear cancels the path's retries so a stale retry cannot resurrect a retired +// reference, and a retry only ever fills absence (onlyIfAbsent), so it cannot overwrite newer +// provenance recorded by a later dispatch or workflow_resume. +const pendingRecordRetryTimersByPath = new Map< + string, + Map> +>(); + +function cancelPendingRecordRetries(filePath: string): void { + const byRunId = pendingRecordRetryTimersByPath.get(filePath); + if (byRunId == null) { + return; + } + for (const timer of byRunId.values()) { + clearTimeout(timer); + } + pendingRecordRetryTimersByPath.delete(filePath); +} + function referencesPath(workspaceSessionDir: string): string { assert(workspaceSessionDir.length > 0, "agent workflow references require session dir"); return path.join(workspaceSessionDir, AGENT_WORKFLOW_RUN_REFERENCES_FILE); @@ -125,11 +146,13 @@ export async function readAgentWorkflowRunReferences( * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded * before the clear indistinguishable from one recorded after it; retiring the references with * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear - * workflow_resume re-records provenance. + * workflow_resume re-records provenance. Pending record retries are cancelled first so a stale + * detached retry cannot recreate a retired reference. */ export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { const filePath = referencesPath(workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { + cancelPendingRecordRetries(filePath); await fs.rm(filePath, { force: true }); }); } @@ -139,6 +162,8 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; + /** Fill-absence mode for detached retries: an existing entry (any newer record) wins. */ + onlyIfAbsent?: boolean; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -154,6 +179,9 @@ export async function recordAgentWorkflowRunReference(input: { // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); + if (input.onlyIfAbsent === true && previous != null) { + return; + } byRunId.set(input.runId, { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after @@ -174,3 +202,75 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } + +// Delays for detached record retries; see scheduleAgentWorkflowRunReferenceRecordRetry. +const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; + +/** + * Retry a failed provenance record in the background. The launching tool has already returned + * and an untouched active run never hits a natural re-record site, so a single failed write + * would permanently supersede the run's terminal wake once storage recovers. Retries reuse the + * launch-time boundary snapshot, only fill absence (a later successful record wins), and are + * cancelled by a full history clear. + */ +export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { + workspaceSessionDir: string; + runId: string; + createdAtMs: number; + afterBoundaryMessageId?: string | null; + retryDelaysMs?: readonly number[] | null; + attempt?: number; +}): void { + const retryDelaysMs = input.retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS; + const attempt = input.attempt ?? 0; + const delayMs = retryDelaysMs[attempt]; + if (delayMs == null) { + log.error("Giving up on agent workflow run reference record after retries", { + runId: input.runId, + attempts: attempt, + }); + return; + } + const filePath = referencesPath(input.workspaceSessionDir); + const timer = setTimeout(() => { + const byRunId = pendingRecordRetryTimersByPath.get(filePath); + // clearTimeout cannot stop a callback Node already dequeued; registry identity is the + // authoritative cancellation signal, so a cancelled-but-raced retry aborts here. + if (byRunId?.get(input.runId) !== timer) { + return; + } + byRunId.delete(input.runId); + if (byRunId.size === 0) { + pendingRecordRetryTimersByPath.delete(filePath); + } + // Detached by design: the launching tool already returned, so only this chain can finish + // the write. Failures reschedule until the bounded delays are exhausted. + void recordAgentWorkflowRunReference({ + workspaceSessionDir: input.workspaceSessionDir, + runId: input.runId, + createdAtMs: input.createdAtMs, + onlyIfAbsent: true, + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), + }).catch((error: unknown) => { + log.warn("Agent workflow run reference record retry failed", { + runId: input.runId, + attempt: attempt + 1, + error, + }); + scheduleAgentWorkflowRunReferenceRecordRetry({ ...input, attempt: attempt + 1 }); + }); + }, delayMs); + timer.unref?.(); + let byRunId = pendingRecordRetryTimersByPath.get(filePath); + if (byRunId == null) { + byRunId = new Map(); + pendingRecordRetryTimersByPath.set(filePath, byRunId); + } + const previousTimer = byRunId.get(input.runId); + if (previousTimer != null) { + clearTimeout(previousTimer); + } + byRunId.set(input.runId, timer); +} diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 23b0f450909..f652dd36f46 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -554,6 +554,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; + getContextMutationEpoch: ReturnType; create: ReturnType; countQueuedAgentPeerMessages: ReturnType; }> @@ -591,6 +592,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; + getContextMutationEpoch: ReturnType; create: ReturnType; } { const sendMessage = @@ -660,6 +662,7 @@ function createWorkspaceServiceMocks( const updateAgentStatus = overrides?.updateAgentStatus ?? mock((): Promise => Promise.resolve()); const isExperimentEnabled = overrides?.isExperimentEnabled ?? mock(() => false); + const getContextMutationEpoch = overrides?.getContextMutationEpoch ?? mock(() => 0); const emitChatEvent = overrides?.emitChatEvent ?? mock((_workspaceId: string, _message: WorkspaceChatMessage) => undefined); @@ -736,6 +739,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, + getContextMutationEpoch, getWorkflowInvocationCurrentness, countQueuedAgentPeerMessages, } as unknown as WorkspaceService, @@ -772,6 +776,7 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, + getContextMutationEpoch, }; } @@ -6355,6 +6360,56 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("drain sends carry a staleness probe that trips after a full clear", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_admission_stale"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + // The prompt is validated against history before the send is admitted; a full clear in + // that window advances the context-mutation epoch. The probe handed to sendMessage must + // observe the live epoch so admission can refuse the stale prompt. + let epoch = 1; + const { workspaceService } = createWorkspaceServiceMocks({ + sendMessage, + getContextMutationEpoch: mock(() => epoch), + }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const internal = sendMessage.mock.calls[0]?.[3] as { admissionStale?: () => boolean }; + expect(typeof internal.admissionStale).toBe("function"); + expect(internal.admissionStale?.()).toBe(false); + epoch = 2; + expect(internal.admissionStale?.()).toBe(true); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 1bc7797e56b..27773cfb84f 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8288,6 +8288,13 @@ export class TaskService { isPersistentChildContinuation ? record.workspaceId : notification.sourceId ); } + // Workflow prompts are validated against history well before the send is admitted; a full + // clear in that window truncates history and retires the sidecar, so the send must refuse + // (admissionStale) rather than inject the stale result into the freshly cleared + // conversation. A refused send leaves the notifications pending for the next drain. + const admissionEpoch = this.workspaceService.getContextMutationEpoch(ownerWorkspaceId); + const sendAdmissionStale = () => + this.workspaceService.getContextMutationEpoch(ownerWorkspaceId) !== admissionEpoch; const workflowNotifications = pending.filter( (notification) => notification.sourceKind === "workflow_run" ); @@ -8394,7 +8401,13 @@ export class TaskService { prompt, sendOptions, // Synthetic, idle-only auto-resume — same flags as the active-work auto-resume path. - { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, requireIdle: true } + { + skipAutoResumeReset: true, + synthetic: true, + agentInitiated: true, + requireIdle: true, + admissionStale: sendAdmissionStale, + } ); if (!sendResult.success && isWorkspaceBusyIdleOnlySend(sendResult.error)) { @@ -8416,6 +8429,7 @@ export class TaskService { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, + admissionStale: sendAdmissionStale, onCanceled: () => { this.scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId); }, diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index bf90c448775..31e423cbffd 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -7,7 +7,10 @@ import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; -import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; +import { + recordAgentWorkflowRunReference, + scheduleAgentWorkflowRunReferenceRecordRetry, +} from "@/node/services/agentWorkflowRunReferences"; import { log } from "@/node/services/log"; import type { TaskService } from "@/node/services/taskService"; @@ -79,51 +82,6 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } -// Bounded background retries for a transiently unreadable sidecar. The only natural re-record -// sites are a new dispatch and workflow_resume, which an untouched active run never hits, so -// giving up after one failed write would permanently supersede the run's terminal wake once -// storage recovers. Retries reuse the launch-time boundary snapshot: provenance describes the -// launch, not the retry moment. -const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; - -function scheduleRecordReferenceRetry(input: { - workspaceSessionDir: string; - runId: string; - createdAtMs: number; - afterBoundaryMessageId: string | null | undefined; - retryDelaysMs: readonly number[]; - attempt: number; -}): void { - const delayMs = input.retryDelaysMs[input.attempt]; - if (delayMs == null) { - log.error("Giving up on agent workflow run reference record after retries", { - runId: input.runId, - attempts: input.attempt, - }); - return; - } - const timer = setTimeout(() => { - // Detached by design: the launching tool already returned, so only this chain can finish - // the write. Failures reschedule until the bounded delays are exhausted. - void recordAgentWorkflowRunReference({ - workspaceSessionDir: input.workspaceSessionDir, - runId: input.runId, - createdAtMs: input.createdAtMs, - ...(input.afterBoundaryMessageId !== undefined - ? { afterBoundaryMessageId: input.afterBoundaryMessageId } - : {}), - }).catch((error: unknown) => { - log.warn("Agent workflow run reference record retry failed", { - runId: input.runId, - attempt: input.attempt + 1, - error: getErrorMessage(error), - }); - scheduleRecordReferenceRetry({ ...input, attempt: input.attempt + 1 }); - }); - }, delayMs); - timer.unref?.(); -} - /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -178,13 +136,12 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); - scheduleRecordReferenceRetry({ + scheduleAgentWorkflowRunReferenceRecordRetry({ workspaceSessionDir, runId, createdAtMs, - afterBoundaryMessageId, - retryDelaysMs: retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS, - attempt: 0, + retryDelaysMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), }); } } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 45322ca77af..176b45d413d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6266,7 +6266,7 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("a failed reference retirement still emits the committed clear to the renderer", async () => { + test("a failed reference retirement aborts the clear before deleting history", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-retire-emit"; const projectPath = path.join(config.rootDir, "project"); @@ -6302,7 +6302,10 @@ describe("WorkspaceService workflow invocation events", () => { }; const session = sessionAccessor.getOrCreateSession(workspaceId); const emitSpy = spyOn(session, "emitChatEvent"); - // A directory at the sidecar path makes retirement fail after the truncation committed. + // A directory at the sidecar path makes retirement fail. Retirement runs BEFORE the + // truncation, so the failure must abort the whole clear: the transcript survives, the + // renderer sees no deletion, and no crash window exists in which the transcript is gone + // while the sidecar lives on. const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); await fsPromises.mkdir(sidecarPath); try { @@ -6311,12 +6314,14 @@ describe("WorkspaceService workflow invocation events", () => { if (!clearResult.success) { expect(clearResult.error).toContain("workflow run references"); } - // The transcript is already gone on disk and the deleted sequences cannot be recovered - // by a retry; the renderer must learn about the deletion even though the cleanup error - // aborts the remaining post-clear steps. expect( emitSpy.mock.calls.some((call) => (call[0] as { type?: string }).type === "delete") - ).toBe(true); + ).toBe(false); + const survivingHistory = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(survivingHistory.success).toBe(true); + if (survivingHistory.success) { + expect(survivingHistory.data).toHaveLength(1); + } } finally { emitSpy.mockRestore(); await fsPromises.rmdir(sidecarPath); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2069613ee69..da118d47116 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3806,6 +3806,14 @@ export class WorkspaceService extends EventEmitter { } } + /** + * Current context-mutation epoch (see contextMutationEpochs). Lets the terminal-attention + * drain refuse a send whose workflow prompt was validated before a full clear committed. + */ + getContextMutationEpoch(workspaceId: string): number { + return this.contextMutationEpochs.get(workspaceId) ?? 0; + } + /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ private advanceContextMutationEpoch(workspaceId: string): void { this.contextMutationEpochs.set( @@ -12902,6 +12910,23 @@ export class WorkspaceService extends EventEmitter { ); } } + // Kernel workflow run references belong to the transcript being discarded: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one recorded + // after it. Retire them durably BEFORE the truncation (like the r41 retry discard above) so + // no crash window exists in which the transcript is gone but the sidecar survives; a crash + // here can only lose a wake for a still-intact conversation, never inject a pre-clear + // result into the cleared one. A post-clear resume re-records provenance, and pending + // record retries are cancelled with the sidecar. + if (isFullClear) { + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `Cannot clear history: stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}); retry once the session storage is writable.` + ); + } + } if (effectivePercentage > 0) { session?.clearUsageState(); } @@ -12918,47 +12943,8 @@ export class WorkspaceService extends EventEmitter { // r41: the discard is durable — sends that entered before it must not be // admitted afterwards (their content references the discarded context). - // The truncation is committed: any early error return below must first emit the deletion, - // or the renderer keeps showing a transcript that no longer exists on disk (the original - // deletedSequences cannot be recovered by a retry). - const deletedSequences = truncateResult.data; - let deletionsEmitted = false; - const emitDeletedSequences = () => { - if (deletionsEmitted || deletedSequences.length === 0) { - return; - } - deletionsEmitted = true; - const deleteMessage: DeleteMessage = { - type: "delete", - historySequences: deletedSequences, - }; - // Emit through the session so ORPC subscriptions receive the event - if (session) { - session.emitChatEvent(deleteMessage); - } else { - // Fallback to direct emit (legacy path) - this.emit("chat", { workspaceId, message: deleteMessage }); - } - }; if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); - // Kernel workflow run references belong to the cleared conversation: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one - // recorded after it, so a surviving reference could inject a pre-clear workflow result - // into the fresh conversation. Retire them immediately after the truncation commits, - // before any later post-clear step that can fail and return early (goal acknowledgment, - // carryover discard), or the stale reference would survive the committed clear. A - // post-clear resume re-records provenance. - try { - await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error) { - emitDeletedSequences(); - return Err( - `History was cleared, but stale workflow run references could not be retired ` + - `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + - `result into the cleared conversation; retry once the session storage is writable.` - ); - } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the @@ -12975,7 +12961,20 @@ export class WorkspaceService extends EventEmitter { await clearPendingBranchSummary(workspaceId); } - emitDeletedSequences(); + const deletedSequences = truncateResult.data; + if (deletedSequences.length > 0) { + const deleteMessage: DeleteMessage = { + type: "delete", + historySequences: deletedSequences, + }; + // Emit through the session so ORPC subscriptions receive the event + if (session) { + session.emitChatEvent(deleteMessage); + } else { + // Fallback to direct emit (legacy path) + this.emit("chat", { workspaceId, message: deleteMessage }); + } + } // On full clear, also delete plan file and clear file change tracking if (isFullClear) { From 62dd29073a7982f7110aae0925807f5b5624e0cf Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:44:16 +0000 Subject: [PATCH 16/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20round-13=20sidecar?= =?UTF-8?q?=20lifecycle=20hardening:=20boundary=20repair,=20removal=20drai?= =?UTF-8?q?n,=20corrupt-dir=20self-heal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A failed record-time boundary snapshot on a decision-free launch now schedules a bounded background repair: if history is still verified empty at repair time it persists the null snapshot the decision-free currentness branch requires (rows cannot disappear outside a full clear, which retires the sidecar), while a decision row seen at repair time keeps the entry boundary-less and fail safe because it may postdate the launch. - Workspace removal cancels and drains detached sidecar maintenance before deleting the session directory, so a late retry cannot mkdir it back into existence; the registry now tracks in-flight writes and the full clear drains through the same path. - clearAgentWorkflowRunReferences removes a directory at the known sidecar path recursively: force alone refuses directories, which previously failed every subsequent full clear identically with no self-heal. --- .../services/agentWorkflowRunReferences.ts | 151 +++++++++++++----- src/node/services/tools/toolUtils.test.ts | 77 +++++++++ src/node/services/tools/toolUtils.ts | 75 +++++++++ src/node/services/workspaceRemoval.test.ts | 34 ++++ src/node/services/workspaceRemoval.ts | 5 + src/node/services/workspaceService.test.ts | 72 ++++++++- 6 files changed, 370 insertions(+), 44 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index b1f55971d6f..9ca4544cece 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -29,24 +29,101 @@ const MAX_FUTURE_SKEW_MS = 60 * 60_000; const referenceFileLocks = new MutexMap(); -// Detached record retries keyed by sidecar path and runId so lifecycle events can govern them: -// a full clear cancels the path's retries so a stale retry cannot resurrect a retired -// reference, and a retry only ever fills absence (onlyIfAbsent), so it cannot overwrite newer -// provenance recorded by a later dispatch or workflow_resume. -const pendingRecordRetryTimersByPath = new Map< +// Detached sidecar maintenance (record retries, boundary-snapshot repairs) keyed by sidecar +// path and a per-run key so lifecycle events can govern it: a full clear or workspace removal +// cancels the path's timers and drains in-flight writes, so stale maintenance can neither +// resurrect a retired reference nor recreate a deleted session directory, and maintenance +// writes only ever fill gaps (onlyIfAbsent / onlyIfBoundaryAbsent), so they cannot overwrite +// newer provenance recorded by a later dispatch or workflow_resume. +const pendingSidecarMaintenanceTimersByPath = new Map< string, Map> >(); +const inFlightSidecarMaintenanceByPath = new Map>>(); -function cancelPendingRecordRetries(filePath: string): void { - const byRunId = pendingRecordRetryTimersByPath.get(filePath); - if (byRunId == null) { - return; +/** Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. */ +export function registerSidecarMaintenanceTimer( + workspaceSessionDir: string, + key: string, + timer: ReturnType +): void { + const filePath = referencesPath(workspaceSessionDir); + let byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); + if (byKey == null) { + byKey = new Map(); + pendingSidecarMaintenanceTimersByPath.set(filePath, byKey); + } + const previous = byKey.get(key); + if (previous != null) { + clearTimeout(previous); + } + byKey.set(key, timer); +} + +/** + * Consume a fired maintenance timer. clearTimeout cannot stop a callback Node already + * dequeued, so registry identity is the authoritative cancellation signal: a + * cancelled-but-raced callback sees a mismatch and must abort. + */ +export function takeSidecarMaintenanceTimer( + workspaceSessionDir: string, + key: string, + timer: ReturnType +): boolean { + const filePath = referencesPath(workspaceSessionDir); + const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); + if (byKey?.get(key) !== timer) { + return false; } - for (const timer of byRunId.values()) { - clearTimeout(timer); + byKey.delete(key); + if (byKey.size === 0) { + pendingSidecarMaintenanceTimersByPath.delete(filePath); + } + return true; +} + +/** Track a maintenance write so cancelAgentWorkflowRunReferenceMaintenance can drain it. */ +export function trackSidecarMaintenanceWrite( + workspaceSessionDir: string, + work: Promise +): void { + const filePath = referencesPath(workspaceSessionDir); + let inFlight = inFlightSidecarMaintenanceByPath.get(filePath); + if (inFlight == null) { + inFlight = new Set(); + inFlightSidecarMaintenanceByPath.set(filePath, inFlight); + } + inFlight.add(work); + const remove = () => { + inFlight.delete(work); + if (inFlight.size === 0) { + inFlightSidecarMaintenanceByPath.delete(filePath); + } + }; + work.then(remove, remove); +} + +/** + * Cancel this sidecar's pending maintenance timers and drain writes already past their + * identity check, so a full clear or workspace removal cannot race a recreation of the file + * or the session directory it is about to delete. Must run BEFORE taking the sidecar file + * lock: an in-flight write acquires that same lock, so draining inside it would deadlock. + */ +export async function cancelAgentWorkflowRunReferenceMaintenance( + workspaceSessionDir: string +): Promise { + const filePath = referencesPath(workspaceSessionDir); + const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); + if (byKey != null) { + for (const timer of byKey.values()) { + clearTimeout(timer); + } + pendingSidecarMaintenanceTimersByPath.delete(filePath); + } + const inFlight = inFlightSidecarMaintenanceByPath.get(filePath); + if (inFlight != null && inFlight.size > 0) { + await Promise.allSettled([...inFlight]); } - pendingRecordRetryTimersByPath.delete(filePath); } function referencesPath(workspaceSessionDir: string): string { @@ -151,9 +228,12 @@ export async function readAgentWorkflowRunReferences( */ export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { const filePath = referencesPath(workspaceSessionDir); + await cancelAgentWorkflowRunReferenceMaintenance(workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - cancelPendingRecordRetries(filePath); - await fs.rm(filePath, { force: true }); + // recursive: a directory at this known sidecar path is corruption (it also fails reads + // with EISDIR), and force alone refuses to remove directories, which would fail every + // subsequent full clear identically. Removing it self-heals the workspace. + await fs.rm(filePath, { force: true, recursive: true }); }); } @@ -164,6 +244,12 @@ export async function recordAgentWorkflowRunReference(input: { afterBoundaryMessageId?: string | null; /** Fill-absence mode for detached retries: an existing entry (any newer record) wins. */ onlyIfAbsent?: boolean; + /** + * Boundary-repair mode: only patch an entry that exists and still lacks a boundary + * snapshot. A missing entry means the reference was retired (clear/removal) and must not be + * resurrected; a present boundary means newer provenance already landed. + */ + onlyIfBoundaryAbsent?: boolean; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -182,6 +268,12 @@ export async function recordAgentWorkflowRunReference(input: { if (input.onlyIfAbsent === true && previous != null) { return; } + if ( + input.onlyIfBoundaryAbsent === true && + (previous == null || previous.afterBoundaryMessageId !== undefined) + ) { + return; + } byRunId.set(input.runId, { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after @@ -203,8 +295,8 @@ export async function recordAgentWorkflowRunReference(input: { }); } -// Delays for detached record retries; see scheduleAgentWorkflowRunReferenceRecordRetry. -const RECORD_REFERENCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; +// Delays for detached sidecar maintenance (record retries, boundary-snapshot repairs). +export const SIDECAR_MAINTENANCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; /** * Retry a failed provenance record in the background. The launching tool has already returned @@ -221,7 +313,7 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { retryDelaysMs?: readonly number[] | null; attempt?: number; }): void { - const retryDelaysMs = input.retryDelaysMs ?? RECORD_REFERENCE_RETRY_DELAYS_MS; + const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; const attempt = input.attempt ?? 0; const delayMs = retryDelaysMs[attempt]; if (delayMs == null) { @@ -231,21 +323,14 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { }); return; } - const filePath = referencesPath(input.workspaceSessionDir); + const key = `record:${input.runId}`; const timer = setTimeout(() => { - const byRunId = pendingRecordRetryTimersByPath.get(filePath); - // clearTimeout cannot stop a callback Node already dequeued; registry identity is the - // authoritative cancellation signal, so a cancelled-but-raced retry aborts here. - if (byRunId?.get(input.runId) !== timer) { + if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { return; } - byRunId.delete(input.runId); - if (byRunId.size === 0) { - pendingRecordRetryTimersByPath.delete(filePath); - } // Detached by design: the launching tool already returned, so only this chain can finish // the write. Failures reschedule until the bounded delays are exhausted. - void recordAgentWorkflowRunReference({ + const work = recordAgentWorkflowRunReference({ workspaceSessionDir: input.workspaceSessionDir, runId: input.runId, createdAtMs: input.createdAtMs, @@ -261,16 +346,8 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { }); scheduleAgentWorkflowRunReferenceRecordRetry({ ...input, attempt: attempt + 1 }); }); + trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); }, delayMs); timer.unref?.(); - let byRunId = pendingRecordRetryTimersByPath.get(filePath); - if (byRunId == null) { - byRunId = new Map(); - pendingRecordRetryTimersByPath.set(filePath, byRunId); - } - const previousTimer = byRunId.get(input.runId); - if (previousTimer != null) { - clearTimeout(previousTimer); - } - byRunId.set(input.runId, timer); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); } diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts index 7552b08a7da..538c2c249a5 100644 --- a/src/node/services/tools/toolUtils.test.ts +++ b/src/node/services/tools/toolUtils.test.ts @@ -49,4 +49,81 @@ describe("recordBackgroundWorkflowRunReference", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("repairs a verified-empty boundary snapshot after a transient read failure", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); + try { + // Launch from a decision-free history whose boundary read fails once: the rediscovery + // entry lands boundary-less, and only the repair can restore the verified-empty (null) + // snapshot the decision-free currentness branch requires. + let calls = 0; + const taskService = { + getWorkflowInvocationBoundaryMessageId: (): Promise => { + calls += 1; + return calls === 1 + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve(null); + }, + }; + await recordBackgroundWorkflowRunReference( + { + workspaceSessionDir, + workspaceId: "ws-boundary-repair", + taskService, + } as unknown as ToolConfiguration, + "wfr_boundary_repair", + 2_000, + [25, 25, 25] + ); + let reference: { afterBoundaryMessageId?: string | null } | undefined; + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + [reference] = await readAgentWorkflowRunReferences(workspaceSessionDir); + if (reference?.afterBoundaryMessageId === null) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(reference).toMatchObject({ + runId: "wfr_boundary_repair", + afterBoundaryMessageId: null, + }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("keeps the entry boundary-less when a decision row exists at repair time", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); + try { + // A decision row seen at repair time may postdate the launch; persisting it would + // overclaim currentness, so the entry must stay boundary-less and fail safe. + let calls = 0; + const taskService = { + getWorkflowInvocationBoundaryMessageId: (): Promise => { + calls += 1; + return calls === 1 + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve("manual-user"); + }, + }; + await recordBackgroundWorkflowRunReference( + { + workspaceSessionDir, + workspaceId: "ws-boundary-unsafe", + taskService, + } as unknown as ToolConfiguration, + "wfr_boundary_unsafe", + 2_000, + [25] + ); + await new Promise((resolve) => setTimeout(resolve, 300)); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]?.runId).toBe("wfr_boundary_unsafe"); + expect(references[0]?.afterBoundaryMessageId).toBeUndefined(); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 31e423cbffd..95b3c8355af 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -8,8 +8,12 @@ import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; import { + SIDECAR_MAINTENANCE_RETRY_DELAYS_MS, recordAgentWorkflowRunReference, + registerSidecarMaintenanceTimer, scheduleAgentWorkflowRunReferenceRecordRetry, + takeSidecarMaintenanceTimer, + trackSidecarMaintenanceWrite, } from "@/node/services/agentWorkflowRunReferences"; import { log } from "@/node/services/log"; import type { TaskService } from "@/node/services/taskService"; @@ -82,6 +86,65 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } +/** + * Repair a missing boundary snapshot in the background. A kernel launch from a decision-free + * history whose record-time boundary read failed persists a boundary-less entry, but the + * decision-free currentness branch accepts only an explicit verified-empty (null) snapshot, + * so without repair the run's terminal wake is permanently superseded once storage recovers. + * Rows never disappear outside a full clear (which retires the sidecar), so a history still + * verified-empty at repair time was also empty at launch and null is faithful launch + * provenance; a decision row seen at repair time may postdate the launch, so persisting it + * would overclaim currentness and the entry stays boundary-less (fail safe). + */ +function scheduleBoundarySnapshotRepair(input: { + workspaceSessionDir: string; + runId: string; + createdAtMs: number; + getBoundary: () => Promise; + retryDelaysMs?: readonly number[] | null; + attempt?: number; +}): void { + const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; + const attempt = input.attempt ?? 0; + const delayMs = retryDelaysMs[attempt]; + if (delayMs == null) { + log.error("Giving up on workflow boundary snapshot repair after retries", { + runId: input.runId, + attempts: attempt, + }); + return; + } + const key = `boundary:${input.runId}`; + const timer = setTimeout(() => { + if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { + return; + } + const work = (async () => { + const boundary = await input.getBoundary(); + if (boundary !== null) { + return; + } + await recordAgentWorkflowRunReference({ + workspaceSessionDir: input.workspaceSessionDir, + runId: input.runId, + createdAtMs: input.createdAtMs, + afterBoundaryMessageId: null, + onlyIfBoundaryAbsent: true, + }); + })().catch((error: unknown) => { + log.warn("Workflow boundary snapshot repair failed", { + runId: input.runId, + attempt: attempt + 1, + error: getErrorMessage(error), + }); + scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1 }); + }); + trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); + }, delayMs); + timer.unref?.(); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); +} + /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -121,6 +184,18 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); + const workspaceId = config.workspaceId; + const getBoundaryMessageId = + taskService?.getWorkflowInvocationBoundaryMessageId?.bind(taskService); + if (workspaceId != null && getBoundaryMessageId != null) { + scheduleBoundarySnapshotRepair({ + workspaceSessionDir, + runId, + createdAtMs, + getBoundary: () => getBoundaryMessageId(workspaceId, runId), + retryDelaysMs, + }); + } } } diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 970ccea6d11..e440984e50b 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -8,6 +8,7 @@ import { targetMutationLockFilePath, withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; +import { scheduleAgentWorkflowRunReferenceRecordRetry } from "@/node/services/agentWorkflowRunReferences"; import { acquireProcessFileLock, getProcessBirth } from "@/node/utils/concurrency/fileLock"; import { healRemovalTombstonesForRegisteredWorkspaces, @@ -21,6 +22,39 @@ import { } from "./workspaceRemoval"; describe("workspaceRemoval", () => { + test("removal cancels pending sidecar record retries so they cannot recreate the session dir", async () => { + using tmp = new DisposableTempDir("workspace-removal-sidecar"); + const rootDir = path.join(tmp.path, "xum-home"); + const workspaceId = "ws-removal-sidecar"; + const sessionDir = path.join(rootDir, "sessions", workspaceId); + await fsPromises.mkdir(sessionDir, { recursive: true }); + + // A detached provenance retry armed before removal; on fire it would mkdir the session + // directory back into existence after the deletion below. + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir: sessionDir, + runId: "wfr_removal_race", + createdAtMs: 1_000, + afterBoundaryMessageId: null, + retryDelaysMs: [100], + }); + + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir, + workspaceId, + attemptId: "test-attempt-sidecar", + }); + await new Promise((resolve) => setTimeout(resolve, 350)); + + expect( + await fsPromises.access(sessionDir).then( + () => true, + () => false + ) + ).toBe(false); + }); + test("deletion waits for a live memory writer, then tombstones and deletes (r61)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 154c7311626..c2fe3b8c4d5 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -27,6 +27,7 @@ * foreign backend's still-running consolidation refuse arbitrarily late. */ +import { cancelAgentWorkflowRunReferenceMaintenance } from "@/node/services/agentWorkflowRunReferences"; import crypto from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -217,6 +218,10 @@ export async function removeSessionDirUnderMemoryLocks(args: { // deleted directory cannot be recreated by a late mutation or // journal append. await publishTombstone(); + // Detached sidecar maintenance (provenance record retries) would survive removal and + // recreate the deleted session directory when its timer fires; cancel and drain it + // like the other session writers before the directory goes away. + await cancelAgentWorkflowRunReferenceMaintenance(args.sessionDir); await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); } ); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 176b45d413d..4994dcc6169 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6302,12 +6302,18 @@ describe("WorkspaceService workflow invocation events", () => { }; const session = sessionAccessor.getOrCreateSession(workspaceId); const emitSpy = spyOn(session, "emitChatEvent"); - // A directory at the sidecar path makes retirement fail. Retirement runs BEFORE the - // truncation, so the failure must abort the whole clear: the transcript survives, the - // renderer sees no deletion, and no crash window exists in which the transcript is gone - // while the sidecar lives on. - const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); - await fsPromises.mkdir(sidecarPath); + // A read-only session directory makes retirement fail (a directory at the sidecar path + // now self-heals instead). Retirement runs BEFORE the truncation, so the failure must + // abort the whole clear: the transcript survives, the renderer sees no deletion, and no + // crash window exists in which the transcript is gone while the sidecar lives on. + const sessionDir = config.getSessionDir(workspaceId); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: sessionDir, + runId: "wfr_retirement_blocked", + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + await fsPromises.chmod(sessionDir, 0o555); try { const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); expect(clearResult.success).toBe(false); @@ -6324,7 +6330,7 @@ describe("WorkspaceService workflow invocation events", () => { } } finally { emitSpy.mockRestore(); - await fsPromises.rmdir(sidecarPath); + await fsPromises.chmod(sessionDir, 0o755); } workspaceService.disposeSession(workspaceId); } finally { @@ -6414,6 +6420,58 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a corrupt sidecar directory self-heals during a full clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-selfheal"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-selfheal", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) + ); + // A directory at the known sidecar path is corruption (reads fail with EISDIR). The + // full clear must remove it and succeed, not fail identically on every retry and leave + // the workspace impossible to clear without manual session-storage repair. + const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); + await fsPromises.mkdir(sidecarPath); + await fsPromises.writeFile(path.join(sidecarPath, "junk.txt"), "junk"); + + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(true); + expect( + await fsPromises.access(sidecarPath).then( + () => true, + () => false + ) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; From 6866c5bcb2ac06a61b0aa31f023c1bd2f462495a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:17:12 +0000 Subject: [PATCH 17/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20round-14=20provenan?= =?UTF-8?q?ce=20integrity:=20supersede-older=20retries,=20generation=20lat?= =?UTF-8?q?ch,=20repair=20ordering,=20sidecar-gated=20finalization,=20pers?= =?UTF-8?q?isted=20drain=20provenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detached record retries skip only when a STRICTLY NEWER record exists, so a failed workflow_resume re-record can supersede the stale dispatch entry it was meant to replace while a newer successful record still wins. - Cancellation bumps a per-path lifecycle generation before clearing timers; registration refuses stale generations, so a retry rescheduled from a failing write's catch handler during the cancellation drain cannot re-arm and recreate retired state. - Boundary repair treats a missing reference as retryable (the record retry lands it later) instead of a satisfied no-op that would leave the eventual entry permanently boundary-less. - listAgentReferencedWorkflowRunIds propagates an unreadable sidecar; the three completion-gating callers defer (assume blockers, skip stream-end reconciliation, keep the task running) instead of finalizing while a kernel workflow may still run. - The drain persists deliveredWorkflowRunIds onto the accepted row (MuxMetadata) and currentness recognizes consumption from that provenance instead of row text, which user-controlled synthetic content (e.g. a heartbeat body) could spoof to suppress a real wake. --- src/common/types/message.ts | 7 ++ src/common/utils/workflowRunMessages.ts | 35 -------- src/node/services/agentSession.ts | 7 ++ .../agentWorkflowRunReferences.test.ts | 60 ++++++++++++++ .../services/agentWorkflowRunReferences.ts | 50 ++++++++++-- src/node/services/messageQueue.ts | 2 + src/node/services/taskService.test.ts | 31 +++++++- src/node/services/taskService.ts | 79 ++++++++++++++----- src/node/services/tools/toolUtils.test.ts | 54 +++++++++++++ src/node/services/tools/toolUtils.ts | 22 +++++- src/node/services/workspaceService.test.ts | 47 +++++------ src/node/services/workspaceService.ts | 24 +++--- 12 files changed, 318 insertions(+), 100 deletions(-) diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 9b16234a106..2ad5017f66a 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -868,6 +868,13 @@ export interface ModelFallbackRecord { // Our custom metadata type export interface MuxMetadata { + /** + * Workflow run IDs whose terminal results this synthetic user row delivered (set by the + * terminal-attention drain). Currentness checks treat the row as consumption for these runs; + * persisted provenance, not text recognition, so repo/user-controlled content quoting a run + * ID (e.g. a heartbeat body) cannot spoof consumption and suppress a real wake. + */ + deliveredWorkflowRunIds?: string[]; /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 66a2b3b5fa9..9fc393e9f7e 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,41 +190,6 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } -/** - * Recognize this run's result payload inside a coalesced terminal-attention prompt from the - * builder's own output format. The drain's synthetic user row can coalesce several runs into - * one message and carries no workflow-result metadata, so currentness checks must read the - * consumption evidence out of the text: each payload block is parsed back and matched on the - * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted - * inside another run's report cannot count as consumption. - */ -export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { - assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); - if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { - return false; - } - const blockPattern = new RegExp( - `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, - "g" - ); - for (const match of text.matchAll(blockPattern)) { - let payload: unknown; - try { - payload = JSON.parse(match[1] ?? ""); - } catch { - continue; - } - if (!isRecordValue(payload)) { - continue; - } - const workflow = payload.workflow; - if (isRecordValue(workflow) && workflow.runId === runId) { - return true; - } - } - return false; -} - export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4422a49a05e..aba92a18eeb 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -2950,6 +2950,8 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; + /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; goalContinuation?: boolean; goalKind?: GoalSyntheticMessageKind; /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ @@ -3545,6 +3547,9 @@ export class AgentSession { ...(internal?.enqueuedAtMs != null ? { enqueuedAtMs: internal.enqueuedAtMs } : {}), // Auto-resume and other system-generated messages are synthetic + UI-visible ...(internal?.synthetic && { synthetic: true, uiVisible: true }), + ...(internal?.deliveredWorkflowRunIds != null && internal.deliveredWorkflowRunIds.length > 0 + ? { deliveredWorkflowRunIds: internal.deliveredWorkflowRunIds } + : {}), }, additionalParts ); @@ -6390,6 +6395,8 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; + /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; /** Request-entry authoring time captured before send preflight awaits (see MessageQueue). */ authoredAtMs?: number; /** True only for a report that continues an existing workspace turn. */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 7f5b66a0636..30769c481ac 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -6,6 +6,7 @@ import { describe, expect, test } from "bun:test"; import { clearAgentWorkflowRunReferences, + getSidecarLifecycleGeneration, readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, scheduleAgentWorkflowRunReferenceRecordRetry, @@ -222,6 +223,65 @@ describe("agent workflow run references", () => { } }); + test("a retry supersedes an older entry for the same run", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A workflow_resume re-record fails transiently while an OLDER dispatch entry exists. + // The retry must replace that stale provenance (its boundary predates the resume) or the + // resumed run's wake is classified not_current; only a strictly newer record wins. + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_supersede", + createdAtMs: 500, + afterBoundaryMessageId: "old-row", + }); + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_supersede", + createdAtMs: 2_000, + afterBoundaryMessageId: "resume-row", + retryDelaysMs: [50], + }); + const deadline = Date.now() + 5_000; + let boundary: string | null | undefined; + while (Date.now() < deadline) { + boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( + (reference) => reference.runId === "wfr_supersede" + )?.afterBoundaryMessageId; + if (boundary === "resume-row") { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(boundary).toBe("resume-row"); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("a chain scheduled before cancellation cannot re-arm after it", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Simulates the reschedule window: a failing in-flight write's catch handler schedules + // the next retry DURING the cancellation drain, carrying the pre-cancel generation. + // Registration must refuse it, or the retry recreates the retired sidecar later. + const staleGeneration = getSidecarLifecycleGeneration(workspaceSessionDir); + await clearAgentWorkflowRunReferences(workspaceSessionDir); + scheduleAgentWorkflowRunReferenceRecordRetry({ + workspaceSessionDir, + runId: "wfr_stale_chain", + createdAtMs: 1_000, + afterBoundaryMessageId: null, + retryDelaysMs: [30], + lifecycleGeneration: staleGeneration, + }); + await new Promise((resolve) => setTimeout(resolve, 250)); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("a full history clear cancels pending record retries", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 9ca4544cece..06a192ca60f 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -40,14 +40,31 @@ const pendingSidecarMaintenanceTimersByPath = new Map< Map> >(); const inFlightSidecarMaintenanceByPath = new Map>>(); +// Bumped by cancellation BEFORE timers are cleared: a maintenance chain captures the +// generation when it is first scheduled, and registration refuses a stale generation, so a +// retry rescheduled from a failing write's catch handler DURING the cancellation drain cannot +// re-arm and later recreate retired state. +const lifecycleGenerationByPath = new Map(); -/** Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. */ +export function getSidecarLifecycleGeneration(workspaceSessionDir: string): number { + return lifecycleGenerationByPath.get(referencesPath(workspaceSessionDir)) ?? 0; +} + +/** + * Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. A + * stale lifecycleGeneration (captured before a cancellation) is refused so the chain dies. + */ export function registerSidecarMaintenanceTimer( workspaceSessionDir: string, key: string, - timer: ReturnType + timer: ReturnType, + lifecycleGeneration: number ): void { const filePath = referencesPath(workspaceSessionDir); + if ((lifecycleGenerationByPath.get(filePath) ?? 0) !== lifecycleGeneration) { + clearTimeout(timer); + return; + } let byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); if (byKey == null) { byKey = new Map(); @@ -113,6 +130,7 @@ export async function cancelAgentWorkflowRunReferenceMaintenance( workspaceSessionDir: string ): Promise { const filePath = referencesPath(workspaceSessionDir); + lifecycleGenerationByPath.set(filePath, (lifecycleGenerationByPath.get(filePath) ?? 0) + 1); const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); if (byKey != null) { for (const timer of byKey.values()) { @@ -242,8 +260,12 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; - /** Fill-absence mode for detached retries: an existing entry (any newer record) wins. */ - onlyIfAbsent?: boolean; + /** + * Detached-retry mode: skip only when a strictly newer record already exists, so a retry + * can supersede the stale entry a failed re-record (e.g. workflow_resume over an old + * dispatch) was meant to replace, while a newer successful record still wins. + */ + skipIfNewerRecordExists?: boolean; /** * Boundary-repair mode: only patch an entry that exists and still lacks a boundary * snapshot. A missing entry means the reference was retired (clear/removal) and must not be @@ -265,7 +287,11 @@ export async function recordAgentWorkflowRunReference(input: { // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); - if (input.onlyIfAbsent === true && previous != null) { + if ( + input.skipIfNewerRecordExists === true && + previous != null && + previous.createdAtMs > createdAtMs + ) { return; } if ( @@ -312,6 +338,8 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { afterBoundaryMessageId?: string | null; retryDelaysMs?: readonly number[] | null; attempt?: number; + /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ + lifecycleGeneration?: number; }): void { const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; const attempt = input.attempt ?? 0; @@ -324,6 +352,8 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { return; } const key = `record:${input.runId}`; + const lifecycleGeneration = + input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); const timer = setTimeout(() => { if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { return; @@ -334,7 +364,7 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { workspaceSessionDir: input.workspaceSessionDir, runId: input.runId, createdAtMs: input.createdAtMs, - onlyIfAbsent: true, + skipIfNewerRecordExists: true, ...(input.afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId: input.afterBoundaryMessageId } : {}), @@ -344,10 +374,14 @@ export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { attempt: attempt + 1, error, }); - scheduleAgentWorkflowRunReferenceRecordRetry({ ...input, attempt: attempt + 1 }); + scheduleAgentWorkflowRunReferenceRecordRetry({ + ...input, + attempt: attempt + 1, + lifecycleGeneration, + }); }); trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); }, delayMs); timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); } diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index dbf71169ca3..82a12421356 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -96,6 +96,8 @@ type QueueDispatchMode = NonNullable; interface QueuedMessageInternalOptions { synthetic?: boolean; agentInitiated?: boolean; + /** Persisted onto the dispatched user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; /** * When the sender authored this message (request entry), before any send * preflight awaits (pricing gate, settings persistence). Goal safety diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index f652dd36f46..6225945a12f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6403,7 +6403,12 @@ describe("TaskService", () => { await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(1); - const internal = sendMessage.mock.calls[0]?.[3] as { admissionStale?: () => boolean }; + const internal = sendMessage.mock.calls[0]?.[3] as { + admissionStale?: () => boolean; + deliveredWorkflowRunIds?: string[]; + }; + // Consumption provenance persisted with the accepted row (crash-replay suppression). + expect(internal.deliveredWorkflowRunIds).toEqual([runId]); expect(typeof internal.admissionStale).toBe("function"); expect(internal.admissionStale?.()).toBe(false); epoch = 2; @@ -6852,6 +6857,30 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); }); + test("an unreadable sidecar defers workspace-turn blocker checks", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const { workspaceService } = createWorkspaceServiceMocks(); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + hasActiveWorkspaceTurnDeferredBlockers(record: { workspaceId: string }): Promise; + }; + + expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( + false + ); + + // The sidecar is the only durable provenance for kernel-launched workflows, and this + // result gates completion decisions: an unreadable sidecar must report blockers (defer), + // not "none" and let the turn finalize while a workflow may still be running. + await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { + recursive: true, + }); + expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( + true + ); + }); + test("initialize recovers terminal notify workspace turns without pending notification", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 27773cfb84f..dec1d8e87dd 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1702,13 +1702,10 @@ export class TaskService { } const runIds = new Set(); - let references: Awaited> = []; - try { - references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error: unknown) { - // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. - log.warn("Failed to read agent workflow run references", { workspaceId, error }); - } + // An unreadable sidecar PROPAGATES: callers gate destructive completion decisions + // (finalizing task reports, ending workspace turns) on this listing, and treating the + // failure as "no references" would let those finalize while a kernel workflow still runs. + const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -8301,6 +8298,10 @@ export class TaskService { const deliverableWorkflowNotificationIds = new Set(); const promptSections: string[] = []; + // Persisted onto the accepted row as consumption provenance (see + // MuxMetadata.deliveredWorkflowRunIds): after a crash between acceptance and the outbox + // delivery mark, restart recovery recognizes the row and does not replay these results. + const deliveredWorkflowRunIds: string[] = []; if (publicAwaitIds.length > 0) { promptSections.push(buildCompletedWorkspaceTurnPrompt(publicAwaitIds)); } @@ -8330,6 +8331,7 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); + deliveredWorkflowRunIds.push(notification.sourceId); promptSections.push(workflowPrompt.prompt); } @@ -8407,6 +8409,7 @@ export class TaskService { agentInitiated: true, requireIdle: true, admissionStale: sendAdmissionStale, + deliveredWorkflowRunIds, } ); @@ -8430,6 +8433,7 @@ export class TaskService { synthetic: true, agentInitiated: true, admissionStale: sendAdmissionStale, + deliveredWorkflowRunIds, onCanceled: () => { this.scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId); }, @@ -12535,10 +12539,21 @@ export class TaskService { return true; } - const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - record.workspaceId, - [] - ); + let referencedWorkflowRunIds: string[]; + try { + referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + record.workspaceId, + [] + ); + } catch (error: unknown) { + // Unreadable sidecar: assume blockers exist so the deferred turn is not finalized while + // a kernel workflow may still be running; the next evaluation retries. + log.warn("Deferring workspace-turn blocker check; sidecar unreadable", { + workspaceId: record.workspaceId, + error, + }); + return true; + } if ( (await this.listActiveBackgroundWorkflowRunIds(record.workspaceId, referencedWorkflowRunIds)) .length > 0 @@ -14155,11 +14170,22 @@ export class TaskService { taskIndex, workspaceId ); - const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); + let referencedWorkflowRunIds: string[]; + try { + referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); + } catch (error: unknown) { + // Unreadable sidecar: neither finalize the turn nor nudge the model about unknown + // runs; leave the stream-end unhandled so deferred recovery re-evaluates later. + log.warn("Skipping parent stream-end workflow reconciliation; sidecar unreadable", { + workspaceId, + error, + }); + return; + } let activeWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, referencedWorkflowRunIds @@ -14446,11 +14472,22 @@ export class TaskService { return; } - const taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); + let taskReferencedWorkflowRunIds: string[]; + try { + taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); + } catch (error: unknown) { + // Unreadable sidecar: defer report finalization like an active blocker instead of + // publishing while a kernel workflow may still be running. + log.warn("Deferring task finalization; sidecar unreadable", { workspaceId, error }); + if (status === "awaiting_report") { + await this.setTaskStatus(workspaceId, "running"); + } + return; + } const activeTaskWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, taskReferencedWorkflowRunIds diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts index 538c2c249a5..9785a1ce04b 100644 --- a/src/node/services/tools/toolUtils.test.ts +++ b/src/node/services/tools/toolUtils.test.ts @@ -93,6 +93,60 @@ describe("recordBackgroundWorkflowRunReference", () => { } }); + test("boundary repair waits for the record retry to land the entry", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); + try { + // Both the boundary read AND the initial sidecar write fail: the reference does not + // exist when the repair first fires. A missing entry must stay retryable, or the record + // retry that lands later creates a permanently boundary-less reference. + let calls = 0; + const taskService = { + getWorkflowInvocationBoundaryMessageId: (): Promise => { + calls += 1; + return calls === 1 + ? Promise.reject(new Error("history unavailable")) + : Promise.resolve(null); + }, + }; + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_seed", + createdAtMs: 1_000, + }); + await fs.chmod(filePath, 0o000); + await recordBackgroundWorkflowRunReference( + { + workspaceSessionDir, + workspaceId: "ws-boundary-late", + taskService, + } as unknown as ToolConfiguration, + "wfr_boundary_late", + 2_000, + [40, 40, 40, 40, 40] + ); + // Storage recovers only after the repair has fired at least once against the missing + // entry; the record retry then lands it and a later repair attempt patches null. + await new Promise((resolve) => setTimeout(resolve, 60)); + await fs.chmod(filePath, 0o600); + + const deadline = Date.now() + 5_000; + let boundary: string | null | undefined; + while (Date.now() < deadline) { + boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( + (reference) => reference.runId === "wfr_boundary_late" + )?.afterBoundaryMessageId; + if (boundary === null) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(boundary).toBe(null); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("keeps the entry boundary-less when a decision row exists at repair time", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); try { diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 95b3c8355af..89e095dce18 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -9,6 +9,8 @@ import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; import { SIDECAR_MAINTENANCE_RETRY_DELAYS_MS, + getSidecarLifecycleGeneration, + readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, registerSidecarMaintenanceTimer, scheduleAgentWorkflowRunReferenceRecordRetry, @@ -103,6 +105,8 @@ function scheduleBoundarySnapshotRepair(input: { getBoundary: () => Promise; retryDelaysMs?: readonly number[] | null; attempt?: number; + /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ + lifecycleGeneration?: number; }): void { const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; const attempt = input.attempt ?? 0; @@ -115,11 +119,25 @@ function scheduleBoundarySnapshotRepair(input: { return; } const key = `boundary:${input.runId}`; + const lifecycleGeneration = + input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); const timer = setTimeout(() => { if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { return; } const work = (async () => { + // The reference may not exist yet: when the initial write also failed, the independent + // record-retry chain lands it later. A missing entry is retryable, not a satisfied + // no-op, or that later record would create a permanently boundary-less reference; + // lifecycle cancellation kills this chain when the reference was retired instead. + const references = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + const entry = references.find((reference) => reference.runId === input.runId); + if (entry == null) { + throw new Error("reference not recorded yet"); + } + if (entry.afterBoundaryMessageId !== undefined) { + return; + } const boundary = await input.getBoundary(); if (boundary !== null) { return; @@ -137,12 +155,12 @@ function scheduleBoundarySnapshotRepair(input: { attempt: attempt + 1, error: getErrorMessage(error), }); - scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1 }); + scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1, lifecycleGeneration }); }); trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); }, delayMs); timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer); + registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); } /** diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4994dcc6169..e92a56be1f6 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6374,44 +6374,47 @@ describe("WorkspaceService workflow invocation events", () => { }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // Another run's payload quoting nothing about this run must not count as consumption. + // A synthetic row whose TEXT reproduces a result payload must not count: synthetic rows + // can carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must + // not spoof consumption and suppress the real wake. await historyService.appendToHistory( workspaceId, createMuxMessage( - "coalesced-other", + "coalesced-spoof", "user", buildWorkflowResultContextMessage({ - rawCommand: "workflow_run other.js", - name: "other.js", - runId: "wfr_currentness_other", + rawCommand: "workflow_run research.js", + name: "research.js", + runId, status: "completed", - result: { reportMarkdown: "other done" }, + result: { reportMarkdown: "spoofed" }, run: null, }), - { timestamp: 1_250, synthetic: true } + { timestamp: 1_240, synthetic: true } ) ); + // Another run's persisted provenance must not count for this run either. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("coalesced-other", "user", "results delivered", { + timestamp: 1_250, + synthetic: true, + deliveredWorkflowRunIds: ["wfr_currentness_other"], + }) + ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a - // crash between durable acceptance and the outbox delivery mark, this row is the only + // The drain persists deliveredWorkflowRunIds onto the accepted row. After a crash + // between durable acceptance and the outbox delivery mark, this provenance is the only // evidence the result already reached history; it must read as consumption or restart // recovery injects the same terminal result again. await historyService.appendToHistory( workspaceId, - createMuxMessage( - "coalesced-result", - "user", - buildWorkflowResultContextMessage({ - rawCommand: "workflow_run research.js", - name: "research.js", - runId, - status: "completed", - result: { reportMarkdown: "done" }, - run: null, - }), - { timestamp: 1_300, synthetic: true } - ) + createMuxMessage("coalesced-result", "user", "results delivered", { + timestamp: 1_300, + synthetic: true, + deliveredWorkflowRunIds: [runId], + }) ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index da118d47116..3b6e9b037b6 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -204,7 +204,6 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, - textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -478,18 +477,16 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) /** * The terminal-attention drain delivers workflow results as one synthetic user prompt that may - * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands - * between the send's durable acceptance and the outbox delivery mark, restart recovery drains - * the notification again; recognizing the accepted row as consumption is what suppresses the - * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is - * classified before this check runs. + * coalesce several runs. If a crash lands between the send's durable acceptance and the outbox + * delivery mark, restart recovery drains the notification again; recognizing the accepted row + * as consumption is what suppresses the replay. Recognition uses the drain's persisted + * provenance (MuxMetadata.deliveredWorkflowRunIds), never the row's text: synthetic rows can + * carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must not spoof + * consumption and suppress a real wake. */ function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { - if (message.role !== "user" || message.metadata?.synthetic !== true) { - return false; - } - return message.parts.some( - (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) + return ( + message.role === "user" && message.metadata?.deliveredWorkflowRunIds?.includes(runId) === true ); } @@ -11254,6 +11251,8 @@ export class WorkspaceService extends EventEmitter { goalId?: string; /** Force Copilot billing classification to "agent" for internal sends. */ agentInitiated?: boolean; + /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ + deliveredWorkflowRunIds?: string[]; onAccepted?: () => Promise | void; onCanceled?: (reason: string) => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; @@ -11516,6 +11515,7 @@ export class WorkspaceService extends EventEmitter { return await session.sendMessage(message, normalizedOptions, { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, + deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, cancelState: internal?.cancelState, @@ -11660,6 +11660,7 @@ export class WorkspaceService extends EventEmitter { { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, + deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, authoredAtMs, workspaceTurnContinuation: internal?.workspaceTurnContinuation, dedupeKey: internal?.queueDedupeKey, @@ -11772,6 +11773,7 @@ export class WorkspaceService extends EventEmitter { onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, + deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, goalContinuation: internal?.goalContinuation, From e14574d8bf2628b68d991ccaeee6e9cc89842860 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:36:47 +0000 Subject: [PATCH 18/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20restore=20the=20cal?= =?UTF-8?q?ler=20tool=20policy=20on=20kernel=20workflow=20wakes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ordinary in-stream workflow continuation carries the live turn's effectiveToolPolicy; the terminal-attention drain's synthetic send starts a fresh turn and omitted it, so a workflow wake could regain tools the caller disabled, with attacker-influenced workflow output choosing the timing. The drain now restores the newest manual user row's persisted caller policy (synthetic rows without one are skipped), fails closed by deferring the wake when history is unreadable, and the agent-level policy recomposes from agentId at send resolution. --- src/node/services/taskService.test.ts | 78 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 48 +++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6225945a12f..79f5369a5af 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6360,6 +6360,84 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_policy_restore"); + await createRun("wfr_policy_lifted"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The launch turn disabled bash; a later synthetic row (an earlier wake) defines no + // policy and must be skipped. Omitting the policy on the wake would let workflow output + // regain the disabled tool at a time the workflow chooses. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("earlier-wake", "user", "results delivered", { + timestamp: 1_100, + synthetic: true, + }) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_restore", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + toolPolicy: restrictedPolicy, + }); + + // A newer manual row without a policy means the caller lifted it: no restoration. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-unrestricted", "user", "carry on", { timestamp: 1_200 }) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_lifted", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + const liftedOptions = sendMessage.mock.calls[1]?.[2] as { toolPolicy?: unknown }; + expect(liftedOptions.toolPolicy).toBeUndefined(); + }); + test("drain sends carry a staleness probe that trips after a full clear", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index dec1d8e87dd..5f431201f42 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -188,6 +188,7 @@ import { isNonRetryableStreamError } from "@/common/utils/messages/retryEligibil import type { SendMessageError, StreamErrorType } from "@/common/types/errors"; import { hasCompletedAgentReport } from "@/common/utils/agentTaskCompletion"; import { isWorkspaceArchived } from "@/common/utils/archive"; +import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; import { DEFAULT_CODER_ARCHIVE_BEHAVIOR } from "@/common/config/coderArchiveBehavior"; import { isSSHRuntime, isWorktreeRuntime } from "@/common/types/runtime"; @@ -7913,6 +7914,34 @@ export class TaskService { this.terminalAttentionDeferRetryTimers.set(ownerWorkspaceId, timer); } + /** + * Caller tool policy to restore on a terminal-attention wake. The newest manual user row + * carries the conversation's persisted caller policy; synthetic rows without one (earlier + * wakes, heartbeat scaffolding) do not define policy and are skipped. Throws when history + * is unreadable so the caller can fail closed instead of waking with unrestricted tools. + */ + private async resolveTerminalWakeCallerToolPolicy( + ownerWorkspaceId: string + ): Promise { + const historyResult = await this.historyService.getLastMessages(ownerWorkspaceId, 50); + if (!historyResult.success) { + throw new Error(`history unavailable: ${historyResult.error}`); + } + for (let i = historyResult.data.length - 1; i >= 0; i--) { + const message = historyResult.data[i]; + if (message?.role !== "user") { + continue; + } + if (message.metadata?.toolPolicy != null) { + return message.metadata.toolPolicy; + } + if (message.metadata?.synthetic !== true) { + return undefined; + } + } + return undefined; + } + private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { const promise = this.workspaceService .waitForIdleAndNoQueuedMessages(ownerWorkspaceId) @@ -8369,11 +8398,30 @@ export class TaskService { const workspaceTurnMuxMetadata = await this.getActiveWorkspaceTurnMuxMetadataForWorkspace(ownerWorkspaceId); + // Security: restore the conversation's active caller tool policy on the wake. The ordinary + // in-stream workflow continuation carries the live turn's effectiveToolPolicy; this + // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake + // regain tools the caller disabled (with attacker-influenced workflow output choosing the + // timing). The agent-level policy recomposes from agentId at send resolution. + let wakeToolPolicy: ToolPolicy | undefined; + try { + wakeToolPolicy = await this.resolveTerminalWakeCallerToolPolicy(ownerWorkspaceId); + } catch (error: unknown) { + // Fail closed: an unknown policy must not fall back to unrestricted tools. + log.warn("Deferring terminal wake; caller tool policy unavailable", { + ownerWorkspaceId, + error, + }); + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); + return; + } + const sendOptions = { model: resumeOptions.model, agentId: resumeOptions.agentId, thinkingLevel: resumeOptions.thinkingLevel, reasoningMode: resumeOptions.reasoningMode, + ...(wakeToolPolicy != null ? { toolPolicy: wakeToolPolicy } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { From 95ca943e9d8468970b3849244c331bdfe7eb5708 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 05:21:55 +0000 Subject: [PATCH 19/63] =?UTF-8?q?=F0=9F=A4=96=20refactor:=20strip=20rounds?= =?UTF-8?q?=2011-14=20retry/repair=20machinery;=20keep=20identity-based=20?= =?UTF-8?q?currentness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per maintainer decision on the review spiral: the detached record retries, generation latches, boundary repair timers, corrupt-sidecar self-heal, removal drains, admission staleness probes, and persisted drain provenance (deliveredWorkflowRunIds) added in rounds 11-14 are removed wholesale. The core design stays: kernel-launched background workflow runs persist a sidecar reference whose boundary-row identity snapshot decides wake currentness, workflow_resume persists terminal consumption, and the terminal drain restores the caller's send restrictions. Folded-in review fixes on the retained core: - The wake restriction walk is unbounded (iterateFullHistory backward), so a long assistant/synthetic tail cannot silently lift the caller's tool policy, and the newest manual turn's disableWorkspaceAgents flag is restored alongside it. - Foreground workflow_resume derives terminal consumption from the dispatch result itself, so a transient refresh read failure cannot leave the delivered result armed for re-injection. --- src/common/types/message.ts | 7 - src/common/utils/workflowRunMessages.ts | 35 +++ src/node/services/agentSession.ts | 7 - .../agentWorkflowRunReferences.test.ts | 112 --------- .../services/agentWorkflowRunReferences.ts | 215 +----------------- src/node/services/messageQueue.ts | 2 - src/node/services/taskService.test.ts | 123 +++------- src/node/services/taskService.ts | 171 ++++++-------- src/node/services/tools/toolUtils.test.ts | 183 --------------- src/node/services/tools/toolUtils.ts | 108 +-------- .../services/tools/workflow_resume.test.ts | 38 ++++ src/node/services/tools/workflow_resume.ts | 16 +- src/node/services/workspaceRemoval.test.ts | 34 --- src/node/services/workspaceRemoval.ts | 5 - src/node/services/workspaceService.test.ts | 171 ++------------ src/node/services/workspaceService.ts | 65 +++--- 16 files changed, 231 insertions(+), 1061 deletions(-) delete mode 100644 src/node/services/tools/toolUtils.test.ts diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 2ad5017f66a..9b16234a106 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -868,13 +868,6 @@ export interface ModelFallbackRecord { // Our custom metadata type export interface MuxMetadata { - /** - * Workflow run IDs whose terminal results this synthetic user row delivered (set by the - * terminal-attention drain). Currentness checks treat the row as consumption for these runs; - * persisted provenance, not text recognition, so repo/user-controlled content quoting a run - * ID (e.g. a heartbeat body) cannot spoof consumption and suppress a real wake. - */ - deliveredWorkflowRunIds?: string[]; /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 9fc393e9f7e..66a2b3b5fa9 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,6 +190,41 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } +/** + * Recognize this run's result payload inside a coalesced terminal-attention prompt from the + * builder's own output format. The drain's synthetic user row can coalesce several runs into + * one message and carries no workflow-result metadata, so currentness checks must read the + * consumption evidence out of the text: each payload block is parsed back and matched on the + * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted + * inside another run's report cannot count as consumption. + */ +export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { + assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); + if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { + return false; + } + const blockPattern = new RegExp( + `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, + "g" + ); + for (const match of text.matchAll(blockPattern)) { + let payload: unknown; + try { + payload = JSON.parse(match[1] ?? ""); + } catch { + continue; + } + if (!isRecordValue(payload)) { + continue; + } + const workflow = payload.workflow; + if (isRecordValue(workflow) && workflow.runId === runId) { + return true; + } + } + return false; +} + export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index aba92a18eeb..4422a49a05e 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -2950,8 +2950,6 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; - /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; goalContinuation?: boolean; goalKind?: GoalSyntheticMessageKind; /** Goal identity persisted alongside goalKind so chat-tail reconciliation can scope the row. */ @@ -3547,9 +3545,6 @@ export class AgentSession { ...(internal?.enqueuedAtMs != null ? { enqueuedAtMs: internal.enqueuedAtMs } : {}), // Auto-resume and other system-generated messages are synthetic + UI-visible ...(internal?.synthetic && { synthetic: true, uiVisible: true }), - ...(internal?.deliveredWorkflowRunIds != null && internal.deliveredWorkflowRunIds.length > 0 - ? { deliveredWorkflowRunIds: internal.deliveredWorkflowRunIds } - : {}), }, additionalParts ); @@ -6395,8 +6390,6 @@ export class AgentSession { internal?: { synthetic?: boolean; agentInitiated?: boolean; - /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; /** Request-entry authoring time captured before send preflight awaits (see MessageQueue). */ authoredAtMs?: number; /** True only for a report that continues an existing workspace turn. */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 30769c481ac..69171c081cf 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -5,11 +5,8 @@ import * as path from "node:path"; import { describe, expect, test } from "bun:test"; import { - clearAgentWorkflowRunReferences, - getSidecarLifecycleGeneration, readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, - scheduleAgentWorkflowRunReferenceRecordRetry, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -193,115 +190,6 @@ describe("agent workflow run references", () => { } }); - test("a pending record retry never overwrites newer provenance", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // The retry carries the stale launch-time snapshot; a workflow_resume records newer - // provenance before the timer fires. Fill-absence semantics must let the newer record win. - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_lifecycle", - createdAtMs: 1_000, - afterBoundaryMessageId: "stale-row", - retryDelaysMs: [150], - }); - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_lifecycle", - createdAtMs: 2_000, - afterBoundaryMessageId: "resume-row", - }); - await new Promise((resolve) => setTimeout(resolve, 400)); - const references = await readAgentWorkflowRunReferences(workspaceSessionDir); - expect(references).toHaveLength(1); - expect(references[0]).toMatchObject({ - runId: "wfr_lifecycle", - afterBoundaryMessageId: "resume-row", - }); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("a retry supersedes an older entry for the same run", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // A workflow_resume re-record fails transiently while an OLDER dispatch entry exists. - // The retry must replace that stale provenance (its boundary predates the resume) or the - // resumed run's wake is classified not_current; only a strictly newer record wins. - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_supersede", - createdAtMs: 500, - afterBoundaryMessageId: "old-row", - }); - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_supersede", - createdAtMs: 2_000, - afterBoundaryMessageId: "resume-row", - retryDelaysMs: [50], - }); - const deadline = Date.now() + 5_000; - let boundary: string | null | undefined; - while (Date.now() < deadline) { - boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( - (reference) => reference.runId === "wfr_supersede" - )?.afterBoundaryMessageId; - if (boundary === "resume-row") { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(boundary).toBe("resume-row"); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("a chain scheduled before cancellation cannot re-arm after it", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // Simulates the reschedule window: a failing in-flight write's catch handler schedules - // the next retry DURING the cancellation drain, carrying the pre-cancel generation. - // Registration must refuse it, or the retry recreates the retired sidecar later. - const staleGeneration = getSidecarLifecycleGeneration(workspaceSessionDir); - await clearAgentWorkflowRunReferences(workspaceSessionDir); - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_stale_chain", - createdAtMs: 1_000, - afterBoundaryMessageId: null, - retryDelaysMs: [30], - lifecycleGeneration: staleGeneration, - }); - await new Promise((resolve) => setTimeout(resolve, 250)); - expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("a full history clear cancels pending record retries", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // A stale detached retry must not resurrect a reference the clear retired; against the - // then decision-free history it would read current and inject the pre-clear result. - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId: "wfr_cleared", - createdAtMs: 1_000, - afterBoundaryMessageId: null, - retryDelaysMs: [100], - }); - await clearAgentWorkflowRunReferences(workspaceSessionDir); - await new Promise((resolve) => setTimeout(resolve, 350)); - expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - test("self-heals unparseable file contents to empty", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 06a192ca60f..43a9d23a778 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -5,7 +5,6 @@ import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; -import { log } from "@/node/services/log"; export interface AgentWorkflowRunReference { runId: string; @@ -29,121 +28,6 @@ const MAX_FUTURE_SKEW_MS = 60 * 60_000; const referenceFileLocks = new MutexMap(); -// Detached sidecar maintenance (record retries, boundary-snapshot repairs) keyed by sidecar -// path and a per-run key so lifecycle events can govern it: a full clear or workspace removal -// cancels the path's timers and drains in-flight writes, so stale maintenance can neither -// resurrect a retired reference nor recreate a deleted session directory, and maintenance -// writes only ever fill gaps (onlyIfAbsent / onlyIfBoundaryAbsent), so they cannot overwrite -// newer provenance recorded by a later dispatch or workflow_resume. -const pendingSidecarMaintenanceTimersByPath = new Map< - string, - Map> ->(); -const inFlightSidecarMaintenanceByPath = new Map>>(); -// Bumped by cancellation BEFORE timers are cleared: a maintenance chain captures the -// generation when it is first scheduled, and registration refuses a stale generation, so a -// retry rescheduled from a failing write's catch handler DURING the cancellation drain cannot -// re-arm and later recreate retired state. -const lifecycleGenerationByPath = new Map(); - -export function getSidecarLifecycleGeneration(workspaceSessionDir: string): number { - return lifecycleGenerationByPath.get(referencesPath(workspaceSessionDir)) ?? 0; -} - -/** - * Arm a maintenance timer; a newer schedule for the same key supersedes the older timer. A - * stale lifecycleGeneration (captured before a cancellation) is refused so the chain dies. - */ -export function registerSidecarMaintenanceTimer( - workspaceSessionDir: string, - key: string, - timer: ReturnType, - lifecycleGeneration: number -): void { - const filePath = referencesPath(workspaceSessionDir); - if ((lifecycleGenerationByPath.get(filePath) ?? 0) !== lifecycleGeneration) { - clearTimeout(timer); - return; - } - let byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); - if (byKey == null) { - byKey = new Map(); - pendingSidecarMaintenanceTimersByPath.set(filePath, byKey); - } - const previous = byKey.get(key); - if (previous != null) { - clearTimeout(previous); - } - byKey.set(key, timer); -} - -/** - * Consume a fired maintenance timer. clearTimeout cannot stop a callback Node already - * dequeued, so registry identity is the authoritative cancellation signal: a - * cancelled-but-raced callback sees a mismatch and must abort. - */ -export function takeSidecarMaintenanceTimer( - workspaceSessionDir: string, - key: string, - timer: ReturnType -): boolean { - const filePath = referencesPath(workspaceSessionDir); - const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); - if (byKey?.get(key) !== timer) { - return false; - } - byKey.delete(key); - if (byKey.size === 0) { - pendingSidecarMaintenanceTimersByPath.delete(filePath); - } - return true; -} - -/** Track a maintenance write so cancelAgentWorkflowRunReferenceMaintenance can drain it. */ -export function trackSidecarMaintenanceWrite( - workspaceSessionDir: string, - work: Promise -): void { - const filePath = referencesPath(workspaceSessionDir); - let inFlight = inFlightSidecarMaintenanceByPath.get(filePath); - if (inFlight == null) { - inFlight = new Set(); - inFlightSidecarMaintenanceByPath.set(filePath, inFlight); - } - inFlight.add(work); - const remove = () => { - inFlight.delete(work); - if (inFlight.size === 0) { - inFlightSidecarMaintenanceByPath.delete(filePath); - } - }; - work.then(remove, remove); -} - -/** - * Cancel this sidecar's pending maintenance timers and drain writes already past their - * identity check, so a full clear or workspace removal cannot race a recreation of the file - * or the session directory it is about to delete. Must run BEFORE taking the sidecar file - * lock: an in-flight write acquires that same lock, so draining inside it would deadlock. - */ -export async function cancelAgentWorkflowRunReferenceMaintenance( - workspaceSessionDir: string -): Promise { - const filePath = referencesPath(workspaceSessionDir); - lifecycleGenerationByPath.set(filePath, (lifecycleGenerationByPath.get(filePath) ?? 0) + 1); - const byKey = pendingSidecarMaintenanceTimersByPath.get(filePath); - if (byKey != null) { - for (const timer of byKey.values()) { - clearTimeout(timer); - } - pendingSidecarMaintenanceTimersByPath.delete(filePath); - } - const inFlight = inFlightSidecarMaintenanceByPath.get(filePath); - if (inFlight != null && inFlight.size > 0) { - await Promise.allSettled([...inFlight]); - } -} - function referencesPath(workspaceSessionDir: string): string { assert(workspaceSessionDir.length > 0, "agent workflow references require session dir"); return path.join(workspaceSessionDir, AGENT_WORKFLOW_RUN_REFERENCES_FILE); @@ -241,17 +125,12 @@ export async function readAgentWorkflowRunReferences( * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded * before the clear indistinguishable from one recorded after it; retiring the references with * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear - * workflow_resume re-records provenance. Pending record retries are cancelled first so a stale - * detached retry cannot recreate a retired reference. + * workflow_resume re-records provenance. */ export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { const filePath = referencesPath(workspaceSessionDir); - await cancelAgentWorkflowRunReferenceMaintenance(workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { - // recursive: a directory at this known sidecar path is corruption (it also fails reads - // with EISDIR), and force alone refuses to remove directories, which would fail every - // subsequent full clear identically. Removing it self-heals the workspace. - await fs.rm(filePath, { force: true, recursive: true }); + await fs.rm(filePath, { force: true }); }); } @@ -260,18 +139,6 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; - /** - * Detached-retry mode: skip only when a strictly newer record already exists, so a retry - * can supersede the stale entry a failed re-record (e.g. workflow_resume over an old - * dispatch) was meant to replace, while a newer successful record still wins. - */ - skipIfNewerRecordExists?: boolean; - /** - * Boundary-repair mode: only patch an entry that exists and still lacks a boundary - * snapshot. A missing entry means the reference was retired (clear/removal) and must not be - * resurrected; a present boundary means newer provenance already landed. - */ - onlyIfBoundaryAbsent?: boolean; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -287,19 +154,6 @@ export async function recordAgentWorkflowRunReference(input: { // Clamp like parseReferences: never persist a future-dated timestamp. const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); - if ( - input.skipIfNewerRecordExists === true && - previous != null && - previous.createdAtMs > createdAtMs - ) { - return; - } - if ( - input.onlyIfBoundaryAbsent === true && - (previous == null || previous.afterBoundaryMessageId !== undefined) - ) { - return; - } byRunId.set(input.runId, { runId: input.runId, // Latest record wins: workflow_resume re-records the reference, and a resume issued after @@ -320,68 +174,3 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } - -// Delays for detached sidecar maintenance (record retries, boundary-snapshot repairs). -export const SIDECAR_MAINTENANCE_RETRY_DELAYS_MS: readonly number[] = [1_000, 10_000, 60_000]; - -/** - * Retry a failed provenance record in the background. The launching tool has already returned - * and an untouched active run never hits a natural re-record site, so a single failed write - * would permanently supersede the run's terminal wake once storage recovers. Retries reuse the - * launch-time boundary snapshot, only fill absence (a later successful record wins), and are - * cancelled by a full history clear. - */ -export function scheduleAgentWorkflowRunReferenceRecordRetry(input: { - workspaceSessionDir: string; - runId: string; - createdAtMs: number; - afterBoundaryMessageId?: string | null; - retryDelaysMs?: readonly number[] | null; - attempt?: number; - /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ - lifecycleGeneration?: number; -}): void { - const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; - const attempt = input.attempt ?? 0; - const delayMs = retryDelaysMs[attempt]; - if (delayMs == null) { - log.error("Giving up on agent workflow run reference record after retries", { - runId: input.runId, - attempts: attempt, - }); - return; - } - const key = `record:${input.runId}`; - const lifecycleGeneration = - input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); - const timer = setTimeout(() => { - if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { - return; - } - // Detached by design: the launching tool already returned, so only this chain can finish - // the write. Failures reschedule until the bounded delays are exhausted. - const work = recordAgentWorkflowRunReference({ - workspaceSessionDir: input.workspaceSessionDir, - runId: input.runId, - createdAtMs: input.createdAtMs, - skipIfNewerRecordExists: true, - ...(input.afterBoundaryMessageId !== undefined - ? { afterBoundaryMessageId: input.afterBoundaryMessageId } - : {}), - }).catch((error: unknown) => { - log.warn("Agent workflow run reference record retry failed", { - runId: input.runId, - attempt: attempt + 1, - error, - }); - scheduleAgentWorkflowRunReferenceRecordRetry({ - ...input, - attempt: attempt + 1, - lifecycleGeneration, - }); - }); - trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); - }, delayMs); - timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); -} diff --git a/src/node/services/messageQueue.ts b/src/node/services/messageQueue.ts index 82a12421356..dbf71169ca3 100644 --- a/src/node/services/messageQueue.ts +++ b/src/node/services/messageQueue.ts @@ -96,8 +96,6 @@ type QueueDispatchMode = NonNullable; interface QueuedMessageInternalOptions { synthetic?: boolean; agentInitiated?: boolean; - /** Persisted onto the dispatched user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; /** * When the sender authored this message (request entry), before any send * preflight awaits (pricing gate, settings persistence). Goal safety diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 79f5369a5af..8367cc47dc3 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -554,7 +554,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; - getContextMutationEpoch: ReturnType; create: ReturnType; countQueuedAgentPeerMessages: ReturnType; }> @@ -592,7 +591,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled: ReturnType; emitChatEvent: ReturnType; isWorkflowInvocationCurrent: ReturnType; - getContextMutationEpoch: ReturnType; create: ReturnType; } { const sendMessage = @@ -662,7 +660,6 @@ function createWorkspaceServiceMocks( const updateAgentStatus = overrides?.updateAgentStatus ?? mock((): Promise => Promise.resolve()); const isExperimentEnabled = overrides?.isExperimentEnabled ?? mock(() => false); - const getContextMutationEpoch = overrides?.getContextMutationEpoch ?? mock(() => 0); const emitChatEvent = overrides?.emitChatEvent ?? mock((_workspaceId: string, _message: WorkspaceChatMessage) => undefined); @@ -739,7 +736,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, - getContextMutationEpoch, getWorkflowInvocationCurrentness, countQueuedAgentPeerMessages, } as unknown as WorkspaceService, @@ -776,7 +772,6 @@ function createWorkspaceServiceMocks( isExperimentEnabled, emitChatEvent, isWorkflowInvocationCurrent, - getContextMutationEpoch, }; } @@ -6438,10 +6433,11 @@ describe("TaskService", () => { expect(liftedOptions.toolPolicy).toBeUndefined(); }); - test("drain sends carry a staleness probe that trips after a full clear", async () => { + test("wake restriction restore walks past a long synthetic tail and carries the agent disable flag", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_admission_stale"; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_policy_long_tail"; const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); await runStore.createRun({ id: runId, @@ -6463,34 +6459,38 @@ describe("TaskService", () => { const sendMessage = mock( (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); - // The prompt is validated against history before the send is admitted; a full clear in - // that window advances the context-mutation epoch. The probe handed to sendMessage must - // observe the live epoch so admission can refuse the stale prompt. - let epoch = 1; - const { workspaceService } = createWorkspaceServiceMocks({ - sendMessage, - getContextMutationEpoch: mock(() => epoch), - }); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }) + ); + // A tail longer than any bounded history read: the launch turn's restrictions must still + // be found, not silently lifted once enough rows accumulate after the manual turn. + for (let i = 0; i < 60; i++) { + await historyService.appendToHistory( + parentId, + createMuxMessage(`assistant-${i}`, "assistant", `progress ${i}`, { timestamp: 1_001 + i }) + ); + } await taskService.enqueueWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", }); await flushTerminalAttentionDrains(taskService); - expect(sendMessage).toHaveBeenCalledTimes(1); - const internal = sendMessage.mock.calls[0]?.[3] as { - admissionStale?: () => boolean; - deliveredWorkflowRunIds?: string[]; - }; - // Consumption provenance persisted with the accepted row (crash-replay suppression). - expect(internal.deliveredWorkflowRunIds).toEqual([runId]); - expect(typeof internal.admissionStale).toBe("function"); - expect(internal.admissionStale?.()).toBe(false); - epoch = 2; - expect(internal.admissionStale?.()).toBe(true); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }); }); test("initialize replays and clears persisted pending task guidance", async () => { @@ -6890,75 +6890,6 @@ describe("TaskService", () => { }); }); - test("initialize recovery keeps indeterminate workflow runs enqueued for a later drain", async () => { - const config = await createTestConfig(rootDir); - const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_recovery_indeterminate"; - const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); - await runStore.createRun({ - id: runId, - workspaceId: parentId, - workflow: { - name: "research", - description: "Research workflow", - scope: "built-in", - executable: true, - }, - source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-06-19T00:00:00.000Z", - }); - await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); - await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - - const terminalAttentionStore = new TerminalAttentionStore(config); - const sendMessage = mock( - (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) - ); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - // History/sidecar unreadable at startup. Recovery is the only reconstruction point for a - // wake that never reached the outbox, and no pending notification exists yet to arm the - // drain's defer retry, so skipping here would strand the run until another restart. The - // boolean wrapper collapses indeterminate to false, which is what recovery must NOT use. - (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = - mock(() => Promise.resolve("indeterminate")); - (workspaceService as unknown as Record).isWorkflowInvocationCurrent = mock( - () => Promise.resolve(false) - ); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); - - await taskService.initialize(); - await flushTerminalAttentionDrains(taskService); - - expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); - }); - - test("an unreadable sidecar defers workspace-turn blocker checks", async () => { - const config = await createTestConfig(rootDir); - const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const { workspaceService } = createWorkspaceServiceMocks(); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); - const internal = taskService as unknown as { - hasActiveWorkspaceTurnDeferredBlockers(record: { workspaceId: string }): Promise; - }; - - expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( - false - ); - - // The sidecar is the only durable provenance for kernel-launched workflows, and this - // result gates completion decisions: an unreadable sidecar must report blockers (defer), - // not "none" and let the turn finalize while a workflow may still be running. - await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { - recursive: true, - }); - expect(await internal.hasActiveWorkspaceTurnDeferredBlockers({ workspaceId: parentId })).toBe( - true - ); - }); - test("initialize recovers terminal notify workspace turns without pending notification", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 5f431201f42..f2177fde1b5 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1703,10 +1703,13 @@ export class TaskService { } const runIds = new Set(); - // An unreadable sidecar PROPAGATES: callers gate destructive completion decisions - // (finalizing task reports, ending workspace turns) on this listing, and treating the - // failure as "no references" would let those finalize while a kernel workflow still runs. - const references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + let references: Awaited> = []; + try { + references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error: unknown) { + // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. + log.warn("Failed to read agent workflow run references", { workspaceId, error }); + } for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -7594,15 +7597,7 @@ export class TaskService { ) { continue; } - const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( - workspace.id, - run.id - ); - // Indeterminate (unreadable history/sidecar) must still enqueue: startup recovery is - // the only reconstruction point for wakes that never reached the outbox, and no - // pending notification exists yet to arm the drain's defer retry. The drain - // re-evaluates currentness and defers or supersedes with full context. - if (currentness === "not_current") { + if (!(await this.workspaceService.isWorkflowInvocationCurrent(workspace.id, run.id))) { continue; } const created = await this.terminalAttentionStore.enqueueIfAbsent({ @@ -7915,31 +7910,50 @@ export class TaskService { } /** - * Caller tool policy to restore on a terminal-attention wake. The newest manual user row - * carries the conversation's persisted caller policy; synthetic rows without one (earlier - * wakes, heartbeat scaffolding) do not define policy and are skipped. Throws when history - * is unreadable so the caller can fail closed instead of waking with unrestricted tools. + * Caller send restrictions (tool policy, workspace-agent disable flag) to restore on a + * terminal-attention wake. The newest manual user row carries the conversation's persisted + * restrictions; synthetic rows without any (earlier wakes, heartbeat scaffolding) do not + * define them and are skipped. The walk is unbounded: a long assistant/synthetic tail after + * the launch turn must not push the defining row out of sight and silently lift the + * restrictions. Throws when history is unreadable so the caller can fail closed instead of + * waking with unrestricted tools. */ - private async resolveTerminalWakeCallerToolPolicy( + private async resolveTerminalWakeCallerSendRestrictions( ownerWorkspaceId: string - ): Promise { - const historyResult = await this.historyService.getLastMessages(ownerWorkspaceId, 50); - if (!historyResult.success) { - throw new Error(`history unavailable: ${historyResult.error}`); - } - for (let i = historyResult.data.length - 1; i >= 0; i--) { - const message = historyResult.data[i]; - if (message?.role !== "user") { - continue; - } - if (message.metadata?.toolPolicy != null) { - return message.metadata.toolPolicy; - } - if (message.metadata?.synthetic !== true) { + ): Promise<{ toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }> { + const state: { + found: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + } = { found: null }; + const historyResult = await this.historyService.iterateFullHistory( + ownerWorkspaceId, + "backward", + (messages) => { + for (const message of messages) { + if (message.role !== "user") { + continue; + } + const metadata = message.metadata; + if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { + state.found = { + ...(metadata.toolPolicy != null ? { toolPolicy: metadata.toolPolicy } : {}), + ...(metadata.disableWorkspaceAgents != null + ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } + : {}), + }; + return false; + } + if (metadata?.synthetic !== true) { + state.found = {}; + return false; + } + } return undefined; } + ); + if (!historyResult.success) { + throw new Error(`history unavailable: ${historyResult.error}`); } - return undefined; + return state.found ?? {}; } private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { @@ -8314,23 +8328,12 @@ export class TaskService { isPersistentChildContinuation ? record.workspaceId : notification.sourceId ); } - // Workflow prompts are validated against history well before the send is admitted; a full - // clear in that window truncates history and retires the sidecar, so the send must refuse - // (admissionStale) rather than inject the stale result into the freshly cleared - // conversation. A refused send leaves the notifications pending for the next drain. - const admissionEpoch = this.workspaceService.getContextMutationEpoch(ownerWorkspaceId); - const sendAdmissionStale = () => - this.workspaceService.getContextMutationEpoch(ownerWorkspaceId) !== admissionEpoch; const workflowNotifications = pending.filter( (notification) => notification.sourceKind === "workflow_run" ); const deliverableWorkflowNotificationIds = new Set(); const promptSections: string[] = []; - // Persisted onto the accepted row as consumption provenance (see - // MuxMetadata.deliveredWorkflowRunIds): after a crash between acceptance and the outbox - // delivery mark, restart recovery recognizes the row and does not replay these results. - const deliveredWorkflowRunIds: string[] = []; if (publicAwaitIds.length > 0) { promptSections.push(buildCompletedWorkspaceTurnPrompt(publicAwaitIds)); } @@ -8360,7 +8363,6 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); - deliveredWorkflowRunIds.push(notification.sourceId); promptSections.push(workflowPrompt.prompt); } @@ -8403,9 +8405,9 @@ export class TaskService { // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake // regain tools the caller disabled (with attacker-influenced workflow output choosing the // timing). The agent-level policy recomposes from agentId at send resolution. - let wakeToolPolicy: ToolPolicy | undefined; + let wakeRestrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }; try { - wakeToolPolicy = await this.resolveTerminalWakeCallerToolPolicy(ownerWorkspaceId); + wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); } catch (error: unknown) { // Fail closed: an unknown policy must not fall back to unrestricted tools. log.warn("Deferring terminal wake; caller tool policy unavailable", { @@ -8421,7 +8423,8 @@ export class TaskService { agentId: resumeOptions.agentId, thinkingLevel: resumeOptions.thinkingLevel, reasoningMode: resumeOptions.reasoningMode, - ...(wakeToolPolicy != null ? { toolPolicy: wakeToolPolicy } : {}), + ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), + ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { @@ -8451,14 +8454,7 @@ export class TaskService { prompt, sendOptions, // Synthetic, idle-only auto-resume — same flags as the active-work auto-resume path. - { - skipAutoResumeReset: true, - synthetic: true, - agentInitiated: true, - requireIdle: true, - admissionStale: sendAdmissionStale, - deliveredWorkflowRunIds, - } + { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, requireIdle: true } ); if (!sendResult.success && isWorkspaceBusyIdleOnlySend(sendResult.error)) { @@ -8480,8 +8476,6 @@ export class TaskService { skipAutoResumeReset: true, synthetic: true, agentInitiated: true, - admissionStale: sendAdmissionStale, - deliveredWorkflowRunIds, onCanceled: () => { this.scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId); }, @@ -12587,21 +12581,10 @@ export class TaskService { return true; } - let referencedWorkflowRunIds: string[]; - try { - referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - record.workspaceId, - [] - ); - } catch (error: unknown) { - // Unreadable sidecar: assume blockers exist so the deferred turn is not finalized while - // a kernel workflow may still be running; the next evaluation retries. - log.warn("Deferring workspace-turn blocker check; sidecar unreadable", { - workspaceId: record.workspaceId, - error, - }); - return true; - } + const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + record.workspaceId, + [] + ); if ( (await this.listActiveBackgroundWorkflowRunIds(record.workspaceId, referencedWorkflowRunIds)) .length > 0 @@ -14218,22 +14201,11 @@ export class TaskService { taskIndex, workspaceId ); - let referencedWorkflowRunIds: string[]; - try { - referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); - } catch (error: unknown) { - // Unreadable sidecar: neither finalize the turn nor nudge the model about unknown - // runs; leave the stream-end unhandled so deferred recovery re-evaluates later. - log.warn("Skipping parent stream-end workflow reconciliation; sidecar unreadable", { - workspaceId, - error, - }); - return; - } + const referencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); let activeWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, referencedWorkflowRunIds @@ -14520,22 +14492,11 @@ export class TaskService { return; } - let taskReferencedWorkflowRunIds: string[]; - try { - taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( - workspaceId, - event.parts, - event.messageId - ); - } catch (error: unknown) { - // Unreadable sidecar: defer report finalization like an active blocker instead of - // publishing while a kernel workflow may still be running. - log.warn("Deferring task finalization; sidecar unreadable", { workspaceId, error }); - if (status === "awaiting_report") { - await this.setTaskStatus(workspaceId, "running"); - } - return; - } + const taskReferencedWorkflowRunIds = await this.listAgentReferencedWorkflowRunIds( + workspaceId, + event.parts, + event.messageId + ); const activeTaskWorkflowRunIds = await this.listActiveBackgroundWorkflowRunIds( workspaceId, taskReferencedWorkflowRunIds diff --git a/src/node/services/tools/toolUtils.test.ts b/src/node/services/tools/toolUtils.test.ts deleted file mode 100644 index 9785a1ce04b..00000000000 --- a/src/node/services/tools/toolUtils.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -import * as fs from "node:fs/promises"; -import * as os from "node:os"; -import * as path from "node:path"; - -import { describe, expect, test } from "bun:test"; - -import type { ToolConfiguration } from "@/common/utils/tools/tools"; -import { - readAgentWorkflowRunReferences, - recordAgentWorkflowRunReference, -} from "@/node/services/agentWorkflowRunReferences"; -import { recordBackgroundWorkflowRunReference } from "@/node/services/tools/toolUtils"; - -describe("recordBackgroundWorkflowRunReference", () => { - test("retries a failed provenance record until storage recovers", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-record-")); - try { - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_existing", - createdAtMs: 1_000, - }); - const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); - // Unreadable at record time: the tool has already returned by the time storage recovers, - // and an untouched active run never hits a natural re-record site, so only the bounded - // background retry can persist provenance for the terminal wake. - await fs.chmod(filePath, 0o000); - await recordBackgroundWorkflowRunReference( - { workspaceSessionDir } as unknown as ToolConfiguration, - "wfr_retry", - 2_000, - [25, 25, 25] - ); - await fs.chmod(filePath, 0o600); - - const deadline = Date.now() + 5_000; - let runIds: string[] = []; - while (Date.now() < deadline) { - runIds = (await readAgentWorkflowRunReferences(workspaceSessionDir)).map( - (reference) => reference.runId - ); - if (runIds.includes("wfr_retry")) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(new Set(runIds)).toEqual(new Set(["wfr_existing", "wfr_retry"])); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("repairs a verified-empty boundary snapshot after a transient read failure", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); - try { - // Launch from a decision-free history whose boundary read fails once: the rediscovery - // entry lands boundary-less, and only the repair can restore the verified-empty (null) - // snapshot the decision-free currentness branch requires. - let calls = 0; - const taskService = { - getWorkflowInvocationBoundaryMessageId: (): Promise => { - calls += 1; - return calls === 1 - ? Promise.reject(new Error("history unavailable")) - : Promise.resolve(null); - }, - }; - await recordBackgroundWorkflowRunReference( - { - workspaceSessionDir, - workspaceId: "ws-boundary-repair", - taskService, - } as unknown as ToolConfiguration, - "wfr_boundary_repair", - 2_000, - [25, 25, 25] - ); - let reference: { afterBoundaryMessageId?: string | null } | undefined; - const deadline = Date.now() + 5_000; - while (Date.now() < deadline) { - [reference] = await readAgentWorkflowRunReferences(workspaceSessionDir); - if (reference?.afterBoundaryMessageId === null) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(reference).toMatchObject({ - runId: "wfr_boundary_repair", - afterBoundaryMessageId: null, - }); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("boundary repair waits for the record retry to land the entry", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); - try { - // Both the boundary read AND the initial sidecar write fail: the reference does not - // exist when the repair first fires. A missing entry must stay retryable, or the record - // retry that lands later creates a permanently boundary-less reference. - let calls = 0; - const taskService = { - getWorkflowInvocationBoundaryMessageId: (): Promise => { - calls += 1; - return calls === 1 - ? Promise.reject(new Error("history unavailable")) - : Promise.resolve(null); - }, - }; - const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_seed", - createdAtMs: 1_000, - }); - await fs.chmod(filePath, 0o000); - await recordBackgroundWorkflowRunReference( - { - workspaceSessionDir, - workspaceId: "ws-boundary-late", - taskService, - } as unknown as ToolConfiguration, - "wfr_boundary_late", - 2_000, - [40, 40, 40, 40, 40] - ); - // Storage recovers only after the repair has fired at least once against the missing - // entry; the record retry then lands it and a later repair attempt patches null. - await new Promise((resolve) => setTimeout(resolve, 60)); - await fs.chmod(filePath, 0o600); - - const deadline = Date.now() + 5_000; - let boundary: string | null | undefined; - while (Date.now() < deadline) { - boundary = (await readAgentWorkflowRunReferences(workspaceSessionDir)).find( - (reference) => reference.runId === "wfr_boundary_late" - )?.afterBoundaryMessageId; - if (boundary === null) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(boundary).toBe(null); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("keeps the entry boundary-less when a decision row exists at repair time", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "toolutils-boundary-")); - try { - // A decision row seen at repair time may postdate the launch; persisting it would - // overclaim currentness, so the entry must stay boundary-less and fail safe. - let calls = 0; - const taskService = { - getWorkflowInvocationBoundaryMessageId: (): Promise => { - calls += 1; - return calls === 1 - ? Promise.reject(new Error("history unavailable")) - : Promise.resolve("manual-user"); - }, - }; - await recordBackgroundWorkflowRunReference( - { - workspaceSessionDir, - workspaceId: "ws-boundary-unsafe", - taskService, - } as unknown as ToolConfiguration, - "wfr_boundary_unsafe", - 2_000, - [25] - ); - await new Promise((resolve) => setTimeout(resolve, 300)); - const references = await readAgentWorkflowRunReferences(workspaceSessionDir); - expect(references).toHaveLength(1); - expect(references[0]?.runId).toBe("wfr_boundary_unsafe"); - expect(references[0]?.afterBoundaryMessageId).toBeUndefined(); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); -}); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 89e095dce18..a56f4574762 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -7,16 +7,7 @@ import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkspaceChatMessage } from "@/common/orpc/types"; import type { ToolConfiguration } from "@/common/utils/tools/tools"; -import { - SIDECAR_MAINTENANCE_RETRY_DELAYS_MS, - getSidecarLifecycleGeneration, - readAgentWorkflowRunReferences, - recordAgentWorkflowRunReference, - registerSidecarMaintenanceTimer, - scheduleAgentWorkflowRunReferenceRecordRetry, - takeSidecarMaintenanceTimer, - trackSidecarMaintenanceWrite, -} from "@/node/services/agentWorkflowRunReferences"; +import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { log } from "@/node/services/log"; import type { TaskService } from "@/node/services/taskService"; @@ -88,81 +79,6 @@ export async function emitWorkflowRunAttachedEvent(input: { await input.config.emitChatEvent(event); } -/** - * Repair a missing boundary snapshot in the background. A kernel launch from a decision-free - * history whose record-time boundary read failed persists a boundary-less entry, but the - * decision-free currentness branch accepts only an explicit verified-empty (null) snapshot, - * so without repair the run's terminal wake is permanently superseded once storage recovers. - * Rows never disappear outside a full clear (which retires the sidecar), so a history still - * verified-empty at repair time was also empty at launch and null is faithful launch - * provenance; a decision row seen at repair time may postdate the launch, so persisting it - * would overclaim currentness and the entry stays boundary-less (fail safe). - */ -function scheduleBoundarySnapshotRepair(input: { - workspaceSessionDir: string; - runId: string; - createdAtMs: number; - getBoundary: () => Promise; - retryDelaysMs?: readonly number[] | null; - attempt?: number; - /** Chain state: the lifecycle generation captured when the chain was first scheduled. */ - lifecycleGeneration?: number; -}): void { - const retryDelaysMs = input.retryDelaysMs ?? SIDECAR_MAINTENANCE_RETRY_DELAYS_MS; - const attempt = input.attempt ?? 0; - const delayMs = retryDelaysMs[attempt]; - if (delayMs == null) { - log.error("Giving up on workflow boundary snapshot repair after retries", { - runId: input.runId, - attempts: attempt, - }); - return; - } - const key = `boundary:${input.runId}`; - const lifecycleGeneration = - input.lifecycleGeneration ?? getSidecarLifecycleGeneration(input.workspaceSessionDir); - const timer = setTimeout(() => { - if (!takeSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer)) { - return; - } - const work = (async () => { - // The reference may not exist yet: when the initial write also failed, the independent - // record-retry chain lands it later. A missing entry is retryable, not a satisfied - // no-op, or that later record would create a permanently boundary-less reference; - // lifecycle cancellation kills this chain when the reference was retired instead. - const references = await readAgentWorkflowRunReferences(input.workspaceSessionDir); - const entry = references.find((reference) => reference.runId === input.runId); - if (entry == null) { - throw new Error("reference not recorded yet"); - } - if (entry.afterBoundaryMessageId !== undefined) { - return; - } - const boundary = await input.getBoundary(); - if (boundary !== null) { - return; - } - await recordAgentWorkflowRunReference({ - workspaceSessionDir: input.workspaceSessionDir, - runId: input.runId, - createdAtMs: input.createdAtMs, - afterBoundaryMessageId: null, - onlyIfBoundaryAbsent: true, - }); - })().catch((error: unknown) => { - log.warn("Workflow boundary snapshot repair failed", { - runId: input.runId, - attempt: attempt + 1, - error: getErrorMessage(error), - }); - scheduleBoundarySnapshotRepair({ ...input, attempt: attempt + 1, lifecycleGeneration }); - }); - trackSidecarMaintenanceWrite(input.workspaceSessionDir, work); - }, delayMs); - timer.unref?.(); - registerSidecarMaintenanceTimer(input.workspaceSessionDir, key, timer, lifecycleGeneration); -} - /** * Persist agent provenance for a workflow run that outlives the current turn (background * start/resume, or a foreground run that backgrounded itself). TaskService reads these @@ -173,8 +89,7 @@ function scheduleBoundarySnapshotRepair(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number, - retryDelaysMs?: readonly number[] | null + createdAtMs: number ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { @@ -202,18 +117,6 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); - const workspaceId = config.workspaceId; - const getBoundaryMessageId = - taskService?.getWorkflowInvocationBoundaryMessageId?.bind(taskService); - if (workspaceId != null && getBoundaryMessageId != null) { - scheduleBoundarySnapshotRepair({ - workspaceSessionDir, - runId, - createdAtMs, - getBoundary: () => getBoundaryMessageId(workspaceId, runId), - retryDelaysMs, - }); - } } } @@ -229,12 +132,5 @@ export async function recordBackgroundWorkflowRunReference( runId, error: getErrorMessage(error), }); - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir, - runId, - createdAtMs, - retryDelaysMs, - ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), - }); } } diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 63df2868bc6..79d9826f211 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -329,6 +329,44 @@ describe("workflow_resume tool", () => { }); }); + test("consumes the foreground terminal result even when the refresh read fails", async () => { + using tempDir = new TestTempDir("test-workflow-resume-refresh-failure-consume"); + // WorkflowService.getRun collapses transient read failures to null. The terminal result is + // still returned to the model, so consumption must derive from the dispatch result or the + // pending terminal attention would re-inject it later. + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? buildRun() : null; + }), + resumeRun: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "resumed" }, + })), + }); + const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + }); + + const result = await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + }); + expect(result).toMatchObject({ status: "completed", runId: "wfr_resume_me", mode: "resume" }); + }); + test("rejects default resume of a failed run with checkpoint retry guidance", async () => { using tempDir = new TestTempDir("test-workflow-resume-failed"); const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 649c1ca90bf..57870d7d78d 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -4,7 +4,7 @@ import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; import { isTerminalWorkflowRunStatus, type WorkflowRunRecord } from "@/common/types/workflow"; import { getWorkflowCheckpointRetryEligibility } from "@/common/utils/workflowRetryEligibility"; -import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; +import { WorkflowRunRecordSchema, WorkflowRunStatusSchema } from "@/common/orpc/schemas"; import { WorkflowResumeToolResultSchema, TOOL_DEFINITIONS, @@ -158,7 +158,9 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // workflow_resume part in history, so the history-walk consumption predicates cannot see // that this turn already received the terminal result. Persist consumption durably so the // terminal-attention drain never re-delivers it. - const markTerminalAttentionConsumed = async (terminalRun: WorkflowRunRecord) => { + const markTerminalAttentionConsumed = async ( + terminalRun: Pick + ) => { if (!isTerminalWorkflowRunStatus(terminalRun.status)) { return; } @@ -249,8 +251,14 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Foreground only: a background dispatch can still observe the stale pre-dispatch terminal // status, and consuming it would tombstone the retried run's future terminal wake. - if (!isBackgroundDispatch && refreshedRun != null) { - await markTerminalAttentionConsumed(refreshedRun); + // Consumption derives from the dispatch result itself, not the refreshed record: the + // terminal result is returned to the model below even when the refresh read fails, and + // skipping the tombstone would let the pending terminal attention re-inject it later. + if (!isBackgroundDispatch) { + const dispatchedStatus = WorkflowRunStatusSchema.safeParse(dispatched.status); + if (dispatchedStatus.success) { + await markTerminalAttentionConsumed({ id: runId, status: dispatchedStatus.data }); + } } return parseToolResult( diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index e440984e50b..970ccea6d11 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -8,7 +8,6 @@ import { targetMutationLockFilePath, withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; -import { scheduleAgentWorkflowRunReferenceRecordRetry } from "@/node/services/agentWorkflowRunReferences"; import { acquireProcessFileLock, getProcessBirth } from "@/node/utils/concurrency/fileLock"; import { healRemovalTombstonesForRegisteredWorkspaces, @@ -22,39 +21,6 @@ import { } from "./workspaceRemoval"; describe("workspaceRemoval", () => { - test("removal cancels pending sidecar record retries so they cannot recreate the session dir", async () => { - using tmp = new DisposableTempDir("workspace-removal-sidecar"); - const rootDir = path.join(tmp.path, "xum-home"); - const workspaceId = "ws-removal-sidecar"; - const sessionDir = path.join(rootDir, "sessions", workspaceId); - await fsPromises.mkdir(sessionDir, { recursive: true }); - - // A detached provenance retry armed before removal; on fire it would mkdir the session - // directory back into existence after the deletion below. - scheduleAgentWorkflowRunReferenceRecordRetry({ - workspaceSessionDir: sessionDir, - runId: "wfr_removal_race", - createdAtMs: 1_000, - afterBoundaryMessageId: null, - retryDelaysMs: [100], - }); - - await removeSessionDirUnderMemoryLocks({ - rootDir, - sessionDir, - workspaceId, - attemptId: "test-attempt-sidecar", - }); - await new Promise((resolve) => setTimeout(resolve, 350)); - - expect( - await fsPromises.access(sessionDir).then( - () => true, - () => false - ) - ).toBe(false); - }); - test("deletion waits for a live memory writer, then tombstones and deletes (r61)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index c2fe3b8c4d5..154c7311626 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -27,7 +27,6 @@ * foreign backend's still-running consolidation refuse arbitrarily late. */ -import { cancelAgentWorkflowRunReferenceMaintenance } from "@/node/services/agentWorkflowRunReferences"; import crypto from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -218,10 +217,6 @@ export async function removeSessionDirUnderMemoryLocks(args: { // deleted directory cannot be recreated by a late mutation or // journal append. await publishTombstone(); - // Detached sidecar maintenance (provenance record retries) would survive removal and - // recreate the deleted session directory when its timer fires; cancel and drain it - // like the other session writers before the directory goes away. - await cancelAgentWorkflowRunReferenceMaintenance(args.sessionDir); await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); } ); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index e92a56be1f6..daac7889e48 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6266,78 +6266,6 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("a failed reference retirement aborts the clear before deleting history", async () => { - const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-currentness-retire-emit"; - const projectPath = path.join(config.rootDir, "project"); - try { - await config.addWorkspace(projectPath, { - id: workspaceId, - name: "workflow-currentness-retire-emit", - projectName: "project", - projectPath, - runtimeConfig: { type: "local" }, - }); - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - aiService: createMockAIService({ - stopStream: mock(() => Promise.resolve(Ok(undefined))), - }), - extensionMetadata: new ExtensionMetadataService( - path.join(config.rootDir, "extensionMetadata.json") - ), - initStateManager: { - ...mockInitStateManager, - off: mock(() => undefined as unknown as InitStateManager), - } as unknown as InitStateManager, - }); - - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) - ); - const sessionAccessor = workspaceService as unknown as { - getOrCreateSession(id: string): { emitChatEvent(message: unknown): void }; - }; - const session = sessionAccessor.getOrCreateSession(workspaceId); - const emitSpy = spyOn(session, "emitChatEvent"); - // A read-only session directory makes retirement fail (a directory at the sidecar path - // now self-heals instead). Retirement runs BEFORE the truncation, so the failure must - // abort the whole clear: the transcript survives, the renderer sees no deletion, and no - // crash window exists in which the transcript is gone while the sidecar lives on. - const sessionDir = config.getSessionDir(workspaceId); - await recordAgentWorkflowRunReference({ - workspaceSessionDir: sessionDir, - runId: "wfr_retirement_blocked", - createdAtMs: 1_150, - afterBoundaryMessageId: "manual-user", - }); - await fsPromises.chmod(sessionDir, 0o555); - try { - const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); - expect(clearResult.success).toBe(false); - if (!clearResult.success) { - expect(clearResult.error).toContain("workflow run references"); - } - expect( - emitSpy.mock.calls.some((call) => (call[0] as { type?: string }).type === "delete") - ).toBe(false); - const survivingHistory = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(survivingHistory.success).toBe(true); - if (survivingHistory.success) { - expect(survivingHistory.data).toHaveLength(1); - } - } finally { - emitSpy.mockRestore(); - await fsPromises.chmod(sessionDir, 0o755); - } - workspaceService.disposeSession(workspaceId); - } finally { - await cleanup(); - } - }); - test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; @@ -6374,47 +6302,44 @@ describe("WorkspaceService workflow invocation events", () => { }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // A synthetic row whose TEXT reproduces a result payload must not count: synthetic rows - // can carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must - // not spoof consumption and suppress the real wake. + // Another run's payload quoting nothing about this run must not count as consumption. await historyService.appendToHistory( workspaceId, createMuxMessage( - "coalesced-spoof", + "coalesced-other", "user", buildWorkflowResultContextMessage({ - rawCommand: "workflow_run research.js", - name: "research.js", - runId, + rawCommand: "workflow_run other.js", + name: "other.js", + runId: "wfr_currentness_other", status: "completed", - result: { reportMarkdown: "spoofed" }, + result: { reportMarkdown: "other done" }, run: null, }), - { timestamp: 1_240, synthetic: true } + { timestamp: 1_250, synthetic: true } ) ); - // Another run's persisted provenance must not count for this run either. - await historyService.appendToHistory( - workspaceId, - createMuxMessage("coalesced-other", "user", "results delivered", { - timestamp: 1_250, - synthetic: true, - deliveredWorkflowRunIds: ["wfr_currentness_other"], - }) - ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - // The drain persists deliveredWorkflowRunIds onto the accepted row. After a crash - // between durable acceptance and the outbox delivery mark, this provenance is the only + // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a + // crash between durable acceptance and the outbox delivery mark, this row is the only // evidence the result already reached history; it must read as consumption or restart // recovery injects the same terminal result again. await historyService.appendToHistory( workspaceId, - createMuxMessage("coalesced-result", "user", "results delivered", { - timestamp: 1_300, - synthetic: true, - deliveredWorkflowRunIds: [runId], - }) + createMuxMessage( + "coalesced-result", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run research.js", + name: "research.js", + runId, + status: "completed", + result: { reportMarkdown: "done" }, + run: null, + }), + { timestamp: 1_300, synthetic: true } + ) ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); @@ -6423,58 +6348,6 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("a corrupt sidecar directory self-heals during a full clear", async () => { - const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-currentness-selfheal"; - const projectPath = path.join(config.rootDir, "project"); - try { - await config.addWorkspace(projectPath, { - id: workspaceId, - name: "workflow-currentness-selfheal", - projectName: "project", - projectPath, - runtimeConfig: { type: "local" }, - }); - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - aiService: createMockAIService({ - stopStream: mock(() => Promise.resolve(Ok(undefined))), - }), - extensionMetadata: new ExtensionMetadataService( - path.join(config.rootDir, "extensionMetadata.json") - ), - initStateManager: { - ...mockInitStateManager, - off: mock(() => undefined as unknown as InitStateManager), - } as unknown as InitStateManager, - }); - - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "hello", { timestamp: 1_000 }) - ); - // A directory at the known sidecar path is corruption (reads fail with EISDIR). The - // full clear must remove it and succeed, not fail identically on every retry and leave - // the workspace impossible to clear without manual session-storage repair. - const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); - await fsPromises.mkdir(sidecarPath); - await fsPromises.writeFile(path.join(sidecarPath, "junk.txt"), "junk"); - - const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); - expect(clearResult.success).toBe(true); - expect( - await fsPromises.access(sidecarPath).then( - () => true, - () => false - ) - ).toBe(false); - workspaceService.disposeSession(workspaceId); - } finally { - await cleanup(); - } - }); - test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-clock"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3b6e9b037b6..bb95cabd55c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -204,6 +204,7 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, + textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -477,16 +478,18 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) /** * The terminal-attention drain delivers workflow results as one synthetic user prompt that may - * coalesce several runs. If a crash lands between the send's durable acceptance and the outbox - * delivery mark, restart recovery drains the notification again; recognizing the accepted row - * as consumption is what suppresses the replay. Recognition uses the drain's persisted - * provenance (MuxMetadata.deliveredWorkflowRunIds), never the row's text: synthetic rows can - * carry user-controlled content (e.g. a heartbeat body), and a quoted run ID must not spoof - * consumption and suppress a real wake. + * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands + * between the send's durable acceptance and the outbox delivery mark, restart recovery drains + * the notification again; recognizing the accepted row as consumption is what suppresses the + * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is + * classified before this check runs. */ function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { - return ( - message.role === "user" && message.metadata?.deliveredWorkflowRunIds?.includes(runId) === true + if (message.role !== "user" || message.metadata?.synthetic !== true) { + return false; + } + return message.parts.some( + (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) ); } @@ -3803,14 +3806,6 @@ export class WorkspaceService extends EventEmitter { } } - /** - * Current context-mutation epoch (see contextMutationEpochs). Lets the terminal-attention - * drain refuse a send whose workflow prompt was validated before a full clear committed. - */ - getContextMutationEpoch(workspaceId: string): number { - return this.contextMutationEpochs.get(workspaceId) ?? 0; - } - /** r41: mark a context-discarding mutation as durably committed (see contextMutationEpochs). */ private advanceContextMutationEpoch(workspaceId: string): void { this.contextMutationEpochs.set( @@ -11251,8 +11246,6 @@ export class WorkspaceService extends EventEmitter { goalId?: string; /** Force Copilot billing classification to "agent" for internal sends. */ agentInitiated?: boolean; - /** Persisted onto the user row; see MuxMetadata.deliveredWorkflowRunIds. */ - deliveredWorkflowRunIds?: string[]; onAccepted?: () => Promise | void; onCanceled?: (reason: string) => Promise | void; onAcceptedPreStreamFailure?: (error: SendMessageError) => Promise | void; @@ -11515,7 +11508,6 @@ export class WorkspaceService extends EventEmitter { return await session.sendMessage(message, normalizedOptions, { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, - deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, cancelState: internal?.cancelState, @@ -11660,7 +11652,6 @@ export class WorkspaceService extends EventEmitter { { synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, - deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, authoredAtMs, workspaceTurnContinuation: internal?.workspaceTurnContinuation, dedupeKey: internal?.queueDedupeKey, @@ -11773,7 +11764,6 @@ export class WorkspaceService extends EventEmitter { onTurnAdmissionCommitted: () => sessionInvisiblePreflight.release(), synthetic: internal?.synthetic, agentInitiated: internal?.agentInitiated, - deliveredWorkflowRunIds: internal?.deliveredWorkflowRunIds, goalKind: internal?.goalKind, goalId: internal?.goalId, goalContinuation: internal?.goalContinuation, @@ -12912,23 +12902,6 @@ export class WorkspaceService extends EventEmitter { ); } } - // Kernel workflow run references belong to the transcript being discarded: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one recorded - // after it. Retire them durably BEFORE the truncation (like the r41 retry discard above) so - // no crash window exists in which the transcript is gone but the sidecar survives; a crash - // here can only lose a wake for a still-intact conversation, never inject a pre-clear - // result into the cleared one. A post-clear resume re-records provenance, and pending - // record retries are cancelled with the sidecar. - if (isFullClear) { - try { - await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error) { - return Err( - `Cannot clear history: stale workflow run references could not be retired ` + - `(${getErrorMessage(error)}); retry once the session storage is writable.` - ); - } - } if (effectivePercentage > 0) { session?.clearUsageState(); } @@ -12947,6 +12920,22 @@ export class WorkspaceService extends EventEmitter { // admitted afterwards (their content references the discarded context). if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them immediately after the truncation commits, + // before any later post-clear step that can fail and return early (goal acknowledgment, + // carryover discard), or the stale reference would survive the committed clear. A + // post-clear resume re-records provenance. + try { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } catch (error) { + return Err( + `History was cleared, but stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + + `result into the cleared conversation; retry once the session storage is writable.` + ); + } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the From d35491c6c96253abbbd530011ba746372df34e9c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 05:47:24 +0000 Subject: [PATCH 20/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20resolve=20the=20wak?= =?UTF-8?q?e's=20agent=20identity=20with=20an=20unbounded=20history=20walk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveParentAutoResumeOptions read only the last 20 rows to find the newest agent-bearing assistant row; agent-less synthetic rows (the drain appends one per pending sub-agent report) could push that row out of the window and silently recompose terminal-wake sends from the exec fallback, lifting a restricted agent's tool policy. Walk full history backward instead, matching the round-16 caller-restriction fix; the exec fallback now applies only to histories that never had an agent turn. --- src/node/services/taskService.test.ts | 7 ++++- src/node/services/taskService.ts | 37 +++++++++++++++------------ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8367cc47dc3..2df7479c391 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6433,7 +6433,7 @@ describe("TaskService", () => { expect(liftedOptions.toolPolicy).toBeUndefined(); }); - test("wake restriction restore walks past a long synthetic tail and carries the agent disable flag", async () => { + test("wake restoration walks a long agent-less tail: caller policy, agent identity, disable flag", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; @@ -6472,6 +6472,10 @@ describe("TaskService", () => { disableWorkspaceAgents: true, }) ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_000, agentId: "plan" }) + ); // A tail longer than any bounded history read: the launch turn's restrictions must still // be found, not silently lifted once enough rows accumulate after the manual turn. for (let i = 0; i < 60; i++) { @@ -6488,6 +6492,7 @@ describe("TaskService", () => { await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index f2177fde1b5..6b9992f0d17 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -2308,26 +2308,29 @@ export class TaskService { // Compaction is internal bookkeeping, not an identity for resuming user work. let agentId = hint?.agentId === "compact" ? undefined : hint?.agentId; - // Durable history preserves the parent identity across process restarts. + // Durable history preserves the parent identity across process restarts. The walk is + // unbounded: synthetic rows without an agent identity (drain-appended sub-agent reports, + // heartbeat scaffolding) can push the newest agent-bearing assistant row past any fixed + // tail, and a truncated read would silently recompose terminal-wake sends from the exec + // fallback, lifting a restricted agent's tool policy. if (!agentId) { - try { - const historyResult = await this.historyService.getLastMessages(parentWorkspaceId, 20); - if (historyResult.success) { - for (let i = historyResult.data.length - 1; i >= 0; i--) { - const msg = historyResult.data[i]; - if ( - msg?.role === "assistant" && - msg.metadata?.agentId && - msg.metadata.agentId !== "compact" - ) { - agentId = msg.metadata.agentId; - break; - } + const found: { agentId?: string } = {}; + await this.historyService.iterateFullHistory(parentWorkspaceId, "backward", (messages) => { + for (const msg of messages) { + if ( + msg.role === "assistant" && + msg.metadata?.agentId && + msg.metadata.agentId !== "compact" + ) { + found.agentId = msg.metadata.agentId; + return false; } } - } catch { - // Best-effort; fall through to defaults - } + return undefined; + }); + // A failed read falls through to defaults (best-effort); the terminal drain separately + // fails closed on unreadable history via resolveTerminalWakeCallerSendRestrictions. + agentId = found.agentId; } // 3) Default From 07923962c415697db7839fcbe25b043a8164e63d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:09:24 +0000 Subject: [PATCH 21/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20sanitize=20persiste?= =?UTF-8?q?d=20wake=20restrictions=20before=20restoring=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A malformed persisted toolPolicy (object, number) copied into the synthetic send throws during agent resolution before the wake is accepted, leaving the terminal notification pending and every later drain blocked on the same corrupt row. Parse the field with ToolPolicySchema and drop invalid values (the intact disable flag on the same row still applies), per the self-healing doctrine for corrupt history rows. The agent-identity walk gets the same-class guard: only non-empty string agentId metadata defines the resume identity. --- src/node/services/taskService.test.ts | 56 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 20 ++++++++-- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 2df7479c391..930dc2d4918 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6498,6 +6498,62 @@ describe("TaskService", () => { }); }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_policy_corrupt"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // Persisted metadata is untrusted disk state: a corrupt toolPolicy shape must be dropped + // (not copied into the send, where it would throw during resolution and permanently block + // the wake), while the intact disable flag on the same row still applies. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-corrupt", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: { bogus: true }, + disableWorkspaceAgents: true, + } as unknown as Parameters[3]) + ); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBe(true); + }); + test("initialize replays and clears persisted pending task guidance", async () => { const config = await createTestConfig(rootDir); const projectPath = path.join(rootDir, "repo"); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 6b9992f0d17..b283bb60934 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -118,6 +118,7 @@ import { import type { ProjectRef, WorkspaceMetadata } from "@/common/types/workspace"; import { getRuntimeType } from "@/node/runtime/initHook"; import { AgentIdSchema } from "@/common/orpc/schemas"; +import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import type { AgentDefinitionScope } from "@/common/types/agentDefinition"; import { normalizeAgentId, @@ -2319,7 +2320,8 @@ export class TaskService { for (const msg of messages) { if ( msg.role === "assistant" && - msg.metadata?.agentId && + typeof msg.metadata?.agentId === "string" && + msg.metadata.agentId.length > 0 && msg.metadata.agentId !== "compact" ) { found.agentId = msg.metadata.agentId; @@ -7937,9 +7939,21 @@ export class TaskService { } const metadata = message.metadata; if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { + // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep + // inside send resolution and leave the wake permanently blocked on the same corrupt + // row. Sanitize instead of trusting the JSON shape; an unparseable policy restores + // nothing while a valid disable flag still applies (self-healing doctrine). + const parsedPolicy = + metadata.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; + if (parsedPolicy != null && !parsedPolicy.success) { + log.warn("Ignoring malformed persisted toolPolicy on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } state.found = { - ...(metadata.toolPolicy != null ? { toolPolicy: metadata.toolPolicy } : {}), - ...(metadata.disableWorkspaceAgents != null + ...(parsedPolicy?.success ? { toolPolicy: parsedPolicy.data } : {}), + ...(typeof metadata.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), }; From 92806142997e6304d7ccd29722862671bfee68e0 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:36:22 +0000 Subject: [PATCH 22/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20persist=20kernel=20?= =?UTF-8?q?workflow=20provenance=20before=20the=20runner=20can=20reach=20t?= =?UTF-8?q?erminal=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both workflow tools recorded the sidecar reference after the dispatch returned, and background execution starts at lease acquisition, so a fast run (or a process exit mid-dispatch) could reach terminal state with no top-level invocation part and no sidecar reference; the terminal drain would then permanently supersede its wake. Explicit background launches now record in the awaited onRunCreated hook (run durable, runner not started) and explicit background resumes record before the dispatch; the post-dispatch records remain only for foreground runs that backgrounded themselves, where the outcome is only knowable after dispatch. Test mocks now honor the onRunCreated contract and probe that the reference is durable at dispatch time. --- .../services/tools/workflow_resume.test.ts | 14 ++++++- src/node/services/tools/workflow_resume.ts | 13 +++++- src/node/services/tools/workflow_run.test.ts | 42 ++++++++++++++----- src/node/services/tools/workflow_run.ts | 13 +++++- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 79d9826f211..b0d90d1858f 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -169,7 +169,18 @@ describe("workflow_resume tool", () => { test("resumes in background and records an agent workflow run reference", async () => { using tempDir = new TestTempDir("test-workflow-resume-bg"); - const workflowService = buildWorkflowService(); + // The reference must be durable BEFORE the dispatch: background execution starts at lease + // acquisition, and a fast run could otherwise reach terminal state with no provenance. + let referenceDurableAtDispatch = false; + const workflowService = buildWorkflowService({ + resumeRunInBackground: mock(async () => { + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableAtDispatch = references.some( + (reference) => reference.runId === "wfr_resume_me" + ); + return { runId: "wfr_resume_me", status: "running" as const, result: null }; + }), + }); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: false, @@ -187,6 +198,7 @@ describe("workflow_resume tool", () => { projectTrusted: false, }); expect(workflowService.resumeRun).not.toHaveBeenCalled(); + expect(referenceDurableAtDispatch).toBe(true); const references = await readAgentWorkflowRunReferences(tempDir.path); expect(references.map((reference) => reference.runId)).toContain("wfr_resume_me"); expect(result).toMatchObject({ status: "running", runId: "wfr_resume_me", mode: "resume" }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 57870d7d78d..70042c94b32 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -204,6 +204,15 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) runId, projectTrusted: config.trusted === true, }; + // Provenance must be durable BEFORE the dispatch: background execution starts at lease + // acquisition, and a fast run (or a process exit mid-dispatch) can reach terminal state + // before any post-dispatch write, permanently superseding its wake. A failed dispatch + // leaves the re-recorded reference behind, which is benign: it mirrors what a + // successful resume would persist for a run this turn explicitly re-engaged with, and + // rediscovery filters against the run store. + if (args.run_in_background === true) { + await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); + } let dispatched: { runId: string; status: string; result: unknown }; try { if (mode === "retry_from_checkpoint") { @@ -234,9 +243,11 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Background-style resumes outlive this turn; persist provenance so the run is // rediscoverable (task_await/task_list) and its terminal result re-engages the agent. + // Explicit background resumes already recorded it pre-dispatch; this covers a + // foreground resume that backgrounded itself. const isBackgroundDispatch = args.run_in_background === true || dispatched.status === "backgrounded"; - if (isBackgroundDispatch) { + if (isBackgroundDispatch && args.run_in_background !== true) { await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); } diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 05064896ffa..d49544850c6 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -14,9 +14,14 @@ import { resolveWorkflowScript } from "@/node/services/workflows/workflowScriptR import { TestTempDir, createTestToolConfig, writeProjectSkill } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import type { TaskService } from "@/node/services/taskService"; +import type { ToolConfiguration } from "@/common/utils/tools/tools"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +type BackgroundStartInput = Parameters< + NonNullable["startWorkflowInBackground"]> +>[0]; + const mockToolCallOptions: ToolExecutionOptions = { toolCallId: "test-call-id", messages: [], @@ -645,11 +650,23 @@ describe("workflow_run tool", () => { const startWorkflow = mock(async () => { throw new Error("foreground start should not be used"); }); - const startWorkflowInBackground = mock(async () => ({ - runId: "wfr_background", - status: "running" as const, - result: null, - })); + // Faithful to WorkflowService: onRunCreated is awaited at run creation, before the runner + // starts executing. The probe checks the sidecar reference is already durable at that + // point, so a fast run (or a crash) after launch cannot lose its terminal wake. + let referenceDurableBeforeRunnerStart = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_background", + status: "pending", + result: null, + run: null, + }); + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableBeforeRunnerStart = references.some( + (reference) => reference.runId === "wfr_background" + ); + return { runId: "wfr_background", status: "running" as const, result: null }; + }); const getRun = mock(async () => null); const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); const tool = createWorkflowRunTool({ @@ -668,6 +685,7 @@ describe("workflow_run tool", () => { mockToolCallOptions ); + expect(referenceDurableBeforeRunnerStart).toBe(true); // The reference must persist the invocation-boundary snapshot: currentness compares row // identity, so a reference recorded without it fails safe and the wake is dropped. const references = await readAgentWorkflowRunReferences(tempDir.path); @@ -697,11 +715,15 @@ describe("workflow_run tool", () => { test("records a rediscovery-only reference when the boundary snapshot fails", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-boundary-error"); const scriptPath = await writeWorkflowScript(tempDir.path); - const startWorkflowInBackground = mock(async () => ({ - runId: "wfr_boundary_error", - status: "running" as const, - result: null, - })); + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_boundary_error", + status: "pending", + result: null, + run: null, + }); + return { runId: "wfr_boundary_error", status: "running" as const, result: null }; + }); const getWorkflowInvocationBoundaryMessageId = mock(async () => { throw new Error("history read failed"); }); diff --git a/src/node/services/tools/workflow_run.ts b/src/node/services/tools/workflow_run.ts index 425b379eae8..075089795cc 100644 --- a/src/node/services/tools/workflow_run.ts +++ b/src/node/services/tools/workflow_run.ts @@ -262,6 +262,7 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => } } const createdRun: { id: string | null } = { id: null }; + const invocationStartedAtMs = Date.now(); const startInput = { script, workspaceId, @@ -271,6 +272,12 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => createdRun.id = event.runId; // The run record is durable now, so a concurrent duplicate launch will see it. releaseAdmission?.(); + // Provenance must be durable BEFORE the runner starts: a fast background run (or a + // process exit mid-dispatch) can reach terminal state before any post-dispatch + // write, and a terminal wake with no sidecar reference is permanently superseded. + if (args.run_in_background === true) { + await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs); + } await emitWorkflowRunAttachedEvent({ config, workspaceId, @@ -280,7 +287,6 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => }); }, }; - const invocationStartedAtMs = Date.now(); let result: { runId: string; status: string; result: unknown }; try { result = @@ -335,7 +341,10 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => releaseAdmission?.(); } - if (isBackgroundWorkflowResult(args, result.status)) { + // Explicit background launches already recorded provenance in onRunCreated; this covers + // a foreground dispatch that backgrounded itself, where the run ID outcome is only + // knowable post-dispatch. + if (args.run_in_background !== true && isBackgroundWorkflowResult(args, result.status)) { await recordBackgroundWorkflowRunReference(config, result.runId, invocationStartedAtMs); } From 42a596beba53933cb598c76273189d4fa4e0e1de Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:59:37 +0000 Subject: [PATCH 23/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20preserve=20the=20ca?= =?UTF-8?q?ller=20tool=20policy=20through=20on-send=20compaction=20follow-?= =?UTF-8?q?ups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pickPreservedSendOptions preserved disableWorkspaceAgents but not toolPolicy, so a restricted send that triggered on-send auto-compaction redispatched its follow-up allow-all; with terminal wakes now restoring the caller policy on synthetic sends, that gap let a compacting wake resume unrestricted. Preserve the policy in the durable follow-up and restore it at redispatch behind ToolPolicySchema validation, since the follow-up crosses the same raw JSON persistence boundary as its other fields. --- src/common/types/message.ts | 4 ++++ .../agentSession.autoCompaction.test.ts | 11 +++++++++++ src/node/services/agentSession.ts | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 9b16234a106..db02c4d6f64 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -63,6 +63,7 @@ type PreservedSendOptions = Pick< | "providerOptions" | "experiments" | "disableWorkspaceAgents" + | "toolPolicy" | "strictAgentResolution" | "allowAgentSetGoal" | "skipAiSettingsPersistence" @@ -82,6 +83,9 @@ export function pickPreservedSendOptions(options: SendMessageOptions): Preserved // can persist across restarts and build versions. experiments: withLegacyPtcExclusiveMirror(options.experiments), disableWorkspaceAgents: options.disableWorkspaceAgents, + // Security: a restricted turn (including a terminal-wake send restoring the caller's + // policy) that triggers on-send compaction must not redispatch its follow-up allow-all. + toolPolicy: options.toolPolicy, // Delegated turns with explicit agent overrides must stay loud across the // compaction replay too — dropping this would let the follow-up silently // fall back to exec if the agent vanished in the meantime. diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 3ef773f3396..ca2a53e8281 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -95,10 +95,12 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { getThreshold: mock(() => 0.85), } as unknown as CompactionMonitor; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; const result = await session.sendMessage("please inspect @foo.ts", { model: "openai:gpt-4o", agentId: "exec", disableWorkspaceAgents: true, + toolPolicy: restrictedPolicy, }); expect(result.success).toBe(true); @@ -120,6 +122,15 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { ); expect(persistedCompactionMessage).toBeDefined(); expect(persistedCompactionMessage?.metadata?.disableWorkspaceAgents).toBe(true); + // The durable follow-up must keep the caller's restrictions: the redispatched turn + // reconstructs its options from this persisted request, and dropping the policy there + // would resume the conversation allow-all after compaction. + const followUpContent = + persistedCompactionMessage?.metadata?.muxMetadata?.type === "compaction-request" + ? persistedCompactionMessage.metadata.muxMetadata.parsed.followUpContent + : undefined; + expect(followUpContent?.toolPolicy).toEqual(restrictedPolicy); + expect(followUpContent?.disableWorkspaceAgents).toBe(true); const emittedSnapshot = events.some( (message) => diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4422a49a05e..0b560f406e8 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -44,6 +44,7 @@ import { SendMessageOptionsSchema, SkillNameSchema, } from "@/common/orpc/schemas"; +import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { buildStreamErrorEventData, @@ -7200,6 +7201,23 @@ export class AgentSession { experiments: aliasLegacyPtcExclusive(followUp.experiments), allowAgentSetGoal: followUp.allowAgentSetGoal, disableWorkspaceAgents: followUp.disableWorkspaceAgents, + // Same raw JSON boundary: a persisted follow-up may carry a malformed toolPolicy, and + // restoring it unvalidated would throw during resolution. Invalid values are dropped + // like any corrupt persisted policy (self-healing doctrine); a restricted turn's + // follow-up must otherwise keep its policy instead of redispatching allow-all. + ...(() => { + if (followUp.toolPolicy == null) { + return {}; + } + const parsed = ToolPolicySchema.safeParse(followUp.toolPolicy); + if (!parsed.success) { + log.warn("Ignoring malformed persisted toolPolicy on compaction follow-up", { + workspaceId: this.workspaceId, + }); + return {}; + } + return { toolPolicy: parsed.data }; + })(), // Explicit-agent turns stay loud on the resumed turn too: the requested agent // may have been removed/hidden/disabled while compaction ran. strictAgentResolution: followUp.strictAgentResolution, From c59964ff7f385fd1c239e6265e227530fc6cd48a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:24:07 +0000 Subject: [PATCH 24/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20record=20resume=20p?= =?UTF-8?q?rovenance=20only=20after=20the=20dispatch=20restarts=20the=20ru?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-19 pre-dispatch record was wrong for workflow_resume: a resumed run sits in an old failed/interrupted state until the dispatch durably restarts it, and a crash in that window left a fresh, current sidecar reference pointing at the stale terminal state, which startup recovery would deliver as this resume's wake. Record after the dispatch again: the crash window now loses the resume's wake instead of replaying a stale one (fail-safe), and with the process alive delivery always waits for the owner to go idle, by which point the record is durable. workflow_run keeps its onRunCreated record, where the freshly created run has no prior terminal state to replay. --- .../services/tools/workflow_resume.test.ts | 7 ++++--- src/node/services/tools/workflow_resume.ts | 19 +++++++------------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index b0d90d1858f..f6433a16512 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -169,8 +169,9 @@ describe("workflow_resume tool", () => { test("resumes in background and records an agent workflow run reference", async () => { using tempDir = new TestTempDir("test-workflow-resume-bg"); - // The reference must be durable BEFORE the dispatch: background execution starts at lease - // acquisition, and a fast run could otherwise reach terminal state with no provenance. + // The reference must NOT be durable before the dispatch: the run still sits in its old + // terminal state until the dispatch restarts it, and a pre-dispatch reference would let a + // crash in that window replay the stale failure/interruption as a current wake. let referenceDurableAtDispatch = false; const workflowService = buildWorkflowService({ resumeRunInBackground: mock(async () => { @@ -198,7 +199,7 @@ describe("workflow_resume tool", () => { projectTrusted: false, }); expect(workflowService.resumeRun).not.toHaveBeenCalled(); - expect(referenceDurableAtDispatch).toBe(true); + expect(referenceDurableAtDispatch).toBe(false); const references = await readAgentWorkflowRunReferences(tempDir.path); expect(references.map((reference) => reference.runId)).toContain("wfr_resume_me"); expect(result).toMatchObject({ status: "running", runId: "wfr_resume_me", mode: "resume" }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 70042c94b32..0fac258ed3c 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -204,15 +204,6 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) runId, projectTrusted: config.trusted === true, }; - // Provenance must be durable BEFORE the dispatch: background execution starts at lease - // acquisition, and a fast run (or a process exit mid-dispatch) can reach terminal state - // before any post-dispatch write, permanently superseding its wake. A failed dispatch - // leaves the re-recorded reference behind, which is benign: it mirrors what a - // successful resume would persist for a run this turn explicitly re-engaged with, and - // rediscovery filters against the run store. - if (args.run_in_background === true) { - await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); - } let dispatched: { runId: string; status: string; result: unknown }; try { if (mode === "retry_from_checkpoint") { @@ -243,11 +234,15 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Background-style resumes outlive this turn; persist provenance so the run is // rediscoverable (task_await/task_list) and its terminal result re-engages the agent. - // Explicit background resumes already recorded it pre-dispatch; this covers a - // foreground resume that backgrounded itself. + // Deliberately AFTER the dispatch, unlike workflow_run's onRunCreated record: the + // resumed run sits in an old terminal state until the dispatch durably restarts it, and + // a pre-dispatch reference would let a crash in that window make startup recovery + // deliver the stale failure/interruption as a current wake. Recording late fails safe + // instead (a crash loses this resume's wake); with the process alive, delivery always + // waits for the owner to go idle, by which point this record is durable. const isBackgroundDispatch = args.run_in_background === true || dispatched.status === "backgrounded"; - if (isBackgroundDispatch && args.run_in_background !== true) { + if (isBackgroundDispatch) { await recordBackgroundWorkflowRunReference(config, runId, invocationStartedAtMs); } From 2befd06b0d71ae4086c485087e94fb5100f22daf Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:34:21 +0000 Subject: [PATCH 25/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20restore=20the=20str?= =?UTF-8?q?ict-agent=20pin=20on=20terminal=20wakes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An explicit agent override persists strictAgentResolution in the manual row's retry snapshot, and startup retries and compaction follow-ups both restore it, but the terminal-wake send dropped it: if the pinned agent definition vanished or was corrupted while a background run executed, the wake would silently recompose from the exec fallback. Restore the pin from the same restriction walk (it also stops at a plain manual row that carries only the pin), keeping vanished-agent failures loud. --- src/node/services/taskService.test.ts | 2 ++ src/node/services/taskService.ts | 43 +++++++++++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 930dc2d4918..8df60ea9b94 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6470,6 +6470,7 @@ describe("TaskService", () => { timestamp: 1_000, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, + retrySendOptions: { model: "openai:gpt-4o", agentId: "exec", strictAgentResolution: true }, }) ); await historyService.appendToHistory( @@ -6493,6 +6494,7 @@ describe("TaskService", () => { expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ agentId: "plan", + strictAgentResolution: true, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index b283bb60934..ea4245f1444 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7915,19 +7915,25 @@ export class TaskService { } /** - * Caller send restrictions (tool policy, workspace-agent disable flag) to restore on a - * terminal-attention wake. The newest manual user row carries the conversation's persisted - * restrictions; synthetic rows without any (earlier wakes, heartbeat scaffolding) do not - * define them and are skipped. The walk is unbounded: a long assistant/synthetic tail after - * the launch turn must not push the defining row out of sight and silently lift the - * restrictions. Throws when history is unreadable so the caller can fail closed instead of - * waking with unrestricted tools. + * Caller send restrictions (tool policy, workspace-agent disable flag, strict-agent pin) to + * restore on a terminal-attention wake. The newest manual user row carries the + * conversation's persisted restrictions; synthetic rows without any (earlier wakes, + * heartbeat scaffolding) do not define them and are skipped. The walk is unbounded: a long + * assistant/synthetic tail after the launch turn must not push the defining row out of + * sight and silently lift the restrictions. Throws when history is unreadable so the caller + * can fail closed instead of waking with unrestricted tools. */ - private async resolveTerminalWakeCallerSendRestrictions( - ownerWorkspaceId: string - ): Promise<{ toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }> { + private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: boolean; + }> { const state: { - found: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + found: { + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: boolean; + } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( ownerWorkspaceId, @@ -7938,6 +7944,11 @@ export class TaskService { continue; } const metadata = message.metadata; + // The strict-agent pin lives in the row's retry snapshot: an explicit agent override + // must stay loud on the wake too, or a vanished/corrupted definition would silently + // recompose the send from the exec fallback (same rule as startup retry and + // compaction follow-ups). + const strictAgentResolution = metadata?.retrySendOptions?.strictAgentResolution === true; if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep // inside send resolution and leave the wake permanently blocked on the same corrupt @@ -7956,11 +7967,12 @@ export class TaskService { ...(typeof metadata.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), + ...(strictAgentResolution ? { strictAgentResolution: true } : {}), }; return false; } if (metadata?.synthetic !== true) { - state.found = {}; + state.found = strictAgentResolution ? { strictAgentResolution: true } : {}; return false; } } @@ -8422,7 +8434,11 @@ export class TaskService { // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake // regain tools the caller disabled (with attacker-influenced workflow output choosing the // timing). The agent-level policy recomposes from agentId at send resolution. - let wakeRestrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean }; + let wakeRestrictions: { + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: boolean; + }; try { wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); } catch (error: unknown) { @@ -8442,6 +8458,7 @@ export class TaskService { reasoningMode: resumeOptions.reasoningMode, ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), + ...(wakeRestrictions.strictAgentResolution === true ? { strictAgentResolution: true } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { From 36bb23ca60f4039072b3ebf976047c1b40a502da Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 07:59:20 +0000 Subject: [PATCH 26/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20forward=20the=20obj?= =?UTF-8?q?ect-form=20strict-agent=20pin=20on=20terminal=20wakes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-20 restore accepted only literal true, dropping the object form ({expectedScope, expectedSource, expectedChain}) that explicit workspace-agent overrides persist; a same-ID replacement definition could then satisfy resolution silently. Validate the persisted value against the SendMessageOptions union schema and forward it verbatim, as the field's design note requires and startup retry already does. --- src/node/services/taskService.test.ts | 8 +++++-- src/node/services/taskService.ts | 34 ++++++++++++++++++++------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8df60ea9b94..7ecf8798210 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6470,7 +6470,11 @@ describe("TaskService", () => { timestamp: 1_000, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, - retrySendOptions: { model: "openai:gpt-4o", agentId: "exec", strictAgentResolution: true }, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "exec", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, }) ); await historyService.appendToHistory( @@ -6494,7 +6498,7 @@ describe("TaskService", () => { expect(sendMessage).toHaveBeenCalledTimes(1); expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ agentId: "plan", - strictAgentResolution: true, + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, toolPolicy: restrictedPolicy, disableWorkspaceAgents: true, }); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index ea4245f1444..c98b7fdd652 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -118,7 +118,7 @@ import { import type { ProjectRef, WorkspaceMetadata } from "@/common/types/workspace"; import { getRuntimeType } from "@/node/runtime/initHook"; import { AgentIdSchema } from "@/common/orpc/schemas"; -import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; +import { SendMessageOptionsSchema, ToolPolicySchema } from "@/common/orpc/schemas/stream"; import type { AgentDefinitionScope } from "@/common/types/agentDefinition"; import { normalizeAgentId, @@ -7926,13 +7926,13 @@ export class TaskService { private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean; - strictAgentResolution?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; }> { const state: { found: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean; - strictAgentResolution?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( @@ -7947,8 +7947,22 @@ export class TaskService { // The strict-agent pin lives in the row's retry snapshot: an explicit agent override // must stay loud on the wake too, or a vanished/corrupted definition would silently // recompose the send from the exec fallback (same rule as startup retry and - // compaction follow-ups). - const strictAgentResolution = metadata?.retrySendOptions?.strictAgentResolution === true; + // compaction follow-ups). Forwarded verbatim per the field's design note: the object + // form pins the validated definition's scope/source/chain provenance, not just + // loudness. Schema-validated like toolPolicy below, since it crosses the same + // persisted-row boundary; invalid shapes are dropped. + const rawStrictPin = metadata?.retrySendOptions?.strictAgentResolution; + const parsedStrictPin = + rawStrictPin != null && rawStrictPin !== false + ? SendMessageOptionsSchema.shape.strictAgentResolution.safeParse(rawStrictPin) + : null; + if (parsedStrictPin != null && !parsedStrictPin.success) { + log.warn("Ignoring malformed persisted strictAgentResolution on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } + const strictAgentResolution = parsedStrictPin?.success ? parsedStrictPin.data : undefined; if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep // inside send resolution and leave the wake permanently blocked on the same corrupt @@ -7967,12 +7981,12 @@ export class TaskService { ...(typeof metadata.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), - ...(strictAgentResolution ? { strictAgentResolution: true } : {}), + ...(strictAgentResolution != null ? { strictAgentResolution } : {}), }; return false; } if (metadata?.synthetic !== true) { - state.found = strictAgentResolution ? { strictAgentResolution: true } : {}; + state.found = strictAgentResolution != null ? { strictAgentResolution } : {}; return false; } } @@ -8437,7 +8451,7 @@ export class TaskService { let wakeRestrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean; - strictAgentResolution?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; }; try { wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); @@ -8458,7 +8472,9 @@ export class TaskService { reasoningMode: resumeOptions.reasoningMode, ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), - ...(wakeRestrictions.strictAgentResolution === true ? { strictAgentResolution: true } : {}), + ...(wakeRestrictions.strictAgentResolution != null + ? { strictAgentResolution: wakeRestrictions.strictAgentResolution } + : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { From cf0323c577cf79a516fba86eb451ae5c300d195d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 08:47:54 +0000 Subject: [PATCH 27/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20stop=20compaction?= =?UTF-8?q?=20recovery=20from=20clobbering=20preserved=20follow-up=20field?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pickPreservedSendOptions emitted every preserved field as an explicit key, so compaction retry (which spreads the pick over the persisted follow-up with storage-derived options that never carry a caller toolPolicy) overwrote the original restriction with undefined and the recovered follow-up resumed with unrestricted caller tools. Omit unset fields from the pick instead; this also stops the same latent clobbering of disableWorkspaceAgents and the other preserved fields on retry. --- src/browser/utils/chatCommands.test.ts | 26 +++++++++++++++++++ src/common/types/message.ts | 35 ++++++++++++++++++-------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/browser/utils/chatCommands.test.ts b/src/browser/utils/chatCommands.test.ts index ed157332b1f..fcd4f8a3034 100644 --- a/src/browser/utils/chatCommands.test.ts +++ b/src/browser/utils/chatCommands.test.ts @@ -1582,6 +1582,32 @@ describe("prepareCompactionMessage", () => { expect(metadata.parsed.followUpContent?.agentId).toBe("exec"); }); + test("compaction recovery keeps the persisted follow-up's restrictions when retry options lack them", () => { + // Retrying a failed compaction passes the already-persisted follow-up together with + // storage-derived send options, which never carry a caller toolPolicy. The preserved + // restrictions must survive that recomposition or the recovered follow-up resumes with + // unrestricted caller tools. + const recoveredFollowUp = { + text: "Keep building", + model: "openai:gpt-4o", + agentId: "code", + toolPolicy: [{ regex_match: "^bash$", action: "disable" as const }], + disableWorkspaceAgents: true, + }; + const { metadata } = prepareCompactionMessage({ + workspaceId: "ws-1", + followUpContent: recoveredFollowUp, + sendMessageOptions: { model: "anthropic:claude-sonnet-4-6", agentId: "exec" }, + }); + + expectCompactionMetadata(metadata); + expect(metadata.parsed.followUpContent?.toolPolicy).toEqual(recoveredFollowUp.toolPolicy); + expect(metadata.parsed.followUpContent?.disableWorkspaceAgents).toBe(true); + // Existing model/agentId still win over the retry-time options. + expect(metadata.parsed.followUpContent?.model).toBe("openai:gpt-4o"); + expect(metadata.parsed.followUpContent?.agentId).toBe("code"); + }); + test("does not create followUpContent when no text or images provided", () => { const sendMessageOptions = createBaseOptions(); const { metadata } = prepareCompactionMessage({ diff --git a/src/common/types/message.ts b/src/common/types/message.ts index db02c4d6f64..2fa770234c9 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -74,24 +74,39 @@ type PreservedSendOptions = Pick< * Use this helper to avoid duplicating the field list when building CompactionFollowUpRequest. */ export function pickPreservedSendOptions(options: SendMessageOptions): PreservedSendOptions { + // Unset fields are OMITTED, not emitted as explicit undefined: compaction recovery spreads + // this pick over an already-persisted follow-up, and an undefined key would clobber the + // original preserved value (e.g. a restricted turn's toolPolicy) instead of leaving it. return { - thinkingLevel: options.thinkingLevel, - reasoningMode: options.reasoningMode, - additionalSystemInstructions: options.additionalSystemInstructions, - providerOptions: options.providerOptions, + ...(options.thinkingLevel !== undefined ? { thinkingLevel: options.thinkingLevel } : {}), + ...(options.reasoningMode !== undefined ? { reasoningMode: options.reasoningMode } : {}), + ...(options.additionalSystemInstructions !== undefined + ? { additionalSystemInstructions: options.additionalSystemInstructions } + : {}), + ...(options.providerOptions !== undefined ? { providerOptions: options.providerOptions } : {}), // Downgrade-compat (see withLegacyPtcExclusiveMirror): preserved options // can persist across restarts and build versions. - experiments: withLegacyPtcExclusiveMirror(options.experiments), - disableWorkspaceAgents: options.disableWorkspaceAgents, + ...(options.experiments !== undefined + ? { experiments: withLegacyPtcExclusiveMirror(options.experiments) } + : {}), + ...(options.disableWorkspaceAgents !== undefined + ? { disableWorkspaceAgents: options.disableWorkspaceAgents } + : {}), // Security: a restricted turn (including a terminal-wake send restoring the caller's // policy) that triggers on-send compaction must not redispatch its follow-up allow-all. - toolPolicy: options.toolPolicy, + ...(options.toolPolicy !== undefined ? { toolPolicy: options.toolPolicy } : {}), // Delegated turns with explicit agent overrides must stay loud across the // compaction replay too — dropping this would let the follow-up silently // fall back to exec if the agent vanished in the meantime. - strictAgentResolution: options.strictAgentResolution, - allowAgentSetGoal: options.allowAgentSetGoal, - skipAiSettingsPersistence: options.skipAiSettingsPersistence, + ...(options.strictAgentResolution !== undefined + ? { strictAgentResolution: options.strictAgentResolution } + : {}), + ...(options.allowAgentSetGoal !== undefined + ? { allowAgentSetGoal: options.allowAgentSetGoal } + : {}), + ...(options.skipAiSettingsPersistence !== undefined + ? { skipAiSettingsPersistence: options.skipAiSettingsPersistence } + : {}), }; } From 9f57ada8516c938a2752f423a177ff7c3dbd462b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:07:17 +0000 Subject: [PATCH 28/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20bind=20workflow=20t?= =?UTF-8?q?erminal=20wakes=20to=20the=20initiating=20agent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wake's agent identity came from the newest agent-bearing assistant row, which a later synthetic heartbeat turn can own without superseding the run, pairing a different agent's tool surface with the launch turn's caller policy. Persist the launching agent with the run reference and prefer it at drain time; legacy references keep the history-walk fallback. --- src/common/utils/tools/tools.ts | 2 + .../agentWorkflowRunReferences.test.ts | 35 +++++++++ .../services/agentWorkflowRunReferences.ts | 14 ++++ src/node/services/aiService.ts | 1 + src/node/services/taskService.test.ts | 73 +++++++++++++++++++ src/node/services/taskService.ts | 38 +++++++++- src/node/services/tools/toolUtils.ts | 1 + src/node/services/tools/workflow_run.test.ts | 3 + 8 files changed, 165 insertions(+), 2 deletions(-) diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index fc286a097c6..5db7e05ba30 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -184,6 +184,8 @@ export interface ToolConfiguration { workspaceSessionDir?: string; /** Workspace ID for tracking background processes and plan storage */ workspaceId?: string; + /** Resolved agent identity of the turn executing the tools (workflow wake provenance). */ + agentId?: string; /** Pre-resolved mux-managed resource scope (global ~/.xum vs project root). */ xumScope?: XumToolScope; /** Memory service for the memory tool (present only when the memory experiment is enabled). */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 69171c081cf..955c3c71b7c 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -58,6 +58,41 @@ describe("agent workflow run references", () => { } }); + test("roundtrips the initiating agent and drops invalid persisted shapes", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + }); + let references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + }); + + // Identity is advisory: a malformed persisted agentId drops the field, not the entry, + // so the run keeps its wake and identity falls back to the history walk. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_agent_number", createdAtMs: 1_000, agentId: 7 }, + { runId: "wfr_agent_empty", createdAtMs: 1_000, agentId: "" }, + ], + }) + ); + references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + test("drops persisted future-dated references and repairs them on the next record", async () => { const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); try { diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 43a9d23a778..f31ca53d72d 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -17,6 +17,13 @@ export interface AgentWorkflowRunReference { * Absent on legacy entries, which fail safe to not-current. */ afterBoundaryMessageId?: string | null; + /** + * Agent identity of the turn that launched/resumed the run. The terminal wake binds to this + * instead of the newest agent-bearing assistant row, which a later synthetic turn (e.g. a + * heartbeat) can own without superseding the run. Advisory: absent on legacy entries, which + * fall back to the history walk. + */ + agentId?: string; } const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; @@ -81,6 +88,10 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { ? boundaryRaw : null : undefined; + // Identity is advisory (the wake falls back to the history walk), so an invalid shape + // drops only the field, not the entry. + const agentId = + typeof record.agentId === "string" && record.agentId.length > 0 ? record.agentId : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. @@ -90,6 +101,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { runId: record.runId, createdAtMs: record.createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(agentId !== undefined ? { agentId } : {}), }); } } @@ -139,6 +151,7 @@ export async function recordAgentWorkflowRunReference(input: { runId: string; createdAtMs?: number; afterBoundaryMessageId?: string | null; + agentId?: string; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -165,6 +178,7 @@ export async function recordAgentWorkflowRunReference(input: { ...(input.afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId: input.afterBoundaryMessageId } : {}), + ...(input.agentId != null && input.agentId.length > 0 ? { agentId: input.agentId } : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index d0d192fd51f..f43478fb251 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -2646,6 +2646,7 @@ export class AIService extends EventEmitter { planFilePath, ancestorPlanFilePaths, workspaceId, + agentId: effectiveAgentId, xumScope, timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 7ecf8798210..3586153cd8b 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6504,6 +6504,79 @@ describe("TaskService", () => { }); }); + test("workflow wakes bind to the initiating agent, not a later synthetic turn's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_initiating_agent"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("launch-turn", "assistant", "starting", { + timestamp: 1_001, + agentId: "plan", + }) + ); + // A heartbeat is synthetic, not a manual supersession boundary: the run stays current, but + // its agent-bearing assistant row is now the newest one in history. The wake must use the + // launch turn's agent from the sidecar, not the heartbeat's. + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat", "user", "heartbeat", { timestamp: 1_002, synthetic: true }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-turn", "assistant", "idle check", { + timestamp: 1_003, + agentId: "exec", + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "plan", + }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index c98b7fdd652..44476e15edb 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8021,7 +8021,13 @@ export class TaskService { ownerWorkspaceId: string, runId: string ): Promise< - { outcome: "deliver"; prompt: string } | { outcome: "superseded" } | { outcome: "defer" } + | { + outcome: "deliver"; + prompt: string; + initiatingAgent?: { agentId: string; createdAtMs: number }; + } + | { outcome: "superseded" } + | { outcome: "defer" } > { assert(ownerWorkspaceId.length > 0, "buildWorkflowTerminalPrompt requires ownerWorkspaceId"); assert(runId.length > 0, "buildWorkflowTerminalPrompt requires runId"); @@ -8058,9 +8064,27 @@ export class TaskService { if (currentness === "not_current") { return { outcome: "superseded" }; } + // Bind the wake to the agent recorded at launch: the newest agent-bearing assistant row + // can belong to an unrelated later synthetic turn (a heartbeat is not a supersession + // boundary), which would pair a different agent's tool surface with the launch turn's + // caller policy. Advisory: legacy references fall back to the history walk. + let initiatingAgent: { agentId: string; createdAtMs: number } | undefined; + try { + const references = await readAgentWorkflowRunReferences( + this.config.getSessionDir(ownerWorkspaceId) + ); + const reference = references.find((candidate) => candidate.runId === run.id); + if (reference?.agentId != null) { + initiatingAgent = { agentId: reference.agentId, createdAtMs: reference.createdAtMs }; + } + } catch { + // Identity is advisory; an unreadable sidecar already deferred delivery above whenever + // currentness itself depended on it. + } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; return { outcome: "deliver", + ...(initiatingAgent != null ? { initiatingAgent } : {}), prompt: buildWorkflowResultContextMessage({ rawCommand: `workflow_run ${scriptPath}`, name: scriptPath, @@ -8375,6 +8399,7 @@ export class TaskService { (notification) => notification.sourceKind === "workflow_run" ); const deliverableWorkflowNotificationIds = new Set(); + let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; const promptSections: string[] = []; if (publicAwaitIds.length > 0) { @@ -8406,6 +8431,14 @@ export class TaskService { continue; } deliverableWorkflowNotificationIds.add(notification.id); + // Newest launch wins when several current runs coalesce into one wake. + if ( + workflowPrompt.initiatingAgent != null && + (workflowInitiatingAgent == null || + workflowPrompt.initiatingAgent.createdAtMs > workflowInitiatingAgent.createdAtMs) + ) { + workflowInitiatingAgent = workflowPrompt.initiatingAgent; + } promptSections.push(workflowPrompt.prompt); } @@ -8438,7 +8471,8 @@ export class TaskService { const resumeOptions = await this.resolveParentAutoResumeOptions( ownerWorkspaceId, entry, - defaultModel + defaultModel, + workflowInitiatingAgent != null ? { agentId: workflowInitiatingAgent.agentId } : undefined ); const workspaceTurnMuxMetadata = await this.getActiveWorkspaceTurnMuxMetadataForWorkspace(ownerWorkspaceId); diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index a56f4574762..24b20398666 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -126,6 +126,7 @@ export async function recordBackgroundWorkflowRunReference( runId, createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(config.agentId != null && config.agentId.length > 0 ? { agentId: config.agentId } : {}), }); } catch (error: unknown) { log.warn("Failed to record agent workflow run reference", { diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index d49544850c6..143a1bcddb6 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -672,6 +672,7 @@ describe("workflow_run tool", () => { const tool = createWorkflowRunTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, + agentId: "plan", taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, @@ -693,6 +694,8 @@ describe("workflow_run tool", () => { expect(references[0]).toMatchObject({ runId: "wfr_background", afterBoundaryMessageId: "boundary-row-1", + // The wake binds to the launching agent, so the reference must carry its identity. + agentId: "plan", }); expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( "workspace-1", From 125aae5a56df7aabdf00ed7486b3fbcd6990b4c0 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:29:07 +0000 Subject: [PATCH 29/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20schema-validate=20p?= =?UTF-8?q?ersisted=20initiating=20agent=20IDs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A malformed but non-empty persisted agentId passed the string check, and stream resolution normalizes an unknown requested agent to exec, so a corrupt sidecar entry could silently swap a restricted agent's wake onto exec's tool surface. Parse the field with AgentIdSchema and drop invalid values, keeping the history-walk fallback. --- src/node/services/agentWorkflowRunReferences.test.ts | 4 ++++ src/node/services/agentWorkflowRunReferences.ts | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 955c3c71b7c..03cb142ed41 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -82,12 +82,16 @@ describe("agent workflow run references", () => { references: [ { runId: "wfr_agent_number", createdAtMs: 1_000, agentId: 7 }, { runId: "wfr_agent_empty", createdAtMs: 1_000, agentId: "" }, + // Non-empty but schema-invalid: stream resolution would normalize it to exec, + // silently swapping a restricted agent's wake onto exec's tool surface. + { runId: "wfr_agent_malformed", createdAtMs: 1_000, agentId: "bad id" }, ], }) ); references = await readAgentWorkflowRunReferences(workspaceSessionDir); expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_malformed", createdAtMs: 1_000 }); } finally { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index f31ca53d72d..f6b52a98ee6 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { AgentIdSchema } from "@/common/schemas/ids"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -89,9 +90,11 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { : null : undefined; // Identity is advisory (the wake falls back to the history walk), so an invalid shape - // drops only the field, not the entry. - const agentId = - typeof record.agentId === "string" && record.agentId.length > 0 ? record.agentId : undefined; + // drops only the field, not the entry. Schema-validate rather than accepting any string: + // stream resolution normalizes an unknown requested agent to exec, so a corrupt persisted + // ID would silently swap a restricted agent's wake onto exec's tool surface. + const agentIdParse = AgentIdSchema.safeParse(record.agentId); + const agentId = agentIdParse.success ? agentIdParse.data : undefined; // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. From e1b19548088169c6c682c3bed1f0d08259bef974 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:36:50 +0000 Subject: [PATCH 30/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20split=20coalesced?= =?UTF-8?q?=20workflow=20wakes=20by=20initiating=20agent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole coalesced prompt is handled under the single agentId passed to sendMessage, so batching current runs from different initiating agents would hand a restricted agent's attacker-influenced output to another agent's tool grants. Deliver one initiating-agent group per drain (newest launch first) and keep the other groups pending on the re-armed retry drain. --- src/node/services/taskService.test.ts | 93 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 46 ++++++++++--- 2 files changed, 131 insertions(+), 8 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 3586153cd8b..3357c018c1f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6577,6 +6577,99 @@ describe("TaskService", () => { }); }); + test("coalesced workflow wakes split by initiating agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_split_plan"); + await createRun("wfr_split_exec"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Two current runs from different initiating agents: one coalesced wake would hand the + // older run's (attacker-influenced) output to the newer agent's tool grants. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_split_exec", + createdAtMs: 1_000, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_split_plan", + createdAtMs: 2_000, + agentId: "plan", + }); + + // Seed the store directly so ONE drain observes both pending notifications; per-enqueue + // drains would deliver them separately without exercising the coalescing path. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_split_plan", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_split_exec", + }); + await drain(parentId); + + // The newest launch's group delivers first, alone, under its own agent. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_split_plan"); + expect(firstPrompt).not.toContain("wfr_split_exec"); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + // The deferred group delivers on a later drain under its own agent. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_split_exec"); + expect(secondPrompt).not.toContain("wfr_split_plan"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 44476e15edb..19378948242 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8399,7 +8399,11 @@ export class TaskService { (notification) => notification.sourceKind === "workflow_run" ); const deliverableWorkflowNotificationIds = new Set(); - let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; + const deliverableWorkflowPrompts: Array<{ + notificationId: string; + prompt: string; + initiatingAgent?: { agentId: string; createdAtMs: number }; + }> = []; const promptSections: string[] = []; if (publicAwaitIds.length > 0) { @@ -8430,16 +8434,42 @@ export class TaskService { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); continue; } - deliverableWorkflowNotificationIds.add(notification.id); - // Newest launch wins when several current runs coalesce into one wake. + deliverableWorkflowPrompts.push({ + notificationId: notification.id, + prompt: workflowPrompt.prompt, + ...(workflowPrompt.initiatingAgent != null + ? { initiatingAgent: workflowPrompt.initiatingAgent } + : {}), + }); + } + // Deliver one initiating-agent group per drain: the whole coalesced prompt is handled + // under the single agentId passed to sendMessage, so batching runs from different agents + // would hand a restricted agent's (attacker-influenced) output to another agent's tool + // grants. The newest launch's group goes first; runs bound to other agents, and the + // history-walk fallback group, stay pending and deliver on the re-armed retry drain. + let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; + for (const candidate of deliverableWorkflowPrompts) { + const agent = candidate.initiatingAgent; if ( - workflowPrompt.initiatingAgent != null && - (workflowInitiatingAgent == null || - workflowPrompt.initiatingAgent.createdAtMs > workflowInitiatingAgent.createdAtMs) + agent != null && + (workflowInitiatingAgent == null || agent.createdAtMs > workflowInitiatingAgent.createdAtMs) ) { - workflowInitiatingAgent = workflowPrompt.initiatingAgent; + workflowInitiatingAgent = agent; } - promptSections.push(workflowPrompt.prompt); + } + const selectedAgentId = workflowInitiatingAgent?.agentId; + const selectedWorkflowPrompts = + selectedAgentId == null + ? deliverableWorkflowPrompts + : deliverableWorkflowPrompts.filter( + (candidate) => candidate.initiatingAgent?.agentId === selectedAgentId + ); + if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { + this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); + } + for (const candidate of selectedWorkflowPrompts) { + deliverableWorkflowNotificationIds.add(candidate.notificationId); + promptSections.push(candidate.prompt); } // Sub-agent reports and failures are already durable user-context messages. Resume from history From cb313080a0a34041c5571e1674f4923a85100582 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:56:37 +0000 Subject: [PATCH 31/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20isolate=20wake=20id?= =?UTF-8?q?entity=20groups=20and=20honor=20synthetic=20launch=20pins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two round-24 findings: (1) mixed drains applied the workflow group's initiating agent to coalesced workspace-turn and sub-agent sections, so those now resume under the conversation's own identity while agent-bound workflow groups defer to their own wake; (2) the restriction walk ignored synthetic launch rows carrying only a strict-agent pin, so the pin and the policy now resolve independently: the newest pin-bearing or manual row defines the pin, the newest policy-bearing or manual row defines the policy. --- src/node/services/taskService.test.ts | 154 ++++++++++++++++++++++++++ src/node/services/taskService.ts | 75 ++++++++----- 2 files changed, 199 insertions(+), 30 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 3357c018c1f..b957f5f06c4 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6670,6 +6670,160 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("mixed drains keep workspace-turn attention off the workflow's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_mixed"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_001, agentId: "plan" }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "exec", + }); + + // A workspace-turn result resumes under the conversation's own identity; sharing its wake + // with an agent-bound workflow group would process it under the workflow's agent instead. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_mixed_handle", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }); + await drain(parentId); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wst_mixed_handle"); + expect(firstPrompt).not.toContain(runId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain(runId); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("wake keeps a synthetic launch row's strict pin without lifting the manual policy", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_synthetic_pin"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + // The kernel workflow launched from a pinned synthetic turn (preserved heartbeat or + // compaction follow-up): its pin must ride the wake without lifting the manual policy. + await historyService.appendToHistory( + parentId, + createMuxMessage("synthetic-launch", "user", "heartbeat", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }, + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "plan", + }); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + toolPolicy: restrictedPolicy, + strictAgentResolution: { expectedScope: "built-in" }, + }); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 19378948242..5a045aefe7f 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7928,13 +7928,14 @@ export class TaskService { disableWorkspaceAgents?: boolean; strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; }> { + // The pin and the policy resolve independently: a synthetic launch row (preserved + // heartbeat, compaction follow-up) can carry only a strict pin, and the wake bound to that + // turn's agent must keep the pin loud without lifting an older manual row's policy. + // Manual rows still define both wholesale (absence means lifted). const state: { - found: { - toolPolicy?: ToolPolicy; - disableWorkspaceAgents?: boolean; - strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; - } | null; - } = { found: null }; + pin: { strictAgentResolution?: SendMessageOptions["strictAgentResolution"] } | null; + restrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + } = { pin: null, restrictions: null }; const historyResult = await this.historyService.iterateFullHistory( ownerWorkspaceId, "backward", @@ -7963,30 +7964,38 @@ export class TaskService { }); } const strictAgentResolution = parsedStrictPin?.success ? parsedStrictPin.data : undefined; - if (metadata?.toolPolicy != null || metadata?.disableWorkspaceAgents != null) { + if ( + state.pin == null && + (strictAgentResolution != null || metadata?.synthetic !== true) + ) { + state.pin = strictAgentResolution != null ? { strictAgentResolution } : {}; + } + if ( + state.restrictions == null && + (metadata?.toolPolicy != null || + metadata?.disableWorkspaceAgents != null || + metadata?.synthetic !== true) + ) { // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep // inside send resolution and leave the wake permanently blocked on the same corrupt // row. Sanitize instead of trusting the JSON shape; an unparseable policy restores // nothing while a valid disable flag still applies (self-healing doctrine). const parsedPolicy = - metadata.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; + metadata?.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; if (parsedPolicy != null && !parsedPolicy.success) { log.warn("Ignoring malformed persisted toolPolicy on terminal wake", { ownerWorkspaceId, messageId: message.id, }); } - state.found = { + state.restrictions = { ...(parsedPolicy?.success ? { toolPolicy: parsedPolicy.data } : {}), - ...(typeof metadata.disableWorkspaceAgents === "boolean" + ...(typeof metadata?.disableWorkspaceAgents === "boolean" ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } : {}), - ...(strictAgentResolution != null ? { strictAgentResolution } : {}), }; - return false; } - if (metadata?.synthetic !== true) { - state.found = strictAgentResolution != null ? { strictAgentResolution } : {}; + if (state.pin != null && state.restrictions != null) { return false; } } @@ -7996,7 +8005,7 @@ export class TaskService { if (!historyResult.success) { throw new Error(`history unavailable: ${historyResult.error}`); } - return state.found ?? {}; + return { ...(state.restrictions ?? {}), ...(state.pin ?? {}) }; } private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { @@ -8445,25 +8454,31 @@ export class TaskService { // Deliver one initiating-agent group per drain: the whole coalesced prompt is handled // under the single agentId passed to sendMessage, so batching runs from different agents // would hand a restricted agent's (attacker-influenced) output to another agent's tool - // grants. The newest launch's group goes first; runs bound to other agents, and the - // history-walk fallback group, stay pending and deliver on the re-armed retry drain. + // grants. The same applies to mixed batches: workspace-turn and sub-agent attention + // resumes under the conversation's own (history-walk) identity, so agent-bound workflow + // groups never share their send. Deferred groups stay pending and deliver on the re-armed + // retry drain; among agent-bound groups the newest launch goes first. + const hasNonWorkflowDeliverables = + deliverableAgentNotificationIds.size > 0 || deliverableWorkspaceTurnNotificationIds.size > 0; let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; - for (const candidate of deliverableWorkflowPrompts) { - const agent = candidate.initiatingAgent; - if ( - agent != null && - (workflowInitiatingAgent == null || agent.createdAtMs > workflowInitiatingAgent.createdAtMs) - ) { - workflowInitiatingAgent = agent; + if (!hasNonWorkflowDeliverables) { + for (const candidate of deliverableWorkflowPrompts) { + const agent = candidate.initiatingAgent; + if ( + agent != null && + (workflowInitiatingAgent == null || + agent.createdAtMs > workflowInitiatingAgent.createdAtMs) + ) { + workflowInitiatingAgent = agent; + } } } const selectedAgentId = workflowInitiatingAgent?.agentId; - const selectedWorkflowPrompts = - selectedAgentId == null - ? deliverableWorkflowPrompts - : deliverableWorkflowPrompts.filter( - (candidate) => candidate.initiatingAgent?.agentId === selectedAgentId - ); + const selectedWorkflowPrompts = deliverableWorkflowPrompts.filter((candidate) => + hasNonWorkflowDeliverables + ? candidate.initiatingAgent == null + : selectedAgentId == null || candidate.initiatingAgent?.agentId === selectedAgentId + ); if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); } From 8e1bcd65500cc4bfed27363a05048fad351eb64c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:04:48 +0000 Subject: [PATCH 32/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20wake=20pro?= =?UTF-8?q?venance=20writes,=20reads,=20and=20pin=20pairing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three round-25 findings: (1) an explicit background launch now fails loudly (run stays pending and resumable) when the pre-launch sidecar write fails, instead of starting a runner whose terminal wake would be permanently superseded; (2) transient run-store read failures (non-ENOENT fs errors) defer the terminal wake for retry instead of tombstoning it, while missing or unparseable runs stay superseded; (3) the launch turn's strict-agent pin is persisted with the run reference (null for verified-unpinned) and the wake re-pins the selected group's own provenance, because the newest pin-bearing history row can belong to a different group's wake and a mismatched pin would reject the wake on every retry. --- src/common/utils/tools/tools.ts | 4 +- .../agentWorkflowRunReferences.test.ts | 27 ++++ .../services/agentWorkflowRunReferences.ts | 44 +++++- src/node/services/aiService.ts | 1 + src/node/services/taskService.test.ts | 138 ++++++++++++++++++ src/node/services/taskService.ts | 56 +++++-- src/node/services/tools/toolUtils.ts | 33 ++++- src/node/services/tools/workflow_run.test.ts | 40 ++++- src/node/services/tools/workflow_run.ts | 4 +- 9 files changed, 329 insertions(+), 18 deletions(-) diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index 5db7e05ba30..a6b69a4d248 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -2,7 +2,7 @@ import { xai } from "@ai-sdk/xai"; import { type LanguageModel, type Tool } from "ai"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; -import type { ProvidersConfigMap } from "@/common/orpc/types"; +import type { ProvidersConfigMap, SendMessageOptions } from "@/common/orpc/types"; import { isGrokFrontierModel } from "@/common/types/thinking"; import type { BackgroundWorkAttentionPolicy } from "@/common/types/backgroundWorkAttention"; import { cloneToolPreservingDescriptors } from "@/common/utils/tools/cloneToolPreservingDescriptors"; @@ -186,6 +186,8 @@ export interface ToolConfiguration { workspaceId?: string; /** Resolved agent identity of the turn executing the tools (workflow wake provenance). */ agentId?: string; + /** The turn's strict-agent pin, persisted with workflow run provenance so wakes re-pin the launch agent. */ + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; /** Pre-resolved mux-managed resource scope (global ~/.xum vs project root). */ xumScope?: XumToolScope; /** Memory service for the memory tool (present only when the memory experiment is enabled). */ diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 03cb142ed41..1939e4f305a 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -66,12 +66,14 @@ describe("agent workflow run references", () => { runId: "wfr_agent", createdAtMs: 1_000, agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, }); let references = await readAgentWorkflowRunReferences(workspaceSessionDir); expect(references).toContainEqual({ runId: "wfr_agent", createdAtMs: 1_000, agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, }); // Identity is advisory: a malformed persisted agentId drops the field, not the entry, @@ -85,6 +87,20 @@ describe("agent workflow run references", () => { // Non-empty but schema-invalid: stream resolution would normalize it to exec, // silently swapping a restricted agent's wake onto exec's tool surface. { runId: "wfr_agent_malformed", createdAtMs: 1_000, agentId: "bad id" }, + // Invalid pin shapes degrade to the legacy walk fallback (field dropped); a + // persisted false means verified-unpinned (null), like absence at record time. + { + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: 42 }, + }, + { + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: false, + }, ], }) ); @@ -92,6 +108,17 @@ describe("agent workflow run references", () => { expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); expect(references).toContainEqual({ runId: "wfr_agent_malformed", createdAtMs: 1_000 }); + expect(references).toContainEqual({ + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + }); + expect(references).toContainEqual({ + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: null, + }); } finally { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index f6b52a98ee6..2f66b634964 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -3,10 +3,18 @@ import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { SendMessageOptionsSchema } from "@/common/orpc/schemas/stream"; +import type { SendMessageOptions } from "@/common/orpc/types"; import { AgentIdSchema } from "@/common/schemas/ids"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; +/** A meaningful strict-agent pin: `false` and absence both mean "not pinned" and persist as null. */ +export type AgentWorkflowRunStrictPin = Exclude< + NonNullable, + false +>; + export interface AgentWorkflowRunReference { runId: string; createdAtMs: number; @@ -25,8 +33,18 @@ export interface AgentWorkflowRunReference { * fall back to the history walk. */ agentId?: string; + /** + * The launch turn's strict-agent pin, paired with agentId: a wake that re-binds this agent + * must re-pin the launch turn's provenance, because the newest pin-bearing history row can + * belong to a different group's wake and a mismatched pin makes resolution reject every + * retry. null records a verified-unpinned launch; absent (legacy or invalid persisted + * shape) falls back to the history-walk pin. + */ + strictAgentResolution?: AgentWorkflowRunStrictPin | null; } +const StrictPinSchema = SendMessageOptionsSchema.shape.strictAgentResolution; + const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; // Backward clock corrections (e.g. an NTP step after booting with a fast clock) can make a @@ -95,6 +113,21 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { // ID would silently swap a restricted agent's wake onto exec's tool surface. const agentIdParse = AgentIdSchema.safeParse(record.agentId); const agentId = agentIdParse.success ? agentIdParse.data : undefined; + // The pin only means anything paired with a surviving identity; false and invalid shapes + // degrade differently (unpinned vs legacy walk fallback), matching the field doc above. + let strictAgentResolution: AgentWorkflowRunStrictPin | null | undefined; + if (agentId !== undefined && "strictAgentResolution" in record) { + const pinRaw = record.strictAgentResolution; + if (pinRaw === null || pinRaw === false) { + strictAgentResolution = null; + } else { + const pinParse = StrictPinSchema.safeParse(pinRaw); + strictAgentResolution = + pinParse.success && pinParse.data != null && pinParse.data !== false + ? pinParse.data + : undefined; + } + } // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run // superseded. The chosen record is kept wholesale, including its boundary snapshot. @@ -105,6 +138,7 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { createdAtMs: record.createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), ...(agentId !== undefined ? { agentId } : {}), + ...(strictAgentResolution !== undefined ? { strictAgentResolution } : {}), }); } } @@ -155,6 +189,7 @@ export async function recordAgentWorkflowRunReference(input: { createdAtMs?: number; afterBoundaryMessageId?: string | null; agentId?: string; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); @@ -181,7 +216,14 @@ export async function recordAgentWorkflowRunReference(input: { ...(input.afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId: input.afterBoundaryMessageId } : {}), - ...(input.agentId != null && input.agentId.length > 0 ? { agentId: input.agentId } : {}), + ...(input.agentId != null && input.agentId.length > 0 + ? { + agentId: input.agentId, + ...(input.strictAgentResolution !== undefined + ? { strictAgentResolution: input.strictAgentResolution } + : {}), + } + : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index f43478fb251..58169e0749b 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -2647,6 +2647,7 @@ export class AIService extends EventEmitter { ancestorPlanFilePaths, workspaceId, agentId: effectiveAgentId, + strictAgentResolution, xumScope, timelineService: timelineExperimentEnabled ? this.timelineService : undefined, workspaceHeartbeatService: this.workspaceHeartbeatService, diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b957f5f06c4..6532f19e836 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6824,6 +6824,144 @@ describe("TaskService", () => { }); }); + test("transient run-store read failures defer the wake instead of tombstoning it", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const terminalAttentionStore = new TerminalAttentionStore(config); + + // run.json exists but is unreadable (EISDIR): potentially transient, so the wake must + // stay pending for a later drain instead of being durably tombstoned. + const unreadableRunId = "wfr_unreadable"; + await fsPromises.mkdir( + path.join(config.getSessionDir(parentId), "workflows", unreadableRunId, "run.json"), + { recursive: true } + ); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: unreadableRunId, + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + + // A definitively missing run (ENOENT) is still tombstoned, not deferred forever. + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_missing", + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.get(parentId, "workflow_run:wfr_missing")).toMatchObject({ + status: "superseded", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + + test("wake re-pins the selected group's recorded launch pin, not the newest row's", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_unpinned"); + await createRun("wfr_pin_recorded"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + // The newest pin-bearing row belongs to a DIFFERENT group's wake: pinning its provenance + // onto this group's agentId would make resolution reject the wake on every retry. + await historyService.appendToHistory( + parentId, + createMuxMessage("other-group-wake", "user", "earlier group results", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, + }) + ); + + // A verified-unpinned launch (null) must suppress the walk pin entirely. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_unpinned", + agentId: "exec", + strictAgentResolution: null, + }); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_unpinned", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const unpinnedOptions = sendMessage.mock.calls[0]?.[2] as { + agentId?: string; + strictAgentResolution?: unknown; + }; + expect(unpinnedOptions.agentId).toBe("exec"); + expect(unpinnedOptions.strictAgentResolution).toBeUndefined(); + + // A recorded launch pin overrides the walk pin exactly. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_recorded", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_recorded", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + }); + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 5a045aefe7f..a1a486d4547 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -210,6 +210,7 @@ import { type TerminalAttentionOutcome, } from "@/node/services/terminalAttentionStore"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { AgentWorkflowRunStrictPin } from "@/node/services/agentWorkflowRunReferences"; import { isWorkflowRunTaskId } from "@/node/services/tools/taskId"; import { normalizeWorkflowAgentReportPayloadForHostSchema } from "@/common/utils/tools/workflowReportPayload"; import { @@ -1051,6 +1052,13 @@ interface ParentAutoResumeHint { agentId?: string; } +/** Launch identity recorded with a workflow run reference; see AgentWorkflowRunReference. */ +interface WorkflowWakeInitiatingAgent { + agentId: string; + createdAtMs: number; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; +} + function isTypedWorkspaceEvent(value: unknown, type: string): boolean { return ( typeof value === "object" && @@ -8030,11 +8038,7 @@ export class TaskService { ownerWorkspaceId: string, runId: string ): Promise< - | { - outcome: "deliver"; - prompt: string; - initiatingAgent?: { agentId: string; createdAtMs: number }; - } + | { outcome: "deliver"; prompt: string; initiatingAgent?: WorkflowWakeInitiatingAgent } | { outcome: "superseded" } | { outcome: "defer" } > { @@ -8047,6 +8051,22 @@ export class TaskService { try { run = await runStore.getRun(runId); } catch (error: unknown) { + // A missing run (ENOENT) or an unparseable record (no fs code; rereading cannot repair + // it) is definitively ineligible. Every other fs failure (EIO, EACCES, EISDIR...) is + // potentially transient, and tombstoning on it would permanently drop the wake over a + // recoverable fault: defer those like indeterminate currentness below. + const code = + error != null && typeof error === "object" && "code" in error + ? (error as { code?: unknown }).code + : undefined; + if (typeof code === "string" && code !== "ENOENT") { + log.warn("Deferring workflow terminal wake-up; run record unreadable", { + ownerWorkspaceId, + runId, + error: getErrorMessage(error), + }); + return { outcome: "defer" }; + } log.warn("Failed to load terminal workflow run for wake-up", { ownerWorkspaceId, runId, @@ -8077,14 +8097,20 @@ export class TaskService { // can belong to an unrelated later synthetic turn (a heartbeat is not a supersession // boundary), which would pair a different agent's tool surface with the launch turn's // caller policy. Advisory: legacy references fall back to the history walk. - let initiatingAgent: { agentId: string; createdAtMs: number } | undefined; + let initiatingAgent: WorkflowWakeInitiatingAgent | undefined; try { const references = await readAgentWorkflowRunReferences( this.config.getSessionDir(ownerWorkspaceId) ); const reference = references.find((candidate) => candidate.runId === run.id); if (reference?.agentId != null) { - initiatingAgent = { agentId: reference.agentId, createdAtMs: reference.createdAtMs }; + initiatingAgent = { + agentId: reference.agentId, + createdAtMs: reference.createdAtMs, + ...(reference.strictAgentResolution !== undefined + ? { strictAgentResolution: reference.strictAgentResolution } + : {}), + }; } } catch { // Identity is advisory; an unreadable sidecar already deferred delivery above whenever @@ -8411,7 +8437,7 @@ export class TaskService { const deliverableWorkflowPrompts: Array<{ notificationId: string; prompt: string; - initiatingAgent?: { agentId: string; createdAtMs: number }; + initiatingAgent?: WorkflowWakeInitiatingAgent; }> = []; const promptSections: string[] = []; @@ -8460,7 +8486,7 @@ export class TaskService { // retry drain; among agent-bound groups the newest launch goes first. const hasNonWorkflowDeliverables = deliverableAgentNotificationIds.size > 0 || deliverableWorkspaceTurnNotificationIds.size > 0; - let workflowInitiatingAgent: { agentId: string; createdAtMs: number } | undefined; + let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; if (!hasNonWorkflowDeliverables) { for (const candidate of deliverableWorkflowPrompts) { const agent = candidate.initiatingAgent; @@ -8544,6 +8570,14 @@ export class TaskService { return; } + // Pair the pin with the selected group: the newest pin-bearing history row can belong to + // a different group's wake (each wake persists its own pin), and pinning another agent's + // provenance onto this group's agentId makes resolution reject the wake on every retry. A + // recorded pin (or a verified-unpinned null) overrides the walk; legacy references + // without the field keep the walk pin. + const groupPin = workflowInitiatingAgent?.strictAgentResolution; + const effectiveStrictPin = + groupPin !== undefined ? (groupPin ?? undefined) : wakeRestrictions.strictAgentResolution; const sendOptions = { model: resumeOptions.model, agentId: resumeOptions.agentId, @@ -8551,9 +8585,7 @@ export class TaskService { reasoningMode: resumeOptions.reasoningMode, ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), - ...(wakeRestrictions.strictAgentResolution != null - ? { strictAgentResolution: wakeRestrictions.strictAgentResolution } - : {}), + ...(effectiveStrictPin != null ? { strictAgentResolution: effectiveStrictPin } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 24b20398666..79f2c8ca758 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -89,10 +89,26 @@ export async function emitWorkflowRunAttachedEvent(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number + createdAtMs: number, + options?: { + /** + * Pre-launch records must not fail soft: the sidecar is the kernel invocation's only + * durable provenance, and starting the runner without it lets a fast terminal run have + * its wake permanently marked superseded. Throwing before dispatch leaves the run + * pending and resumable (workflow_resume re-records), so the caller surfaces a loud, + * recoverable launch failure instead. Post-dispatch records stay best-effort because the + * run already started and failing the tool would strand it. + */ + propagateWriteFailure?: boolean; + } ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { + if (options?.propagateWriteFailure === true) { + throw new Error( + `Cannot record workflow run provenance without a workspace session dir: ${runId}` + ); + } log.warn("Skipping agent workflow run reference without workspace session dir", { runId }); return; } @@ -126,9 +142,22 @@ export async function recordBackgroundWorkflowRunReference( runId, createdAtMs, ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), - ...(config.agentId != null && config.agentId.length > 0 ? { agentId: config.agentId } : {}), + ...(config.agentId != null && config.agentId.length > 0 + ? { + agentId: config.agentId, + // The pin pairs with the identity: null records a verified-unpinned launch so the + // wake never inherits another row's pin (see AgentWorkflowRunReference). + strictAgentResolution: + config.strictAgentResolution != null && config.strictAgentResolution !== false + ? config.strictAgentResolution + : null, + } + : {}), }); } catch (error: unknown) { + if (options?.propagateWriteFailure === true) { + throw error; + } log.warn("Failed to record agent workflow run reference", { runId, error: getErrorMessage(error), diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 143a1bcddb6..3e394d5c35f 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -673,6 +673,7 @@ describe("workflow_run tool", () => { ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, @@ -694,8 +695,10 @@ describe("workflow_run tool", () => { expect(references[0]).toMatchObject({ runId: "wfr_background", afterBoundaryMessageId: "boundary-row-1", - // The wake binds to the launching agent, so the reference must carry its identity. + // The wake binds to the launching agent, so the reference must carry its identity + // and the launch turn's provenance pin. agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, }); expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( "workspace-1", @@ -757,6 +760,41 @@ describe("workflow_run tool", () => { expect(references[0] != null && "afterBoundaryMessageId" in references[0]).toBe(false); }); + test("a sidecar write failure aborts the background launch before the runner starts", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-sidecar-write-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + // The reference path exists as a directory, so every sidecar write fails. + await fs.mkdir(path.join(tempDir.path, "agent-workflow-runs.json")); + let runnerStarted = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_sidecar_write_error", + status: "pending", + result: null, + run: null, + }); + runnerStarted = true; + return { runId: "wfr_sidecar_write_error", status: "running" as const, result: null }; + }); + const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + agentId: "plan", + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { startWorkflowInBackground, getRun }, + }); + + // Starting the runner without durable provenance would let a fast terminal run have its + // wake permanently superseded; the launch must fail loudly instead, leaving the run + // pending and resumable (workflow_resume re-records provenance). + await expect( + tool.execute!({ script_path: scriptPath, run_in_background: true }, mockToolCallOptions) + ).rejects.toThrow(/created durable run wfr_sidecar_write_error/); + expect(runnerStarted).toBe(false); + }); + test("requires the workflow service", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-missing"); const scriptPath = await writeWorkflowScript(tempDir.path); diff --git a/src/node/services/tools/workflow_run.ts b/src/node/services/tools/workflow_run.ts index 075089795cc..300da0f60fc 100644 --- a/src/node/services/tools/workflow_run.ts +++ b/src/node/services/tools/workflow_run.ts @@ -276,7 +276,9 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => // process exit mid-dispatch) can reach terminal state before any post-dispatch // write, and a terminal wake with no sidecar reference is permanently superseded. if (args.run_in_background === true) { - await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs); + await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs, { + propagateWriteFailure: true, + }); } await emitWorkflowRunAttachedEvent({ config, From d53ac1ad2efe70b6470c2f502371f014477149ee Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:41:01 +0000 Subject: [PATCH 33/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20defer=20boundaryles?= =?UTF-8?q?s=20workflow=20references=20instead=20of=20wall-clock=20orderin?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/agentWorkflowRunReferences.ts | 6 +-- src/node/services/tools/toolUtils.ts | 2 +- src/node/services/workspaceService.test.ts | 27 +++++------ src/node/services/workspaceService.ts | 46 ++++++++----------- 4 files changed, 36 insertions(+), 45 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 2f66b634964..6e2ff131194 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -92,9 +92,9 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { const hasBoundary = "afterBoundaryMessageId" in record; const boundaryRaw = record.afterBoundaryMessageId; // A present-but-invalid snapshot ("" or a non-string) is corruption, not a legacy record: - // migrating it into the wall-clock fallback could let a stale reference outrank a newer - // boundary within the tolerated clock skew. Reject the entry; absence stays reserved for - // records that genuinely predate the field. + // demoting it to a boundaryless entry would misclassify a recorded boundary as unknowable + // provenance (parking its wake as indeterminate). Reject the entry; absence stays reserved + // for records that genuinely predate the field. if ( hasBoundary && boundaryRaw !== null && diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index 79f2c8ca758..d842b9ab224 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -119,7 +119,7 @@ export async function recordBackgroundWorkflowRunReference( // failure must not be persisted as a verified-empty boundary (null): record without the // field instead, so the run stays rediscoverable (listAgentReferencedWorkflowRunIds) and a // later workflow_resume re-record can repair provenance, while the unverifiable boundary - // fails safe for wake delivery. + // defers wake delivery (indeterminate) instead of guessing from wall-clock order. let afterBoundaryMessageId: string | null | undefined; const taskService = config.taskService; if (config.workspaceId != null && taskService?.getWorkflowInvocationBoundaryMessageId != null) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index daac7889e48..6ee594182cc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6406,7 +6406,7 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("migrates legacy sidecar references through the wall-clock fallback", async () => { + test("defers boundaryless sidecar references instead of trusting wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-legacy"; const runId = "wfr_currentness_legacy"; @@ -6438,30 +6438,31 @@ describe("WorkspaceService workflow invocation events", () => { workspaceId, createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); - // Entries written before boundary snapshots existed carry only a timestamp. An in-flight - // run recorded after the newest boundary must keep its wake across the upgrade. + // A reference without a boundary snapshot (pre-upgrade entry or record-time history read + // failure) cannot be ordered against the decision row by identity: the wake defers + // instead of delivering, and the boolean caller stays fail-safe. await recordAgentWorkflowRunReference({ workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, }); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); - // A newer boundary still supersedes a legacy entry. + // A backward clock correction gives the newer superseding turn an OLDER timestamp than + // the reference. Wall-clock ordering would resurrect the superseded reference as current + // and deliver its output under the newer turn's tool policy; it must stay deferred. await historyService.appendToHistory( workspaceId, createMuxMessage("manual-user-2", "user", "never mind, answer something else", { - timestamp: 1_200, + timestamp: 1_100, }) ); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); - - // An undatable boundary cannot be ordered against a legacy timestamp: fail safe. - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user-undated", "user", "another instruction", {}) + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" ); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); workspaceService.disposeSession(workspaceId); } finally { await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index bb95cabd55c..43688444a46 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -10959,11 +10959,11 @@ export class WorkspaceService extends EventEmitter { } /** - * Three-state currentness: "indeterminate" means history could not be read, so the answer is - * unknown rather than no. Callers that would permanently drop a terminal wake on a negative - * answer (the terminal-attention drain tombstones notifications) must retain and retry on - * "indeterminate" instead; boolean callers treat it as not-current, the pre-existing - * fail-safe for non-destructive decisions. + * Three-state currentness: "indeterminate" means history/provenance could not be read or + * ordered, so the answer is unknown rather than no. Callers that would permanently drop a + * terminal wake on a negative answer (the terminal-attention drain tombstones notifications) + * must retain and retry on "indeterminate" instead; boolean callers treat it as not-current, + * the pre-existing fail-safe for non-destructive decisions. */ async getWorkflowInvocationCurrentness( workspaceId: string, @@ -10990,7 +10990,8 @@ export class WorkspaceService extends EventEmitter { // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a // consumed boundary, equality means a background resume/retry was recorded after the prior // result was delivered. References without a boundary snapshot (pre-upgrade entries, - // record-time read failures) take a wall-clock migration fallback below. + // record-time read failures) cannot be ordered against the decision row at all and defer + // as indeterminate below. let references: AgentWorkflowRunReference[]; try { references = await readAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); @@ -11021,15 +11022,13 @@ export class WorkspaceService extends EventEmitter { return "not_current"; } if (reference.afterBoundaryMessageId === undefined) { - // Migration fallback: entries written before boundary snapshots existed (or after a - // record-time history read failure) carry only a timestamp, and an in-flight run must not - // lose its wake across the upgrade. Fall back to wall-clock ordering against a datable - // boundary; an undatable boundary fails safe. All new records take the identity path - // above, so clock-correction edge cases are confined to this shrinking population. - if (decision.timestampMs == null) { - return "not_current"; - } - return reference.createdAtMs > decision.timestampMs ? "current" : "not_current"; + // No boundary snapshot (pre-upgrade entry or record-time history read failure): row + // identity cannot be verified, and wall-clock ordering is the exact hole the identity + // path exists to close (a backward clock correction would let a pre-supersession + // reference outrank a newer manual turn and deliver its output under that turn's tool + // policy). Defer like an unreadable history: the wake stays pending, a workflow_resume + // re-record repairs provenance, and an explicit resume/await still consumes the run. + return "indeterminate"; } if (reference.afterBoundaryMessageId === null) { // Verified-empty snapshot: a decision row now exists, so it appeared after the record. @@ -11052,7 +11051,6 @@ export class WorkspaceService extends EventEmitter { status: "found"; outcome: "invocation" | "consumed" | "superseded"; messageId: string; - timestampMs: number | null; } | { status: "none" } | { status: "error" } @@ -11061,7 +11059,6 @@ export class WorkspaceService extends EventEmitter { found: { outcome: "invocation" | "consumed" | "superseded"; messageId: string; - timestampMs: number | null; } | null; } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( @@ -11069,10 +11066,8 @@ export class WorkspaceService extends EventEmitter { "backward", (messages) => { for (const message of messages) { - const timestamp = message.metadata?.timestamp; - const timestampMs = typeof timestamp === "number" ? timestamp : null; if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { - state.found = { outcome: "superseded", messageId: message.id, timestampMs }; + state.found = { outcome: "superseded", messageId: message.id }; return false; } if ( @@ -11081,11 +11076,11 @@ export class WorkspaceService extends EventEmitter { isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - state.found = { outcome: "consumed", messageId: message.id, timestampMs }; + state.found = { outcome: "consumed", messageId: message.id }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - state.found = { outcome: "invocation", messageId: message.id, timestampMs }; + state.found = { outcome: "invocation", messageId: message.id }; return false; } } @@ -11101,12 +11096,7 @@ export class WorkspaceService extends EventEmitter { return { status: "error" }; } return state.found != null - ? { - status: "found", - outcome: state.found.outcome, - messageId: state.found.messageId, - timestampMs: state.found.timestampMs, - } + ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } : { status: "none" }; } From cd1b200807b857cc4fe7a2c69c40c301bd0ca7a6 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:15:17 +0000 Subject: [PATCH 34/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20split=20wakes=20by?= =?UTF-8?q?=20launch=20pin=20and=20repair=20downgrade-stripped=20provenanc?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/orpc/router.ts | 2 + src/node/services/taskService.test.ts | 95 +++++++++++++++++++ src/node/services/taskService.ts | 32 +++++-- .../workflows/WorkflowService.test.ts | 56 +++++++++++ .../services/workflows/WorkflowService.ts | 21 ++++ src/node/services/workspaceService.test.ts | 90 +++++++++++++++++- src/node/services/workspaceService.ts | 40 ++++++++ 7 files changed, 326 insertions(+), 10 deletions(-) diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index c425ea2be02..327db8336f9 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -584,6 +584,8 @@ export async function resolveWorkflowContext( skillStorageContext, }), onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), + onRunCrashResumed: (event) => + context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), ...(options.onBackgroundRunTerminal != null ? { onBackgroundRunTerminal: options.onBackgroundRunTerminal } : {}), diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6532f19e836..0316e4089f1 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6670,6 +6670,101 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("coalesced workflow wakes split by strict pin within one agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_split_pinned"); + await createRun("wfr_pin_split_unpinned"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Same agentId, different launch pins (an agent definition replaced between synthetic + // launches): one coalesced wake would process the pinned run's output under the newer + // verified-unpinned launch. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_split_pinned", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: "wfr_pin_split_unpinned", + createdAtMs: 2_000, + agentId: "plan", + strictAgentResolution: null, + }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_pin_split_pinned", + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_pin_split_unpinned", + }); + await drain(parentId); + + // The newest launch delivers first, alone, without the other launch's pin. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_pin_split_unpinned"); + expect(firstPrompt).not.toContain("wfr_pin_split_pinned"); + const firstOptions = sendMessage.mock.calls[0]?.[2] as Record; + expect(firstOptions.agentId).toBe("plan"); + expect(firstOptions.strictAgentResolution).toBeUndefined(); + + // The pinned launch delivers on the retry drain under its own recorded pin. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_pin_split_pinned"); + expect(secondPrompt).not.toContain("wfr_pin_split_unpinned"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("mixed drains keep workspace-turn attention off the workflow's agent", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index a1a486d4547..89bafb64a7a 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1059,6 +1059,15 @@ interface WorkflowWakeInitiatingAgent { strictAgentResolution?: AgentWorkflowRunStrictPin | null; } +// Coalescing key for terminal workflow wakes: the pin is part of the launch identity, so an +// agentId alone must not merge a pinned launch with an unpinned (or differently pinned) one. +// undefined (legacy walk fallback), null (verified unpinned), and each concrete pin are +// distinct groups; over-splitting structurally equal pins is safe, merging them is not. +function workflowWakeGroupKey(agent: WorkflowWakeInitiatingAgent): string { + const pin = agent.strictAgentResolution; + return `${agent.agentId}\u0000${pin === undefined ? "walk" : JSON.stringify(pin)}`; +} + function isTypedWorkspaceEvent(value: unknown, type: string): boolean { return ( typeof value === "object" && @@ -8477,13 +8486,15 @@ export class TaskService { : {}), }); } - // Deliver one initiating-agent group per drain: the whole coalesced prompt is handled - // under the single agentId passed to sendMessage, so batching runs from different agents - // would hand a restricted agent's (attacker-influenced) output to another agent's tool - // grants. The same applies to mixed batches: workspace-turn and sub-agent attention - // resumes under the conversation's own (history-walk) identity, so agent-bound workflow - // groups never share their send. Deferred groups stay pending and deliver on the re-armed - // retry drain; among agent-bound groups the newest launch goes first. + // Deliver one launch-identity group per drain, keyed by agentId AND recorded strict pin: + // the whole coalesced prompt is handled under the single agentId/pin passed to + // sendMessage, so batching runs from different agents (or runs sharing an agentId but + // launched under different pins, e.g. an agent definition replaced between synthetic + // turns) would hand a restricted launch's (attacker-influenced) output to another + // launch's tool grants. The same applies to mixed batches: workspace-turn and sub-agent + // attention resumes under the conversation's own (history-walk) identity, so agent-bound + // workflow groups never share their send. Deferred groups stay pending and deliver on the + // re-armed retry drain; among agent-bound groups the newest launch goes first. const hasNonWorkflowDeliverables = deliverableAgentNotificationIds.size > 0 || deliverableWorkspaceTurnNotificationIds.size > 0; let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; @@ -8499,11 +8510,14 @@ export class TaskService { } } } - const selectedAgentId = workflowInitiatingAgent?.agentId; + const selectedGroupKey = + workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : undefined; const selectedWorkflowPrompts = deliverableWorkflowPrompts.filter((candidate) => hasNonWorkflowDeliverables ? candidate.initiatingAgent == null - : selectedAgentId == null || candidate.initiatingAgent?.agentId === selectedAgentId + : selectedGroupKey == null || + (candidate.initiatingAgent != null && + workflowWakeGroupKey(candidate.initiatingAgent) === selectedGroupKey) ); if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index 008acd2a5ce..03ccf20442f 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -1007,3 +1007,59 @@ describe("WorkflowRunStore.listActiveRunSummaries", () => { expect(summaries.map((summary) => summary.runId)).toEqual(["wfr_healthy"]); }); }); + +describe("WorkflowService crash recovery", () => { + test("crash resume fires provenance repair before the run reaches terminal", async () => { + using tmp = new DisposableTempDir("workflow-service-crash-repair"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_crash", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, + source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + // Orphaned by a crash: durable status says running, but no runner holds the lease. + await runStore.appendStatus("wfr_crash", "running", "2026-05-29T00:00:01.000Z"); + + const events: string[] = []; + let resolveCompleted: (() => void) | undefined; + const completed = new Promise((resolve) => { + resolveCompleted = resolve; + }); + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + generateRunId: () => "wfr_unused", + runnerId: "runner-a", + onRunCrashResumed: (event) => { + events.push(`repair:${event.workspaceId}:${event.runId}`); + }, + onRunStatusChanged: (event) => { + events.push(`status:${event.status}`); + if (event.status === "completed") { + resolveCompleted?.(); + } + }, + }); + + const resumed = await service.resumeCrashedRuns({ + workspaceId: "workspace-1", + projectTrusted: true, + }); + expect(resumed).toEqual(["wfr_crash"]); + await completed; + // The repair hook is awaited before the runner restarts, so even an instantly completing + // run cannot reach terminal with unrepaired provenance. + expect(events[0]).toBe("repair:workspace-1:wfr_crash"); + expect(events).toContain("status:completed"); + await expect(runStore.getRun("wfr_crash")).resolves.toMatchObject({ status: "completed" }); + }); +}); diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 70b8fb35f2a..94c90f8e528 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -54,6 +54,12 @@ export interface WorkflowServiceOptions { resolveWorkflowScript?: (scriptPath: string) => Promise; onBackgroundRunTerminal?: (event: WorkflowBackgroundRunTerminalEvent) => Promise | void; onRunStatusChanged?: (event: WorkflowRunStatusChangedEvent) => Promise | void; + /** + * Fired when crash recovery is about to resume an orphaned run, before the runner restarts. + * Used to repair wake provenance a pre-boundary build stripped from the sidecar; awaited so + * a fast run cannot reach terminal before the repair lands. + */ + onRunCrashResumed?: (event: { workspaceId: string; runId: string }) => Promise | void; /** When true, background terminal notifications also fire for interrupted runs. */ notifyInterruptedBackgroundRunTerminal?: boolean; generateRunId?: () => string; @@ -127,6 +133,10 @@ export class WorkflowService { private readonly onBackgroundRunTerminal?: ( event: WorkflowBackgroundRunTerminalEvent ) => Promise | void; + private readonly onRunCrashResumed?: (event: { + workspaceId: string; + runId: string; + }) => Promise | void; private readonly onRunStatusChanged?: ( event: WorkflowRunStatusChangedEvent ) => Promise | void; @@ -150,6 +160,7 @@ export class WorkflowService { this.taskAdapterFactory = options.taskAdapterFactory; this.resolveWorkflowScript = options.resolveWorkflowScript; this.onBackgroundRunTerminal = options.onBackgroundRunTerminal; + this.onRunCrashResumed = options.onRunCrashResumed; this.onRunStatusChanged = options.onRunStatusChanged; this.notifyInterruptedBackgroundRunTerminal = options.notifyInterruptedBackgroundRunTerminal === true; @@ -573,6 +584,16 @@ export class WorkflowService { return false; } + if (this.onRunCrashResumed != null) { + try { + await this.onRunCrashResumed({ workspaceId: run.workspaceId, runId: run.id }); + } catch (error) { + // Best-effort: an unrepaired reference defers its wake as indeterminate rather than + // losing it, so a failed repair must not block the resume itself. + console.error("Workflow crash-resume provenance repair failed:", error); + } + } + const retryDelayMs = await this.runStore.getLeaseRetryDelayMs( input.runId, this.clock?.nowMs() ?? Date.now() diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 6ee594182cc..72bb510eaf4 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -59,7 +59,10 @@ import { WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowResultContextMessage, } from "@/common/utils/workflowRunMessages"; -import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; +import { + readAgentWorkflowRunReferences, + recordAgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; import * as todoStorageModule from "@/node/services/todos/todoStorage"; import * as runtimeFactory from "@/node/runtime/runtimeFactory"; @@ -6469,6 +6472,91 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("crash-resume repair re-snapshots only boundaryless references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-crash-repair"; + const strippedRunId = "wfr_crash_repair_stripped"; + const anchoredRunId = "wfr_crash_repair_anchored"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-crash-repair", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // A downgrade rewrote the sidecar without boundary fields (or a record-time read failure + // omitted them): crash-resume repair re-snapshots the boundary, keeping the recorded + // launch identity, and the deferred wake becomes deliverable again. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: strippedRunId, + createdAtMs: 1_150, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, strippedRunId); + const repaired = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect(repaired.find((reference) => reference.runId === strippedRunId)).toMatchObject({ + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, strippedRunId) + ).toBe("current"); + + // A reference that still carries its boundary may record a pre-supersession launch: + // repair must not refresh it into the current context. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: anchoredRunId, + createdAtMs: 1_050, + afterBoundaryMessageId: "older-row", + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, anchoredRunId); + const untouched = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect( + untouched.find((reference) => reference.runId === anchoredRunId)?.afterBoundaryMessageId + ).toBe("older-row"); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, anchoredRunId) + ).toBe("not_current"); + + // Unknown run: nothing to repair, nothing recorded. + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, "wfr_unknown"); + const after = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + expect(after.map((reference) => reference.runId).sort()).toEqual([ + anchoredRunId, + strippedRunId, + ]); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("treats an unreadable history as indeterminate, not superseded", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-io-error"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 43688444a46..faaecf2b352 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6,6 +6,7 @@ import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; import { clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, + recordAgentWorkflowRunReference, type AgentWorkflowRunReference, } from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; @@ -11122,6 +11123,45 @@ export class WorkspaceService extends EventEmitter { return decision.status === "found" ? decision.messageId : null; } + /** + * Re-snapshot the boundary for a run reference that lost it: a pre-boundary build rewrites + * the sidecar with only runId/createdAtMs on any record (upgrade -> downgrade -> upgrade + * strips the field), and a boundaryless reference defers its terminal wake as indeterminate + * until provenance is re-established. Crash recovery calls this before restarting an + * orphaned run: the run is verifiably non-terminal there, so a resume-time snapshot is + * legitimate launch provenance for the continued execution, mirroring the workflow_resume + * tool's re-record. References that still carry a boundary (including verified-empty null) + * are left untouched: refreshing them would forgive manual supersessions on every restart. + */ + async repairWorkflowRunReferenceBoundary(workspaceId: string, runId: string): Promise { + assert(workspaceId.length > 0, "repairWorkflowRunReferenceBoundary requires workspaceId"); + assert(runId.length > 0, "repairWorkflowRunReferenceBoundary requires runId"); + const sessionDir = this.config.getSessionDir(workspaceId); + const references = await readAgentWorkflowRunReferences(sessionDir); + const reference = references.find((candidate) => candidate.runId === runId); + if (reference == null || reference.afterBoundaryMessageId !== undefined) { + return; + } + const afterBoundaryMessageId = await this.getWorkflowInvocationBoundaryMessageId( + workspaceId, + runId + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: sessionDir, + runId, + createdAtMs: reference.createdAtMs, + afterBoundaryMessageId, + ...(reference.agentId != null + ? { + agentId: reference.agentId, + ...(reference.strictAgentResolution !== undefined + ? { strictAgentResolution: reference.strictAgentResolution } + : {}), + } + : {}), + }); + } + /** * Increment a preflight admission counter in the caller's synchronous entry block and * return a disposable releasing it. Pairs renderer-initiated workspace activity with the From 578d72bf2b2385a91d7c02a6ee6347bfe034421c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:32:50 +0000 Subject: [PATCH 35/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20gate=20crash-resume?= =?UTF-8?q?=20provenance=20repair=20on=20supersession-free=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/workspaceService.test.ts | 39 +++++++++++++++++----- src/node/services/workspaceService.ts | 29 +++++++++++----- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 72bb510eaf4..9efd4e1d6cc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6472,10 +6472,11 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("crash-resume repair re-snapshots only boundaryless references", async () => { + test("crash-resume repair restores provenance only on supersession-free evidence", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-crash-repair"; const strippedRunId = "wfr_crash_repair_stripped"; + const supersededRunId = "wfr_crash_repair_superseded"; const anchoredRunId = "wfr_crash_repair_anchored"; const projectPath = path.join(config.rootDir, "project"); try { @@ -6501,13 +6502,9 @@ describe("WorkspaceService workflow invocation events", () => { } as unknown as InitStateManager, }); - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) - ); - // A downgrade rewrote the sidecar without boundary fields (or a record-time read failure - // omitted them): crash-resume repair re-snapshots the boundary, keeping the recorded - // launch identity, and the deferred wake becomes deliverable again. + // A downgrade rewrote the sidecar without boundary fields. With a decision-free history + // the repair has supersession-free evidence: it records a verified-empty boundary, + // keeps the recorded launch identity, and the deferred wake becomes deliverable. await recordAgentWorkflowRunReference({ workspaceSessionDir: config.getSessionDir(workspaceId), runId: strippedRunId, @@ -6519,7 +6516,7 @@ describe("WorkspaceService workflow invocation events", () => { const repaired = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); expect(repaired.find((reference) => reference.runId === strippedRunId)).toMatchObject({ createdAtMs: 1_150, - afterBoundaryMessageId: "manual-user", + afterBoundaryMessageId: null, agentId: "plan", strictAgentResolution: { expectedScope: "built-in" }, }); @@ -6527,6 +6524,29 @@ describe("WorkspaceService workflow invocation events", () => { await workspaceService.getWorkflowInvocationCurrentness(workspaceId, strippedRunId) ).toBe("current"); + // Once a manual row is the newest decision row, a stripped launch cannot be ordered + // against it: the run may predate the supersession, so repair must refuse and the wake + // must stay deferred rather than resurrect a possibly superseded result. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, do something else", { + timestamp: 1_200, + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId: supersededRunId, + createdAtMs: 1_100, + }); + await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, supersededRunId); + const refused = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); + const refusedReference = refused.find((reference) => reference.runId === supersededRunId); + expect(refusedReference).toBeDefined(); + expect(refusedReference != null && "afterBoundaryMessageId" in refusedReference).toBe(false); + expect( + await workspaceService.getWorkflowInvocationCurrentness(workspaceId, supersededRunId) + ).toBe("indeterminate"); + // A reference that still carries its boundary may record a pre-supersession launch: // repair must not refresh it into the current context. await recordAgentWorkflowRunReference({ @@ -6550,6 +6570,7 @@ describe("WorkspaceService workflow invocation events", () => { expect(after.map((reference) => reference.runId).sort()).toEqual([ anchoredRunId, strippedRunId, + supersededRunId, ]); workspaceService.disposeSession(workspaceId); } finally { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index faaecf2b352..f5c7c552492 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -11128,10 +11128,15 @@ export class WorkspaceService extends EventEmitter { * the sidecar with only runId/createdAtMs on any record (upgrade -> downgrade -> upgrade * strips the field), and a boundaryless reference defers its terminal wake as indeterminate * until provenance is re-established. Crash recovery calls this before restarting an - * orphaned run: the run is verifiably non-terminal there, so a resume-time snapshot is - * legitimate launch provenance for the continued execution, mirroring the workflow_resume - * tool's re-record. References that still carry a boundary (including verified-empty null) - * are left untouched: refreshing them would forgive manual supersessions on every restart. + * orphaned run, but the repair proceeds only on supersession-free evidence: a decision-free + * history (recorded as a verified-empty boundary) or a newest decision row that belongs to + * this run. A newest manual/reset row is refused: the stripped launch cannot be ordered + * against it by identity, and snapshotting it would resurrect a possibly pre-supersession + * result into the newer conversation (the same reference with a surviving boundary would + * stay not_current). Those wakes stay deferred until an explicit workflow_resume, which + * carries current-context intent. References that still carry a boundary (including + * verified-empty null) are left untouched: refreshing them would forgive manual + * supersessions on every restart. */ async repairWorkflowRunReferenceBoundary(workspaceId: string, runId: string): Promise { assert(workspaceId.length > 0, "repairWorkflowRunReferenceBoundary requires workspaceId"); @@ -11142,15 +11147,21 @@ export class WorkspaceService extends EventEmitter { if (reference == null || reference.afterBoundaryMessageId !== undefined) { return; } - const afterBoundaryMessageId = await this.getWorkflowInvocationBoundaryMessageId( - workspaceId, - runId - ); + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + if (decision.status === "error") { + throw new Error("workflow invocation boundary unavailable: history read failed"); + } + if (decision.status === "found" && decision.outcome === "superseded") { + return; + } + // Supersession-free evidence only: no decision row at all (verified-empty null), or the + // newest decision row is this run's own invocation/consumed row, which no manual row can + // postdate (the backward walk would have found that manual row first). await recordAgentWorkflowRunReference({ workspaceSessionDir: sessionDir, runId, createdAtMs: reference.createdAtMs, - afterBoundaryMessageId, + afterBoundaryMessageId: decision.status === "found" ? decision.messageId : null, ...(reference.agentId != null ? { agentId: reference.agentId, From 56171b714c4b5bffc59fdf81a8c7da97932d0ec1 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:54:32 +0000 Subject: [PATCH 36/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20make=20crash-resume?= =?UTF-8?q?=20boundary=20repair=20a=20compare-and-set=20under=20the=20side?= =?UTF-8?q?car=20lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentWorkflowRunReferences.test.ts | 77 +++++++++++++++++++ .../services/agentWorkflowRunReferences.ts | 34 ++++++++ src/node/services/workspaceService.ts | 18 ++--- 3 files changed, 117 insertions(+), 12 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 1939e4f305a..7ecab498c72 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -7,6 +7,7 @@ import { describe, expect, test } from "bun:test"; import { readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, + repairAgentWorkflowRunReferenceBoundary, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -298,4 +299,80 @@ describe("agent workflow run references", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("boundary repair is a compare-and-set on a surviving boundaryless reference", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_repairable"; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_bystander", + createdAtMs: 1_100, + afterBoundaryMessageId: "row-1", + }); + + // Repairs in place, preserving the rest of the entry and its neighbors. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId, + afterBoundaryMessageId: null, + }) + ).toBe(true); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references.find((reference) => reference.runId === runId)).toEqual({ + runId, + createdAtMs: 1_000, + afterBoundaryMessageId: null, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(references.find((reference) => reference.runId === "wfr_bystander")).toEqual({ + runId: "wfr_bystander", + createdAtMs: 1_100, + afterBoundaryMessageId: "row-1", + }); + + // A reference that already carries a boundary (here the one just repaired) is never + // overwritten: a concurrent explicit re-record must win over a stale repair. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId, + afterBoundaryMessageId: "stale-row", + }) + ).toBe(false); + const unchanged = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(unchanged.find((reference) => reference.runId === runId)?.afterBoundaryMessageId).toBe( + null + ); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("boundary repair refuses to recreate a cleared sidecar", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A full-history clear deleted the sidecar between the repair's reads and its write: + // the stale repair must not resurrect the retired reference as verified-empty current. + expect( + await repairAgentWorkflowRunReferenceBoundary({ + workspaceSessionDir, + runId: "wfr_cleared", + afterBoundaryMessageId: null, + }) + ).toBe(false); + expect(await fs.readdir(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 6e2ff131194..01d5cba36ae 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -233,3 +233,37 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } + +/** + * Compare-and-set boundary repair for a surviving boundaryless reference. The reference is + * re-validated under the sidecar file lock: a concurrent full-history clear deletes the file + * (retiring every reference), and an unconditional write would recreate it with a + * verified-empty boundary, resurrecting the retired pre-clear result as "current" in the + * freshly cleared conversation. A reference that concurrently gained a boundary (explicit + * workflow_resume re-record) is also left untouched. Returns false without writing when the + * reference is gone or already carries a boundary. + */ +export async function repairAgentWorkflowRunReferenceBoundary(input: { + workspaceSessionDir: string; + runId: string; + afterBoundaryMessageId: string | null; +}): Promise { + assert(input.runId.length > 0, "agent workflow reference repair requires runId"); + const filePath = referencesPath(input.workspaceSessionDir); + + return referenceFileLocks.withLock(filePath, async () => { + const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); + const reference = existing.find((candidate) => candidate.runId === input.runId); + if (reference == null || reference.afterBoundaryMessageId !== undefined) { + return false; + } + const references = existing.map((candidate) => + candidate.runId === input.runId + ? { ...candidate, afterBoundaryMessageId: input.afterBoundaryMessageId } + : candidate + ); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await writeFileAtomic(filePath, JSON.stringify({ references }, null, 2)); + return true; + }); +} diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index f5c7c552492..40dbef1620e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6,7 +6,7 @@ import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; import { clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, - recordAgentWorkflowRunReference, + repairAgentWorkflowRunReferenceBoundary, type AgentWorkflowRunReference, } from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; @@ -11156,20 +11156,14 @@ export class WorkspaceService extends EventEmitter { } // Supersession-free evidence only: no decision row at all (verified-empty null), or the // newest decision row is this run's own invocation/consumed row, which no manual row can - // postdate (the backward walk would have found that manual row first). - await recordAgentWorkflowRunReference({ + // postdate (the backward walk would have found that manual row first). The write is a + // compare-and-set under the sidecar lock: a full clear landing after the reads above + // deletes the sidecar, and an unconditional record would recreate it with a + // verified-empty boundary, resurrecting the retired pre-clear result as "current". + await repairAgentWorkflowRunReferenceBoundary({ workspaceSessionDir: sessionDir, runId, - createdAtMs: reference.createdAtMs, afterBoundaryMessageId: decision.status === "found" ? decision.messageId : null, - ...(reference.agentId != null - ? { - agentId: reference.agentId, - ...(reference.strictAgentResolution !== undefined - ? { strictAgentResolution: reference.strictAgentResolution } - : {}), - } - : {}), }); } From 6776f483e89c00e35a192343e7969392dee28671 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:04:03 +0000 Subject: [PATCH 37/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20defer=20identity-le?= =?UTF-8?q?ss=20wakes=20and=20wire=20terminal=20attention=20into=20crash-r?= =?UTF-8?q?esumed=20services?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/orpc/router.test.ts | 39 ++++++++++++++++++++ src/node/orpc/router.ts | 21 +++++++++-- src/node/services/taskService.test.ts | 53 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 8 +++- 4 files changed, 116 insertions(+), 5 deletions(-) diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index 62db8bbac95..5d164ce80ee 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -951,6 +951,45 @@ export default function workflow() { return { reportMarkdown: "should not run" } expect(result.result).toBeNull(); await waitForRouterWorkflowStatus(client, "workspace-1", result.runId, "completed"); }); + + test("crash-resumed background runs enqueue terminal attention on settle", async () => { + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); + await runStore.createRun({ + id: "wfr_crash_wake", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + // Orphaned by a crash: durable status says running, but no live runner. + await runStore.appendStatus("wfr_crash_wake", "running", "2026-05-29T00:00:01.000Z"); + + const enqueueWorkflowRunTerminalAttention = mock(async () => undefined); + const context = createContext({ enabled: true }); + (context as unknown as Record).taskService = { + enqueueWorkflowRunTerminalAttention, + }; + ( + context.workspaceService as unknown as Record + ).repairWorkflowRunReferenceBoundary = mock(async () => undefined); + const client = createRouterClient(router(), { context }); + + // A read path triggers crash recovery; the resumed run's settle must land in the + // terminal-attention outbox instead of waiting for the next restart's sweep. + await client.workflows.listRuns({ workspaceId: "workspace-1" }); + await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_crash_wake", "completed"); + const deadline = Date.now() + 5_000; + while (enqueueWorkflowRunTerminalAttention.mock.calls.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(enqueueWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_crash_wake", + status: "completed", + }); + }); }); describe("router config.saveConfig", () => { diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index 327db8336f9..f809a63e7be 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -586,9 +586,24 @@ export async function resolveWorkflowContext( onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), onRunCrashResumed: (event) => context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), - ...(options.onBackgroundRunTerminal != null - ? { onBackgroundRunTerminal: options.onBackgroundRunTerminal } - : {}), + // Read paths (listRuns / stream subscribe) create services purely to observe runs, but + // crash recovery can resume an orphaned background run on them: without a terminal + // callback the settled run would enqueue no terminal attention until the next restart's + // sweep. Default to the standard outbox enqueue (idempotent via enqueueIfAbsent); + // explicit callbacks (slash-command continuations, retry) keep their custom behavior. + onBackgroundRunTerminal: + options.onBackgroundRunTerminal ?? + (async (event) => { + // Nested runs surface through their parent workflow, not their own wake. + if (event.run.parentWorkflow != null) { + return; + } + await context.taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: workspaceId, + runId: event.runId, + status: event.status, + }); + }), getCurrentProjectTrusted: resolveWorkflowProjectTrusted, runnerId: `workflow-runner:${workspaceId}`, }), diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 0316e4089f1..8bb07ce4e29 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6964,6 +6964,59 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); }); + test("wake defers when the launch-identity read fails after currentness succeeds", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_identity_unreadable"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Currentness succeeds without the sidecar (e.g. a direct invocation row)... + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const terminalAttentionStore = new TerminalAttentionStore(config); + + // ...but the launch-identity read fails transiently (EISDIR). Delivering without the + // recorded identity would bind the wake to the newest agent-bearing history row, so the + // wake must stay pending for the retry drain. + await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { + recursive: true, + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }); + await drain(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + }); + test("wake re-pins the selected group's recorded launch pin, not the newest row's", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 89bafb64a7a..3f399b09f42 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8122,8 +8122,12 @@ export class TaskService { }; } } catch { - // Identity is advisory; an unreadable sidecar already deferred delivery above whenever - // currentness itself depended on it. + // Currentness can succeed (e.g. a direct invocation row) and this identity read still + // fail transiently. Delivering without the recorded identity would bind the wake to the + // newest agent-bearing history row, handing the run's output to an unrelated later + // synthetic turn's agent; defer to the bounded retry instead, like an unreadable run + // record. + return { outcome: "defer" }; } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; return { From ce9085a89c48202e0c0349e898b109d08e73f75c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:00:35 +0000 Subject: [PATCH 38/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20retain=20and=20retr?= =?UTF-8?q?y=20failed=20workflow=20terminal=20attention=20enqueues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 129 ++++++++++++++++++++++++++ src/node/services/taskService.ts | 69 ++++++++++++-- 2 files changed, 192 insertions(+), 6 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8bb07ce4e29..de36f151129 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6355,6 +6355,135 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("failed terminal attention enqueue is retained and retried on the bounded timer", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_retry"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + ( + taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } + ).terminalAttentionDeferRetryDelayMs = 10; + + // The workflow terminal callback driving this enqueue is single-attempt, so when the + // first outbox write fails only the retained in-process retry can persist the wake. + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( + new Error("EIO: outbox write failed") + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + + // Real timers: poll until the armed retry re-enqueues and the follow-up drain delivers. + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + await flushTerminalAttentionDrains(taskService); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("reset drops a retained terminal attention enqueue", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_reset"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( + new Error("EIO: outbox write failed") + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + // The run was reset (e.g. resumed) before the retry fired; the retained stale wake + // must be dropped instead of resurrected by the retry. + await taskService.resetWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId }); + + // Drive the armed retry directly so the outcome is deterministic under real timers. + await ( + taskService as unknown as { + retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise; + } + ).retryRetainedWorkflowTerminalEnqueues(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 3f399b09f42..41d75da60c1 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1569,6 +1569,19 @@ export class TaskService { ReturnType >(); private terminalAttentionDeferRetryDelayMs = TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS; + // Workflow terminal callbacks are single-attempt (WorkflowService swallows callback + // rejections), so a transient outbox write failure would otherwise silence the wake for + // the life of the process. Retain failed enqueue params (owner -> runId -> status) and + // retry on the defer-retry cadence; reset drops the entry so a resumed run cannot + // resurrect its stale terminal wake. + private readonly retainedWorkflowTerminalEnqueues = new Map< + string, + Map + >(); + private readonly workflowTerminalEnqueueRetryTimers = new Map< + string, + ReturnType + >(); private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -7748,12 +7761,55 @@ export class TaskService { if (!isTerminalWorkflowRunStatus(params.status)) { return; } - await this.enqueueTerminalAttention({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - terminalOutcome: terminalAttentionOutcome(params.status), - sourceId: params.runId, - }); + try { + await this.enqueueTerminalAttention({ + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + terminalOutcome: terminalAttentionOutcome(params.status), + sourceId: params.runId, + }); + } catch (error) { + log.error("Workflow terminal attention enqueue failed; retrying on bounded timer", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + error, + }); + this.retainWorkflowTerminalEnqueue(params); + } + } + + private retainWorkflowTerminalEnqueue(params: { + ownerWorkspaceId: string; + runId: string; + status: WorkflowRunStatus; + }): void { + let byRun = this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId); + if (byRun == null) { + byRun = new Map(); + this.retainedWorkflowTerminalEnqueues.set(params.ownerWorkspaceId, byRun); + } + byRun.set(params.runId, params.status); + if (this.workflowTerminalEnqueueRetryTimers.has(params.ownerWorkspaceId)) { + return; + } + const timer = setTimeout(() => { + this.workflowTerminalEnqueueRetryTimers.delete(params.ownerWorkspaceId); + void this.retryRetainedWorkflowTerminalEnqueues(params.ownerWorkspaceId); + }, this.terminalAttentionDeferRetryDelayMs); + timer.unref?.(); + this.workflowTerminalEnqueueRetryTimers.set(params.ownerWorkspaceId, timer); + } + + private async retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise { + const byRun = this.retainedWorkflowTerminalEnqueues.get(ownerWorkspaceId); + if (byRun == null) { + return; + } + this.retainedWorkflowTerminalEnqueues.delete(ownerWorkspaceId); + for (const [runId, status] of byRun) { + // A re-attempt that fails again re-retains the entry and re-arms the timer. + await this.enqueueWorkflowRunTerminalAttention({ ownerWorkspaceId, runId, status }); + } } async resetWorkflowRunTerminalAttention(params: { @@ -7765,6 +7821,7 @@ export class TaskService { "resetWorkflowRunTerminalAttention requires ownerWorkspaceId" ); assert(params.runId.length > 0, "resetWorkflowRunTerminalAttention requires runId"); + this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId)?.delete(params.runId); await this.terminalAttentionStore.delete( params.ownerWorkspaceId, TerminalAttentionStore.notificationId("workflow_run", params.runId) From 7edfe81c193e6e7b29f29f8a9465ac0a477cbf46 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:49:06 +0000 Subject: [PATCH 39/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20workflow?= =?UTF-8?q?=20wake=20recovery=20(resume=20reset,=20repair=20retry,=20reset?= =?UTF-8?q?-boundary=20stop)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/orpc/router.test.ts | 41 ++++++++++- src/node/orpc/router.ts | 15 +++- src/node/services/taskService.test.ts | 72 +++++++++++++++++++ src/node/services/taskService.ts | 12 +++- .../workflows/WorkflowService.test.ts | 58 +++++++++++++++ .../services/workflows/WorkflowService.ts | 40 ++++++++++- 6 files changed, 233 insertions(+), 5 deletions(-) diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index 5d164ce80ee..ccfc19616b7 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -362,7 +362,11 @@ describe("router workflow routes", () => { getWorkflowContinuationSendOptions: mock(() => null), sendMessage: mock(async () => ({ success: true, data: undefined })), }, - taskService: {}, + // Nonterminal run status changes reset any stale terminal notification; the stub keeps + // that call observable without wiring a full TaskService. + taskService: { + resetWorkflowRunTerminalAttention: mock(async () => undefined), + }, experimentsService: { isExperimentEnabled: mock(() => options.enabled), }, @@ -970,6 +974,7 @@ export default function workflow() { return { reportMarkdown: "should not run" } const context = createContext({ enabled: true }); (context as unknown as Record).taskService = { enqueueWorkflowRunTerminalAttention, + resetWorkflowRunTerminalAttention: mock(async () => undefined), }; ( context.workspaceService as unknown as Record @@ -990,6 +995,40 @@ export default function workflow() { return { reportMarkdown: "should not run" } status: "completed", }); }); + + test("router-managed resume resets a stale terminal notification before restart", async () => { + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); + await runStore.createRun({ + id: "wfr_resume_reset", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + + const context = createContext({ enabled: true }); + const resetWorkflowRunTerminalAttention = ( + context.taskService as unknown as { + resetWorkflowRunTerminalAttention: ReturnType; + } + ).resetWorkflowRunTerminalAttention; + const client = createRouterClient(router(), { context }); + + await client.workflows.interrupt({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); + resetWorkflowRunTerminalAttention.mockClear(); + + // The prior run's notification survives under the stable workflow_run: id as + // delivered/superseded; without a reset on restart, enqueueIfAbsent preserves that + // record and the resumed run's terminal wake is silently dropped. + await client.workflows.resume({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); + expect(resetWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_reset", + }); + await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_resume_reset", "completed"); + }); }); describe("router config.saveConfig", () => { diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index f809a63e7be..02feeed6332 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -116,6 +116,7 @@ import * as path from "node:path"; import type { DevToolsEvent } from "@/common/types/devtools"; import type { WorkflowRunStreamEvent } from "@/common/types/workflow"; +import { isTerminalWorkflowRunStatus } from "@/common/types/workflow"; import type { WorkflowRunLivenessEntry } from "@/common/orpc/schemas/api"; import type { MuxMessage } from "@/common/types/message"; import { coerceThinkingLevel } from "@/common/types/thinking"; @@ -583,7 +584,19 @@ export async function resolveWorkflowContext( includeAgentPlugins, skillStorageContext, }), - onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), + onRunStatusChanged: async (event) => { + // Router-managed restarts (resume / retry / crash recovery) must clear a prior + // delivered or superseded notification when the run leaves terminal state, or + // enqueueIfAbsent would preserve the stale record and silently drop the resumed + // run's next terminal wake (mirrors the AIService-owned service). + if (!isTerminalWorkflowRunStatus(event.status)) { + await context.taskService.resetWorkflowRunTerminalAttention({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } + await context.workspaceService.emitWorkflowRunActivity(event); + }, onRunCrashResumed: (event) => context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), // Read paths (listRuns / stream subscribe) create services purely to observe runs, but diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index de36f151129..6162817435f 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6484,6 +6484,78 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_policy_reset_boundary"; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The pre-reset manual row disabled bash, but the context reset discarded that + // conversation. The workflow launched from a post-reset synthetic turn (heartbeat), so + // its wake must use fresh defaults instead of resurrecting the discarded restriction. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("reset-boundary", "assistant", "Context reset", { + timestamp: 2_000, + contextBoundaryKind: "reset", + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-launch", "user", "[heartbeat] launch the workflow", { + timestamp: 3_000, + synthetic: true, + }) + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBeUndefined(); + }); + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 41d75da60c1..55f46a1245e 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7994,8 +7994,10 @@ export class TaskService { * conversation's persisted restrictions; synthetic rows without any (earlier wakes, * heartbeat scaffolding) do not define them and are skipped. The walk is unbounded: a long * assistant/synthetic tail after the launch turn must not push the defining row out of - * sight and silently lift the restrictions. Throws when history is unreadable so the caller - * can fail closed instead of waking with unrestricted tools. + * sight and silently lift the restrictions. It stops at the newest context reset boundary, + * since rows from the discarded context must not re-disable tools available to the reset + * context. Throws when history is unreadable so the caller can fail closed instead of + * waking with unrestricted tools. */ private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ toolPolicy?: ToolPolicy; @@ -8015,6 +8017,12 @@ export class TaskService { "backward", (messages) => { for (const message of messages) { + // A context reset discards everything before it: pre-reset rows must not define + // the wake's restrictions or pin. Stopping here leaves undefined fields as fresh + // defaults, matching a manual send in the post-reset context. + if (isResetBoundaryMessage(message)) { + return false; + } if (message.role !== "user") { continue; } diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index 03ccf20442f..bfcb2ce0814 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -1062,4 +1062,62 @@ describe("WorkflowService crash recovery", () => { expect(events).toContain("status:completed"); await expect(runStore.getRun("wfr_crash")).resolves.toMatchObject({ status: "completed" }); }); + + test("failed crash-resume provenance repair retries on a bounded timer", async () => { + using tmp = new DisposableTempDir("workflow-service-crash-repair-retry"); + const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); + await runStore.createRun({ + id: "wfr_crash_repair_retry", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, + source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_crash_repair_retry", "running", "2026-05-29T00:00:01.000Z"); + + // The repair hook only fires at resume time; if its transient failure were terminal, the + // reference would stay boundaryless after the run settles and every drain would defer the + // wake as indeterminate with nothing left to repair it. + const repairCalls: string[] = []; + let failFirstRepair = true; + const service = new WorkflowService({ + runStore, + runtimeFactory: new QuickJSRuntimeFactory(), + taskAdapter: { + async runAgent() { + throw new Error("No agent steps expected"); + }, + }, + generateRunId: () => "wfr_unused", + runnerId: "runner-a", + onRunCrashResumed: (event) => { + repairCalls.push(`${event.workspaceId}:${event.runId}`); + if (failFirstRepair) { + failFirstRepair = false; + throw new Error("EIO: sidecar write failed"); + } + }, + }); + ( + service as unknown as { crashResumeRepairRetryDelayMs: number } + ).crashResumeRepairRetryDelayMs = 10; + + const resumed = await service.resumeCrashedRuns({ + workspaceId: "workspace-1", + projectTrusted: true, + }); + expect(resumed).toEqual(["wfr_crash_repair_retry"]); + expect(repairCalls).toEqual(["workspace-1:wfr_crash_repair_retry"]); + + const deadline = Date.now() + 5_000; + while (Date.now() < deadline && repairCalls.length < 2) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(repairCalls).toEqual([ + "workspace-1:wfr_crash_repair_retry", + "workspace-1:wfr_crash_repair_retry", + ]); + }); }); diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 94c90f8e528..83fa80f363e 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -118,6 +118,13 @@ const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES = new Set([ // oRPC creates a WorkflowService per request, so workflow lifecycle state that spans requests // needs process-wide registries. const pendingCrashResumeTimers = new Map>(); +// Crash-resume provenance repair only fires at resume time: once the run settles, nothing +// else re-records the boundary and its terminal wake stays indeterminate on every drain. +// Retry a failed repair on a bounded timer. Module-level like pendingCrashResumeTimers +// because WorkflowService instances are per-request. +const CRASH_RESUME_REPAIR_RETRY_DELAY_MS = 30_000; +const CRASH_RESUME_REPAIR_MAX_ATTEMPTS = 5; +const pendingCrashResumeRepairTimers = new Map>(); const activeWorkflowInterruptStatusWrites = new Map>(); const activeWorkflowRunnerAbortControllers = new Map(); @@ -137,6 +144,8 @@ export class WorkflowService { workspaceId: string; runId: string; }) => Promise | void; + // Field, not the constant, so tests can shrink the repair retry backoff. + private crashResumeRepairRetryDelayMs = CRASH_RESUME_REPAIR_RETRY_DELAY_MS; private readonly onRunStatusChanged?: ( event: WorkflowRunStatusChangedEvent ) => Promise | void; @@ -589,8 +598,11 @@ export class WorkflowService { await this.onRunCrashResumed({ workspaceId: run.workspaceId, runId: run.id }); } catch (error) { // Best-effort: an unrepaired reference defers its wake as indeterminate rather than - // losing it, so a failed repair must not block the resume itself. + // losing it, so a failed repair must not block the resume itself. Retry off-path: + // repair is CAS-guarded and refuses once a boundary exists, so late success (even + // after the run settles) only unblocks the deferred wake. console.error("Workflow crash-resume provenance repair failed:", error); + this.scheduleCrashResumeRepairRetry({ workspaceId: run.workspaceId, runId: run.id }, 1); } } @@ -651,6 +663,32 @@ export class WorkflowService { pendingCrashResumeTimers.set(input.runId, timer); } + private scheduleCrashResumeRepairRetry( + input: { workspaceId: string; runId: string }, + attempt: number + ): void { + const repairHook = this.onRunCrashResumed; + if (repairHook == null || attempt > CRASH_RESUME_REPAIR_MAX_ATTEMPTS) { + return; + } + if (pendingCrashResumeRepairTimers.has(input.runId)) { + return; + } + const timer = setTimeout(() => { + pendingCrashResumeRepairTimers.delete(input.runId); + void (async () => { + try { + await repairHook({ workspaceId: input.workspaceId, runId: input.runId }); + } catch (error) { + console.error("Workflow crash-resume provenance repair retry failed:", error); + this.scheduleCrashResumeRepairRetry(input, attempt + 1); + } + })(); + }, this.crashResumeRepairRetryDelayMs); + unrefTimer(timer); + pendingCrashResumeRepairTimers.set(input.runId, timer); + } + private registerActiveRunnerAbortController( runId: string, workspaceId: string, From 8b30f29d9bd4d83291b16dfec39a30f21d2f3889 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:30:29 +0000 Subject: [PATCH 40/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20complete=20failed?= =?UTF-8?q?=20workflow=20notification=20resets=20on=20the=20next=20termina?= =?UTF-8?q?l=20enqueue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 64 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 42 ++++++++++++++++-- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 6162817435f..c0d96c1a736 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6484,6 +6484,70 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("a failed reset is completed by the next terminal enqueue", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_reset_retained"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + // First settle delivers normally, leaving a delivered record under the stable id. + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + + // The restart's reset fails transiently; callers swallow the rejection, so only the + // retained pending reset can stop the delivered record from absorbing the next enqueue. + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "delete").mockRejectedValueOnce(new Error("EIO: outbox delete failed")); + await taskService.resetWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId }); + + // The resumed run settles again: the enqueue completes the reset and delivers fresh. + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 55f46a1245e..523c44fa6b6 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1582,6 +1582,12 @@ export class TaskService { string, ReturnType >(); + // A reset whose store delete fails is retained here (owner -> runIds) instead of being + // swallowed: the stale delivered/superseded record only matters when the run settles + // again, so the next terminal enqueue completes the reset (delete before enqueue) and a + // failure there flows into the retained-enqueue retry above. No timer needed: with no + // later terminal transition the stale record is inert. + private readonly pendingWorkflowNotificationResets = new Map>(); private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -7762,6 +7768,16 @@ export class TaskService { return; } try { + const pendingResets = this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId); + if (pendingResets?.has(params.runId)) { + // Marker cleared only after the delete succeeds so a failure retries the full + // reset-then-enqueue sequence instead of preserving the stale record. + await this.terminalAttentionStore.delete( + params.ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", params.runId) + ); + pendingResets.delete(params.runId); + } await this.enqueueTerminalAttention({ ownerWorkspaceId: params.ownerWorkspaceId, sourceKind: "workflow_run", @@ -7822,10 +7838,28 @@ export class TaskService { ); assert(params.runId.length > 0, "resetWorkflowRunTerminalAttention requires runId"); this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId)?.delete(params.runId); - await this.terminalAttentionStore.delete( - params.ownerWorkspaceId, - TerminalAttentionStore.notificationId("workflow_run", params.runId) - ); + try { + await this.terminalAttentionStore.delete( + params.ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", params.runId) + ); + this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId)?.delete(params.runId); + } catch (error) { + // Restart callers swallow this rejection, so a transiently failed delete would leave + // the stale delivered/superseded record absorbing the resumed run's next terminal + // enqueue. Retain the reset; the next enqueue completes it before enqueuing fresh. + log.error("Workflow terminal attention reset failed; retained for the next enqueue", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + error, + }); + let pendingResets = this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId); + if (pendingResets == null) { + pendingResets = new Set(); + this.pendingWorkflowNotificationResets.set(params.ownerWorkspaceId, pendingResets); + } + pendingResets.add(params.runId); + } } /** From 40dc6e02c30a2d341f1d6df5411d40c1b3dab417 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:32:00 +0000 Subject: [PATCH 41/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20guard=20workflow=20?= =?UTF-8?q?terminal=20wakes=20against=20removed=20owners=20and=20stale=20r?= =?UTF-8?q?ecovery=20records?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review round on PR #3990: - enqueueWorkflowRunTerminalAttention drops the write and retained retry state when the owner workspace is no longer registered, so the bounded retry cannot recreate a removed workspace's session directory. - Startup recovery replaces a stale delivered/superseded workflow terminal record from an older run generation (crash-lost in-memory reset) instead of letting enqueueIfAbsent absorb the newer terminal's wake. - Sidecar repair doc comment no longer overclaims a cross-process file lock; single-instance backends are the supported deployment. --- .../services/agentWorkflowRunReferences.ts | 4 +- src/node/services/taskService.test.ts | 139 ++++++++++++++++++ src/node/services/taskService.ts | 31 +++- 3 files changed, 172 insertions(+), 2 deletions(-) diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 01d5cba36ae..76bc9c406fa 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -236,7 +236,9 @@ export async function recordAgentWorkflowRunReference(input: { /** * Compare-and-set boundary repair for a surviving boundaryless reference. The reference is - * re-validated under the sidecar file lock: a concurrent full-history clear deletes the file + * re-validated under the process-local sidecar mutex (single-instance backends are the + * supported deployment; see requestSingleInstanceLock in desktop/main.ts): a concurrent + * full-history clear deletes the file * (retiring every reference), and an unconditional write would recreate it with a * verified-empty boundary, resurrecting the retired pre-clear result as "current" in the * freshly cleared conversation. A reference that concurrently gained a boundary (explicit diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 32aaaf38be4..8faaddc5073 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -1,5 +1,6 @@ import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from "bun:test"; import * as fsPromises from "fs/promises"; +import { existsSync } from "fs"; import * as path from "path"; import * as os from "os"; import { execSync } from "node:child_process"; @@ -6215,6 +6216,144 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("workspace removal drops retained terminal enqueues without recreating the outbox", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_removed_owner"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internalStore = ( + taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } + ).terminalAttentionStore; + spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( + new Error("EIO: outbox write failed") + ); + + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + + // The owner is removed (config entry gone, session directory deleted) before the bounded + // retry fires; the retry must not recreate the deleted session directory or leave state + // for a future workspace reusing the ID. + const cfg = config.loadConfigOrDefault(); + for (const project of cfg.projects.values()) { + project.workspaces = project.workspaces.filter((workspace) => workspace.id !== parentId); + } + await config.editConfig(() => cfg); + const sessionDir = config.getSessionDir(parentId); + await fsPromises.rm(sessionDir, { recursive: true, force: true }); + + await ( + taskService as unknown as { + retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise; + } + ).retryRetainedWorkflowTerminalEnqueues(parentId); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(existsSync(sessionDir)).toBe(false); + const retained = ( + taskService as unknown as { + retainedWorkflowTerminalEnqueues: Map>; + } + ).retainedWorkflowTerminalEnqueues; + expect(retained.has(parentId)).toBe(false); + }); + + test("startup recovery replaces a stale terminal record from an older run generation", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_recovery_stale_generation"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const recover = () => + ( + taskService as unknown as { + recoverTerminalWorkflowRunAttentionNotifications(): Promise; + } + ).recoverTerminalWorkflowRunAttentionNotifications(); + + // First generation delivers normally, leaving a delivered record newer than the run's + // terminal transition; recovery must not duplicate a record representing this outcome. + await taskService.enqueueWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await recover()).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + + // The resumed run reaches terminal again after the delivered record was written, and the + // process crashed before reset-then-enqueue ran (no in-memory reset marker survives the + // restart). Recovery must replace the stale delivered record instead of letting + // enqueueIfAbsent silently absorb the newer generation's wake. + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + + expect(await recover()).toBe(1); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + }); + test("a failed reset is completed by the next terminal enqueue", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 598b5443eb4..b5551710dd9 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7806,11 +7806,31 @@ export class TaskService implements AgentTaskIntegration { if (!(await this.workspaceService.isWorkflowInvocationCurrent(workspace.id, run.id))) { continue; } + const outcome = terminalAttentionOutcome(run.status); + const notificationId = TerminalAttentionStore.notificationId("workflow_run", run.id); + const existing = await this.terminalAttentionStore.get(workspace.id, notificationId); + const existingCreatedAt = existing != null ? Date.parse(existing.createdAt) : Number.NaN; + const runUpdatedAt = Date.parse(run.updatedAt); + const existingRepresentsCurrentOutcome = + existing?.terminalOutcome === outcome && + Number.isFinite(existingCreatedAt) && + Number.isFinite(runUpdatedAt) && + existingCreatedAt >= runUpdatedAt; + if (existing != null && !existingRepresentsCurrentOutcome) { + // A crash can lose the retained in-memory reset (resetWorkflowRunTerminalAttention + // failure path), leaving a stale delivered/superseded record from an older run + // generation that would absorb this terminal's wake via enqueueIfAbsent. The record + // predates the run's latest terminal transition and the currentness gate above + // verified the wake is owed, so replace it; worst case is one redundant wake (fail + // toward notify, never a lost wake). + await this.terminalAttentionStore.delete(workspace.id, notificationId); + this.pendingWorkflowNotificationResets.get(workspace.id)?.delete(run.id); + } const created = await this.terminalAttentionStore.enqueueIfAbsent({ ownerWorkspaceId: workspace.id, sourceKind: "workflow_run", sourceId: run.id, - terminalOutcome: terminalAttentionOutcome(run.status), + terminalOutcome: outcome, }); if (created != null) { this.scheduleTerminalAttentionDrain(workspace.id); @@ -7935,6 +7955,15 @@ export class TaskService implements AgentTaskIntegration { if (!isTerminalWorkflowRunStatus(params.status)) { return; } + if (findWorkspaceEntry(this.config.loadConfigOrDefault(), params.ownerWorkspaceId) == null) { + // Owner workspace was removed: an outbox write would recreate the deleted session + // directory and leak a stale record to a future workspace reusing the ID. Applies to + // live terminal callbacks and to the bounded retry timer, whose retained state is + // dropped here so nothing re-arms the write. + this.retainedWorkflowTerminalEnqueues.delete(params.ownerWorkspaceId); + this.pendingWorkflowNotificationResets.delete(params.ownerWorkspaceId); + return; + } try { const pendingResets = this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId); if (pendingResets?.has(params.runId)) { From 93311359cb1397d112154e102f554efd99d10c1f Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:58:46 +0000 Subject: [PATCH 42/63] =?UTF-8?q?=F0=9F=A4=96=20test:=20make=20the=20crash?= =?UTF-8?q?-repair=20retry=20test=20robust=20to=20slow=20runners?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 10ms retry timer can fire while resumeCrashedRuns is still awaiting the runner restart on a loaded CI machine, so the pre-poll assertion must not demand exactly one repair attempt. Assert the deterministic initial attempt only; the exact two-attempt sequence stays asserted after the poll. Reproduced the CI failure signature deterministically with a forced yield before the assertion. --- src/node/services/workflows/WorkflowService.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index bfcb2ce0814..f784b7a0f6c 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -1109,7 +1109,10 @@ describe("WorkflowService crash recovery", () => { projectTrusted: true, }); expect(resumed).toEqual(["wfr_crash_repair_retry"]); - expect(repairCalls).toEqual(["workspace-1:wfr_crash_repair_retry"]); + // The shrunk retry timer can fire while resumeCrashedRuns is still awaiting the runner + // restart on a slow machine, so only the initial attempt's ordering is asserted here; + // the exact two-attempt sequence is asserted after the poll below. + expect(repairCalls[0]).toBe("workspace-1:wfr_crash_repair_retry"); const deadline = Date.now() + 5_000; while (Date.now() < deadline && repairCalls.length < 2) { From 89e0428b65de50f8777cbf8078634c19bef81706 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:18:16 +0000 Subject: [PATCH 43/63] =?UTF-8?q?=F0=9F=A4=96=20refactor:=20rework=20workf?= =?UTF-8?q?low=20terminal=20wakes=20as=20level-triggered=20reconciliation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workflow run's own record (notify_on_terminal + terminal status) is the durable mark; the drain re-derives owed wakes from it plus generation-scoped settled markers and history evidence, poked by settlement callbacks, stream ends, startup, and a periodic sweep. Deletes the edge-triggered delivery machinery: outbox pending records for workflows, retained-enqueue retries, reset bookkeeping, defer-retry timers, and crash-resume boundary repair. Boundaryless sidecar references now fail quiet (not_current) instead of deferring forever behind a repair hook; the run result stays retrievable via workflow_resume. --- src/node/orpc/router.ts | 28 +- .../services/agentWorkflowRunReferences.ts | 36 -- src/node/services/aiService.ts | 11 +- src/node/services/taskService.ts | 426 ++++++++---------- src/node/services/terminalAttentionStore.ts | 37 ++ src/node/services/tools/task_await.ts | 4 +- src/node/services/tools/workflow_resume.ts | 23 +- .../services/workflows/WorkflowService.ts | 59 --- src/node/services/workspaceService.ts | 53 +-- 9 files changed, 247 insertions(+), 430 deletions(-) diff --git a/src/node/orpc/router.ts b/src/node/orpc/router.ts index 02feeed6332..57694afaae9 100644 --- a/src/node/orpc/router.ts +++ b/src/node/orpc/router.ts @@ -116,7 +116,6 @@ import * as path from "node:path"; import type { DevToolsEvent } from "@/common/types/devtools"; import type { WorkflowRunStreamEvent } from "@/common/types/workflow"; -import { isTerminalWorkflowRunStatus } from "@/common/types/workflow"; import type { WorkflowRunLivenessEntry } from "@/common/orpc/schemas/api"; import type { MuxMessage } from "@/common/types/message"; import { coerceThinkingLevel } from "@/common/types/thinking"; @@ -584,34 +583,21 @@ export async function resolveWorkflowContext( includeAgentPlugins, skillStorageContext, }), - onRunStatusChanged: async (event) => { - // Router-managed restarts (resume / retry / crash recovery) must clear a prior - // delivered or superseded notification when the run leaves terminal state, or - // enqueueIfAbsent would preserve the stale record and silently drop the resumed - // run's next terminal wake (mirrors the AIService-owned service). - if (!isTerminalWorkflowRunStatus(event.status)) { - await context.taskService.resetWorkflowRunTerminalAttention({ - ownerWorkspaceId: event.workspaceId, - runId: event.runId, - }); - } - await context.workspaceService.emitWorkflowRunActivity(event); - }, - onRunCrashResumed: (event) => - context.workspaceService.repairWorkflowRunReferenceBoundary(event.workspaceId, event.runId), + // No reset bookkeeping on restarts: settled markers are keyed by the run's terminal + // generation, so a resumed run's next terminal transition re-arms attention by itself. + onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), // Read paths (listRuns / stream subscribe) create services purely to observe runs, but // crash recovery can resume an orphaned background run on them: without a terminal - // callback the settled run would enqueue no terminal attention until the next restart's - // sweep. Default to the standard outbox enqueue (idempotent via enqueueIfAbsent); - // explicit callbacks (slash-command continuations, retry) keep their custom behavior. + // callback the settled run would owe its wake to the next sweep. Explicit callbacks + // (slash-command continuations, retry) keep their custom behavior. onBackgroundRunTerminal: options.onBackgroundRunTerminal ?? - (async (event) => { + ((event) => { // Nested runs surface through their parent workflow, not their own wake. if (event.run.parentWorkflow != null) { return; } - await context.taskService.enqueueWorkflowRunTerminalAttention({ + context.taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: workspaceId, runId: event.runId, status: event.status, diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 76bc9c406fa..6e2ff131194 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -233,39 +233,3 @@ export async function recordAgentWorkflowRunReference(input: { ); }); } - -/** - * Compare-and-set boundary repair for a surviving boundaryless reference. The reference is - * re-validated under the process-local sidecar mutex (single-instance backends are the - * supported deployment; see requestSingleInstanceLock in desktop/main.ts): a concurrent - * full-history clear deletes the file - * (retiring every reference), and an unconditional write would recreate it with a - * verified-empty boundary, resurrecting the retired pre-clear result as "current" in the - * freshly cleared conversation. A reference that concurrently gained a boundary (explicit - * workflow_resume re-record) is also left untouched. Returns false without writing when the - * reference is gone or already carries a boundary. - */ -export async function repairAgentWorkflowRunReferenceBoundary(input: { - workspaceSessionDir: string; - runId: string; - afterBoundaryMessageId: string | null; -}): Promise { - assert(input.runId.length > 0, "agent workflow reference repair requires runId"); - const filePath = referencesPath(input.workspaceSessionDir); - - return referenceFileLocks.withLock(filePath, async () => { - const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); - const reference = existing.find((candidate) => candidate.runId === input.runId); - if (reference == null || reference.afterBoundaryMessageId !== undefined) { - return false; - } - const references = existing.map((candidate) => - candidate.runId === input.runId - ? { ...candidate, afterBoundaryMessageId: input.afterBoundaryMessageId } - : candidate - ); - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await writeFileAtomic(filePath, JSON.stringify({ references }, null, 2)); - return true; - }); -} diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index ad8eae5027e..c299b621e22 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -200,7 +200,6 @@ import { createKernelFileLoader } from "@/node/services/tools/kernelFileLoad"; import { eventSpine, type RequestAssembleContext } from "@/node/services/events/eventSpine"; import { getErrorMessage } from "@/common/utils/errors"; import { validateJsonSchemaSubsetSchema } from "@/common/utils/jsonSchemaSubset"; -import { isTerminalWorkflowRunStatus } from "@/common/types/workflow"; import { WORKFLOW_RESULT_METADATA_TYPE, buildWorkflowResultContextMessage, @@ -2329,13 +2328,9 @@ export class AIService extends EventEmitter { runStore: new WorkflowRunStore({ sessionDir: this.config.getSessionDir(workspaceId), }), + // No reset bookkeeping on restarts: settled markers are keyed by the run's + // terminal generation, so a resumed run re-arms attention by itself. onRunStatusChanged: async (event) => { - if (!isTerminalWorkflowRunStatus(event.status)) { - await this.taskService?.resetWorkflowRunTerminalAttention({ - ownerWorkspaceId: event.workspaceId, - runId: event.runId, - }); - } await this.onWorkflowRunStatusChanged?.(event); }, runtimeFactory: new QuickJSRuntimeFactory(), @@ -2379,7 +2374,7 @@ export class AIService extends EventEmitter { return; } if (this.taskService != null) { - await this.taskService.enqueueWorkflowRunTerminalAttention({ + this.taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: workspaceId, runId, status, diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index b5551710dd9..27707225a18 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -157,6 +157,7 @@ import type { ErrorEvent, StreamAbortEvent, StreamEndEvent } from "@/common/type import { isActiveWorkflowRunStatus, isTerminalWorkflowRunStatus, + type WorkflowRunRecord, type WorkflowRunStatus, } from "@/common/types/workflow"; import { isDynamicToolPart, type DynamicToolPart } from "@/common/types/toolParts"; @@ -874,10 +875,11 @@ function isWorkspaceBusyIdleOnlySend(error: unknown): boolean { const REMOVED_AGENT_TASKS_DIR = "removed-agent-tasks"; const COMPLETED_REPORT_CACHE_MAX_ENTRIES = 128; -// Retry cadence for terminal-attention drains deferred by indeterminate workflow currentness -// (unreadable history/sidecar). There is no deterministic "storage recovered" signal, so a -// bounded timer is the re-trigger; each retry that defers again arms the next one. -const TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS = 30_000; +// Level-triggered backstop for workflow terminal wakes: the sweep re-derives owed wakes from +// durable state (run records + settled markers), so lost terminal callbacks, deferred +// (transiently unreadable) evaluations, and crashes all recover here without any per-failure +// retry bookkeeping. +const WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS = 5 * 60_000; /** Maximum consecutive auto-resumes before stopping. Prevents infinite loops when descendants are stuck. */ // Task-recovery paths must stay deterministic and editing-capable even when @@ -1675,32 +1677,12 @@ export class TaskService implements AgentTaskIntegration { // tests and shutdown can await them; drains are idempotent and re-triggered on owner idle events. private readonly pendingTerminalAttentionDrainsByOwner = new Map>(); private readonly pendingTerminalAttentionDrains = new Set>(); - // One armed defer-retry timer per owner (see scheduleTerminalAttentionDeferRetry). The delay - // is a field, not a constant, so tests can shrink it without waiting out the real backoff. - private readonly terminalAttentionDeferRetryTimers = new Map< - string, - ReturnType - >(); - private terminalAttentionDeferRetryDelayMs = TERMINAL_ATTENTION_DEFER_RETRY_DELAY_MS; - // Workflow terminal callbacks are single-attempt (WorkflowService swallows callback - // rejections), so a transient outbox write failure would otherwise silence the wake for - // the life of the process. Retain failed enqueue params (owner -> runId -> status) and - // retry on the defer-retry cadence; reset drops the entry so a resumed run cannot - // resurrect its stale terminal wake. - private readonly retainedWorkflowTerminalEnqueues = new Map< - string, - Map - >(); - private readonly workflowTerminalEnqueueRetryTimers = new Map< - string, - ReturnType - >(); - // A reset whose store delete fails is retained here (owner -> runIds) instead of being - // swallowed: the stale delivered/superseded record only matters when the run settles - // again, so the next terminal enqueue completes the reset (delete before enqueue) and a - // failure there flows into the retained-enqueue retry above. No timer needed: with no - // later terminal transition the stale record is inert. - private readonly pendingWorkflowNotificationResets = new Map>(); + // Owed workflow terminal wakes (owner -> runIds believed terminal and not yet settled). An + // in-memory work queue over durable state, not a delivery record: entries are (re)derived + // from run records + settled markers at startup and on the periodic sweep and added by live + // terminal callbacks, so losing the map merely delays a wake until the next sweep. + private readonly pendingWorkflowRunAttention = new Map>(); + private workflowAttentionSweepTimer: ReturnType | null = null; private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -3219,8 +3201,15 @@ export class TaskService implements AgentTaskIntegration { log.error("Startup workflow task archive sweep failed", { error }); } - const recoveredTerminalWorkflowRunNotificationCount = - await this.recoverTerminalWorkflowRunAttentionNotifications(); + const queuedTerminalWorkflowRunAttentionCount = await this.sweepWorkflowRunTerminalAttention(); + if (this.workflowAttentionSweepTimer == null) { + this.workflowAttentionSweepTimer = setInterval(() => { + void this.sweepWorkflowRunTerminalAttention().catch((error: unknown) => { + log.warn("Workflow terminal attention sweep failed", { error }); + }); + }, WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS); + this.workflowAttentionSweepTimer.unref?.(); + } const recoveredTerminalWorkspaceTurnNotificationCount = await this.recoverTerminalWorkspaceTurnAttentionNotifications(); const terminalAttentionDrainStartedAt = Date.now(); @@ -3246,7 +3235,7 @@ export class TaskService implements AgentTaskIntegration { patchGenerationRecoveryMs, bestOfParentRecoveryCount: bestOfParentWorkspaceIds.size, bestOfRecoveryMs, - recoveredTerminalWorkflowRunNotificationCount, + queuedTerminalWorkflowRunAttentionCount, recoveredTerminalWorkspaceTurnNotificationCount, pendingTerminalAttentionOwnerWorkspaceCount: pendingTerminalAttentionOwnerWorkspaceIds.length, terminalAttentionDrainMs, @@ -7773,12 +7762,24 @@ export class TaskService implements AgentTaskIntegration { }); } - private async recoverTerminalWorkflowRunAttentionNotifications(): Promise { + /** + * Level-triggered reconciliation scan: re-derive owed workflow terminal wakes from durable + * state (top-level notify_on_terminal run records in a terminal status, minus + * generation-scoped settled markers) into the in-memory queue and poke the drain. Runs at + * startup and on a fixed sweep interval, so missed terminal callbacks, crashes, and + * deferred (transiently unreadable) evaluations always get re-evaluated without any + * per-failure retry bookkeeping. Archived workspaces are skipped, which parks their wakes + * unsettled: unarchiving re-queues them on the next sweep instead of dropping them. + */ + private async sweepWorkflowRunTerminalAttention(): Promise { const cfg = this.config.loadConfigOrDefault(); - let recoveredCount = 0; + let queuedCount = 0; for (const project of cfg.projects.values()) { for (const workspace of project.workspaces) { - if (workspace.id == null) { + if ( + workspace.id == null || + isWorkspaceArchived(workspace.archivedAt, workspace.unarchivedAt) + ) { continue; } const runStore = new WorkflowRunStore({ @@ -7788,12 +7789,13 @@ export class TaskService implements AgentTaskIntegration { try { runs = await runStore.listRuns(); } catch (error: unknown) { - log.warn("Failed to recover workflow terminal notifications", { + log.warn("Failed to sweep workflow terminal attention", { workspaceId: workspace.id, error: getErrorMessage(error), }); continue; } + let queuedForWorkspace = false; for (const run of runs) { if ( run.workspaceId !== workspace.id || @@ -7803,43 +7805,30 @@ export class TaskService implements AgentTaskIntegration { ) { continue; } - if (!(await this.workspaceService.isWorkflowInvocationCurrent(workspace.id, run.id))) { + const marker = await this.terminalAttentionStore.get( + workspace.id, + TerminalAttentionStore.notificationId("workflow_run", run.id, run.updatedAt) + ); + if (marker != null && marker.status !== "pending") { continue; } - const outcome = terminalAttentionOutcome(run.status); - const notificationId = TerminalAttentionStore.notificationId("workflow_run", run.id); - const existing = await this.terminalAttentionStore.get(workspace.id, notificationId); - const existingCreatedAt = existing != null ? Date.parse(existing.createdAt) : Number.NaN; - const runUpdatedAt = Date.parse(run.updatedAt); - const existingRepresentsCurrentOutcome = - existing?.terminalOutcome === outcome && - Number.isFinite(existingCreatedAt) && - Number.isFinite(runUpdatedAt) && - existingCreatedAt >= runUpdatedAt; - if (existing != null && !existingRepresentsCurrentOutcome) { - // A crash can lose the retained in-memory reset (resetWorkflowRunTerminalAttention - // failure path), leaving a stale delivered/superseded record from an older run - // generation that would absorb this terminal's wake via enqueueIfAbsent. The record - // predates the run's latest terminal transition and the currentness gate above - // verified the wake is owed, so replace it; worst case is one redundant wake (fail - // toward notify, never a lost wake). - await this.terminalAttentionStore.delete(workspace.id, notificationId); - this.pendingWorkflowNotificationResets.get(workspace.id)?.delete(run.id); + let runIds = this.pendingWorkflowRunAttention.get(workspace.id); + if (runIds == null) { + runIds = new Set(); + this.pendingWorkflowRunAttention.set(workspace.id, runIds); } - const created = await this.terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: workspace.id, - sourceKind: "workflow_run", - sourceId: run.id, - terminalOutcome: outcome, - }); - if (created != null) { - this.scheduleTerminalAttentionDrain(workspace.id); - recoveredCount += 1; + if (!runIds.has(run.id)) { + runIds.add(run.id); + queuedCount += 1; } + queuedForWorkspace = true; + } + if (queuedForWorkspace) { + this.scheduleTerminalAttentionDrain(workspace.id); } } } - return recoveredCount; + return queuedCount; } private async recoverTerminalWorkspaceTurnAttentionNotifications(): Promise { @@ -7932,131 +7921,38 @@ export class TaskService implements AgentTaskIntegration { } // ---- Terminal attention notifier ------------------------------------------------------------ - // Deep module for delivering terminal wake-ups for notify_on_terminal work. Settlement paths - // enqueue a persisted notification (outside any settlement lock); the notifier drains pending - // notifications when the owner is idle, sends one coalesced synthetic wake-up, and marks each - // delivered only after an accepted send. Crash/restart safe via the persisted store. + // Deep module for delivering terminal wake-ups for notify_on_terminal work. Sub-agent and + // workspace-turn settlements enqueue a persisted outbox notification (outside any settlement + // lock); workflow wakes are level-triggered instead, re-derived from run records + settled + // markers (see sweepWorkflowRunTerminalAttention). The drain fires when the owner is idle, + // sends one coalesced synthetic wake-up, and records delivery only after an accepted send. /** - * Persist a pending terminal wake-up for the owner workspace and schedule an async drain. - * Idempotent by source kind/id. Must NOT be called while holding settlement/event locks; only - * the persisted enqueue happens synchronously inside callers, the drain is deferred. + * Note a top-level background workflow run's terminal transition and poke the drain. Purely + * an accelerator over the durable state the sweep re-derives (run records + settled + * markers): a lost poke, a removed workspace, or a later resume needs no compensation here, + * so this writes nothing to disk. */ - async enqueueWorkflowRunTerminalAttention(params: { + noteWorkflowRunTerminalAttention(params: { ownerWorkspaceId: string; runId: string; status: WorkflowRunStatus; - }): Promise { + }): void { assert( params.ownerWorkspaceId.length > 0, - "enqueueWorkflowRunTerminalAttention requires ownerWorkspaceId" + "noteWorkflowRunTerminalAttention requires ownerWorkspaceId" ); - assert(params.runId.length > 0, "enqueueWorkflowRunTerminalAttention requires runId"); + assert(params.runId.length > 0, "noteWorkflowRunTerminalAttention requires runId"); if (!isTerminalWorkflowRunStatus(params.status)) { return; } - if (findWorkspaceEntry(this.config.loadConfigOrDefault(), params.ownerWorkspaceId) == null) { - // Owner workspace was removed: an outbox write would recreate the deleted session - // directory and leak a stale record to a future workspace reusing the ID. Applies to - // live terminal callbacks and to the bounded retry timer, whose retained state is - // dropped here so nothing re-arms the write. - this.retainedWorkflowTerminalEnqueues.delete(params.ownerWorkspaceId); - this.pendingWorkflowNotificationResets.delete(params.ownerWorkspaceId); - return; - } - try { - const pendingResets = this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId); - if (pendingResets?.has(params.runId)) { - // Marker cleared only after the delete succeeds so a failure retries the full - // reset-then-enqueue sequence instead of preserving the stale record. - await this.terminalAttentionStore.delete( - params.ownerWorkspaceId, - TerminalAttentionStore.notificationId("workflow_run", params.runId) - ); - pendingResets.delete(params.runId); - } - await this.enqueueTerminalAttention({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - terminalOutcome: terminalAttentionOutcome(params.status), - sourceId: params.runId, - }); - } catch (error) { - log.error("Workflow terminal attention enqueue failed; retrying on bounded timer", { - ownerWorkspaceId: params.ownerWorkspaceId, - runId: params.runId, - error, - }); - this.retainWorkflowTerminalEnqueue(params); - } - } - - private retainWorkflowTerminalEnqueue(params: { - ownerWorkspaceId: string; - runId: string; - status: WorkflowRunStatus; - }): void { - let byRun = this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId); - if (byRun == null) { - byRun = new Map(); - this.retainedWorkflowTerminalEnqueues.set(params.ownerWorkspaceId, byRun); - } - byRun.set(params.runId, params.status); - if (this.workflowTerminalEnqueueRetryTimers.has(params.ownerWorkspaceId)) { - return; - } - const timer = setTimeout(() => { - this.workflowTerminalEnqueueRetryTimers.delete(params.ownerWorkspaceId); - void this.retryRetainedWorkflowTerminalEnqueues(params.ownerWorkspaceId); - }, this.terminalAttentionDeferRetryDelayMs); - timer.unref?.(); - this.workflowTerminalEnqueueRetryTimers.set(params.ownerWorkspaceId, timer); - } - - private async retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise { - const byRun = this.retainedWorkflowTerminalEnqueues.get(ownerWorkspaceId); - if (byRun == null) { - return; - } - this.retainedWorkflowTerminalEnqueues.delete(ownerWorkspaceId); - for (const [runId, status] of byRun) { - // A re-attempt that fails again re-retains the entry and re-arms the timer. - await this.enqueueWorkflowRunTerminalAttention({ ownerWorkspaceId, runId, status }); - } - } - - async resetWorkflowRunTerminalAttention(params: { - ownerWorkspaceId: string; - runId: string; - }): Promise { - assert( - params.ownerWorkspaceId.length > 0, - "resetWorkflowRunTerminalAttention requires ownerWorkspaceId" - ); - assert(params.runId.length > 0, "resetWorkflowRunTerminalAttention requires runId"); - this.retainedWorkflowTerminalEnqueues.get(params.ownerWorkspaceId)?.delete(params.runId); - try { - await this.terminalAttentionStore.delete( - params.ownerWorkspaceId, - TerminalAttentionStore.notificationId("workflow_run", params.runId) - ); - this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId)?.delete(params.runId); - } catch (error) { - // Restart callers swallow this rejection, so a transiently failed delete would leave - // the stale delivered/superseded record absorbing the resumed run's next terminal - // enqueue. Retain the reset; the next enqueue completes it before enqueuing fresh. - log.error("Workflow terminal attention reset failed; retained for the next enqueue", { - ownerWorkspaceId: params.ownerWorkspaceId, - runId: params.runId, - error, - }); - let pendingResets = this.pendingWorkflowNotificationResets.get(params.ownerWorkspaceId); - if (pendingResets == null) { - pendingResets = new Set(); - this.pendingWorkflowNotificationResets.set(params.ownerWorkspaceId, pendingResets); - } - pendingResets.add(params.runId); + let runIds = this.pendingWorkflowRunAttention.get(params.ownerWorkspaceId); + if (runIds == null) { + runIds = new Set(); + this.pendingWorkflowRunAttention.set(params.ownerWorkspaceId, runIds); } + runIds.add(params.runId); + this.scheduleTerminalAttentionDrain(params.ownerWorkspaceId); } /** @@ -8070,29 +7966,37 @@ export class TaskService implements AgentTaskIntegration { return this.workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); } - async markWorkflowRunTerminalAttentionConsumed(params: { + /** + * Durable "this terminal generation needs no further wake" marker, keyed by the run's + * terminal updatedAt: a later resume produces a new generation and thereby re-arms + * attention without any reset bookkeeping. Write-once and best-effort by design: if the + * write fails or never happens, the next drain evaluation re-derives the same answer from + * run + history evidence and merely re-attempts the marker. + */ + async markWorkflowRunTerminalAttentionSettled(params: { ownerWorkspaceId: string; runId: string; status: WorkflowRunStatus; + runUpdatedAt: string; + settledAs: "delivered" | "superseded"; }): Promise { assert( params.ownerWorkspaceId.length > 0, - "markWorkflowRunTerminalAttentionConsumed requires ownerWorkspaceId" + "markWorkflowRunTerminalAttentionSettled requires ownerWorkspaceId" ); - assert(params.runId.length > 0, "markWorkflowRunTerminalAttentionConsumed requires runId"); + assert(params.runId.length > 0, "markWorkflowRunTerminalAttentionSettled requires runId"); if (!isTerminalWorkflowRunStatus(params.status)) { return; } - await this.terminalAttentionStore.enqueueIfAbsent({ + await this.terminalAttentionStore.recordSettled({ ownerWorkspaceId: params.ownerWorkspaceId, sourceKind: "workflow_run", - terminalOutcome: terminalAttentionOutcome(params.status), sourceId: params.runId, + generationId: params.runUpdatedAt, + terminalOutcome: terminalAttentionOutcome(params.status), + status: params.settledAs, }); - await this.terminalAttentionStore.markDelivered( - params.ownerWorkspaceId, - TerminalAttentionStore.notificationId("workflow_run", params.runId) - ); + this.pendingWorkflowRunAttention.get(params.ownerWorkspaceId)?.delete(params.runId); } async markWorkspaceTurnTerminalAttentionConsumed(params: { @@ -8201,24 +8105,6 @@ export class TaskService implements AgentTaskIntegration { this.pendingTerminalAttentionDrains.add(promise); } - /** - * A deferred (indeterminate) terminal wake has no deterministic "storage recovered" signal - * to re-trigger the drain, and an idle-wait would resolve immediately on an already-idle - * owner and busy-loop while the fault persists. Retry on a bounded timer instead, one armed - * timer per owner; each retry that defers again arms the next one. - */ - private scheduleTerminalAttentionDeferRetry(ownerWorkspaceId: string): void { - if (this.terminalAttentionDeferRetryTimers.has(ownerWorkspaceId)) { - return; - } - const timer = setTimeout(() => { - this.terminalAttentionDeferRetryTimers.delete(ownerWorkspaceId); - this.scheduleTerminalAttentionDrain(ownerWorkspaceId); - }, this.terminalAttentionDeferRetryDelayMs); - timer.unref?.(); - this.terminalAttentionDeferRetryTimers.set(ownerWorkspaceId, timer); - } - /** * Caller send restrictions (tool policy, workspace-agent disable flag, strict-agent pin) to * restore on a terminal-attention wake. The newest manual user row carries the @@ -8343,8 +8229,17 @@ export class TaskService implements AgentTaskIntegration { ownerWorkspaceId: string, runId: string ): Promise< - | { outcome: "deliver"; prompt: string; initiatingAgent?: WorkflowWakeInitiatingAgent } - | { outcome: "superseded" } + | { + outcome: "deliver"; + prompt: string; + run: WorkflowRunRecord; + initiatingAgent?: WorkflowWakeInitiatingAgent; + } + // settle: superseded or already consumed; record the generation marker so scans stop here. + | { outcome: "settle"; run: WorkflowRunRecord } + // drop: not (or no longer) a wake candidate; just dequeue, nothing durable to mark. + | { outcome: "drop" } + // defer: state transiently unreadable; keep queued for the next drain trigger or sweep. | { outcome: "defer" } > { assert(ownerWorkspaceId.length > 0, "buildWorkflowTerminalPrompt requires ownerWorkspaceId"); @@ -8358,8 +8253,8 @@ export class TaskService implements AgentTaskIntegration { } catch (error: unknown) { // A missing run (ENOENT) or an unparseable record (no fs code; rereading cannot repair // it) is definitively ineligible. Every other fs failure (EIO, EACCES, EISDIR...) is - // potentially transient, and tombstoning on it would permanently drop the wake over a - // recoverable fault: defer those like indeterminate currentness below. + // potentially transient, and dropping on it would delay the wake to the next sweep's + // re-derivation for no reason: defer those like indeterminate currentness below. const code = error != null && typeof error === "object" && "code" in error ? (error as { code?: unknown }).code @@ -8377,26 +8272,27 @@ export class TaskService implements AgentTaskIntegration { runId, error: getErrorMessage(error), }); - return { outcome: "superseded" }; + return { outcome: "drop" }; } if ( run.workspaceId !== ownerWorkspaceId || run.parentWorkflow != null || + // A resumed run left terminal state; its next terminal transition re-queues it. !isTerminalWorkflowRunStatus(run.status) ) { - return { outcome: "superseded" }; + return { outcome: "drop" }; } const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( ownerWorkspaceId, run.id ); - // Indeterminate means history was unreadable, not that the run was superseded: tombstoning + // Indeterminate means history was unreadable, not that the run was superseded: settling // now would permanently drop the wake over a transient fault, so defer and retry instead. if (currentness === "indeterminate") { return { outcome: "defer" }; } if (currentness === "not_current") { - return { outcome: "superseded" }; + return { outcome: "settle", run }; } // Bind the wake to the agent recorded at launch: the newest agent-bearing assistant row // can belong to an unrelated later synthetic turn (a heartbeat is not a supersession @@ -8428,6 +8324,7 @@ export class TaskService implements AgentTaskIntegration { const scriptPath = run.workflow.sourcePath ?? run.workflow.name; return { outcome: "deliver", + run, ...(initiatingAgent != null ? { initiatingAgent } : {}), prompt: buildWorkflowResultContextMessage({ rawCommand: `workflow_run ${scriptPath}`, @@ -8649,8 +8546,20 @@ export class TaskService implements AgentTaskIntegration { * drained notifications delivered. Stale (deleted-workspace) notifications are marked superseded. */ private async drainTerminalAttention(ownerWorkspaceId: string): Promise { - const pending = await this.terminalAttentionStore.listPending(ownerWorkspaceId); - if (pending.length === 0) { + const allPending = await this.terminalAttentionStore.listPending(ownerWorkspaceId); + // Legacy pre-reconciler outbox records for workflow runs (no generation suffix) are dead + // state now that workflow wakes are re-derived from run records + settled markers: delete + // them so they cannot hold the drain hot forever. + for (const notification of allPending) { + if (notification.sourceKind === "workflow_run") { + await this.terminalAttentionStore.delete(ownerWorkspaceId, notification.id); + } + } + const pending = allPending.filter((notification) => notification.sourceKind !== "workflow_run"); + const queuedWorkflowRunIds = Array.from( + this.pendingWorkflowRunAttention.get(ownerWorkspaceId) ?? [] + ); + if (pending.length === 0 && queuedWorkflowRunIds.length === 0) { return; } @@ -8658,6 +8567,8 @@ export class TaskService implements AgentTaskIntegration { const entry = findWorkspaceEntry(cfg, ownerWorkspaceId); if (entry == null) { // Owner workspace no longer exists: the terminal artifacts remain retrievable elsewhere. + // Queue only, no markers: a settled-marker write would recreate the deleted session dir. + this.pendingWorkflowRunAttention.delete(ownerWorkspaceId); for (const notification of pending) { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); } @@ -8665,6 +8576,9 @@ export class TaskService implements AgentTaskIntegration { } if (isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt)) { + // Workflow wakes stay unsettled while archived: the sweep skips archived workspaces, so + // dropping the queue parks them until an unarchive-time sweep re-derives the entries. + this.pendingWorkflowRunAttention.delete(ownerWorkspaceId); for (const notification of pending) { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); } @@ -8741,44 +8655,47 @@ export class TaskService implements AgentTaskIntegration { publicAwaitId: isPersistentChildContinuation ? record.workspaceId : notification.sourceId, }); } - const workflowNotifications = pending.filter( - (notification) => notification.sourceKind === "workflow_run" - ); - const deliverableWorkflowNotificationIds = new Set(); const deliverableWorkflowPrompts: Array<{ - notificationId: string; + runId: string; + run: WorkflowRunRecord; prompt: string; initiatingAgent?: WorkflowWakeInitiatingAgent; }> = []; const workflowPromptSections: string[] = []; - for (const notification of workflowNotifications) { - const workflowPrompt = await this.buildWorkflowTerminalPrompt( - ownerWorkspaceId, - notification.sourceId - ); + for (const runId of queuedWorkflowRunIds) { + const workflowPrompt = await this.buildWorkflowTerminalPrompt(ownerWorkspaceId, runId); if (workflowPrompt.outcome === "defer") { - // Currentness was indeterminate (history or sidecar unreadable): keep the notification - // pending rather than permanently dropping the wake, and arm a bounded retry, because an - // already-idle owner produces no further drain trigger on its own. - log.warn("Deferring workflow terminal attention; history unavailable", { + // History, sidecar, or the run record was transiently unreadable: the run stays + // queued and the next drain trigger or sweep re-evaluates it. + log.warn("Deferring workflow terminal attention; state unavailable", { ownerWorkspaceId, - runId: notification.sourceId, + runId, }); - this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); continue; } - if (workflowPrompt.outcome === "superseded") { + if (workflowPrompt.outcome === "drop") { + this.pendingWorkflowRunAttention.get(ownerWorkspaceId)?.delete(runId); + continue; + } + if (workflowPrompt.outcome === "settle") { // Dropping a notify_on_terminal wake strands the run's owner; keep the drop diagnosable. - log.warn("Dropping superseded workflow terminal attention", { + log.warn("Settling superseded workflow terminal attention", { ownerWorkspaceId, - runId: notification.sourceId, + runId, + }); + await this.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId, + runId, + status: workflowPrompt.run.status, + runUpdatedAt: workflowPrompt.run.updatedAt, + settledAs: "superseded", }); - await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); continue; } deliverableWorkflowPrompts.push({ - notificationId: notification.id, + runId, + run: workflowPrompt.run, prompt: workflowPrompt.prompt, ...(workflowPrompt.initiatingAgent != null ? { initiatingAgent: workflowPrompt.initiatingAgent } @@ -8822,11 +8739,9 @@ export class TaskService implements AgentTaskIntegration { (candidate.initiatingAgent != null && workflowWakeGroupKey(candidate.initiatingAgent) === selectedGroupKey) ); - if (selectedWorkflowPrompts.length < deliverableWorkflowPrompts.length) { - this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); - } + // Unselected groups stay queued: the selected group's wake turn ends with a streamEnded + // drain (and the sweep backstops an aborted one), which delivers the next group. for (const candidate of selectedWorkflowPrompts) { - deliverableWorkflowNotificationIds.add(candidate.notificationId); workflowPromptSections.push(candidate.prompt); } @@ -8852,12 +8767,12 @@ export class TaskService implements AgentTaskIntegration { try { wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); } catch (error: unknown) { - // Fail closed: an unknown policy must not fall back to unrestricted tools. + // Fail closed: an unknown policy must not fall back to unrestricted tools. Everything + // stays pending/queued for the next drain trigger or sweep. log.warn("Deferring terminal wake; caller tool policy unavailable", { ownerWorkspaceId, error, }); - this.scheduleTerminalAttentionDeferRetry(ownerWorkspaceId); return; } @@ -8917,12 +8832,9 @@ export class TaskService implements AgentTaskIntegration { if (notification.sourceKind === "agent_task") { return deliverableAgentNotificationIds.has(notification.id); } - if (notification.sourceKind === "workflow_run") { - return deliverableWorkflowNotificationIds.has(notification.id); - } return deliverableWorkspaceTurnNotificationIds.has(notification.id); }); - if (effectivePending.length === 0) { + if (effectivePending.length === 0 && selectedWorkflowPrompts.length === 0) { await markSuppressedSuperseded(); return; } @@ -8931,6 +8843,26 @@ export class TaskService implements AgentTaskIntegration { for (const notification of effectivePending) { await this.terminalAttentionStore.markDelivered(ownerWorkspaceId, notification.id); } + for (const candidate of selectedWorkflowPrompts) { + try { + await this.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId, + runId: candidate.runId, + status: candidate.run.status, + runUpdatedAt: candidate.run.updatedAt, + settledAs: "delivered", + }); + } catch (error: unknown) { + // Best-effort: the delivered wake itself is durable history evidence, so the next + // evaluation settles this run as consumed and re-attempts the marker. + log.warn("Failed to record delivered workflow wake marker", { + ownerWorkspaceId, + runId: candidate.runId, + error, + }); + this.pendingWorkflowRunAttention.get(ownerWorkspaceId)?.delete(candidate.runId); + } + } }; const markPendingForRetry = async () => { diff --git a/src/node/services/terminalAttentionStore.ts b/src/node/services/terminalAttentionStore.ts index 3370fa8081d..61cb36a0d90 100644 --- a/src/node/services/terminalAttentionStore.ts +++ b/src/node/services/terminalAttentionStore.ts @@ -138,6 +138,43 @@ export class TerminalAttentionStore { return record; } + /** + * Write-once settlement marker: records a notification directly in a terminal status with a + * single write (no pending intermediate a concurrent reader could misread as an owed wake). + * An existing record for the same id is left untouched. + */ + async recordSettled( + notification: Omit< + TerminalAttentionNotification, + "id" | "status" | "createdAt" | "outputDelivery" + > & { + generationId: string; + status: "delivered" | "superseded"; + } + ): Promise { + const id = TerminalAttentionStore.notificationId( + notification.sourceKind, + notification.sourceId, + notification.generationId + ); + const existing = await this.get(notification.ownerWorkspaceId, id); + if (existing != null && existing.status !== "pending") { + return; + } + await this.write({ + id, + ownerWorkspaceId: notification.ownerWorkspaceId, + sourceKind: notification.sourceKind, + sourceId: notification.sourceId, + generationId: notification.generationId, + outputDelivery: outputDeliveryForSource(notification.sourceKind), + terminalOutcome: notification.terminalOutcome, + status: notification.status, + createdAt: new Date().toISOString(), + ...(notification.status === "delivered" ? { deliveredAt: new Date().toISOString() } : {}), + }); + } + async get(ownerWorkspaceId: string, id: string): Promise { let raw: string; try { diff --git a/src/node/services/tools/task_await.ts b/src/node/services/tools/task_await.ts index 0c8c8b13e03..4ecc2954ceb 100644 --- a/src/node/services/tools/task_await.ts +++ b/src/node/services/tools/task_await.ts @@ -411,10 +411,12 @@ export const createTaskAwaitTool: ToolFactory = (config: ToolConfiguration) => { if (!isTerminalWorkflowRunStatus(run.status)) { return; } - await taskService.markWorkflowRunTerminalAttentionConsumed?.({ + await taskService.markWorkflowRunTerminalAttentionSettled?.({ ownerWorkspaceId: workspaceId, status: run.status, runId: run.id, + runUpdatedAt: run.updatedAt, + settledAs: "delivered", }); }; diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 0fac258ed3c..4b1f8527cb4 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -159,15 +159,17 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // that this turn already received the terminal result. Persist consumption durably so the // terminal-attention drain never re-delivers it. const markTerminalAttentionConsumed = async ( - terminalRun: Pick + terminalRun: Pick ) => { if (!isTerminalWorkflowRunStatus(terminalRun.status)) { return; } - await config.taskService?.markWorkflowRunTerminalAttentionConsumed?.({ + await config.taskService?.markWorkflowRunTerminalAttentionSettled?.({ ownerWorkspaceId: workspaceId, runId: terminalRun.id, status: terminalRun.status, + runUpdatedAt: terminalRun.updatedAt, + settledAs: "delivered", }); }; @@ -255,15 +257,16 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const refreshedRunIsStale = isBackgroundDispatch && refreshedRun != null && refreshedRun.status === run.status; - // Foreground only: a background dispatch can still observe the stale pre-dispatch terminal - // status, and consuming it would tombstone the retried run's future terminal wake. - // Consumption derives from the dispatch result itself, not the refreshed record: the - // terminal result is returned to the model below even when the refresh read fails, and - // skipping the tombstone would let the pending terminal attention re-inject it later. - if (!isBackgroundDispatch) { + // Foreground only: a background dispatch can still observe the stale pre-dispatch + // terminal status, and settling it would absorb the retried run's future terminal wake. + // The settled marker binds to the run's terminal generation (updatedAt), so it needs a + // refreshed record that reflects the dispatched terminal status; when the refresh failed + // or lags, skip the marker and the wake settles as consumed from the tool result in + // history on the next scan (worst case one redundant wake for a kernel-nested resume). + if (!isBackgroundDispatch && refreshedRun != null) { const dispatchedStatus = WorkflowRunStatusSchema.safeParse(dispatched.status); - if (dispatchedStatus.success) { - await markTerminalAttentionConsumed({ id: runId, status: dispatchedStatus.data }); + if (dispatchedStatus.success && refreshedRun.status === dispatchedStatus.data) { + await markTerminalAttentionConsumed(refreshedRun); } } diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 83fa80f363e..70b8fb35f2a 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -54,12 +54,6 @@ export interface WorkflowServiceOptions { resolveWorkflowScript?: (scriptPath: string) => Promise; onBackgroundRunTerminal?: (event: WorkflowBackgroundRunTerminalEvent) => Promise | void; onRunStatusChanged?: (event: WorkflowRunStatusChangedEvent) => Promise | void; - /** - * Fired when crash recovery is about to resume an orphaned run, before the runner restarts. - * Used to repair wake provenance a pre-boundary build stripped from the sidecar; awaited so - * a fast run cannot reach terminal before the repair lands. - */ - onRunCrashResumed?: (event: { workspaceId: string; runId: string }) => Promise | void; /** When true, background terminal notifications also fire for interrupted runs. */ notifyInterruptedBackgroundRunTerminal?: boolean; generateRunId?: () => string; @@ -118,13 +112,6 @@ const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES = new Set([ // oRPC creates a WorkflowService per request, so workflow lifecycle state that spans requests // needs process-wide registries. const pendingCrashResumeTimers = new Map>(); -// Crash-resume provenance repair only fires at resume time: once the run settles, nothing -// else re-records the boundary and its terminal wake stays indeterminate on every drain. -// Retry a failed repair on a bounded timer. Module-level like pendingCrashResumeTimers -// because WorkflowService instances are per-request. -const CRASH_RESUME_REPAIR_RETRY_DELAY_MS = 30_000; -const CRASH_RESUME_REPAIR_MAX_ATTEMPTS = 5; -const pendingCrashResumeRepairTimers = new Map>(); const activeWorkflowInterruptStatusWrites = new Map>(); const activeWorkflowRunnerAbortControllers = new Map(); @@ -140,12 +127,6 @@ export class WorkflowService { private readonly onBackgroundRunTerminal?: ( event: WorkflowBackgroundRunTerminalEvent ) => Promise | void; - private readonly onRunCrashResumed?: (event: { - workspaceId: string; - runId: string; - }) => Promise | void; - // Field, not the constant, so tests can shrink the repair retry backoff. - private crashResumeRepairRetryDelayMs = CRASH_RESUME_REPAIR_RETRY_DELAY_MS; private readonly onRunStatusChanged?: ( event: WorkflowRunStatusChangedEvent ) => Promise | void; @@ -169,7 +150,6 @@ export class WorkflowService { this.taskAdapterFactory = options.taskAdapterFactory; this.resolveWorkflowScript = options.resolveWorkflowScript; this.onBackgroundRunTerminal = options.onBackgroundRunTerminal; - this.onRunCrashResumed = options.onRunCrashResumed; this.onRunStatusChanged = options.onRunStatusChanged; this.notifyInterruptedBackgroundRunTerminal = options.notifyInterruptedBackgroundRunTerminal === true; @@ -593,19 +573,6 @@ export class WorkflowService { return false; } - if (this.onRunCrashResumed != null) { - try { - await this.onRunCrashResumed({ workspaceId: run.workspaceId, runId: run.id }); - } catch (error) { - // Best-effort: an unrepaired reference defers its wake as indeterminate rather than - // losing it, so a failed repair must not block the resume itself. Retry off-path: - // repair is CAS-guarded and refuses once a boundary exists, so late success (even - // after the run settles) only unblocks the deferred wake. - console.error("Workflow crash-resume provenance repair failed:", error); - this.scheduleCrashResumeRepairRetry({ workspaceId: run.workspaceId, runId: run.id }, 1); - } - } - const retryDelayMs = await this.runStore.getLeaseRetryDelayMs( input.runId, this.clock?.nowMs() ?? Date.now() @@ -663,32 +630,6 @@ export class WorkflowService { pendingCrashResumeTimers.set(input.runId, timer); } - private scheduleCrashResumeRepairRetry( - input: { workspaceId: string; runId: string }, - attempt: number - ): void { - const repairHook = this.onRunCrashResumed; - if (repairHook == null || attempt > CRASH_RESUME_REPAIR_MAX_ATTEMPTS) { - return; - } - if (pendingCrashResumeRepairTimers.has(input.runId)) { - return; - } - const timer = setTimeout(() => { - pendingCrashResumeRepairTimers.delete(input.runId); - void (async () => { - try { - await repairHook({ workspaceId: input.workspaceId, runId: input.runId }); - } catch (error) { - console.error("Workflow crash-resume provenance repair retry failed:", error); - this.scheduleCrashResumeRepairRetry(input, attempt + 1); - } - })(); - }, this.crashResumeRepairRetryDelayMs); - unrefTimer(timer); - pendingCrashResumeRepairTimers.set(input.runId, timer); - } - private registerActiveRunnerAbortController( runId: string, workspaceId: string, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index defdc1212ec..3c43258f6cd 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6,7 +6,6 @@ import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; import { clearAgentWorkflowRunReferences, readAgentWorkflowRunReferences, - repairAgentWorkflowRunReferenceBoundary, type AgentWorkflowRunReference, } from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; @@ -11065,9 +11064,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // identity cannot be verified, and wall-clock ordering is the exact hole the identity // path exists to close (a backward clock correction would let a pre-supersession // reference outrank a newer manual turn and deliver its output under that turn's tool - // policy). Defer like an unreadable history: the wake stays pending, a workflow_resume - // re-record repairs provenance, and an explicit resume/await still consumes the run. - return "indeterminate"; + // policy). Fail quiet rather than deliver or defer forever: this is a deliberately + // accepted narrow window (downgrade-stripped or snapshot-failed launches), and the + // run's result stays retrievable via an explicit workflow_resume, which re-records the + // reference with a fresh boundary. + return "not_current"; } if (reference.afterBoundaryMessageId === null) { // Verified-empty snapshot: a decision row now exists, so it appeared after the record. @@ -11161,50 +11162,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return decision.status === "found" ? decision.messageId : null; } - /** - * Re-snapshot the boundary for a run reference that lost it: a pre-boundary build rewrites - * the sidecar with only runId/createdAtMs on any record (upgrade -> downgrade -> upgrade - * strips the field), and a boundaryless reference defers its terminal wake as indeterminate - * until provenance is re-established. Crash recovery calls this before restarting an - * orphaned run, but the repair proceeds only on supersession-free evidence: a decision-free - * history (recorded as a verified-empty boundary) or a newest decision row that belongs to - * this run. A newest manual/reset row is refused: the stripped launch cannot be ordered - * against it by identity, and snapshotting it would resurrect a possibly pre-supersession - * result into the newer conversation (the same reference with a surviving boundary would - * stay not_current). Those wakes stay deferred until an explicit workflow_resume, which - * carries current-context intent. References that still carry a boundary (including - * verified-empty null) are left untouched: refreshing them would forgive manual - * supersessions on every restart. - */ - async repairWorkflowRunReferenceBoundary(workspaceId: string, runId: string): Promise { - assert(workspaceId.length > 0, "repairWorkflowRunReferenceBoundary requires workspaceId"); - assert(runId.length > 0, "repairWorkflowRunReferenceBoundary requires runId"); - const sessionDir = this.config.getSessionDir(workspaceId); - const references = await readAgentWorkflowRunReferences(sessionDir); - const reference = references.find((candidate) => candidate.runId === runId); - if (reference == null || reference.afterBoundaryMessageId !== undefined) { - return; - } - const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); - if (decision.status === "error") { - throw new Error("workflow invocation boundary unavailable: history read failed"); - } - if (decision.status === "found" && decision.outcome === "superseded") { - return; - } - // Supersession-free evidence only: no decision row at all (verified-empty null), or the - // newest decision row is this run's own invocation/consumed row, which no manual row can - // postdate (the backward walk would have found that manual row first). The write is a - // compare-and-set under the sidecar lock: a full clear landing after the reads above - // deletes the sidecar, and an unconditional record would recreate it with a - // verified-empty boundary, resurrecting the retired pre-clear result as "current". - await repairAgentWorkflowRunReferenceBoundary({ - workspaceSessionDir: sessionDir, - runId, - afterBoundaryMessageId: decision.status === "found" ? decision.messageId : null, - }); - } - /** * Increment a preflight admission counter in the caller's synchronous entry block and * return a disposable releasing it. Pairs renderer-initiated workspace activity with the From b1f89dd484a95e5c0a5c420965f297d19b51b56a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:16:48 +0000 Subject: [PATCH 44/63] =?UTF-8?q?=F0=9F=A4=96=20test:=20rework=20workflow?= =?UTF-8?q?=20terminal=20wake=20tests=20for=20level-triggered=20reconcilia?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wake candidates now come from the in-memory queue re-derived by the sweep, not persisted outbox records, so drain tests seed the queue (or call noteWorkflowRunTerminalAttention) instead of enqueueing workflow outbox records. Deferred wakes assert queued state and the absence of settlement markers; definitively missing runs are dropped without durable tombstones. Deleted tests for removed machinery (retained enqueues, reset bookkeeping, defer retry timers, crash-resume boundary repair) and added coverage for legacy pending workflow outbox record cleanup. --- src/node/orpc/router.test.ts | 56 +-- .../agentWorkflowRunReferences.test.ts | 77 --- src/node/services/taskService.test.ts | 443 ++++++------------ src/node/services/tools/task_await.test.ts | 8 +- .../services/tools/workflow_resume.test.ts | 49 +- .../workflows/WorkflowService.test.ts | 117 ----- src/node/services/workspaceService.test.ts | 123 +---- 7 files changed, 176 insertions(+), 697 deletions(-) diff --git a/src/node/orpc/router.test.ts b/src/node/orpc/router.test.ts index ccfc19616b7..6123c98e75f 100644 --- a/src/node/orpc/router.test.ts +++ b/src/node/orpc/router.test.ts @@ -362,10 +362,8 @@ describe("router workflow routes", () => { getWorkflowContinuationSendOptions: mock(() => null), sendMessage: mock(async () => ({ success: true, data: undefined })), }, - // Nonterminal run status changes reset any stale terminal notification; the stub keeps - // that call observable without wiring a full TaskService. taskService: { - resetWorkflowRunTerminalAttention: mock(async () => undefined), + noteWorkflowRunTerminalAttention: mock(() => undefined), }, experimentsService: { isExperimentEnabled: mock(() => options.enabled), @@ -956,7 +954,7 @@ export default function workflow() { return { reportMarkdown: "should not run" } await waitForRouterWorkflowStatus(client, "workspace-1", result.runId, "completed"); }); - test("crash-resumed background runs enqueue terminal attention on settle", async () => { + test("crash-resumed background runs note terminal attention on settle", async () => { const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); await runStore.createRun({ id: "wfr_crash_wake", @@ -970,65 +968,27 @@ export default function workflow() { return { reportMarkdown: "should not run" } // Orphaned by a crash: durable status says running, but no live runner. await runStore.appendStatus("wfr_crash_wake", "running", "2026-05-29T00:00:01.000Z"); - const enqueueWorkflowRunTerminalAttention = mock(async () => undefined); + const noteWorkflowRunTerminalAttention = mock(() => undefined); const context = createContext({ enabled: true }); (context as unknown as Record).taskService = { - enqueueWorkflowRunTerminalAttention, - resetWorkflowRunTerminalAttention: mock(async () => undefined), + noteWorkflowRunTerminalAttention, }; - ( - context.workspaceService as unknown as Record - ).repairWorkflowRunReferenceBoundary = mock(async () => undefined); const client = createRouterClient(router(), { context }); - // A read path triggers crash recovery; the resumed run's settle must land in the - // terminal-attention outbox instead of waiting for the next restart's sweep. + // A read path triggers crash recovery; the resumed run's settle must poke the drain + // instead of waiting for the next sweep. await client.workflows.listRuns({ workspaceId: "workspace-1" }); await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_crash_wake", "completed"); const deadline = Date.now() + 5_000; - while (enqueueWorkflowRunTerminalAttention.mock.calls.length === 0 && Date.now() < deadline) { + while (noteWorkflowRunTerminalAttention.mock.calls.length === 0 && Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 25)); } - expect(enqueueWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + expect(noteWorkflowRunTerminalAttention).toHaveBeenCalledWith({ ownerWorkspaceId: "workspace-1", runId: "wfr_crash_wake", status: "completed", }); }); - - test("router-managed resume resets a stale terminal notification before restart", async () => { - const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); - await runStore.createRun({ - id: "wfr_resume_reset", - workspaceId: "workspace-1", - workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, - source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-05-29T00:00:00.000Z", - }); - - const context = createContext({ enabled: true }); - const resetWorkflowRunTerminalAttention = ( - context.taskService as unknown as { - resetWorkflowRunTerminalAttention: ReturnType; - } - ).resetWorkflowRunTerminalAttention; - const client = createRouterClient(router(), { context }); - - await client.workflows.interrupt({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); - resetWorkflowRunTerminalAttention.mockClear(); - - // The prior run's notification survives under the stable workflow_run: id as - // delivered/superseded; without a reset on restart, enqueueIfAbsent preserves that - // record and the resumed run's terminal wake is silently dropped. - await client.workflows.resume({ workspaceId: "workspace-1", runId: "wfr_resume_reset" }); - expect(resetWorkflowRunTerminalAttention).toHaveBeenCalledWith({ - ownerWorkspaceId: "workspace-1", - runId: "wfr_resume_reset", - }); - await waitForRouterWorkflowStatus(client, "workspace-1", "wfr_resume_reset", "completed"); - }); }); describe("router config.saveConfig", () => { diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 7ecab498c72..1939e4f305a 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -7,7 +7,6 @@ import { describe, expect, test } from "bun:test"; import { readAgentWorkflowRunReferences, recordAgentWorkflowRunReference, - repairAgentWorkflowRunReferenceBoundary, } from "@/node/services/agentWorkflowRunReferences"; describe("agent workflow run references", () => { @@ -299,80 +298,4 @@ describe("agent workflow run references", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); - - test("boundary repair is a compare-and-set on a surviving boundaryless reference", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - const runId = "wfr_repairable"; - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId, - createdAtMs: 1_000, - agentId: "plan", - strictAgentResolution: { expectedScope: "built-in" }, - }); - await recordAgentWorkflowRunReference({ - workspaceSessionDir, - runId: "wfr_bystander", - createdAtMs: 1_100, - afterBoundaryMessageId: "row-1", - }); - - // Repairs in place, preserving the rest of the entry and its neighbors. - expect( - await repairAgentWorkflowRunReferenceBoundary({ - workspaceSessionDir, - runId, - afterBoundaryMessageId: null, - }) - ).toBe(true); - const references = await readAgentWorkflowRunReferences(workspaceSessionDir); - expect(references.find((reference) => reference.runId === runId)).toEqual({ - runId, - createdAtMs: 1_000, - afterBoundaryMessageId: null, - agentId: "plan", - strictAgentResolution: { expectedScope: "built-in" }, - }); - expect(references.find((reference) => reference.runId === "wfr_bystander")).toEqual({ - runId: "wfr_bystander", - createdAtMs: 1_100, - afterBoundaryMessageId: "row-1", - }); - - // A reference that already carries a boundary (here the one just repaired) is never - // overwritten: a concurrent explicit re-record must win over a stale repair. - expect( - await repairAgentWorkflowRunReferenceBoundary({ - workspaceSessionDir, - runId, - afterBoundaryMessageId: "stale-row", - }) - ).toBe(false); - const unchanged = await readAgentWorkflowRunReferences(workspaceSessionDir); - expect(unchanged.find((reference) => reference.runId === runId)?.afterBoundaryMessageId).toBe( - null - ); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); - - test("boundary repair refuses to recreate a cleared sidecar", async () => { - const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); - try { - // A full-history clear deleted the sidecar between the repair's reads and its write: - // the stale repair must not resurrect the retired reference as verified-empty current. - expect( - await repairAgentWorkflowRunReferenceBoundary({ - workspaceSessionDir, - runId: "wfr_cleared", - afterBoundaryMessageId: null, - }) - ).toBe(false); - expect(await fs.readdir(workspaceSessionDir)).toEqual([]); - } finally { - await fs.rm(workspaceSessionDir, { recursive: true, force: true }); - } - }); }); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 8faaddc5073..2aed76f75d7 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -5965,7 +5965,7 @@ describe("TaskService", () => { const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); const { taskService } = createTaskServiceHarness(config, { workspaceService }); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", @@ -6007,13 +6007,13 @@ describe("TaskService", () => { (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - // History unreadable at drain time: currentness is indeterminate, so the notification must - // stay pending for a later drain instead of being tombstoned as superseded. + // History unreadable at drain time: currentness is indeterminate, so the run must stay + // queued for a later drain or sweep instead of being settled as superseded. (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = mock(() => Promise.resolve("indeterminate")); const { taskService } = createTaskServiceHarness(config, { workspaceService }); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", @@ -6021,10 +6021,16 @@ describe("TaskService", () => { await flushTerminalAttentionDrains(taskService); expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + const run = await runStore.getRun(runId); + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ) + ).toBeNull(); }); - test("deferred terminal wake-up retries on the bounded timer", async () => { + test("a deferred wake delivers on the next drain trigger", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); const runId = "wfr_terminal_defer_retry"; @@ -6056,8 +6062,8 @@ describe("TaskService", () => { (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - // The first drain sees a transient storage fault that clears before the retry fires. An - // already-idle owner produces no other drain trigger, so only the bounded retry delivers. + // The first drain sees a transient storage fault: the run stays queued with no timer + // bookkeeping, and any later drain trigger (stream end, sweep) re-evaluates and delivers. let currentnessCalls = 0; (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = mock(() => { @@ -6065,11 +6071,8 @@ describe("TaskService", () => { return Promise.resolve(currentnessCalls === 1 ? "indeterminate" : "current"); }); const { taskService } = createTaskServiceHarness(config, { workspaceService }); - ( - taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } - ).terminalAttentionDeferRetryDelayMs = 10; - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", @@ -6077,87 +6080,24 @@ describe("TaskService", () => { await flushTerminalAttentionDrains(taskService); expect(sendMessage).not.toHaveBeenCalled(); - // Real timers: poll until the armed retry fires and the follow-up drain delivers. - const deadline = Date.now() + 5_000; - while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { - await new Promise((resolve) => setTimeout(resolve, 10)); - await flushTerminalAttentionDrains(taskService); - } - expect(sendMessage).toHaveBeenCalledTimes(1); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); - }); - - test("failed terminal attention enqueue is retained and retried on the bounded timer", async () => { - const config = await createTestConfig(rootDir); - const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_terminal_enqueue_retry"; - const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); - await runStore.createRun({ - id: runId, - workspaceId: parentId, - workflow: { - name: "research", - description: "Research workflow", - scope: "built-in", - executable: true, - }, - source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-06-19T00:00:00.000Z", - }); - await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); - await runStore.appendNextEvent(runId, { - type: "result", - at: "2026-06-19T00:00:02.000Z", - result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, - }); - await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - - const terminalAttentionStore = new TerminalAttentionStore(config); - const sendMessage = mock( - (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) - ); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = - mock(() => Promise.resolve("current")); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); ( - taskService as unknown as { terminalAttentionDeferRetryDelayMs: number } - ).terminalAttentionDeferRetryDelayMs = 10; - - // The workflow terminal callback driving this enqueue is single-attempt, so when the - // first outbox write fails only the retained in-process retry can persist the wake. - const internalStore = ( - taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } - ).terminalAttentionStore; - spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( - new Error("EIO: outbox write failed") - ); - - await taskService.enqueueWorkflowRunTerminalAttention({ - ownerWorkspaceId: parentId, - runId, - status: "completed", - }); + taskService as unknown as { scheduleTerminalAttentionDrain(id: string): void } + ).scheduleTerminalAttentionDrain(parentId); await flushTerminalAttentionDrains(taskService); - expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); - - // Real timers: poll until the armed retry re-enqueues and the follow-up drain delivers. - const deadline = Date.now() + 5_000; - while (Date.now() < deadline && sendMessage.mock.calls.length === 0) { - await new Promise((resolve) => setTimeout(resolve, 10)); - await flushTerminalAttentionDrains(taskService); - } expect(sendMessage).toHaveBeenCalledTimes(1); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + const run = await runStore.getRun(runId); + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ) + ).toMatchObject({ status: "delivered" }); }); - test("reset drops a retained terminal attention enqueue", async () => { + test("drains for a removed workspace drop queued workflow wakes without touching disk", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_terminal_enqueue_reset"; + const runId = "wfr_terminal_enqueue_removed_owner"; const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); await runStore.createRun({ id: runId, @@ -6181,7 +6121,6 @@ describe("TaskService", () => { }); await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - const terminalAttentionStore = new TerminalAttentionStore(config); const sendMessage = mock( (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); @@ -6189,109 +6128,60 @@ describe("TaskService", () => { (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = mock(() => Promise.resolve("current")); const { taskService } = createTaskServiceHarness(config, { workspaceService }); - const internalStore = ( - taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } - ).terminalAttentionStore; - spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( - new Error("EIO: outbox write failed") - ); - await taskService.enqueueWorkflowRunTerminalAttention({ + // The owner is removed (config entry gone, session directory deleted) before the drain + // runs; the drain must not recreate the deleted session directory or leave queued state + // for a future workspace reusing the ID. + const cfg = config.loadConfigOrDefault(); + for (const project of cfg.projects.values()) { + project.workspaces = project.workspaces.filter((workspace) => workspace.id !== parentId); + } + await config.editConfig(() => cfg); + const sessionDir = config.getSessionDir(parentId); + await fsPromises.rm(sessionDir, { recursive: true, force: true }); + + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", }); - // The run was reset (e.g. resumed) before the retry fired; the retained stale wake - // must be dropped instead of resurrected by the retry. - await taskService.resetWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId }); + await flushTerminalAttentionDrains(taskService); - // Drive the armed retry directly so the outcome is deterministic under real timers. - await ( + expect(sendMessage).not.toHaveBeenCalled(); + expect(existsSync(sessionDir)).toBe(false); + const queued = ( taskService as unknown as { - retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise; + pendingWorkflowRunAttention: Map>; } - ).retryRetainedWorkflowTerminalEnqueues(parentId); - await flushTerminalAttentionDrains(taskService); - expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + ).pendingWorkflowRunAttention; + expect(queued.has(parentId)).toBe(false); }); - test("workspace removal drops retained terminal enqueues without recreating the outbox", async () => { + test("a legacy pending workflow outbox record is deleted by the next drain", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_terminal_enqueue_removed_owner"; - const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); - await runStore.createRun({ - id: runId, - workspaceId: parentId, - workflow: { - name: "research", - description: "Research workflow", - scope: "built-in", - executable: true, - }, - source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-06-19T00:00:00.000Z", - }); - await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); - await runStore.appendNextEvent(runId, { - type: "result", - at: "2026-06-19T00:00:02.000Z", - result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, - }); - await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - - const sendMessage = mock( - (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) - ); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = - mock(() => Promise.resolve("current")); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); - const internalStore = ( - taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } - ).terminalAttentionStore; - spyOn(internalStore, "enqueueIfAbsent").mockRejectedValueOnce( - new Error("EIO: outbox write failed") - ); - - await taskService.enqueueWorkflowRunTerminalAttention({ + const { taskService } = createTaskServiceHarness(config); + const terminalAttentionStore = new TerminalAttentionStore(config); + const legacy = await terminalAttentionStore.enqueueIfAbsent({ ownerWorkspaceId: parentId, - runId, - status: "completed", + sourceKind: "workflow_run", + sourceId: "wfr_legacy_outbox", }); - - // The owner is removed (config entry gone, session directory deleted) before the bounded - // retry fires; the retry must not recreate the deleted session directory or leave state - // for a future workspace reusing the ID. - const cfg = config.loadConfigOrDefault(); - for (const project of cfg.projects.values()) { - project.workspaces = project.workspaces.filter((workspace) => workspace.id !== parentId); - } - await config.editConfig(() => cfg); - const sessionDir = config.getSessionDir(parentId); - await fsPromises.rm(sessionDir, { recursive: true, force: true }); + assert(legacy, "legacy workflow attention must enqueue"); await ( taskService as unknown as { - retryRetainedWorkflowTerminalEnqueues(ownerWorkspaceId: string): Promise; + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; } - ).retryRetainedWorkflowTerminalEnqueues(parentId); - await flushTerminalAttentionDrains(taskService); + ).drainTerminalAttention(parentId); - expect(sendMessage).not.toHaveBeenCalled(); - expect(existsSync(sessionDir)).toBe(false); - const retained = ( - taskService as unknown as { - retainedWorkflowTerminalEnqueues: Map>; - } - ).retainedWorkflowTerminalEnqueues; - expect(retained.has(parentId)).toBe(false); + // Deleted outright rather than superseded: workflow wakes are re-derived from run + // records now, so a pre-reconciler pending record is dead state that would otherwise + // hold the drain hot forever. + expect(await terminalAttentionStore.get(parentId, legacy.id)).toBeNull(); }); - test("startup recovery replaces a stale terminal record from an older run generation", async () => { + test("the sweep re-queues a resumed run's new terminal generation past the old delivered marker", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); const runId = "wfr_recovery_stale_generation"; @@ -6320,104 +6210,39 @@ describe("TaskService", () => { (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = mock(() => Promise.resolve("current")); const { taskService } = createTaskServiceHarness(config, { workspaceService }); - const recover = () => + const sweep = () => ( taskService as unknown as { - recoverTerminalWorkflowRunAttentionNotifications(): Promise; + sweepWorkflowRunTerminalAttention(): Promise; } - ).recoverTerminalWorkflowRunAttentionNotifications(); + ).sweepWorkflowRunTerminalAttention(); - // First generation delivers normally, leaving a delivered record newer than the run's - // terminal transition; recovery must not duplicate a record representing this outcome. - await taskService.enqueueWorkflowRunTerminalAttention({ + // First generation delivers normally, leaving a delivered marker bound to that terminal + // generation; the sweep must not re-queue an already-settled generation. + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "failed", }); await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(1); - expect(await recover()).toBe(0); + expect(await sweep()).toBe(0); await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(1); - // The resumed run reaches terminal again after the delivered record was written, and the - // process crashed before reset-then-enqueue ran (no in-memory reset marker survives the - // restart). Recovery must replace the stale delivered record instead of letting - // enqueueIfAbsent silently absorb the newer generation's wake. + // The resumed run reaches terminal again with a newer updatedAt; the old delivered marker + // belongs to the previous generation, so the sweep re-queues the wake without any reset + // bookkeeping having run. await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { allowFailedCheckpointRetry: true, }); await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); - expect(await recover()).toBe(1); + expect(await sweep()).toBe(1); await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(2); }); - test("a failed reset is completed by the next terminal enqueue", async () => { - const config = await createTestConfig(rootDir); - const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_reset_retained"; - const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); - await runStore.createRun({ - id: runId, - workspaceId: parentId, - workflow: { - name: "research", - description: "Research workflow", - scope: "built-in", - executable: true, - }, - source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-06-19T00:00:00.000Z", - }); - await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); - await runStore.appendNextEvent(runId, { - type: "result", - at: "2026-06-19T00:00:02.000Z", - result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, - }); - await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - - const terminalAttentionStore = new TerminalAttentionStore(config); - const sendMessage = mock( - (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) - ); - const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = - mock(() => Promise.resolve("current")); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); - - // First settle delivers normally, leaving a delivered record under the stable id. - await taskService.enqueueWorkflowRunTerminalAttention({ - ownerWorkspaceId: parentId, - runId, - status: "completed", - }); - await flushTerminalAttentionDrains(taskService); - expect(sendMessage).toHaveBeenCalledTimes(1); - - // The restart's reset fails transiently; callers swallow the rejection, so only the - // retained pending reset can stop the delivered record from absorbing the next enqueue. - const internalStore = ( - taskService as unknown as { terminalAttentionStore: TerminalAttentionStore } - ).terminalAttentionStore; - spyOn(internalStore, "delete").mockRejectedValueOnce(new Error("EIO: outbox delete failed")); - await taskService.resetWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId }); - - // The resumed run settles again: the enqueue completes the reset and delivers fresh. - await taskService.enqueueWorkflowRunTerminalAttention({ - ownerWorkspaceId: parentId, - runId, - status: "completed", - }); - await flushTerminalAttentionDrains(taskService); - expect(sendMessage).toHaveBeenCalledTimes(2); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); - }); - test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); @@ -6475,7 +6300,7 @@ describe("TaskService", () => { }) ); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", @@ -6541,7 +6366,7 @@ describe("TaskService", () => { synthetic: true, }) ); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId: "wfr_policy_restore", status: "completed", @@ -6557,7 +6382,7 @@ describe("TaskService", () => { parentId, createMuxMessage("manual-unrestricted", "user", "carry on", { timestamp: 1_200 }) ); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId: "wfr_policy_lifted", status: "completed", @@ -6624,7 +6449,7 @@ describe("TaskService", () => { createMuxMessage(`assistant-${i}`, "assistant", `progress ${i}`, { timestamp: 1_001 + i }) ); } - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", @@ -6700,7 +6525,7 @@ describe("TaskService", () => { agentId: "plan", }); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", @@ -6769,19 +6594,12 @@ describe("TaskService", () => { agentId: "plan", }); - // Seed the store directly so ONE drain observes both pending notifications; per-enqueue - // drains would deliver them separately without exercising the coalescing path. + // Seed the in-memory queue directly so ONE drain observes both runs; per-note drains + // would deliver them separately without exercising the coalescing path. const terminalAttentionStore = new TerminalAttentionStore(config); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: "wfr_split_plan", - }); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: "wfr_split_exec", - }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set(["wfr_split_plan", "wfr_split_exec"])); await drain(parentId); // The newest launch's group delivers first, alone, under its own agent. @@ -6866,16 +6684,13 @@ describe("TaskService", () => { }); const terminalAttentionStore = new TerminalAttentionStore(config); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: "wfr_pin_split_pinned", - }); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: "wfr_pin_split_unpinned", - }); + // Seed the in-memory queue directly so ONE drain observes both runs. + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set( + parentId, + new Set(["wfr_pin_split_pinned", "wfr_pin_split_unpinned"]) + ); await drain(parentId); // The newest launch delivers first, alone, without the other launch's pin. @@ -6957,11 +6772,9 @@ describe("TaskService", () => { sourceKind: "workspace_turn", sourceId: "wst_mixed_handle", }); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: runId, - }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); await drain(parentId); expect(sendMessage).toHaveBeenCalledTimes(1); @@ -7040,7 +6853,7 @@ describe("TaskService", () => { agentId: "plan", }); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", @@ -7062,41 +6875,39 @@ describe("TaskService", () => { ); const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); const { taskService } = createTaskServiceHarness(config, { workspaceService }); - const drain = ( - taskService as unknown as { - drainTerminalAttention: (ownerWorkspaceId: string) => Promise; - } - ).drainTerminalAttention.bind(taskService); const terminalAttentionStore = new TerminalAttentionStore(config); + const queued = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention; // run.json exists but is unreadable (EISDIR): potentially transient, so the wake must - // stay pending for a later drain instead of being durably tombstoned. + // stay queued for a later drain or sweep instead of being dropped. const unreadableRunId = "wfr_unreadable"; await fsPromises.mkdir( path.join(config.getSessionDir(parentId), "workflows", unreadableRunId, "run.json"), { recursive: true } ); - await terminalAttentionStore.enqueueIfAbsent({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: unreadableRunId, + runId: unreadableRunId, + status: "completed", }); - await drain(parentId); + await flushTerminalAttentionDrains(taskService); expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + expect(queued.get(parentId)?.has(unreadableRunId)).toBe(true); - // A definitively missing run (ENOENT) is still tombstoned, not deferred forever. - await terminalAttentionStore.enqueueIfAbsent({ + // A definitively missing run (ENOENT) is dropped from the queue: the sweep re-derives + // owed wakes from run records, so nothing durable is needed to keep it away. + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: "wfr_missing", + runId: "wfr_missing", + status: "completed", }); - await drain(parentId); + await flushTerminalAttentionDrains(taskService); expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.get(parentId, "workflow_run:wfr_missing")).toMatchObject({ - status: "superseded", - }); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + expect(queued.get(parentId)?.has("wfr_missing")).toBe(false); + expect(queued.get(parentId)?.has(unreadableRunId)).toBe(true); + expect(await terminalAttentionStore.get(parentId, "workflow_run:wfr_missing")).toBeNull(); }); test("wake defers when the launch-identity read fails after currentness succeeds", async () => { @@ -7129,27 +6940,35 @@ describe("TaskService", () => { (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = mock(() => Promise.resolve("current")); const { taskService } = createTaskServiceHarness(config, { workspaceService }); - const drain = ( - taskService as unknown as { - drainTerminalAttention: (ownerWorkspaceId: string) => Promise; - } - ).drainTerminalAttention.bind(taskService); const terminalAttentionStore = new TerminalAttentionStore(config); // ...but the launch-identity read fails transiently (EISDIR). Delivering without the // recorded identity would bind the wake to the newest agent-bearing history row, so the - // wake must stay pending for the retry drain. + // wake must stay queued for the retry drain. await fsPromises.mkdir(path.join(config.getSessionDir(parentId), "agent-workflow-runs.json"), { recursive: true, }); - await terminalAttentionStore.enqueueIfAbsent({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, - sourceKind: "workflow_run", - sourceId: runId, + runId, + status: "completed", }); - await drain(parentId); + await flushTerminalAttentionDrains(taskService); expect(sendMessage).not.toHaveBeenCalled(); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(1); + expect( + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(runId) + ).toBe(true); + const run = await runStore.getRun(runId); + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ) + ).toBeNull(); }); test("wake re-pins the selected group's recorded launch pin, not the newest row's", async () => { @@ -7211,7 +7030,7 @@ describe("TaskService", () => { agentId: "exec", strictAgentResolution: null, }); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId: "wfr_pin_unpinned", status: "completed", @@ -7232,7 +7051,7 @@ describe("TaskService", () => { agentId: "plan", strictAgentResolution: { expectedScope: "built-in" }, }); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId: "wfr_pin_recorded", status: "completed", @@ -7286,7 +7105,7 @@ describe("TaskService", () => { disableWorkspaceAgents: true, } as unknown as Parameters[3]) ); - await taskService.enqueueWorkflowRunTerminalAttention({ + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", diff --git a/src/node/services/tools/task_await.test.ts b/src/node/services/tools/task_await.test.ts index 4863fcd303a..ddcc5a35a9b 100644 --- a/src/node/services/tools/task_await.test.ts +++ b/src/node/services/tools/task_await.test.ts @@ -1194,11 +1194,11 @@ describe("task_await tool", () => { }, ]); - const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); const taskService = { listActiveDescendantAgentTaskIds: mock(() => []), isDescendantAgentTask: mock(() => Promise.resolve(false)), - markWorkflowRunTerminalAttentionConsumed, + markWorkflowRunTerminalAttentionSettled, waitForAgentReport: mock(() => { throw new Error("workflow run IDs should not be treated as agent tasks"); }), @@ -1233,10 +1233,12 @@ describe("task_await tool", () => { workspaceId: "parent-workspace", runId: "wfr_demo", }); - expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + expect(markWorkflowRunTerminalAttentionSettled).toHaveBeenCalledWith({ ownerWorkspaceId: "parent-workspace", status: "completed", runId: "wfr_demo", + runUpdatedAt: "2026-01-01T00:00:05.000Z", + settledAs: "delivered", }); }); diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index f6433a16512..c1dcc63b5cd 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -243,7 +243,7 @@ describe("workflow_resume tool", () => { }); }); - test("marks terminal attention consumed when returning an already-completed run's result", async () => { + test("marks terminal attention settled when returning an already-completed run's result", async () => { using tempDir = new TestTempDir("test-workflow-resume-consumed"); const completedRun = buildRun({ status: "completed", @@ -259,12 +259,12 @@ describe("workflow_resume tool", () => { ], }); const workflowService = buildWorkflowService({ getRun: mock(async () => completedRun) }); - const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, workflowService, - taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, }); await tool.execute!( @@ -272,24 +272,26 @@ describe("workflow_resume tool", () => { mockToolCallOptions ); - expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + expect(markWorkflowRunTerminalAttentionSettled).toHaveBeenCalledWith({ ownerWorkspaceId: "workspace-1", runId: "wfr_resume_me", status: "completed", + runUpdatedAt: completedRun.updatedAt, + settledAs: "delivered", }); }); - test("does not mark terminal attention consumed for background dispatches", async () => { + test("does not mark terminal attention settled for background dispatches", async () => { using tempDir = new TestTempDir("test-workflow-resume-background-no-consume"); // The refresh after a background dispatch can still observe the stale pre-dispatch failed - // status; consuming it would tombstone the retried run's future terminal wake. + // status; settling it would absorb the retried run's future terminal wake. const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); - const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, workflowService, - taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, }); await tool.execute!( @@ -297,10 +299,10 @@ describe("workflow_resume tool", () => { mockToolCallOptions ); - expect(markWorkflowRunTerminalAttentionConsumed).not.toHaveBeenCalled(); + expect(markWorkflowRunTerminalAttentionSettled).not.toHaveBeenCalled(); }); - test("marks terminal attention consumed when a foreground retry finishes terminal", async () => { + test("marks terminal attention settled when a foreground retry finishes terminal", async () => { using tempDir = new TestTempDir("test-workflow-resume-foreground-consume"); const failedRun = buildFailedRun(); const completedRun = buildRun({ @@ -322,12 +324,12 @@ describe("workflow_resume tool", () => { result: { reportMarkdown: "retried" }, })), }); - const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, workflowService, - taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, }); await tool.execute!( @@ -335,18 +337,21 @@ describe("workflow_resume tool", () => { mockToolCallOptions ); - expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + expect(markWorkflowRunTerminalAttentionSettled).toHaveBeenCalledWith({ ownerWorkspaceId: "workspace-1", runId: "wfr_resume_me", status: "completed", + runUpdatedAt: completedRun.updatedAt, + settledAs: "delivered", }); }); - test("consumes the foreground terminal result even when the refresh read fails", async () => { + test("skips the settled marker when the refresh read fails but still returns the result", async () => { using tempDir = new TestTempDir("test-workflow-resume-refresh-failure-consume"); - // WorkflowService.getRun collapses transient read failures to null. The terminal result is - // still returned to the model, so consumption must derive from the dispatch result or the - // pending terminal attention would re-inject it later. + // WorkflowService.getRun collapses transient read failures to null. The marker binds to the + // run's terminal generation (updatedAt), which is unknowable without the refreshed record: + // the terminal result still returns to the model, and the wake settles as consumed from + // history evidence on the next scan instead of being marked here. let getRunCalls = 0; const workflowService = buildWorkflowService({ getRun: mock(async () => { @@ -359,12 +364,12 @@ describe("workflow_resume tool", () => { result: { reportMarkdown: "resumed" }, })), }); - const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, workflowService, - taskService: { markWorkflowRunTerminalAttentionConsumed } as unknown as TaskService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, }); const result = await tool.execute!( @@ -372,11 +377,7 @@ describe("workflow_resume tool", () => { mockToolCallOptions ); - expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ - ownerWorkspaceId: "workspace-1", - runId: "wfr_resume_me", - status: "completed", - }); + expect(markWorkflowRunTerminalAttentionSettled).not.toHaveBeenCalled(); expect(result).toMatchObject({ status: "completed", runId: "wfr_resume_me", mode: "resume" }); }); diff --git a/src/node/services/workflows/WorkflowService.test.ts b/src/node/services/workflows/WorkflowService.test.ts index f784b7a0f6c..008acd2a5ce 100644 --- a/src/node/services/workflows/WorkflowService.test.ts +++ b/src/node/services/workflows/WorkflowService.test.ts @@ -1007,120 +1007,3 @@ describe("WorkflowRunStore.listActiveRunSummaries", () => { expect(summaries.map((summary) => summary.runId)).toEqual(["wfr_healthy"]); }); }); - -describe("WorkflowService crash recovery", () => { - test("crash resume fires provenance repair before the run reaches terminal", async () => { - using tmp = new DisposableTempDir("workflow-service-crash-repair"); - const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); - await runStore.createRun({ - id: "wfr_crash", - workspaceId: "workspace-1", - workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, - source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-05-29T00:00:00.000Z", - }); - // Orphaned by a crash: durable status says running, but no runner holds the lease. - await runStore.appendStatus("wfr_crash", "running", "2026-05-29T00:00:01.000Z"); - - const events: string[] = []; - let resolveCompleted: (() => void) | undefined; - const completed = new Promise((resolve) => { - resolveCompleted = resolve; - }); - const service = new WorkflowService({ - runStore, - runtimeFactory: new QuickJSRuntimeFactory(), - taskAdapter: { - async runAgent() { - throw new Error("No agent steps expected"); - }, - }, - generateRunId: () => "wfr_unused", - runnerId: "runner-a", - onRunCrashResumed: (event) => { - events.push(`repair:${event.workspaceId}:${event.runId}`); - }, - onRunStatusChanged: (event) => { - events.push(`status:${event.status}`); - if (event.status === "completed") { - resolveCompleted?.(); - } - }, - }); - - const resumed = await service.resumeCrashedRuns({ - workspaceId: "workspace-1", - projectTrusted: true, - }); - expect(resumed).toEqual(["wfr_crash"]); - await completed; - // The repair hook is awaited before the runner restarts, so even an instantly completing - // run cannot reach terminal with unrepaired provenance. - expect(events[0]).toBe("repair:workspace-1:wfr_crash"); - expect(events).toContain("status:completed"); - await expect(runStore.getRun("wfr_crash")).resolves.toMatchObject({ status: "completed" }); - }); - - test("failed crash-resume provenance repair retries on a bounded timer", async () => { - using tmp = new DisposableTempDir("workflow-service-crash-repair-retry"); - const runStore = new WorkflowRunStore({ sessionDir: tmp.path }); - await runStore.createRun({ - id: "wfr_crash_repair_retry", - workspaceId: "workspace-1", - workflow: { name: "demo", description: "Demo workflow", scope: "built-in", executable: true }, - source: 'export default function workflow() { return { reportMarkdown: "done" }; }\n', - args: {}, - attentionPolicy: "notify_on_terminal", - now: "2026-05-29T00:00:00.000Z", - }); - await runStore.appendStatus("wfr_crash_repair_retry", "running", "2026-05-29T00:00:01.000Z"); - - // The repair hook only fires at resume time; if its transient failure were terminal, the - // reference would stay boundaryless after the run settles and every drain would defer the - // wake as indeterminate with nothing left to repair it. - const repairCalls: string[] = []; - let failFirstRepair = true; - const service = new WorkflowService({ - runStore, - runtimeFactory: new QuickJSRuntimeFactory(), - taskAdapter: { - async runAgent() { - throw new Error("No agent steps expected"); - }, - }, - generateRunId: () => "wfr_unused", - runnerId: "runner-a", - onRunCrashResumed: (event) => { - repairCalls.push(`${event.workspaceId}:${event.runId}`); - if (failFirstRepair) { - failFirstRepair = false; - throw new Error("EIO: sidecar write failed"); - } - }, - }); - ( - service as unknown as { crashResumeRepairRetryDelayMs: number } - ).crashResumeRepairRetryDelayMs = 10; - - const resumed = await service.resumeCrashedRuns({ - workspaceId: "workspace-1", - projectTrusted: true, - }); - expect(resumed).toEqual(["wfr_crash_repair_retry"]); - // The shrunk retry timer can fire while resumeCrashedRuns is still awaiting the runner - // restart on a slow machine, so only the initial attempt's ordering is asserted here; - // the exact two-attempt sequence is asserted after the poll below. - expect(repairCalls[0]).toBe("workspace-1:wfr_crash_repair_retry"); - - const deadline = Date.now() + 5_000; - while (Date.now() < deadline && repairCalls.length < 2) { - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(repairCalls).toEqual([ - "workspace-1:wfr_crash_repair_retry", - "workspace-1:wfr_crash_repair_retry", - ]); - }); -}); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index b917a7f426b..32429cee4db 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -59,10 +59,7 @@ import { WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowResultContextMessage, } from "@/common/utils/workflowRunMessages"; -import { - readAgentWorkflowRunReferences, - recordAgentWorkflowRunReference, -} from "@/node/services/agentWorkflowRunReferences"; +import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; import * as todoStorageModule from "@/node/services/todos/todoStorage"; import * as runtimeFactory from "@/node/runtime/runtimeFactory"; @@ -9232,7 +9229,7 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("defers boundaryless sidecar references instead of trusting wall-clock order", async () => { + test("fails boundaryless sidecar references quiet instead of trusting wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-legacy"; const runId = "wfr_currentness_legacy"; @@ -9265,21 +9262,21 @@ describe("WorkspaceService workflow invocation events", () => { createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); // A reference without a boundary snapshot (pre-upgrade entry or record-time history read - // failure) cannot be ordered against the decision row by identity: the wake defers - // instead of delivering, and the boolean caller stays fail-safe. + // failure) cannot be ordered against the decision row by identity: the wake fails quiet + // (not_current) rather than delivering or deferring forever. await recordAgentWorkflowRunReference({ workspaceSessionDir: config.getSessionDir(workspaceId), runId, createdAtMs: 1_150, }); expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( - "indeterminate" + "not_current" ); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); // A backward clock correction gives the newer superseding turn an OLDER timestamp than // the reference. Wall-clock ordering would resurrect the superseded reference as current - // and deliver its output under the newer turn's tool policy; it must stay deferred. + // and deliver its output under the newer turn's tool policy; it must stay quiet. await historyService.appendToHistory( workspaceId, createMuxMessage("manual-user-2", "user", "never mind, answer something else", { @@ -9287,7 +9284,7 @@ describe("WorkspaceService workflow invocation events", () => { }) ); expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( - "indeterminate" + "not_current" ); workspaceService.disposeSession(workspaceId); } finally { @@ -9295,112 +9292,6 @@ describe("WorkspaceService workflow invocation events", () => { } }); - test("crash-resume repair restores provenance only on supersession-free evidence", async () => { - const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-crash-repair"; - const strippedRunId = "wfr_crash_repair_stripped"; - const supersededRunId = "wfr_crash_repair_superseded"; - const anchoredRunId = "wfr_crash_repair_anchored"; - const projectPath = path.join(config.rootDir, "project"); - try { - await config.addWorkspace(projectPath, { - id: workspaceId, - name: "workflow-crash-repair", - projectName: "project", - projectPath, - runtimeConfig: { type: "local" }, - }); - const workspaceService = createWorkspaceServiceForTest({ - config, - historyService, - aiService: createMockAIService({ - stopStream: mock(() => Promise.resolve(Ok(undefined))), - }), - extensionMetadata: new ExtensionMetadataService( - path.join(config.rootDir, "extensionMetadata.json") - ), - initStateManager: { - ...mockInitStateManager, - off: mock(() => undefined as unknown as InitStateManager), - } as unknown as InitStateManager, - }); - - // A downgrade rewrote the sidecar without boundary fields. With a decision-free history - // the repair has supersession-free evidence: it records a verified-empty boundary, - // keeps the recorded launch identity, and the deferred wake becomes deliverable. - await recordAgentWorkflowRunReference({ - workspaceSessionDir: config.getSessionDir(workspaceId), - runId: strippedRunId, - createdAtMs: 1_150, - agentId: "plan", - strictAgentResolution: { expectedScope: "built-in" }, - }); - await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, strippedRunId); - const repaired = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); - expect(repaired.find((reference) => reference.runId === strippedRunId)).toMatchObject({ - createdAtMs: 1_150, - afterBoundaryMessageId: null, - agentId: "plan", - strictAgentResolution: { expectedScope: "built-in" }, - }); - expect( - await workspaceService.getWorkflowInvocationCurrentness(workspaceId, strippedRunId) - ).toBe("current"); - - // Once a manual row is the newest decision row, a stripped launch cannot be ordered - // against it: the run may predate the supersession, so repair must refuse and the wake - // must stay deferred rather than resurrect a possibly superseded result. - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "never mind, do something else", { - timestamp: 1_200, - }) - ); - await recordAgentWorkflowRunReference({ - workspaceSessionDir: config.getSessionDir(workspaceId), - runId: supersededRunId, - createdAtMs: 1_100, - }); - await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, supersededRunId); - const refused = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); - const refusedReference = refused.find((reference) => reference.runId === supersededRunId); - expect(refusedReference).toBeDefined(); - expect(refusedReference != null && "afterBoundaryMessageId" in refusedReference).toBe(false); - expect( - await workspaceService.getWorkflowInvocationCurrentness(workspaceId, supersededRunId) - ).toBe("indeterminate"); - - // A reference that still carries its boundary may record a pre-supersession launch: - // repair must not refresh it into the current context. - await recordAgentWorkflowRunReference({ - workspaceSessionDir: config.getSessionDir(workspaceId), - runId: anchoredRunId, - createdAtMs: 1_050, - afterBoundaryMessageId: "older-row", - }); - await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, anchoredRunId); - const untouched = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); - expect( - untouched.find((reference) => reference.runId === anchoredRunId)?.afterBoundaryMessageId - ).toBe("older-row"); - expect( - await workspaceService.getWorkflowInvocationCurrentness(workspaceId, anchoredRunId) - ).toBe("not_current"); - - // Unknown run: nothing to repair, nothing recorded. - await workspaceService.repairWorkflowRunReferenceBoundary(workspaceId, "wfr_unknown"); - const after = await readAgentWorkflowRunReferences(config.getSessionDir(workspaceId)); - expect(after.map((reference) => reference.runId).sort()).toEqual([ - anchoredRunId, - strippedRunId, - supersededRunId, - ]); - workspaceService.disposeSession(workspaceId); - } finally { - await cleanup(); - } - }); - test("treats an unreadable history as indeterminate, not superseded", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-io-error"; From ecff903d0fc5ab3200bcfca664c5bc97b042b472 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:28:06 +0000 Subject: [PATCH 45/63] =?UTF-8?q?=F0=9F=A4=96=20refactor:=20clean=20up=20l?= =?UTF-8?q?eftovers=20from=20the=20wake=20reconciliation=20redesign?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix comments still describing the removed outbox/tombstone/retry-timer machinery, drop defensive branches the write-once generation markers make impossible, replace the compaction follow-up toolPolicy IIFE spread with a hoisted parse, deduplicate the pending-attention queue insert, and align workflow_resume's settlement helper naming (dropping a redundant Zod re-parse of an already-typed status). --- src/node/services/agentSession.ts | 30 ++++++------- .../services/agentWorkflowRunReferences.ts | 12 ++--- src/node/services/taskService.test.ts | 2 +- src/node/services/taskService.ts | 45 +++++++++---------- src/node/services/terminalAttentionStore.ts | 2 +- src/node/services/tools/toolUtils.ts | 2 +- src/node/services/tools/workflow_resume.ts | 17 +++---- src/node/services/workspaceService.test.ts | 11 ++--- src/node/services/workspaceService.ts | 13 +++--- 9 files changed, 66 insertions(+), 68 deletions(-) diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index e598d7dfbfc..1f8f58124a5 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -7244,6 +7244,18 @@ export class AgentSession { followUp.muxMetadata ); + // Same raw JSON boundary as below: a persisted follow-up may carry a malformed toolPolicy, + // and restoring it unvalidated would throw during resolution. Invalid values are dropped + // like any corrupt persisted policy (self-healing doctrine); a restricted turn's + // follow-up must otherwise keep its policy instead of redispatching allow-all. + const persistedToolPolicy = + followUp.toolPolicy != null ? ToolPolicySchema.safeParse(followUp.toolPolicy) : undefined; + if (persistedToolPolicy != null && !persistedToolPolicy.success) { + log.warn("Ignoring malformed persisted toolPolicy on compaction follow-up", { + workspaceId: this.workspaceId, + }); + } + // Build options for the follow-up message from the preserved send settings captured // when the compaction handoff was staged. Avoid forwarding internal-only recovery flags. const options: SendMessageOptions & { @@ -7265,23 +7277,7 @@ export class AgentSession { experiments: aliasLegacyPtcExclusive(followUp.experiments), allowAgentSetGoal: followUp.allowAgentSetGoal, disableWorkspaceAgents: followUp.disableWorkspaceAgents, - // Same raw JSON boundary: a persisted follow-up may carry a malformed toolPolicy, and - // restoring it unvalidated would throw during resolution. Invalid values are dropped - // like any corrupt persisted policy (self-healing doctrine); a restricted turn's - // follow-up must otherwise keep its policy instead of redispatching allow-all. - ...(() => { - if (followUp.toolPolicy == null) { - return {}; - } - const parsed = ToolPolicySchema.safeParse(followUp.toolPolicy); - if (!parsed.success) { - log.warn("Ignoring malformed persisted toolPolicy on compaction follow-up", { - workspaceId: this.workspaceId, - }); - return {}; - } - return { toolPolicy: parsed.data }; - })(), + ...(persistedToolPolicy?.success ? { toolPolicy: persistedToolPolicy.data } : {}), // Explicit-agent turns stay loud on the resumed turn too: the requested agent // may have been removed/hidden/disabled while compaction ran. strictAgentResolution: followUp.strictAgentResolution, diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index 6e2ff131194..9322f7dc0ef 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -92,9 +92,9 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { const hasBoundary = "afterBoundaryMessageId" in record; const boundaryRaw = record.afterBoundaryMessageId; // A present-but-invalid snapshot ("" or a non-string) is corruption, not a legacy record: - // demoting it to a boundaryless entry would misclassify a recorded boundary as unknowable - // provenance (parking its wake as indeterminate). Reject the entry; absence stays reserved - // for records that genuinely predate the field. + // demoting it to a boundaryless entry would misclassify a recorded boundary as legacy + // provenance (silently settling its wake as superseded). Reject the entry; absence stays + // reserved for records that genuinely predate the field. if ( hasBoundary && boundaryRaw !== null && @@ -157,9 +157,9 @@ export async function readAgentWorkflowRunReferences( } // For kernel-launched runs this file is the only durable invocation evidence, and callers // deciding wake delivery must distinguish "no reference" from "cannot know right now": - // flattening a transient read failure into [] would let the terminal drain tombstone the - // run's wake. Corrupted contents below stay self-healing because rereading cannot repair - // them, while a failed read can succeed later. + // flattening a transient read failure into [] would let the terminal drain settle the + // run's wake as superseded. Corrupted contents below stay self-healing because rereading + // cannot repair them, while a failed read can succeed later. throw error; } try { diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 2aed76f75d7..ee62e5a2c8b 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6867,7 +6867,7 @@ describe("TaskService", () => { }); }); - test("transient run-store read failures defer the wake instead of tombstoning it", async () => { + test("transient run-store read failures defer the wake instead of dropping it", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); const sendMessage = mock( diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 27707225a18..6f859a92359 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7809,16 +7809,10 @@ export class TaskService implements AgentTaskIntegration { workspace.id, TerminalAttentionStore.notificationId("workflow_run", run.id, run.updatedAt) ); - if (marker != null && marker.status !== "pending") { + if (marker != null) { continue; } - let runIds = this.pendingWorkflowRunAttention.get(workspace.id); - if (runIds == null) { - runIds = new Set(); - this.pendingWorkflowRunAttention.set(workspace.id, runIds); - } - if (!runIds.has(run.id)) { - runIds.add(run.id); + if (this.queueWorkflowRunAttention(workspace.id, run.id)) { queuedCount += 1; } queuedForWorkspace = true; @@ -7946,13 +7940,22 @@ export class TaskService implements AgentTaskIntegration { if (!isTerminalWorkflowRunStatus(params.status)) { return; } - let runIds = this.pendingWorkflowRunAttention.get(params.ownerWorkspaceId); + this.queueWorkflowRunAttention(params.ownerWorkspaceId, params.runId); + this.scheduleTerminalAttentionDrain(params.ownerWorkspaceId); + } + + /** Returns true when the run was newly queued for this owner. */ + private queueWorkflowRunAttention(ownerWorkspaceId: string, runId: string): boolean { + let runIds = this.pendingWorkflowRunAttention.get(ownerWorkspaceId); if (runIds == null) { runIds = new Set(); - this.pendingWorkflowRunAttention.set(params.ownerWorkspaceId, runIds); + this.pendingWorkflowRunAttention.set(ownerWorkspaceId, runIds); } - runIds.add(params.runId); - this.scheduleTerminalAttentionDrain(params.ownerWorkspaceId); + if (runIds.has(runId)) { + return false; + } + runIds.add(runId); + return true; } /** @@ -8317,8 +8320,7 @@ export class TaskService implements AgentTaskIntegration { // Currentness can succeed (e.g. a direct invocation row) and this identity read still // fail transiently. Delivering without the recorded identity would bind the wake to the // newest agent-bearing history row, handing the run's output to an unrelated later - // synthetic turn's agent; defer to the bounded retry instead, like an unreadable run - // record. + // synthetic turn's agent; defer like an unreadable run record. return { outcome: "defer" }; } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; @@ -8704,17 +8706,14 @@ export class TaskService implements AgentTaskIntegration { } // Deliver one launch-identity group per drain, keyed by agentId AND recorded strict pin: // the whole coalesced prompt is handled under the single agentId/pin passed to - // sendMessage, so batching runs from different agents (or runs sharing an agentId but - // launched under different pins, e.g. an agent definition replaced between synthetic - // turns) would hand a restricted launch's (attacker-influenced) output to another - // launch's tool grants. The same applies to mixed batches: workspace-turn and sub-agent - // attention resumes under the conversation's own (history-walk) identity, so agent-bound - // workflow groups never share their send. Deferred groups stay pending and deliver on the - // re-armed retry drain; among agent-bound groups the newest launch goes first. + // sendMessage, so batching runs from different launch identities would hand a restricted + // launch's (attacker-influenced) output to another launch's tool grants. Mixed batches + // too: workspace-turn and sub-agent attention resumes under the conversation's own + // (history-walk) identity, so agent-bound workflow groups never share their send. + // Unselected groups stay queued for a later drain; the newest launch goes first. // Suppression revalidation below can only shrink the workspace-turn set, so gating on // pre-suppression candidates over-approximates non-workflow deliverables: the safe - // direction, deferring agent-bound groups to the retry drain rather than ever mixing - // launch identities in one send. + // direction, deferring agent-bound groups rather than ever mixing identities in one send. const hasNonWorkflowDeliverables = deliverableAgentNotificationIds.size > 0 || workspaceTurnCandidates.length > 0; let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; diff --git a/src/node/services/terminalAttentionStore.ts b/src/node/services/terminalAttentionStore.ts index 61cb36a0d90..8fd76afda20 100644 --- a/src/node/services/terminalAttentionStore.ts +++ b/src/node/services/terminalAttentionStore.ts @@ -158,7 +158,7 @@ export class TerminalAttentionStore { notification.generationId ); const existing = await this.get(notification.ownerWorkspaceId, id); - if (existing != null && existing.status !== "pending") { + if (existing != null) { return; } await this.write({ diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index d842b9ab224..562b993b489 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -119,7 +119,7 @@ export async function recordBackgroundWorkflowRunReference( // failure must not be persisted as a verified-empty boundary (null): record without the // field instead, so the run stays rediscoverable (listAgentReferencedWorkflowRunIds) and a // later workflow_resume re-record can repair provenance, while the unverifiable boundary - // defers wake delivery (indeterminate) instead of guessing from wall-clock order. + // fails safe to a not-current wake instead of guessing from wall-clock order. let afterBoundaryMessageId: string | null | undefined; const taskService = config.taskService; if (config.workspaceId != null && taskService?.getWorkflowInvocationBoundaryMessageId != null) { diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index 4b1f8527cb4..1463c36d04e 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -4,7 +4,7 @@ import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; import { isTerminalWorkflowRunStatus, type WorkflowRunRecord } from "@/common/types/workflow"; import { getWorkflowCheckpointRetryEligibility } from "@/common/utils/workflowRetryEligibility"; -import { WorkflowRunRecordSchema, WorkflowRunStatusSchema } from "@/common/orpc/schemas"; +import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import { WorkflowResumeToolResultSchema, TOOL_DEFINITIONS, @@ -158,7 +158,7 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // workflow_resume part in history, so the history-walk consumption predicates cannot see // that this turn already received the terminal result. Persist consumption durably so the // terminal-attention drain never re-delivers it. - const markTerminalAttentionConsumed = async ( + const markTerminalAttentionSettled = async ( terminalRun: Pick ) => { if (!isTerminalWorkflowRunStatus(terminalRun.status)) { @@ -176,7 +176,7 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Idempotent success: the work is already done, so hand back the durable result instead // of failing the agent's recovery loop (e.g. resuming after a crash that actually finished). if (run.status === "completed" && mode === "resume") { - await markTerminalAttentionConsumed(run); + await markTerminalAttentionSettled(run); return parseToolResult( WorkflowResumeToolResultSchema, { @@ -263,11 +263,12 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // refreshed record that reflects the dispatched terminal status; when the refresh failed // or lags, skip the marker and the wake settles as consumed from the tool result in // history on the next scan (worst case one redundant wake for a kernel-nested resume). - if (!isBackgroundDispatch && refreshedRun != null) { - const dispatchedStatus = WorkflowRunStatusSchema.safeParse(dispatched.status); - if (dispatchedStatus.success && refreshedRun.status === dispatchedStatus.data) { - await markTerminalAttentionConsumed(refreshedRun); - } + if ( + !isBackgroundDispatch && + refreshedRun != null && + refreshedRun.status === dispatched.status + ) { + await markTerminalAttentionSettled(refreshedRun); } return parseToolResult( diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 32429cee4db..f9d1cb41c66 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9145,9 +9145,9 @@ describe("WorkspaceService workflow invocation events", () => { expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a - // crash between durable acceptance and the outbox delivery mark, this row is the only - // evidence the result already reached history; it must read as consumption or restart - // recovery injects the same terminal result again. + // crash between durable acceptance and the settled-marker write, this row is the only + // evidence the result already reached history; it must read as consumption or the next + // sweep injects the same terminal result again. await historyService.appendToHistory( workspaceId, createMuxMessage( @@ -9337,7 +9337,8 @@ describe("WorkspaceService workflow invocation events", () => { ); try { // The drain distinguishes a read failure (retain and retry) from supersession - // (tombstone); the boolean view stays fail-safe false for non-destructive callers. + // (settle as superseded); the boolean view stays fail-safe false for non-destructive + // callers. expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( "indeterminate" ); @@ -9405,7 +9406,7 @@ describe("WorkspaceService workflow invocation events", () => { // The sidecar is the only invocation evidence for kernel-launched runs: an unreadable // file must read as "cannot know right now", not "no reference", or the drain would - // tombstone the wake on a transient storage fault. + // settle the wake as superseded on a transient storage fault. const sidecarPath = path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json"); await fsPromises.rm(sidecarPath); await fsPromises.mkdir(sidecarPath); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3c43258f6cd..c2729a00721 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -487,9 +487,9 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) /** * The terminal-attention drain delivers workflow results as one synthetic user prompt that may * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands - * between the send's durable acceptance and the outbox delivery mark, restart recovery drains - * the notification again; recognizing the accepted row as consumption is what suppresses the - * replay. Only synthetic rows qualify: a manual user message is a supersession boundary and is + * between the send's durable acceptance and the settled-marker write, the next sweep re-queues + * the run; recognizing the accepted row as consumption is what settles it without a re-send. + * Only synthetic rows qualify: a manual user message is a supersession boundary and is * classified before this check runs. */ function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { @@ -10998,9 +10998,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { /** * Three-state currentness: "indeterminate" means history/provenance could not be read or - * ordered, so the answer is unknown rather than no. Callers that would permanently drop a - * terminal wake on a negative answer (the terminal-attention drain tombstones notifications) - * must retain and retry on "indeterminate" instead; boolean callers treat it as not-current, + * ordered, so the answer is unknown rather than no. Callers that would permanently settle a + * terminal wake on a negative answer (the terminal-attention drain records a superseded + * settlement marker) must retain and retry on "indeterminate" instead; boolean callers treat + * it as not-current, * the pre-existing fail-safe for non-destructive decisions. */ async getWorkflowInvocationCurrentness( From 0ad9f9beac0c55af7880859ea2621f132fd3d5ff Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:47:46 +0000 Subject: [PATCH 46/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20never=20let=20a=20d?= =?UTF-8?q?amaged=20settlement=20marker=20abort=20the=20attention=20sweep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TaskService.initialize() awaits the workflow terminal attention sweep, so an unreadable generation marker (EACCES/EIO/EISDIR) rejected the whole sweep and could block app startup. Catch marker read failures per run (skip the run; the next sweep retries) and wrap the startup sweep call. Regression test verified red-green against the unguarded read. --- src/node/services/taskService.test.ts | 58 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 29 +++++++++++--- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index ee62e5a2c8b..1a3b90b6b9d 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6243,6 +6243,64 @@ describe("TaskService", () => { expect(sendMessage).toHaveBeenCalledTimes(2); }); + test("an unreadable settlement marker skips only that run and never rejects the sweep", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + for (const runId of ["wfr_sweep_marker_a", "wfr_sweep_marker_b"]) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + } + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Keep the queue observable: indeterminate currentness defers every drain delivery. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + }; + const realGet = internal.terminalAttentionStore.get.bind(internal.terminalAttentionStore); + const getSpy = spyOn(internal.terminalAttentionStore, "get") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's unhandled-rejection + // detector on this host before the sweep consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("EACCES: marker unreadable"))) + .mockImplementation(realGet); + + try { + // Startup awaits this sweep: one damaged marker must skip its run, not abort the sweep. + expect(await internal.sweepWorkflowRunTerminalAttention()).toBe(1); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.size).toBe(1); + + // The skipped run is re-derived once the marker read recovers. + expect(await internal.sweepWorkflowRunTerminalAttention()).toBe(1); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.size).toBe(2); + } finally { + getSpy.mockRestore(); + } + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 6f859a92359..a7c64385087 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -3201,7 +3201,13 @@ export class TaskService implements AgentTaskIntegration { log.error("Startup workflow task archive sweep failed", { error }); } - const queuedTerminalWorkflowRunAttentionCount = await this.sweepWorkflowRunTerminalAttention(); + let queuedTerminalWorkflowRunAttentionCount = 0; + try { + queuedTerminalWorkflowRunAttentionCount = await this.sweepWorkflowRunTerminalAttention(); + } catch (error: unknown) { + // Startup-time initialization must never crash the app; the interval sweep retries. + log.warn("Startup workflow terminal attention sweep failed", { error }); + } if (this.workflowAttentionSweepTimer == null) { this.workflowAttentionSweepTimer = setInterval(() => { void this.sweepWorkflowRunTerminalAttention().catch((error: unknown) => { @@ -7805,10 +7811,23 @@ export class TaskService implements AgentTaskIntegration { ) { continue; } - const marker = await this.terminalAttentionStore.get( - workspace.id, - TerminalAttentionStore.notificationId("workflow_run", run.id, run.updatedAt) - ); + let marker: Awaited>; + try { + marker = await this.terminalAttentionStore.get( + workspace.id, + TerminalAttentionStore.notificationId("workflow_run", run.id, run.updatedAt) + ); + } catch (error: unknown) { + // Startup awaits this sweep, so one unreadable marker must not abort it (or app + // init). Skip the run: an unreadable marker cannot prove the wake is owed, and + // the next sweep retries, so delivery is delayed, never crashed or duplicated. + log.warn("Failed to read workflow terminal settlement marker; skipping run", { + workspaceId: workspace.id, + runId: run.id, + error: getErrorMessage(error), + }); + continue; + } if (marker != null) { continue; } From 7fe39e57a012ce42d70de2971c2835d9ee5a11f2 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:22:46 +0000 Subject: [PATCH 47/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20retire=20kernel=20w?= =?UTF-8?q?orkflow=20references=20before=20a=20full=20clear=20commits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full history clear retired the agent-workflow-runs sidecar only after the truncation committed, so a failed deletion left a readable null-boundary reference against the emptied transcript; the next terminal callback or sweep would then read it as current and inject the pre-clear run's output into the fresh conversation. Retire the references first: a failed retirement now aborts the clear with the conversation intact, and a failed truncation only drops wakes (still retrievable via resume). Ordering verified red-green. --- src/node/services/workspaceService.test.ts | 76 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 39 ++++++----- 2 files changed, 99 insertions(+), 16 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index f9d1cb41c66..876ce415170 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9089,6 +9089,82 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a failed reference retirement aborts the clear before truncation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-abort"; + const runId = "wfr_currentness_retire_abort"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-abort", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + + // Retirement failing after a committed truncation would leave the null-snapshot + // reference reading current against the emptied history (pre-clear output injected + // into the fresh conversation), so the clear must abort with the transcript intact. + const truncateSpy = spyOn(historyService, "truncateHistory"); + const internal = workspaceService as unknown as { + retireKernelWorkflowRunReferences(id: string): Promise; + }; + const retireSpy = spyOn(internal, "retireKernelWorkflowRunReferences") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's + // unhandled-rejection detector on this host before the clear consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("read-only session storage"))); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + if (!clearResult.success) { + expect(clearResult.error).toContain("could not be retired"); + } + expect(truncateSpy).not.toHaveBeenCalled(); + } finally { + retireSpy.mockRestore(); + truncateSpy.mockRestore(); + } + + // A retry once storage recovers clears normally and retires the sidecar. + const retryResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(retryResult.success).toBe(true); + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c2729a00721..90947c191ec 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -11141,6 +11141,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { : { status: "none" }; } + /** Testable seam for the pre-truncation retirement in truncateHistory. */ + private async retireKernelWorkflowRunReferences(workspaceId: string): Promise { + await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); + } + /** * Boundary snapshot for the agent-workflow-runs sidecar: the message ID of the newest * invocation-decision row for this run, or null when history has none. Recorded at @@ -12888,6 +12893,24 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + // Kernel workflow run references belong to the cleared conversation: a verified-empty + // (null) boundary snapshot recorded before the clear is indistinguishable from one + // recorded after it, so a surviving reference could inject a pre-clear workflow result + // into the fresh conversation. Retire them BEFORE the truncation commits so both fault + // directions fail safe: a failed retirement aborts with the conversation intact, and a + // failed truncation leaves reference-less runs settling superseded (dropped wake, still + // retrievable via resume) rather than a committed clear racing a live null-boundary + // reference it could no longer delete. A post-clear resume re-records provenance. + if (isFullClear) { + try { + await this.retireKernelWorkflowRunReferences(workspaceId); + } catch (error) { + return Err( + `Cannot clear history: stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). Retry once the session storage is writable.` + ); + } + } if (effectivePercentage > 0) { session?.clearUsageState(); } @@ -12906,22 +12929,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // admitted afterwards (their content references the discarded context). if (isFullClear) { this.advanceContextMutationEpoch(workspaceId); - // Kernel workflow run references belong to the cleared conversation: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one - // recorded after it, so a surviving reference could inject a pre-clear workflow result - // into the fresh conversation. Retire them immediately after the truncation commits, - // before any later post-clear step that can fail and return early (goal acknowledgment, - // carryover discard), or the stale reference would survive the committed clear. A - // post-clear resume re-records provenance. - try { - await clearAgentWorkflowRunReferences(this.config.getSessionDir(workspaceId)); - } catch (error) { - return Err( - `History was cleared, but stale workflow run references could not be retired ` + - `(${getErrorMessage(error)}). A finished background workflow may re-inject its ` + - `result into the cleared conversation; retry once the session storage is writable.` - ); - } } // r43: a fork's settled branch-summary registration stays consumable // until the first send; its row was just deleted, so drop the From 75d5b64a21a07aa69df1c40bd72337423648bb4c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:43:15 +0000 Subject: [PATCH 48/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20re-poke=20terminal?= =?UTF-8?q?=20attention=20drain=20when=20suppression=20empties=20the=20bat?= =?UTF-8?q?ch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workspace-turn candidate can exclude agent-bound workflow groups from a drain batch and then be superseded during the last-moment reread, leaving the batch empty. Nothing is sent, so no streamEnded drain follows and the queued workflow wake waited up to five minutes for the periodic sweep. Schedule another drain in that branch when deliverable workflow prompts remain. The suppressed notifications were just durably marked superseded, so the re-drain selects the workflow group without spinning. --- src/node/services/taskService.test.ts | 85 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 7 +++ 2 files changed, 92 insertions(+) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 1a3b90b6b9d..c0e0fddf774 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6853,6 +6853,91 @@ describe("TaskService", () => { expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); }); + test("a fully suppressed batch re-pokes the drain for unselected workflow groups", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_repoke"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "exec", + }); + + // A pending workspace-turn notification whose handle already carries an owner-follow-up + // supersede: the pre-suppression batch counts it (excluding the agent-bound workflow + // group from the send), then the last-moment reread drops it, emptying the batch. + const taskHandleStore = new TaskHandleStore(config); + await taskHandleStore.upsertWorkspaceTurn({ + kind: "workspace_turn", + handleId: "wst_repoke_suppressed", + ownerWorkspaceId: parentId, + workspaceId: parentId, + turnId: "repoke-suppressed", + status: "interrupted", + error: + "Workspace turn superseded by follow-up turn wst_repoke_successor from the same owner workspace", + createdAt: "2026-08-11T00:00:00.000Z", + updatedAt: "2026-08-11T00:00:01.000Z", + createdWorkspace: false, + disposableWorkspace: false, + }); + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_repoke_suppressed", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + // The empty suppressed batch must re-poke the drain, not park the wake on the sweep. + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const prompt = String(sendMessage.mock.calls[0]?.[1]); + expect(prompt).toContain(runId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + test("wake keeps a synthetic launch row's strict pin without lifting the manual policy", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index a7c64385087..7dd031780f1 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8854,6 +8854,13 @@ export class TaskService implements AgentTaskIntegration { }); if (effectivePending.length === 0 && selectedWorkflowPrompts.length === 0) { await markSuppressedSuperseded(); + // Suppression can empty the very batch that excluded agent-bound workflow groups; with + // nothing sent there is no streamEnded drain, so re-poke instead of parking the queued + // wake on the sweep. No spin: the suppressed notifications were just durably marked + // superseded, so the re-drain sees no non-workflow deliverables and selects a group. + if (deliverableWorkflowPrompts.length > 0) { + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + } return; } From b2bf4c315e93449df580d62569b93a1c88c32e01 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:14:16 +0000 Subject: [PATCH 49/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20terminal?= =?UTF-8?q?=20settlement=20markers=20and=20emptying=20truncations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings on the level-triggered reconciliation: - A sub-100% truncation whose token budget removes every message takes historyService's full-delete fast path but skipped the full-clear guards, so a kernel workflow reference with a verified-empty boundary snapshot could wake a pre-truncation result into the cleared conversation. A read-only emptiness preflight now routes emptying truncations through the full-clear path (admission guard, refine drain, sandbox discard, reference retirement); preflight failures count as emptying so the guarded path fails safe. - Settlement marker I/O is contained inside markWorkflowRunTerminalAttentionSettled per its documented best-effort contract: workflow_resume and task_await return the run's durable result even when the marker write fails, the drain keeps draining, and the retained queue entry re-attempts the marker. - TerminalAttentionStore.recordSettled no longer mkdirs a removed owner's session directory back into existence when a drain's settlement write races workspace removal; the re-derivable marker is dropped instead. Pending-notification enqueues keep directory creation (live-owner event paths). Two truncation tests relied on 50% of a one-message transcript acting as a partial truncation; that fixture actually empties history, so they now use two messages to stay genuinely partial. --- src/node/services/historyService.ts | 77 ++++++++++++------- src/node/services/taskService.test.ts | 38 +++++++++ src/node/services/taskService.ts | 57 +++++++------- .../services/terminalAttentionStore.test.ts | 28 +++++++ src/node/services/terminalAttentionStore.ts | 56 ++++++++++---- src/node/services/workspaceService.test.ts | 68 ++++++++++++++++ src/node/services/workspaceService.ts | 22 +++++- 7 files changed, 279 insertions(+), 67 deletions(-) diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index dc4f212051d..6ba35659166 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2688,6 +2688,56 @@ export class HistoryService { * @param percentage Percentage to truncate (0.0 to 1.0). 1.0 = delete all * @returns Result containing array of deleted historySequence numbers */ + /** + * Token-proportional prefix length that truncateHistory removes at this percentage. Messages + * are stringified whole for counting; only relative weights matter. + */ + private async computeTruncationRemoveCount( + messages: MuxMessage[], + percentage: number + ): Promise { + const tokenizer = await getTokenizerForModel(KNOWN_MODELS.SONNET.id); + const messageTokens = await Promise.all( + messages.map((msg) => tokenizer.countTokens(safeStringifyForCounting(msg))) + ); + const totalTokens = messageTokens.reduce((sum, tokens) => sum + tokens, 0); + const tokensToRemove = Math.floor(totalTokens * percentage); + let tokensRemoved = 0; + let removeCount = 0; + for (const tokens of messageTokens) { + if (tokensRemoved >= tokensToRemove) { + break; + } + tokensRemoved += tokens; + removeCount++; + } + return removeCount; + } + + /** + * Preflight for truncateHistory: whether this percentage takes the full-delete fast path and + * removes every message. The requested percentage alone cannot distinguish an emptying + * truncation from a partial one, and callers that must apply full-clear semantics before the + * rewrite commits (workspaceService retires kernel workflow run references) need the answer + * up front. + */ + async willTruncateHistoryRemoveAllMessages( + workspaceId: string, + percentage: number + ): Promise { + if (percentage >= 1.0) { + return true; + } + const archivedMessages = await this.readArchivedHistory(workspaceId); + const chatMessages = await this.readChatHistory(workspaceId); + const messages = [...archivedMessages, ...chatMessages]; + if (messages.length === 0) { + return false; + } + const removeCount = await this.computeTruncationRemoveCount(messages, percentage); + return removeCount >= messages.length; + } + async truncateHistory( workspaceId: string, percentage: number @@ -2717,32 +2767,7 @@ export class HistoryService { return Ok([]); // Nothing to truncate } - // Get tokenizer for counting (use a default model) - const tokenizer = await getTokenizerForModel(KNOWN_MODELS.SONNET.id); - - // Count tokens for each message - // We stringify the entire message for simplicity - only relative weights matter - const messageTokens: Array<{ message: MuxMessage; tokens: number }> = await Promise.all( - messages.map(async (msg) => { - const tokens = await tokenizer.countTokens(safeStringifyForCounting(msg)); - return { message: msg, tokens }; - }) - ); - - // Calculate total tokens and target to remove - const totalTokens = messageTokens.reduce((sum, mt) => sum + mt.tokens, 0); - const tokensToRemove = Math.floor(totalTokens * percentage); - - // Remove messages from beginning until we've removed enough tokens - let tokensRemoved = 0; - let removeCount = 0; - for (const mt of messageTokens) { - if (tokensRemoved >= tokensToRemove) { - break; - } - tokensRemoved += mt.tokens; - removeCount++; - } + const removeCount = await this.computeTruncationRemoveCount(messages, percentage); // No-op truncation (percentage 0 or rounding to zero tokens) must not // rewrite anything — collapsing the archive back into chat.jsonl would diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index c6c582e290f..4be81276022 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6186,6 +6186,44 @@ describe("TaskService", () => { expect(sendMessage).not.toHaveBeenCalled(); }); + test("a failed settlement marker write never rejects and keeps the queue entry", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const { taskService } = createTaskServiceHarness(config); + const internal = taskService as unknown as { + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + }; + internal.pendingWorkflowRunAttention.set(parentId, new Set(["wfr_marker_soft_fail"])); + const settleSpy = spyOn(internal.terminalAttentionStore, "recordSettled") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's unhandled-rejection + // detector on this host before the call consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("EACCES: marker dir unwritable"))); + + const settleParams = { + ownerWorkspaceId: parentId, + runId: "wfr_marker_soft_fail", + status: "completed" as const, + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered" as const, + }; + try { + // Marker I/O must stay contained (workflow_resume/task_await return durable results + // through this call), and the queue entry must survive so the next drain re-attempts. + await taskService.markWorkflowRunTerminalAttentionSettled(settleParams); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has("wfr_marker_soft_fail")).toBe( + true + ); + + await taskService.markWorkflowRunTerminalAttentionSettled(settleParams); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has("wfr_marker_soft_fail")).toBe( + false + ); + } finally { + settleSpy.mockRestore(); + } + }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 05b2acbcc31..665089008e2 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7855,14 +7855,28 @@ export class TaskService implements AgentTaskIntegration { if (!isTerminalWorkflowRunStatus(params.status)) { return; } - await this.terminalAttentionStore.recordSettled({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - sourceId: params.runId, - generationId: params.runUpdatedAt, - terminalOutcome: terminalAttentionOutcome(params.status), - status: params.settledAs, - }); + try { + await this.terminalAttentionStore.recordSettled({ + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + sourceId: params.runId, + generationId: params.runUpdatedAt, + terminalOutcome: terminalAttentionOutcome(params.status), + status: params.settledAs, + }); + } catch (error: unknown) { + // Contain marker I/O here so no caller fails on bookkeeping: workflow_resume and + // task_await must still return the run's durable result, and the drain must move on to + // its other candidates. Keep the queue entry so the next drain re-evaluates from run + + // history evidence and re-attempts the marker; at worst a truthful wake re-delivers. + log.warn("Failed to record workflow terminal settlement marker", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + settledAs: params.settledAs, + error, + }); + return; + } this.pendingWorkflowRunAttention.get(params.ownerWorkspaceId)?.delete(params.runId); } @@ -8714,24 +8728,15 @@ export class TaskService implements AgentTaskIntegration { await this.terminalAttentionStore.markDelivered(ownerWorkspaceId, notification.id); } for (const candidate of selectedWorkflowPrompts) { - try { - await this.markWorkflowRunTerminalAttentionSettled({ - ownerWorkspaceId, - runId: candidate.runId, - status: candidate.run.status, - runUpdatedAt: candidate.run.updatedAt, - settledAs: "delivered", - }); - } catch (error: unknown) { - // Best-effort: the delivered wake itself is durable history evidence, so the next - // evaluation settles this run as consumed and re-attempts the marker. - log.warn("Failed to record delivered workflow wake marker", { - ownerWorkspaceId, - runId: candidate.runId, - error, - }); - this.pendingWorkflowRunAttention.get(ownerWorkspaceId)?.delete(candidate.runId); - } + // Marker failures are contained inside the settle method; the delivered wake itself is + // durable history evidence, so the next evaluation settles this run as consumed. + await this.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId, + runId: candidate.runId, + status: candidate.run.status, + runUpdatedAt: candidate.run.updatedAt, + settledAs: "delivered", + }); } }; diff --git a/src/node/services/terminalAttentionStore.test.ts b/src/node/services/terminalAttentionStore.test.ts index 9009e4be9b1..825cf26aa56 100644 --- a/src/node/services/terminalAttentionStore.test.ts +++ b/src/node/services/terminalAttentionStore.test.ts @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import * as fsPromises from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; @@ -57,6 +58,33 @@ describe("TerminalAttentionStore", () => { expect(pending.map((n) => n.sourceId)).toEqual(["wst_abc"]); }); + test("recordSettled never recreates a removed owner session dir", async () => { + const config = makeConfig(rootDir); + const store = new TerminalAttentionStore(config); + const settled = { + sourceKind: "workflow_run" as const, + sourceId: "wfr_removed", + generationId: "2026-06-19T00:00:03.000Z", + terminalOutcome: "completed" as const, + status: "superseded" as const, + }; + + // No session dir: a workspace removal racing an in-flight drain settlement. The marker is + // re-derivable dedupe, so it must be dropped rather than resurrecting orphaned session + // state. + await store.recordSettled({ ...settled, ownerWorkspaceId: "owner-removed" }); + expect(existsSync(config.getSessionDir("owner-removed"))).toBe(false); + + // A live owner still gets the terminal-attention subdir created and the marker written. + await fsPromises.mkdir(config.getSessionDir("owner-live"), { recursive: true }); + await store.recordSettled({ ...settled, ownerWorkspaceId: "owner-live" }); + const record = await store.get( + "owner-live", + TerminalAttentionStore.notificationId("workflow_run", "wfr_removed", settled.generationId) + ); + expect(record?.status).toBe("superseded"); + }); + test("loads pending notifications written with legacy derived fields", async () => { const config = makeConfig(rootDir); const dir = path.join(config.getSessionDir("owner-1"), TERMINAL_ATTENTION_DIR); diff --git a/src/node/services/terminalAttentionStore.ts b/src/node/services/terminalAttentionStore.ts index 8fd76afda20..fe3c8725862 100644 --- a/src/node/services/terminalAttentionStore.ts +++ b/src/node/services/terminalAttentionStore.ts @@ -161,18 +161,24 @@ export class TerminalAttentionStore { if (existing != null) { return; } - await this.write({ - id, - ownerWorkspaceId: notification.ownerWorkspaceId, - sourceKind: notification.sourceKind, - sourceId: notification.sourceId, - generationId: notification.generationId, - outputDelivery: outputDeliveryForSource(notification.sourceKind), - terminalOutcome: notification.terminalOutcome, - status: notification.status, - createdAt: new Date().toISOString(), - ...(notification.status === "delivered" ? { deliveredAt: new Date().toISOString() } : {}), - }); + await this.write( + { + id, + ownerWorkspaceId: notification.ownerWorkspaceId, + sourceKind: notification.sourceKind, + sourceId: notification.sourceId, + generationId: notification.generationId, + outputDelivery: outputDeliveryForSource(notification.sourceKind), + terminalOutcome: notification.terminalOutcome, + status: notification.status, + createdAt: new Date().toISOString(), + ...(notification.status === "delivered" ? { deliveredAt: new Date().toISOString() } : {}), + }, + // Settlement markers are re-derivable dedupe over run + history evidence, and reconcilers + // write them while workspace removal may be deleting the session directory: never mkdir + // the owner dir back into existence for one (orphaned session state); drop it instead. + { createOwnerDir: false } + ); } async get(ownerWorkspaceId: string, id: string): Promise { @@ -267,9 +273,31 @@ export class TerminalAttentionStore { }); } - private async write(record: TerminalAttentionNotification): Promise { + private async write( + record: TerminalAttentionNotification, + options?: { createOwnerDir?: boolean } + ): Promise { const dir = this.dir(record.ownerWorkspaceId); - await fsPromises.mkdir(dir, { recursive: true }); + if (options?.createOwnerDir === false) { + try { + // Non-recursive: creates only the terminal-attention subdir under an owner session dir + // that still exists; a missing parent means the owner was removed. + await fsPromises.mkdir(dir); + } catch (error) { + if (isErrnoWithCode(error, "ENOENT")) { + log.debug("Dropping terminal attention write for removed owner session dir", { + ownerWorkspaceId: record.ownerWorkspaceId, + id: record.id, + }); + return; + } + if (!isErrnoWithCode(error, "EEXIST")) { + throw error; + } + } + } else { + await fsPromises.mkdir(dir, { recursive: true }); + } await fsPromises.writeFile( this.file(record.ownerWorkspaceId, record.id), JSON.stringify(record, null, 2), diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 876ce415170..66500829f50 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9165,6 +9165,62 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a partial truncation that empties history retires kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-partial-empty"; + const runId = "wfr_currentness_partial_empty"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-partial-empty", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // Half of a single-message transcript crosses the whole-history removal budget, so the + // token-proportional truncation takes historyService's full-delete fast path. The + // emptied transcript must retire the null-snapshot reference exactly like an explicit + // clear, or the pre-truncation workflow result would read current against the emptied + // decision-free history. + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(truncateResult.success).toBe(true); + expect(await historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual(Ok([])); + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; @@ -10793,10 +10849,16 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { projectPath: "/tmp/full-clear-sandbox-project", runtimeConfig: { type: "local" }, }); + // Two similar-size messages: 50% removes only the first, keeping the truncation genuinely + // partial (a one-message 50% empties history and routes as a full clear). await historyService.appendToHistory( workspaceId, createMuxMessage("pre-clear-user", "user", "before clear", {}) ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("pre-clear-user-b", "user", "still here", {}) + ); const discardSpy = spyOn(sandboxHostService, "discardScope").mockImplementation(() => Promise.resolve() ); @@ -10847,10 +10909,16 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { projectPath: "/tmp/clear-drains-refine-project", runtimeConfig: { type: "local" }, }); + // Two similar-size messages keep the 50% truncation genuinely partial (see the sandbox + // discard test above). await historyService.appendToHistory( workspaceId, createMuxMessage("pre-clear-user", "user", "before clear", {}) ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("pre-clear-user-b", "user", "still here", {}) + ); const drained: string[] = []; workspaceService.setRefinePassCanceller({ cancelInFlightRefinePass: (id) => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 90947c191ec..7b215d538e1 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12825,7 +12825,27 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async truncateHistory(workspaceId: string, percentage?: number): Promise> { const effectivePercentage = percentage ?? 1.0; - const isFullClear = effectivePercentage >= 1.0; + // A token-proportional truncation below 100% still empties the transcript when the removal + // budget reaches the final message (historyService's full-delete fast path), and an emptied + // transcript carries every full-clear hazard: most critically, a kernel workflow reference + // with a verified-empty (null) boundary snapshot reads decision-free history as current, so + // a surviving reference could wake a pre-truncation workflow result into the cleared + // conversation. Decide up front and route emptying truncations through the full-clear path + // (admission guard, refine drain, reference retirement). A preflight read failure counts as + // emptying: the guarded path fails safe (references retired first, wakes dropped but + // resumable) even if the truncation itself later fails. + const isFullClear = + effectivePercentage >= 1.0 || + (effectivePercentage > 0 && + (await this.historyService + .willTruncateHistoryRemoveAllMessages(workspaceId, effectivePercentage) + .catch((error: unknown) => { + log.warn("History truncation emptiness preflight failed; treating as full clear", { + workspaceId, + error, + }); + return true; + }))); // A full clear holds the admission guard across the refine drain/lock // awaits below: without it, a send admitted during those awaits could // snapshot the pre-clear transcript and stream across the truncation, From 631e0824b00356bbd8a06ce5406e8457b282dd7d Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:39:31 +0000 Subject: [PATCH 50/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20revalidate=20emptin?= =?UTF-8?q?ess=20and=20currentness=20at=20their=20commit=20points?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on the truncation preflight and the drain: - The emptiness preflight reads history outside the write lock, so two overlapping partial truncations could both classify as non-emptying while their serialized rewrites empty history, skipping the full-clear guards. historyService.truncateHistory now takes refuseFullDelete and refuses, under the lock, a partial-classified request whose recomputed budget removes every message; workspaceService passes refuseFullDelete for non-full-clear requests, and a retry re-runs the preflight and routes through the full-clear path. - A full history clear completing between drain classification and dispatch could not retract already-materialized workflow candidates, so the requireIdle send would inject a pre-clear result into the cleared conversation. Selected workflow candidates now get the same last-moment revalidation as the workspace-turn batch: currentness is reread in the same parallel batch directly before sendMessage, not_current settles the generation superseded, indeterminate stays queued. The empty-batch re-poke fires only after a durable state change so an all-indeterminate batch parks for the sweep instead of spinning. --- src/node/services/historyService.test.ts | 20 ++++++ src/node/services/historyService.ts | 14 ++++- src/node/services/taskService.test.ts | 73 ++++++++++++++++++++++ src/node/services/taskService.ts | 65 ++++++++++++++----- src/node/services/workspaceService.test.ts | 70 +++++++++++++++++++++ src/node/services/workspaceService.ts | 8 ++- 6 files changed, 233 insertions(+), 17 deletions(-) diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 0ab1bbec8dd..7aeb4a4b8b9 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2321,6 +2321,26 @@ describe("HistoryService", () => { expect(await fs.readFile(archivePath(wsId), "utf-8")).toBe(archiveBefore); }); + it("refuseFullDelete refuses a partial truncation that would remove every message", async () => { + await appendNumberedMessages(service, wsId, 1); + const chatBefore = await fs.readFile(chatPath(wsId), "utf-8"); + + // The caller classified this request as non-emptying; the locked recomputation says it + // empties (an overlapping truncation shrank history in between). Refuse instead of + // taking the full-delete fast path without the caller's full-clear guards. + const refused = await service.truncateHistory(wsId, 0.9, { refuseFullDelete: true }); + expect(refused.success).toBe(false); + if (!refused.success) { + expect(refused.error).toContain("full clear"); + } + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + + // Without the guard the same request takes the fast path and empties history. + const emptied = await service.truncateHistory(wsId, 0.9); + expect(emptied.success).toBe(true); + expect(await service.getHistoryFromLatestBoundary(wsId)).toEqual({ success: true, data: [] }); + }); + it("does not reseed usage from before a partial prefix truncation", async () => { await appendNumberedMessages(service, wsId, 8); await service.appendToHistory( diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index f3caae228f9..0010c56b509 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2994,7 +2994,8 @@ export class HistoryService { async truncateHistory( workspaceId: string, - percentage: number + percentage: number, + options?: { refuseFullDelete?: boolean } ): Promise> { return this.withRecoveredHistoryWriteResultLock( workspaceId, @@ -3032,6 +3033,17 @@ export class HistoryService { // If we're removing all messages, use fast path if (removeCount >= messages.length) { + // Serialized revalidation of the caller's emptiness preflight: an overlapping + // truncation can shrink history between that unserialized read and this locked + // rewrite, turning a partial-classified request into a full delete that skipped + // the caller's full-clear guards (most critically kernel workflow reference + // retirement). Refuse instead of emptying; a retry re-runs the preflight against + // the settled history and routes through the full-clear path. + if (options?.refuseFullDelete === true) { + return Err( + "Truncation would remove every remaining message; retry to run it as a full clear." + ); + } await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4be81276022..b0829eefa75 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6224,6 +6224,79 @@ describe("TaskService", () => { } }); + test("a history clear between classification and dispatch settles the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_clear_race"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Classification sees a current invocation; a full clear then retires the sidecar before + // the batch reaches sendMessage, so the last-moment reread must see not_current. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("not_current")).mockImplementationOnce(() => + Promise.resolve("current") + ); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + // The pre-clear result must not wake the freshly cleared conversation; the run settles + // superseded for this terminal generation and stays retrievable via workflow_resume. + expect(sendMessage).not.toHaveBeenCalled(); + const terminalAttentionStore = new TerminalAttentionStore(config); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("superseded"); + expect( + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(runId) ?? false + ).toBe(false); + }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 04591543216..5db0c541977 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8561,7 +8561,6 @@ export class TaskService implements AgentTaskIntegration { initiatingAgent?: WorkflowWakeInitiatingAgent; }> = []; - const workflowPromptSections: string[] = []; for (const runId of queuedWorkflowRunIds) { const workflowPrompt = await this.buildWorkflowTerminalPrompt(ownerWorkspaceId, runId); if (workflowPrompt.outcome === "defer") { @@ -8637,9 +8636,6 @@ export class TaskService implements AgentTaskIntegration { ); // Unselected groups stay queued: the selected group's wake turn ends with a streamEnded // drain (and the sweep backstops an aborted one), which delivers the next group. - for (const candidate of selectedWorkflowPrompts) { - workflowPromptSections.push(candidate.prompt); - } const resumeOptions = await this.resolveParentAutoResumeOptions( ownerWorkspaceId, @@ -8693,11 +8689,37 @@ export class TaskService implements AgentTaskIntegration { // closing it would require holding settlement locks across delivery, // which the drain must not do; worst case is one redundant wake (fail // toward notify, never a lost wake). - const candidateRecords = await Promise.all( - workspaceTurnCandidates.map((candidate) => - this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, candidate.notification.sourceId) - ) - ); + const [candidateRecords, selectedWorkflowCurrentness] = await Promise.all([ + Promise.all( + workspaceTurnCandidates.map((candidate) => + this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, candidate.notification.sourceId) + ) + ), + // Workflow candidates get the same last-moment treatment: a full history clear that + // completes after buildWorkflowTerminalPrompt classified these runs retires the sidecar + // but cannot retract the materialized candidates, and once the clear releases its + // admission guard the owner is idle again, so this requireIdle send would inject a + // pre-clear workflow result into the freshly cleared conversation. The clear retires + // references before truncating, so this reread sees not_current and settles; unreadable + // state stays queued for the next drain or sweep instead of settling. + Promise.all( + selectedWorkflowPrompts.map((candidate) => + this.workspaceService + .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) + .catch(() => "indeterminate" as const) + ) + ), + ]); + const currentWorkflowPrompts: typeof selectedWorkflowPrompts = []; + const supersededWorkflowPrompts: typeof selectedWorkflowPrompts = []; + selectedWorkflowPrompts.forEach((candidate, index) => { + const currentness = selectedWorkflowCurrentness[index]; + if (currentness === "current") { + currentWorkflowPrompts.push(candidate); + } else if (currentness === "not_current") { + supersededWorkflowPrompts.push(candidate); + } + }); const deliverableWorkspaceTurnNotificationIds = new Set(); const publicAwaitIds: string[] = []; const suppressedNotificationIds: string[] = []; @@ -8714,6 +8736,15 @@ export class TaskService implements AgentTaskIntegration { for (const id of suppressedNotificationIds) { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, id); } + for (const candidate of supersededWorkflowPrompts) { + await this.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId, + runId: candidate.runId, + status: candidate.run.status, + runUpdatedAt: candidate.run.updatedAt, + settledAs: "superseded", + }); + } }; // Sub-agent reports and failures are already durable user-context messages. Resume from history @@ -8722,7 +8753,7 @@ export class TaskService implements AgentTaskIntegration { if (publicAwaitIds.length > 0) { promptSections.push(buildCompletedWorkspaceTurnPrompt(publicAwaitIds)); } - promptSections.push(...workflowPromptSections); + promptSections.push(...currentWorkflowPrompts.map((candidate) => candidate.prompt)); const prompt = promptSections.join("\n\n"); const effectivePending = pending.filter((notification) => { if (notification.sourceKind === "agent_task") { @@ -8730,13 +8761,17 @@ export class TaskService implements AgentTaskIntegration { } return deliverableWorkspaceTurnNotificationIds.has(notification.id); }); - if (effectivePending.length === 0 && selectedWorkflowPrompts.length === 0) { + if (effectivePending.length === 0 && currentWorkflowPrompts.length === 0) { await markSuppressedSuperseded(); // Suppression can empty the very batch that excluded agent-bound workflow groups; with // nothing sent there is no streamEnded drain, so re-poke instead of parking the queued - // wake on the sweep. No spin: the suppressed notifications were just durably marked - // superseded, so the re-drain sees no non-workflow deliverables and selects a group. - if (deliverableWorkflowPrompts.length > 0) { + // wake on the sweep. No spin: re-poke only when this drain durably changed state (a + // suppressed turn or superseded workflow was just marked), so the re-drain sees a + // different candidate set; an all-indeterminate batch parks for the sweep instead. + if ( + deliverableWorkflowPrompts.length > 0 && + (suppressedNotificationIds.length > 0 || supersededWorkflowPrompts.length > 0) + ) { this.scheduleTerminalAttentionDrain(ownerWorkspaceId); } return; @@ -8746,7 +8781,7 @@ export class TaskService implements AgentTaskIntegration { for (const notification of effectivePending) { await this.terminalAttentionStore.markDelivered(ownerWorkspaceId, notification.id); } - for (const candidate of selectedWorkflowPrompts) { + for (const candidate of currentWorkflowPrompts) { // Marker failures are contained inside the settle method; the delivered wake itself is // durable history evidence, so the next evaluation settles this run as consumed. await this.markWorkflowRunTerminalAttentionSettled({ diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 66500829f50..eab77c107af 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9221,6 +9221,76 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("an overlapping truncation that would empty history is refused, not silently cleared", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-preflight-race"; + const runId = "wfr_currentness_preflight_race"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-preflight-race", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // Simulate the preflight racing an overlapping truncation: it classifies this request + // as non-emptying, but the locked rewrite's own recomputation would empty history. The + // serialized revalidation must refuse rather than skip the full-clear guards. + const preflightSpy = spyOn( + historyService, + "willTruncateHistoryRemoveAllMessages" + ).mockImplementationOnce(() => Promise.resolve(false)); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("full clear"); + } + } finally { + preflightSpy.mockRestore(); + } + // Nothing was cleared: the transcript and the kernel reference survive intact. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 903029e4a40..e0893429a3f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12941,7 +12941,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (effectivePercentage > 0) { session?.clearUsageState(); } - const truncate = () => this.historyService.truncateHistory(workspaceId, effectivePercentage); + // refuseFullDelete makes historyService revalidate the emptiness preflight under the + // history write lock: an overlapping truncation can empty history after the preflight + // above said this one would not, silently skipping the full-clear guards. + const truncate = () => + this.historyService.truncateHistory(workspaceId, effectivePercentage, { + refuseFullDelete: !isFullClear, + }); const truncateResult = effectivePercentage > 0 ? await this.clearHistoryWithRetiredBashMonitorWakes(workspaceId, truncate, { From ef5474d1fe957d1b21354dbf3c719da3f25ea837 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:56:40 +0000 Subject: [PATCH 51/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20retire=20kernel=20w?= =?UTF-8?q?orkflow=20references=20on=20every=20conversation-mutating=20cle?= =?UTF-8?q?ar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings extending the retirement class: - A destructive non-compaction replaceHistory leaves a decision-free transcript that a verified-empty (null) boundary reference reads as current, injecting the pre-replacement workflow result. The destructive branch now retires references before the clear commits, with truncateHistory's ordering and failure posture. Compaction replaces keep their references because compaction preserves conversation identity. - A partial prefix truncation can delete the launch turn's restriction-bearing rows without appending a supersession decision, letting the terminal wake recompose from unrestricted defaults. Rather than persisting a second policy surface in the sidecar, retirement now covers every row-removing truncation: the wake settles superseded and the result stays retrievable via resume, which re-records provenance under the surviving context. --- src/node/services/workspaceService.test.ts | 123 ++++++++++++++++++++- src/node/services/workspaceService.ts | 35 ++++-- 2 files changed, 147 insertions(+), 11 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index eab77c107af..ffb98c9e2f0 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9276,7 +9276,9 @@ describe("WorkspaceService workflow invocation events", () => { } finally { preflightSpy.mockRestore(); } - // Nothing was cleared: the transcript and the kernel reference survive intact. + // The transcript is intact; the reference was already retired before the refused + // rewrite (retirement precedes every row-removing truncation), which is the fail-safe + // direction: a dropped wake, with the result still retrievable via resume. const history = await historyService.getHistoryFromLatestBoundary(workspaceId); expect(history.success).toBe(true); if (history.success) { @@ -9284,7 +9286,124 @@ describe("WorkspaceService workflow invocation events", () => { } expect( existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) - ).toBe(true); + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a partial prefix truncation retires kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-prefix-retire"; + const runId = "wfr_currentness_prefix_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-prefix-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "before truncation", { timestamp: 1_200 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-b", "user", "still here after", { timestamp: 1_300 }) + ); + + // A genuinely partial prefix cut can delete the launch turn's restriction-bearing rows + // without adding a supersession decision, so the reference must not survive to + // recompose the wake from unrestricted defaults; the run stays retrievable via resume. + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(truncateResult.success).toBe(true); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a destructive history replacement retires kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-replace-retire"; + const runId = "wfr_currentness_replace_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-replace-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "before replacement", { timestamp: 1_200 }) + ); + + // A destructive non-compaction replacement leaves a decision-free transcript that a + // null-boundary reference would read as current, injecting the pre-replacement result. + const replaceResult = await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement-summary", "assistant", "Replacement summary", {}) + ); + expect(replaceResult.success).toBe(true); + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(false); workspaceService.disposeSession(workspaceId); } finally { await cleanup(); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e0893429a3f..830c1e5ac44 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12920,15 +12920,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } - // Kernel workflow run references belong to the cleared conversation: a verified-empty - // (null) boundary snapshot recorded before the clear is indistinguishable from one - // recorded after it, so a surviving reference could inject a pre-clear workflow result - // into the fresh conversation. Retire them BEFORE the truncation commits so both fault - // directions fail safe: a failed retirement aborts with the conversation intact, and a - // failed truncation leaves reference-less runs settling superseded (dropped wake, still - // retrievable via resume) rather than a committed clear racing a live null-boundary - // reference it could no longer delete. A post-clear resume re-records provenance. - if (isFullClear) { + // Kernel workflow run references belong to the conversation this truncation mutates: a + // full clear leaves a verified-empty (null) boundary snapshot reading the fresh + // conversation as current, and even a prefix truncation can delete the launch turn's + // restriction-bearing rows without appending any supersession decision, letting the wake + // recompose from unrestricted defaults. Retire the references on every row-removing + // truncation, BEFORE it commits, so both fault directions fail safe: a failed retirement + // aborts with the conversation intact, and a failed or refused truncation leaves + // reference-less runs settling superseded (dropped wake, still retrievable via resume, + // which re-records provenance under the surviving context). + if (effectivePercentage > 0) { try { await this.retireKernelWorkflowRunReferences(workspaceId); } catch (error) { @@ -13379,6 +13380,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + // A destructive non-compaction replacement discards the conversation the kernel + // workflow references belong to, exactly like a full clear: a verified-empty (null) + // boundary snapshot reads the decision-free replacement history as current and would + // inject a pre-replacement workflow result into it. Same ordering and failure posture + // as truncateHistory: retire before the clear commits, abort when retirement fails. + // Compaction replaces preserve conversation identity, so their references stay live. + if (!isCompaction) { + try { + await this.retireKernelWorkflowRunReferences(workspaceId); + } catch (error) { + return Err( + `Cannot replace history: stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). Retry once the session storage is writable.` + ); + } + } this.sessions.get(workspaceId)?.clearUsageState(); const clearResult = await this.clearHistoryWithRetiredBashMonitorWakes( workspaceId, From 9c8e0e04974c010f92e34dbb1bca9d8469bf39fc Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 01:15:03 +0000 Subject: [PATCH 52/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20classify=20truncati?= =?UTF-8?q?on=20scope=20and=20retry=20stalled=20wake=20groups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 7 Codex findings: - A no-op truncation (tiny percentage whose token budget rounds to zero removals) no longer retires kernel workflow references. Truncation is preflight-classified as none/partial/all; scope none passes refuseRowRemoval so a racing history change is refused under the write lock instead of removing rows without retirement guards. - An all-indeterminate newest launch group no longer stalls older deliverable groups until the next sweep: the drain retries groups newest-first, settling not_current candidates and delivering the first group with current candidates. --- src/node/services/historyService.test.ts | 19 +++ src/node/services/historyService.ts | 38 ++++-- src/node/services/taskService.test.ts | 89 +++++++++++++ src/node/services/taskService.ts | 146 +++++++++++---------- src/node/services/workspaceService.test.ts | 135 ++++++++++++++++++- src/node/services/workspaceService.ts | 58 ++++---- 6 files changed, 378 insertions(+), 107 deletions(-) diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 7aeb4a4b8b9..65c4d17baca 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2341,6 +2341,25 @@ describe("HistoryService", () => { expect(await service.getHistoryFromLatestBoundary(wsId)).toEqual({ success: true, data: [] }); }); + it("refuseRowRemoval refuses a truncation whose recomputed budget removes messages", async () => { + await appendNumberedMessages(service, wsId, 1); + const chatBefore = await fs.readFile(chatPath(wsId), "utf-8"); + + // The caller classified this request as a no-op (and skipped its row-removal guards), + // but the locked recomputation reaches real rows. Refuse instead of removing them. + const refused = await service.truncateHistory(wsId, 0.9, { refuseRowRemoval: true }); + expect(refused.success).toBe(false); + if (!refused.success) { + expect(refused.error).toContain("no-op"); + } + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + + // A genuine no-op stays a silent success under the same flag. + const noop = await service.truncateHistory(wsId, 0.0001, { refuseRowRemoval: true }); + expect(noop.success).toBe(true); + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + }); + it("does not reseed usage from before a partial prefix truncation", async () => { await appendNumberedMessages(service, wsId, 8); await service.appendToHistory( diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 0010c56b509..82771e16394 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2969,33 +2969,40 @@ export class HistoryService { } /** - * Preflight for truncateHistory: whether this percentage takes the full-delete fast path and - * removes every message. The requested percentage alone cannot distinguish an emptying - * truncation from a partial one, and callers that must apply full-clear semantics before the - * rewrite commits (workspaceService retires kernel workflow run references) need the answer + * Preflight for truncateHistory: whether this percentage removes no rows, a proper prefix, + * or every message (the full-delete fast path). The requested percentage alone cannot + * distinguish these, and callers that must apply per-scope semantics before the rewrite + * commits (workspaceService retires kernel workflow run references only when rows will + * actually be removed, and applies full-clear guards when everything will) need the answer * up front. */ - async willTruncateHistoryRemoveAllMessages( + async classifyTruncationRemoval( workspaceId: string, percentage: number - ): Promise { + ): Promise<"none" | "partial" | "all"> { if (percentage >= 1.0) { - return true; + return "all"; + } + if (percentage <= 0) { + return "none"; } const archivedMessages = await this.readArchivedHistory(workspaceId); const chatMessages = await this.readChatHistory(workspaceId); const messages = [...archivedMessages, ...chatMessages]; if (messages.length === 0) { - return false; + return "none"; } const removeCount = await this.computeTruncationRemoveCount(messages, percentage); - return removeCount >= messages.length; + if (removeCount === 0) { + return "none"; + } + return removeCount >= messages.length ? "all" : "partial"; } async truncateHistory( workspaceId: string, percentage: number, - options?: { refuseFullDelete?: boolean } + options?: { refuseFullDelete?: boolean; refuseRowRemoval?: boolean } ): Promise> { return this.withRecoveredHistoryWriteResultLock( workspaceId, @@ -3024,6 +3031,17 @@ export class HistoryService { const removeCount = await this.computeTruncationRemoveCount(messages, percentage); + // Mirror of refuseFullDelete for the opposite drift direction: the caller + // classified this request as a no-op (and so skipped its row-removal guards, e.g. + // kernel workflow reference retirement), but history grew enough between that + // unserialized read and this locked rewrite for the budget to reach real rows. + // Refuse instead of removing them unguarded; a retry re-classifies. + if (options?.refuseRowRemoval === true && removeCount > 0) { + return Err( + "Truncation classified as a no-op would remove messages; retry to re-run it." + ); + } + // No-op truncation (percentage 0 or rounding to zero tokens) must not // rewrite anything — collapsing the archive back into chat.jsonl would // undo rotation and put lifetime history back on the hot path. diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index b0829eefa75..25eec44bc25 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6297,6 +6297,95 @@ describe("TaskService", () => { ).toBe(false); }); + test("an indeterminate newest group does not stall an older deliverable group", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const oldRunId = "wfr_group_old"; + const newRunId = "wfr_group_new"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + for (const runId of [oldRunId, newRunId]) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + } + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Classification (first call per run) sees both runs current; the last-moment reread then + // fails transiently for the NEWEST group only. The drain must fall through to the older + // group in the same cycle instead of parking every wake on the sweep. + const currentnessCalls = new Map(); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock((_workspaceId: string, runId: string) => { + const count = (currentnessCalls.get(runId) ?? 0) + 1; + currentnessCalls.set(runId, count); + if (count === 1) { + return Promise.resolve("current"); + } + return Promise.resolve(runId === newRunId ? "indeterminate" : "current"); + }); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: oldRunId, + createdAtMs: 1_100, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: newRunId, + createdAtMs: 1_500, + agentId: "plan", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([oldRunId, newRunId])); + + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const prompt = String(sendMessage.mock.calls[0]?.[1]); + expect(prompt).toContain(oldRunId); + expect(prompt).not.toContain(newRunId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + // The unreadable group stays queued for the next drain or sweep, never settled. + expect( + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(newRunId) + ).toBe(true); + }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 5db0c541977..05edd01ea30 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8612,37 +8612,8 @@ export class TaskService implements AgentTaskIntegration { // direction, deferring agent-bound groups rather than ever mixing identities in one send. const hasNonWorkflowDeliverables = deliverableAgentNotificationIds.size > 0 || workspaceTurnCandidates.length > 0; - let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; - if (!hasNonWorkflowDeliverables) { - for (const candidate of deliverableWorkflowPrompts) { - const agent = candidate.initiatingAgent; - if ( - agent != null && - (workflowInitiatingAgent == null || - agent.createdAtMs > workflowInitiatingAgent.createdAtMs) - ) { - workflowInitiatingAgent = agent; - } - } - } - const selectedGroupKey = - workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : undefined; - const selectedWorkflowPrompts = deliverableWorkflowPrompts.filter((candidate) => - hasNonWorkflowDeliverables - ? candidate.initiatingAgent == null - : selectedGroupKey == null || - (candidate.initiatingAgent != null && - workflowWakeGroupKey(candidate.initiatingAgent) === selectedGroupKey) - ); - // Unselected groups stay queued: the selected group's wake turn ends with a streamEnded + // Unselected groups stay queued: the delivered group's wake turn ends with a streamEnded // drain (and the sweep backstops an aborted one), which delivers the next group. - - const resumeOptions = await this.resolveParentAutoResumeOptions( - ownerWorkspaceId, - entry, - defaultModel, - workflowInitiatingAgent != null ? { agentId: workflowInitiatingAgent.agentId } : undefined - ); const workspaceTurnMuxMetadata = await this.getActiveWorkspaceTurnMuxMetadataForWorkspace(ownerWorkspaceId); @@ -8668,15 +8639,6 @@ export class TaskService implements AgentTaskIntegration { return; } - // Pair the pin with the selected group: the newest pin-bearing history row can belong to - // a different group's wake (each wake persists its own pin), and pinning another agent's - // provenance onto this group's agentId makes resolution reject the wake on every retry. A - // recorded pin (or a verified-unpinned null) overrides the walk; legacy references - // without the field keep the walk pin. - const groupPin = workflowInitiatingAgent?.strictAgentResolution; - const effectiveStrictPin = - groupPin !== undefined ? (groupPin ?? undefined) : wakeRestrictions.strictAgentResolution; - // Last-moment suppression revalidation: a quiet owner-follow-up resettle // deletes its notification files, but cannot retract this drain's // already-taken listPending() snapshot, so the handle records are the @@ -8685,41 +8647,87 @@ export class TaskService implements AgentTaskIntegration { // later candidate's await; suppressed handles are dropped instead of // waking the owner, and their notifications are marked superseded only // after the delivery decision. The residual window is the batch read → - // sendMessage gap below (no awaits in between besides delivery itself) — - // closing it would require holding settlement locks across delivery, - // which the drain must not do; worst case is one redundant wake (fail - // toward notify, never a lost wake). - const [candidateRecords, selectedWorkflowCurrentness] = await Promise.all([ - Promise.all( - workspaceTurnCandidates.map((candidate) => - this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, candidate.notification.sourceId) - ) - ), - // Workflow candidates get the same last-moment treatment: a full history clear that - // completes after buildWorkflowTerminalPrompt classified these runs retires the sidecar - // but cannot retract the materialized candidates, and once the clear releases its - // admission guard the owner is idle again, so this requireIdle send would inject a - // pre-clear workflow result into the freshly cleared conversation. The clear retires - // references before truncating, so this reread sees not_current and settles; unreadable - // state stays queued for the next drain or sweep instead of settling. - Promise.all( - selectedWorkflowPrompts.map((candidate) => + // sendMessage gap below (no awaits in between besides the group-scoped + // resume-option resolution and delivery itself) — closing it would + // require holding settlement locks across delivery, which the drain must + // not do; worst case is one redundant wake (fail toward notify, never a + // lost wake). + const candidateRecords = await Promise.all( + workspaceTurnCandidates.map((candidate) => + this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, candidate.notification.sourceId) + ) + ); + // Workflow candidates get the same last-moment treatment, one launch-identity group at a + // time: a full history clear that completes after buildWorkflowTerminalPrompt classified + // these runs retires the sidecar but cannot retract the materialized candidates, and once + // the clear releases its admission guard the owner is idle again, so this requireIdle + // send would inject a pre-clear workflow result into the freshly cleared conversation. + // The clear retires references before truncating, so a reread sees not_current and + // settles; unreadable state stays queued for the next drain or sweep. Groups are tried + // newest-first until one revalidates, so one unreadable group cannot stall independent + // wakes behind the sweep. Bounded: every iteration permanently removes one group from + // this drain's consideration. + const supersededWorkflowPrompts: typeof deliverableWorkflowPrompts = []; + let currentWorkflowPrompts: typeof deliverableWorkflowPrompts = []; + let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; + let remainingWorkflowPrompts = hasNonWorkflowDeliverables + ? deliverableWorkflowPrompts.filter((candidate) => candidate.initiatingAgent == null) + : deliverableWorkflowPrompts; + while (remainingWorkflowPrompts.length > 0) { + let groupAgent: WorkflowWakeInitiatingAgent | undefined; + for (const candidate of remainingWorkflowPrompts) { + const agent = candidate.initiatingAgent; + if (agent != null && (groupAgent == null || agent.createdAtMs > groupAgent.createdAtMs)) { + groupAgent = agent; + } + } + const groupKey = groupAgent != null ? workflowWakeGroupKey(groupAgent) : undefined; + const groupCandidates = remainingWorkflowPrompts.filter((candidate) => + groupKey == null + ? candidate.initiatingAgent == null + : candidate.initiatingAgent != null && + workflowWakeGroupKey(candidate.initiatingAgent) === groupKey + ); + const groupCurrentness = await Promise.all( + groupCandidates.map((candidate) => this.workspaceService .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) .catch(() => "indeterminate" as const) ) - ), - ]); - const currentWorkflowPrompts: typeof selectedWorkflowPrompts = []; - const supersededWorkflowPrompts: typeof selectedWorkflowPrompts = []; - selectedWorkflowPrompts.forEach((candidate, index) => { - const currentness = selectedWorkflowCurrentness[index]; - if (currentness === "current") { - currentWorkflowPrompts.push(candidate); - } else if (currentness === "not_current") { - supersededWorkflowPrompts.push(candidate); + ); + const groupCurrent: typeof deliverableWorkflowPrompts = []; + groupCandidates.forEach((candidate, index) => { + const currentness = groupCurrentness[index]; + if (currentness === "current") { + groupCurrent.push(candidate); + } else if (currentness === "not_current") { + supersededWorkflowPrompts.push(candidate); + } + }); + if (groupCurrent.length > 0) { + currentWorkflowPrompts = groupCurrent; + workflowInitiatingAgent = groupAgent; + break; } - }); + remainingWorkflowPrompts = remainingWorkflowPrompts.filter( + (candidate) => !groupCandidates.includes(candidate) + ); + } + + const resumeOptions = await this.resolveParentAutoResumeOptions( + ownerWorkspaceId, + entry, + defaultModel, + workflowInitiatingAgent != null ? { agentId: workflowInitiatingAgent.agentId } : undefined + ); + // Pair the pin with the delivered group: the newest pin-bearing history row can belong to + // a different group's wake (each wake persists its own pin), and pinning another agent's + // provenance onto this group's agentId makes resolution reject the wake on every retry. A + // recorded pin (or a verified-unpinned null) overrides the walk; legacy references + // without the field keep the walk pin. + const groupPin = workflowInitiatingAgent?.strictAgentResolution; + const effectiveStrictPin = + groupPin !== undefined ? (groupPin ?? undefined) : wakeRestrictions.strictAgentResolution; const deliverableWorkspaceTurnNotificationIds = new Set(); const publicAwaitIds: string[] = []; const suppressedNotificationIds: string[] = []; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ffb98c9e2f0..39a8dfd460d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9265,8 +9265,8 @@ describe("WorkspaceService workflow invocation events", () => { // serialized revalidation must refuse rather than skip the full-clear guards. const preflightSpy = spyOn( historyService, - "willTruncateHistoryRemoveAllMessages" - ).mockImplementationOnce(() => Promise.resolve(false)); + "classifyTruncationRemoval" + ).mockImplementationOnce(() => Promise.resolve("partial" as const)); try { const result = await workspaceService.truncateHistory(workspaceId, 0.5); expect(result.success).toBe(false); @@ -9410,6 +9410,137 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a truncation that removes no rows preserves kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-noop-preserve"; + const runId = "wfr_currentness_noop_preserve"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-noop-preserve", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "first message", { timestamp: 1_200 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-b", "user", "second message", { timestamp: 1_300 }) + ); + + // A tiny percentage rounds to a zero removal budget: the transcript is unchanged, so + // the run's reference must survive or its terminal wake would settle superseded under + // a conversation that never lost a row. + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.0001); + expect(truncateResult.success).toBe(true); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(2); + } + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a no-op-classified truncation that would remove rows is refused with references intact", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-noop-race"; + const runId = "wfr_currentness_noop_race"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-noop-race", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // Simulate the scope preflight racing history growth: classified a no-op (so reference + // retirement was skipped), but the locked recomputation reaches real rows. The + // serialized guard must refuse rather than remove rows with live references. + const preflightSpy = spyOn( + historyService, + "classifyTruncationRemoval" + ).mockImplementationOnce(() => Promise.resolve("none" as const)); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.9); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("no-op"); + } + } finally { + preflightSpy.mockRestore(); + } + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-coalesced"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 830c1e5ac44..7915714b013 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12832,27 +12832,31 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async truncateHistory(workspaceId: string, percentage?: number): Promise> { const effectivePercentage = percentage ?? 1.0; - // A token-proportional truncation below 100% still empties the transcript when the removal - // budget reaches the final message (historyService's full-delete fast path), and an emptied - // transcript carries every full-clear hazard: most critically, a kernel workflow reference - // with a verified-empty (null) boundary snapshot reads decision-free history as current, so - // a surviving reference could wake a pre-truncation workflow result into the cleared - // conversation. Decide up front and route emptying truncations through the full-clear path - // (admission guard, refine drain, reference retirement). A preflight read failure counts as - // emptying: the guarded path fails safe (references retired first, wakes dropped but - // resumable) even if the truncation itself later fails. - const isFullClear = - effectivePercentage >= 1.0 || - (effectivePercentage > 0 && - (await this.historyService - .willTruncateHistoryRemoveAllMessages(workspaceId, effectivePercentage) - .catch((error: unknown) => { - log.warn("History truncation emptiness preflight failed; treating as full clear", { - workspaceId, - error, - }); - return true; - }))); + // A token-proportional truncation below 100% can remove nothing (budget rounds to zero), + // a proper prefix, or everything (historyService's full-delete fast path), and each scope + // carries different obligations: an emptied transcript needs every full-clear guard, a + // prefix cut still needs kernel workflow reference retirement (it can delete the launch + // turn's restriction rows without a supersession decision), and a no-op must retire + // nothing, or active runs' wakes would settle superseded under an unchanged transcript. + // Decide up front; historyService revalidates the dangerous drift directions under the + // history write lock (refuseFullDelete / refuseRowRemoval below). A preflight read failure + // counts as emptying: the guarded path fails safe (references retired first, wakes dropped + // but resumable) even if the truncation itself later fails. + const truncationScope = + effectivePercentage >= 1.0 + ? ("all" as const) + : effectivePercentage <= 0 + ? ("none" as const) + : await this.historyService + .classifyTruncationRemoval(workspaceId, effectivePercentage) + .catch((error: unknown) => { + log.warn("History truncation scope preflight failed; treating as full clear", { + workspaceId, + error, + }); + return "all" as const; + }); + const isFullClear = truncationScope === "all"; // A full clear holds the admission guard across the refine drain/lock // awaits below: without it, a send admitted during those awaits could // snapshot the pre-clear transcript and stream across the truncation, @@ -12929,7 +12933,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // aborts with the conversation intact, and a failed or refused truncation leaves // reference-less runs settling superseded (dropped wake, still retrievable via resume, // which re-records provenance under the surviving context). - if (effectivePercentage > 0) { + if (truncationScope !== "none") { try { await this.retireKernelWorkflowRunReferences(workspaceId); } catch (error) { @@ -12942,12 +12946,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (effectivePercentage > 0) { session?.clearUsageState(); } - // refuseFullDelete makes historyService revalidate the emptiness preflight under the - // history write lock: an overlapping truncation can empty history after the preflight - // above said this one would not, silently skipping the full-clear guards. + // historyService revalidates the scope preflight under the history write lock: an + // overlapping truncation can shift this one across a scope boundary in either dangerous + // direction (a partial cut becoming a full delete skips the full-clear guards; a no-op + // becoming a real cut skips reference retirement). const truncate = () => this.historyService.truncateHistory(workspaceId, effectivePercentage, { - refuseFullDelete: !isFullClear, + refuseFullDelete: truncationScope === "partial", + refuseRowRemoval: truncationScope === "none", }); const truncateResult = effectivePercentage > 0 From 6a60bdf69c150e0fccd871805437a20daf3ae41f Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 01:52:18 +0000 Subject: [PATCH 53/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20harden=20terminal?= =?UTF-8?q?=20wake=20liveness,=20dedupe,=20and=20downgrade=20compat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 8 Codex findings: - The drain classifier now reads the durable settled marker and drops an already-settled generation, so a kernel-nested task_await settling a run after the terminal callback queued it cannot cause a duplicate wake (kernel consumption leaves no history evidence). - The five-minute interval also re-pokes owners with pending outbox records, so sub-agent/workspace-turn attention stuck behind a transient drain failure retries on the sweep cadence instead of waiting for restart. - A non-busy send rejection backs the selected workflow wake group off until the sweep cadence and re-pokes the drain, so a persistently unresolvable group no longer starves older deliverable groups. - Workflow settlement dual-writes the stable un-suffixed marker the previous build dedupes recovery on, so downgrading after consumption does not re-deliver the result. --- src/node/services/taskService.test.ts | 230 ++++++++++++++++++++ src/node/services/taskService.ts | 105 ++++++++- src/node/services/terminalAttentionStore.ts | 6 +- 3 files changed, 331 insertions(+), 10 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 25eec44bc25..4bbb14e2719 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6386,6 +6386,236 @@ describe("TaskService", () => { ).toBe(true); }); + test("a generation settled during the owner's stream is not redelivered by the terminal callback", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_settled_requeue"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + // A kernel-nested task_await consumed the durable result and settled the generation while + // the owner was still streaming, before WorkflowService reached its terminal callback. + await taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered", + }); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + // Kernel consumption leaves no history evidence, so only the durable marker can stop the + // re-queued entry from waking the owner with a duplicate result. + expect(sendMessage).not.toHaveBeenCalled(); + expect( + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(runId) ?? false + ).toBe(false); + }); + + test("stuck pending outbox attention is re-poked by the sweep-cadence reconciler", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_stuck", + }); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { historyService, taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + schedulePendingTerminalAttentionOwnerDrains: () => Promise; + }; + + // Transient restriction read failure: the drain fails closed, leaving the durable record + // pending with no later stream or task event to retry it. + const iterateSpy = spyOn(historyService, "iterateFullHistory") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's unhandled-rejection + // detector on this host before the drain consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("EIO: history unreadable"))); + try { + await internal.drainTerminalAttention(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + + expect(await internal.schedulePendingTerminalAttentionOwnerDrains()).toBe(1); + await flushTerminalAttentionDrains(taskService); + } finally { + iterateSpy.mockRestore(); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(String(sendMessage.mock.calls[0]?.[1])).toContain("wst_stuck"); + expect(await terminalAttentionStore.get(parentId, "workspace_turn:wst_stuck")).toMatchObject({ + status: "delivered", + }); + }); + + test("a rejected group send backs off and lets an older group deliver in the same cycle", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const oldRunId = "wfr_backoff_old"; + const newRunId = "wfr_backoff_new"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + for (const runId of [oldRunId, newRunId]) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + } + + // The newest group's send is persistently rejected (its pinned agent cannot resolve); + // the older group's send succeeds. + const sendMessage = mock((..._args: unknown[]): Promise> => { + const options = _args[2] as { agentId?: string } | undefined; + return options?.agentId === "plan" + ? Promise.resolve(Err({ type: "unknown", raw: "agent not resolvable" })) + : Promise.resolve(Ok(undefined)); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: oldRunId, + createdAtMs: 1_100, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId: newRunId, + createdAtMs: 1_500, + agentId: "plan", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([oldRunId, newRunId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // First attempt selects the newest group and is rejected; the re-poked drain skips the + // backed-off group and delivers the older one instead of parking it on the sweep. + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[0]?.[2]).toMatchObject({ agentId: "plan" }); + expect(sendMessage.mock.calls[1]?.[2]).toMatchObject({ agentId: "exec" }); + expect(String(sendMessage.mock.calls[1]?.[1])).toContain(oldRunId); + const queued = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.get(parentId); + // The rejected group stays queued for the sweep-cadence retry, never settled. + expect(queued?.has(newRunId)).toBe(true); + expect(queued?.has(oldRunId) ?? false).toBe(false); + }); + + test("settlement writes the stable marker the previous build dedupes recovery on", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_downgrade_stable"; + // A real run record also materializes the owner session dir: settlement markers refuse to + // recreate a removed owner dir by design, so the fixture must exist like in production. + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const { taskService } = createTaskServiceHarness(config); + await taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered", + }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + // The previous build recovers by enqueueIfAbsent on the stable un-suffixed id: an existing + // record must block it from re-creating a pending wake for the consumed result. + expect( + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }) + ).toBeNull(); + // This build's generation marker is written alongside it. + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, "2026-06-19T00:00:03.000Z") + ) + ).toMatchObject({ status: "delivered" }); + }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 05edd01ea30..fd7341bae4f 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1656,6 +1656,12 @@ export class TaskService implements AgentTaskIntegration { // from run records + settled markers at startup and on the periodic sweep and added by live // terminal callbacks, so losing the map merely delays a wake until the next sweep. private readonly pendingWorkflowRunAttention = new Map>(); + // Owner -> wake-group key -> epoch ms before which the drain skips selecting the group. A + // group whose send was rejected for group-specific reasons (for example an unresolvable + // strictly pinned agent) would otherwise be re-selected newest-first by every drain and + // sweep, starving older deliverable groups. In-memory only: a restart retries every group, + // and entries expire on read. + private readonly workflowWakeGroupSendBackoffUntilMs = new Map>(); private workflowAttentionSweepTimer: ReturnType | null = null; private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); @@ -3173,17 +3179,17 @@ export class TaskService implements AgentTaskIntegration { void this.sweepWorkflowRunTerminalAttention().catch((error: unknown) => { log.warn("Workflow terminal attention sweep failed", { error }); }); + void this.schedulePendingTerminalAttentionOwnerDrains().catch((error: unknown) => { + log.warn("Pending terminal attention re-poke failed", { error }); + }); }, WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS); this.workflowAttentionSweepTimer.unref?.(); } const recoveredTerminalWorkspaceTurnNotificationCount = await this.recoverTerminalWorkspaceTurnAttentionNotifications(); const terminalAttentionDrainStartedAt = Date.now(); - const pendingTerminalAttentionOwnerWorkspaceIds = - await this.terminalAttentionStore.listPendingOwnerWorkspaceIds(); - for (const ownerWorkspaceId of pendingTerminalAttentionOwnerWorkspaceIds) { - this.scheduleTerminalAttentionDrain(ownerWorkspaceId); - } + const pendingTerminalAttentionOwnerWorkspaceCount = + await this.schedulePendingTerminalAttentionOwnerDrains(); const terminalAttentionDrainMs = Date.now() - terminalAttentionDrainStartedAt; log.info("[startup] TaskService.initialize completed", { @@ -3203,7 +3209,7 @@ export class TaskService implements AgentTaskIntegration { bestOfRecoveryMs, queuedTerminalWorkflowRunAttentionCount, recoveredTerminalWorkspaceTurnNotificationCount, - pendingTerminalAttentionOwnerWorkspaceCount: pendingTerminalAttentionOwnerWorkspaceIds.length, + pendingTerminalAttentionOwnerWorkspaceCount, terminalAttentionDrainMs, cleanupReportedTasksMs, }); @@ -7827,6 +7833,22 @@ export class TaskService implements AgentTaskIntegration { this.scheduleTerminalAttentionDrain(params.ownerWorkspaceId); } + /** + * Level-triggered retry for outbox (sub-agent / workspace-turn) attention: pending records + * are the durable "wake owed" state, but unlike workflow runs they have no periodic + * re-derivation of their own, so a drain that failed transiently (for example an unreadable + * history for caller restrictions) would otherwise leave them stuck until restart. Startup + * and the sweep interval both re-poke their owners; drains are idempotent and no-op when + * nothing is deliverable. + */ + private async schedulePendingTerminalAttentionOwnerDrains(): Promise { + const ownerWorkspaceIds = await this.terminalAttentionStore.listPendingOwnerWorkspaceIds(); + for (const ownerWorkspaceId of ownerWorkspaceIds) { + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + } + return ownerWorkspaceIds.length; + } + /** Returns true when the run was newly queued for this owner. */ private queueWorkflowRunAttention(ownerWorkspaceId: string, runId: string): boolean { let runIds = this.pendingWorkflowRunAttention.get(ownerWorkspaceId); @@ -7875,6 +7897,18 @@ export class TaskService implements AgentTaskIntegration { return; } try { + // Downgrade compatibility: the previous build dedupes its startup re-derivation on the + // stable un-suffixed workflow_run id, so settling only the generation marker would let + // a downgraded build re-create a pending wake for a result the user already consumed. + // Stable-first ordering keeps the generation marker (this build's authority) retryable: + // if either write fails, the queue entry survives and the next drain re-settles both. + await this.terminalAttentionStore.recordSettled({ + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + sourceId: params.runId, + terminalOutcome: terminalAttentionOutcome(params.status), + status: params.settledAs, + }); await this.terminalAttentionStore.recordSettled({ ownerWorkspaceId: params.ownerWorkspaceId, sourceKind: "workflow_run", @@ -8182,6 +8216,28 @@ export class TaskService implements AgentTaskIntegration { ) { return { outcome: "drop" }; } + // The durable settled marker outranks the in-memory queue entry: a kernel-nested + // task_await or workflow_resume can settle this generation after the terminal callback + // queued it, and that consumption leaves no history evidence for the classification + // below, so skipping this check would deliver a duplicate wake. + let settledMarker: Awaited>; + try { + settledMarker = await this.terminalAttentionStore.get( + ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", run.id, run.updatedAt) + ); + } catch (error: unknown) { + // An unreadable marker cannot prove the wake is owed; defer like indeterminate currentness. + log.warn("Deferring workflow terminal wake-up; settlement marker unreadable", { + ownerWorkspaceId, + runId, + error: getErrorMessage(error), + }); + return { outcome: "defer" }; + } + if (settledMarker != null) { + return { outcome: "drop" }; + } const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( ownerWorkspaceId, run.id @@ -8688,6 +8744,23 @@ export class TaskService implements AgentTaskIntegration { : candidate.initiatingAgent != null && workflowWakeGroupKey(candidate.initiatingAgent) === groupKey ); + // Group keys embed \u0000, so the empty string safely keys the unpinned group. + const backoffKey = groupKey ?? ""; + const ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); + const backoffUntil = ownerBackoff?.get(backoffKey); + if (backoffUntil != null) { + if (backoffUntil > Date.now()) { + // Recently rejected send: leave the group queued and give the next group its turn. + remainingWorkflowPrompts = remainingWorkflowPrompts.filter( + (candidate) => !groupCandidates.includes(candidate) + ); + continue; + } + ownerBackoff?.delete(backoffKey); + if (ownerBackoff?.size === 0) { + this.workflowWakeGroupSendBackoffUntilMs.delete(ownerWorkspaceId); + } + } const groupCurrentness = await Promise.all( groupCandidates.map((candidate) => this.workspaceService @@ -8895,7 +8968,25 @@ export class TaskService implements AgentTaskIntegration { } if (!sendResult.success) { - // Owner became busy between the idle check and the send: leave pending and retry next drain. + if (currentWorkflowPrompts.length > 0 && !isWorkspaceBusyIdleOnlySend(sendResult.error)) { + // A non-busy rejection is likely group-specific (an unresolvable pinned agent, a model + // or provider gate). Back the selected group off until the sweep cadence retries it + // and re-poke so the remaining groups get their send this cycle instead of starving + // behind newest-first selection. Bounded: each re-poked drain either delivers or backs + // off one more group, and with every group backed off it selects nothing. + let ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); + if (ownerBackoff == null) { + ownerBackoff = new Map(); + this.workflowWakeGroupSendBackoffUntilMs.set(ownerWorkspaceId, ownerBackoff); + } + ownerBackoff.set( + workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : "", + Date.now() + WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS + ); + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + } + // Busy rejection: the owner started work between the idle check and the send; leave + // pending and retry on the next drain trigger. log.debug("Terminal attention wake-up not accepted; leaving pending", { ownerWorkspaceId, error: sendResult.error, diff --git a/src/node/services/terminalAttentionStore.ts b/src/node/services/terminalAttentionStore.ts index fe3c8725862..d038d42d584 100644 --- a/src/node/services/terminalAttentionStore.ts +++ b/src/node/services/terminalAttentionStore.ts @@ -141,14 +141,14 @@ export class TerminalAttentionStore { /** * Write-once settlement marker: records a notification directly in a terminal status with a * single write (no pending intermediate a concurrent reader could misread as an owed wake). - * An existing record for the same id is left untouched. + * An existing record for the same id is left untouched. Omitting generationId settles the + * stable (un-suffixed) id, which older builds use for whole-source dedupe. */ async recordSettled( notification: Omit< TerminalAttentionNotification, "id" | "status" | "createdAt" | "outputDelivery" > & { - generationId: string; status: "delivered" | "superseded"; } ): Promise { @@ -167,7 +167,7 @@ export class TerminalAttentionStore { ownerWorkspaceId: notification.ownerWorkspaceId, sourceKind: notification.sourceKind, sourceId: notification.sourceId, - generationId: notification.generationId, + ...(notification.generationId != null ? { generationId: notification.generationId } : {}), outputDelivery: outputDeliveryForSource(notification.sourceKind), terminalOutcome: notification.terminalOutcome, status: notification.status, From 165667ccfb95c4a6468a68cfb22e61e55246a25a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 02:20:08 +0000 Subject: [PATCH 54/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20align=20wake=20poli?= =?UTF-8?q?cy,=20admission,=20and=20downgrade=20reset=20with=20terminal=20?= =?UTF-8?q?semantics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 9 Codex findings: - The sweep derives candidates with the terminal callback's continuation policy (completed/failed), so an intentionally interrupted run is no longer re-queued into a continuation prompt after the user stopped it. - Run-status wiring clears the stable downgrade settlement marker when a run leaves terminal state, so downgrading after a resume does not lose the newer result behind the stale whole-run marker. - Partial (row-removing) truncations hold the context-mutation admission guard across reference retirement, so a racing send is refused instead of streaming across the cut with retired workflow provenance. --- src/common/types/workflow.ts | 10 ++ src/node/services/aiService.ts | 12 ++- src/node/services/taskService.test.ts | 99 +++++++++++++++++++ src/node/services/taskService.ts | 37 ++++++- .../workflows/WorkflowService.context.test.ts | 36 +++++++ .../services/workflows/WorkflowService.ts | 22 +++-- src/node/services/workspaceService.test.ts | 80 +++++++++++++++ src/node/services/workspaceService.ts | 13 +-- 8 files changed, 292 insertions(+), 17 deletions(-) diff --git a/src/common/types/workflow.ts b/src/common/types/workflow.ts index 564ecc1ad94..50f70d90441 100644 --- a/src/common/types/workflow.ts +++ b/src/common/types/workflow.ts @@ -51,6 +51,16 @@ export function isTerminalWorkflowRunStatus(status: WorkflowRunStatus): boolean return status === "completed" || status === "failed" || status === "interrupted"; } +/** + * Terminal statuses that owe a proactive background continuation (terminal wake). An + * interrupted run was stopped deliberately, so neither the terminal callback (unless the + * service opts in) nor the level-triggered attention sweep may notify it. + */ +export const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES: ReadonlySet = new Set([ + "completed", + "failed", +]); + /** * Status of a nested-workflow ("child") run event embedded in a parent run's event stream. * Distinct from {@link WorkflowRunStatus}: an in-progress child event reports "started" rather diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index c299b621e22..2e82b53a522 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -23,6 +23,7 @@ import { EXPERIMENT_IDS, type ExperimentId } from "@/common/constants/experiment import type { GoalRecordV1 } from "@/common/types/goal"; import type { ModelMessage, MuxMessage, MuxMessageMetadata } from "@/common/types/message"; import { createMuxMessage } from "@/common/types/message"; +import { isTerminalWorkflowRunStatus } from "@/common/types/workflow"; import type { Config } from "@/node/config"; import { StreamManager, @@ -2328,9 +2329,16 @@ export class AIService extends EventEmitter { runStore: new WorkflowRunStore({ sessionDir: this.config.getSessionDir(workspaceId), }), - // No reset bookkeeping on restarts: settled markers are keyed by the run's - // terminal generation, so a resumed run re-arms attention by itself. + // This build's settled markers are keyed by the run's terminal generation, so + // a resumed run re-arms attention by itself; only the downgrade-compat stable + // marker needs clearing when the run leaves terminal state. onRunStatusChanged: async (event) => { + if (!isTerminalWorkflowRunStatus(event.status)) { + await this.taskService?.clearWorkflowRunDowngradeSettlement({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } await this.onWorkflowRunStatusChanged?.(event); }, runtimeFactory: new QuickJSRuntimeFactory(), diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4bbb14e2719..192dd8ef7bd 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6616,6 +6616,105 @@ describe("TaskService", () => { ).toMatchObject({ status: "delivered" }); }); + test("the sweep does not queue an intentionally interrupted run", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + for (const [runId, status] of [ + ["wfr_sweep_interrupted", "interrupted"], + ["wfr_sweep_completed", "completed"], + ] as const) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, status, "2026-06-19T00:00:03.000Z"); + } + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Keep the queue observable: indeterminate currentness defers every drain delivery. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + pendingWorkflowRunAttention: Map>; + }; + + // The user stopped the interrupted run: re-deriving a continuation wake for it would undo + // the stop with new agent actions. Only the completed run owes attention. + expect(await internal.sweepWorkflowRunTerminalAttention()).toBe(1); + const queued = internal.pendingWorkflowRunAttention.get(parentId); + expect(queued?.has("wfr_sweep_completed")).toBe(true); + expect(queued?.has("wfr_sweep_interrupted") ?? false).toBe(false); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + }); + + test("a restarted run clears the stable downgrade marker but keeps generation markers", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_restart_compat"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const { taskService } = createTaskServiceHarness(config); + await taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered", + }); + + await taskService.clearWorkflowRunDowngradeSettlement({ ownerWorkspaceId: parentId, runId }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + // The previous build re-arms a restarted run by deleting the stable id; after the clear + // its recovery probe can enqueue the run's next result again instead of dropping it. + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId) + ) + ).toBeNull(); + // This build's generation marker is untouched: the old generation stays settled here. + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, "2026-06-19T00:00:03.000Z") + ) + ).toMatchObject({ status: "delivered" }); + }); + test("workflow wake restriction recovery stops at a context reset boundary", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index fd7341bae4f..74ea1a80e9f 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -138,6 +138,7 @@ import type { ErrorEvent, StreamAbortEvent, StreamEndEvent } from "@/common/type import { isActiveWorkflowRunStatus, isTerminalWorkflowRunStatus, + WORKFLOW_BACKGROUND_CONTINUATION_STATUSES, type WorkflowRunRecord, type WorkflowRunStatus, } from "@/common/types/workflow"; @@ -7677,7 +7678,10 @@ export class TaskService implements AgentTaskIntegration { run.workspaceId !== workspace.id || run.parentWorkflow != null || resolveBackgroundWorkAttentionPolicy(run.attentionPolicy) !== "notify_on_terminal" || - !isTerminalWorkflowRunStatus(run.status) + // Interrupted runs were stopped deliberately: the terminal callback only notifies + // them under an explicit service opt-in, and this re-derivation must not undo the + // user's stop by injecting a continuation prompt. Opt-in callbacks queue directly. + !WORKFLOW_BACKGROUND_CONTINUATION_STATUSES.has(run.status) ) { continue; } @@ -7933,6 +7937,37 @@ export class TaskService implements AgentTaskIntegration { this.pendingWorkflowRunAttention.get(params.ownerWorkspaceId)?.delete(params.runId); } + /** + * Downgrade-compat bookkeeping only: settlement dual-writes a stable un-suffixed marker for + * the previous build's whole-run dedupe (see markWorkflowRunTerminalAttentionSettled), and a + * restarted run invalidates it. Without this delete, downgrading after a resume would leave + * the old build refusing to enqueue the run's newer result behind the stale stable marker. + * This build never reads the stable marker (generation markers are the authority), so the + * delete is best-effort and must never fail the status transition. + */ + async clearWorkflowRunDowngradeSettlement(params: { + ownerWorkspaceId: string; + runId: string; + }): Promise { + assert( + params.ownerWorkspaceId.length > 0, + "clearWorkflowRunDowngradeSettlement requires ownerWorkspaceId" + ); + assert(params.runId.length > 0, "clearWorkflowRunDowngradeSettlement requires runId"); + try { + await this.terminalAttentionStore.delete( + params.ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", params.runId) + ); + } catch (error: unknown) { + log.warn("Failed to clear stale workflow downgrade settlement marker", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + error, + }); + } + } + async markWorkspaceTurnTerminalAttentionConsumed(params: { ownerWorkspaceId: string; consumingWorkspaceId: string; diff --git a/src/node/services/workflows/WorkflowService.context.test.ts b/src/node/services/workflows/WorkflowService.context.test.ts index 10ff08ce311..5a8f797ddba 100644 --- a/src/node/services/workflows/WorkflowService.context.test.ts +++ b/src/node/services/workflows/WorkflowService.context.test.ts @@ -10,6 +10,7 @@ import { WorkflowRunStore } from "./WorkflowRunStore"; import { listWorkflowRuns, listWorkflowScripts, + resumeWorkflowRun, startWorkflowRun, type WorkflowServiceContext, } from "./WorkflowService"; @@ -97,6 +98,7 @@ describe("WorkflowService request orchestration", () => { workspaceService, taskService: { noteWorkflowRunTerminalAttention: mock(() => undefined), + clearWorkflowRunDowngradeSettlement: mock(async () => undefined), }, experimentsService: { isExperimentEnabled: mock(() => options.enabled ?? true), @@ -268,6 +270,40 @@ describe("WorkflowService request orchestration", () => { }); }); + test("resuming a run clears the stale downgrade settlement marker", async () => { + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir("workspace-1") }); + await runStore.createRun({ + id: "wfr_resume_compat", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_resume_compat", "running", "2026-05-29T00:00:01.000Z"); + await runStore.appendStatus("wfr_resume_compat", "interrupted", "2026-05-29T00:00:02.000Z"); + + const clearWorkflowRunDowngradeSettlement = mock(async () => undefined); + const { context } = createContext(); + (context as unknown as Record).taskService = { + noteWorkflowRunTerminalAttention: mock(() => undefined), + clearWorkflowRunDowngradeSettlement, + }; + + // Leaving terminal state invalidates the stable downgrade marker written at settlement; + // without the clear, a downgraded build would refuse to enqueue the resumed result. + await resumeWorkflowRun(context, { workspaceId: "workspace-1", runId: "wfr_resume_compat" }); + const deadline = Date.now() + 5_000; + while (clearWorkflowRunDowngradeSettlement.mock.calls.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(clearWorkflowRunDowngradeSettlement).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_compat", + }); + }); + test("rejects disabled dynamic workflows before workspace initialization", async () => { const { context, waitForInit } = createContext({ enabled: false }); try { diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index f904c7d79d1..1db3b42b325 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import { isTerminalWorkflowRunStatus, + WORKFLOW_BACKGROUND_CONTINUATION_STATUSES, type WorkflowScriptDescriptor, type WorkflowRunRecord, type WorkflowRunStatus, @@ -133,11 +134,6 @@ export interface StartNamedWorkflowResult { result: unknown; } -const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES = new Set([ - "completed", - "failed", -]); - // oRPC creates a WorkflowService per request, so workflow lifecycle state that spans requests // needs process-wide registries. const pendingCrashResumeTimers = new Map>(); @@ -1076,9 +1072,19 @@ export async function resolveWorkflowContext( includeAgentPlugins, skillStorageContext, }), - // No reset bookkeeping on restarts: settled markers are keyed by the run's terminal - // generation, so a resumed run's next terminal transition re-arms attention by itself. - onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), + // Settled markers are keyed by the run's terminal generation, so a resumed run's next + // terminal transition re-arms attention by itself; only the downgrade-compat stable + // marker needs clearing when the run leaves terminal state (see + // clearWorkflowRunDowngradeSettlement). + onRunStatusChanged: async (event) => { + if (!isTerminalWorkflowRunStatus(event.status)) { + await context.taskService.clearWorkflowRunDowngradeSettlement({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } + await context.workspaceService.emitWorkflowRunActivity(event); + }, // Read paths (listRuns / stream subscribe) create services purely to observe runs, but // crash recovery can resume an orphaned background run on them: without a terminal // callback the settled run would owe its wake to the next sweep. Explicit callbacks diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 39a8dfd460d..551d3541cfa 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3,6 +3,7 @@ import { WorkspaceService, generateForkBranchName, generateForkTitle } from "./w import { registerInProcessWorkflowRun } from "@/node/services/workflows/workflowArchiveAdmission"; import type { IdleCompactionOutcome } from "./idleCompactionService"; import type { AgentSession } from "./agentSession"; +import { CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE } from "./agentSession"; import { createAgentSessionHarness, createStartedTurnHandle } from "./agentSession.testHarness"; import type { AutoCompactionUsageState } from "@/common/utils/compaction/autoCompactionCheck"; import { createDisplayUsage } from "@/common/utils/tokens/displayUsage"; @@ -9355,6 +9356,85 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a partial truncation blocks send admission across reference retirement", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-partial-admission"; + const runId = "wfr_currentness_partial_admission"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-partial-admission", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "before truncation", { timestamp: 1_200 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-b", "user", "still here after", { timestamp: 1_300 }) + ); + + // A send racing the partial truncation during the retirement await must be refused: + // admitted, it would snapshot the pre-truncation transcript and lose its turn's + // workflow provenance to the reference retirement. + const internal = workspaceService as unknown as { + retireKernelWorkflowRunReferences: (id: string) => Promise; + }; + const originalRetire = internal.retireKernelWorkflowRunReferences.bind(workspaceService); + let raceSendOutcome: string | null = null; + const retireSpy = spyOn(internal, "retireKernelWorkflowRunReferences").mockImplementationOnce( + async (id: string) => { + const sendResult = await workspaceService.sendMessage(workspaceId, "race the cut", { + model: "openai:gpt-4o", + agentId: "exec", + }); + raceSendOutcome = sendResult.success ? "accepted" : JSON.stringify(sendResult.error); + await originalRetire(id); + } + ); + try { + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(truncateResult.success).toBe(true); + } finally { + retireSpy.mockRestore(); + } + expect(raceSendOutcome ?? "").toContain(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a destructive history replacement retires kernel workflow references", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-replace-retire"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7915714b013..5b67a3c8901 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12857,13 +12857,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return "all" as const; }); const isFullClear = truncationScope === "all"; - // A full clear holds the admission guard across the refine drain/lock - // awaits below: without it, a send admitted during those awaits could - // snapshot the pre-clear transcript and stream across the truncation, - // repopulating the cleared context. Partial truncation keeps the plain - // pre-check — no awaits sit between it and the truncation. + // Every row-removing truncation holds the admission guard across its awaits (full clear: + // the refine drain/lock below; partial: kernel workflow reference retirement): without + // it, a send admitted during those awaits could snapshot the pre-truncation transcript + // and stream across the mutation, or lose its turn's workflow provenance to the + // retirement. Scope "none" keeps the plain pre-check: it retires nothing, and + // historyService refuses row-removing drift under the write lock. let admissionGuard: Disposable | null = null; - if (isFullClear) { + if (truncationScope !== "none") { const guardResult = this.acquireContextMutationAdmissionGuard( workspaceId, "truncate history" From f46c7a265b1e585cfbcb88cb69bffcdb174d8f75 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 04:47:52 +0000 Subject: [PATCH 55/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20revalidate=20wake?= =?UTF-8?q?=20groups=20after=20identity=20resolution=20and=20back=20off=20?= =?UTF-8?q?rejected=20non-workflow=20batches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 167 ++++++++++++++++++++++++++ src/node/services/taskService.ts | 126 +++++++++++++++---- 2 files changed, 267 insertions(+), 26 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index ac2464b031e..47cbcd87488 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6301,6 +6301,79 @@ describe("TaskService", () => { ).toBe(false); }); + test("a history clear during resume-option resolution settles the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_resolve_clear_race"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Classification sees a current invocation; the clear completes while the drain resolves + // resume options, so only a currentness reread taken AFTER that resolution observes it. + let cleared = false; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve(cleared ? "not_current" : "current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const svc = taskService as unknown as { + resolveParentAutoResumeOptions: (...args: unknown[]) => Promise; + }; + const originalResolve = svc.resolveParentAutoResumeOptions.bind(taskService); + svc.resolveParentAutoResumeOptions = async (...args: unknown[]) => { + const resolved = await originalResolve(...args); + cleared = true; + return resolved; + }; + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + // The pre-clear result must not wake the freshly cleared conversation. + expect(sendMessage).not.toHaveBeenCalled(); + const terminalAttentionStore = new TerminalAttentionStore(config); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("superseded"); + }); + test("an indeterminate newest group does not stall an older deliverable group", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); @@ -6569,6 +6642,100 @@ describe("TaskService", () => { expect(queued?.has(oldRunId) ?? false).toBe(false); }); + test("a rejected non-workflow send backs off and lets an agent-bound group deliver", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_nonworkflow_backoff"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + // The workspace-turn batch's conversation-identity send is persistently rejected; the + // agent-bound group's own pinned identity can still send. + const sendMessage = mock((..._args: unknown[]): Promise> => { + const options = _args[2] as { agentId?: string } | undefined; + return options?.agentId === "plan" + ? Promise.resolve(Ok(undefined)) + : Promise.resolve(Err({ type: "unknown", raw: "agent not resolvable" })); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(parentId), + runId, + agentId: "plan", + }); + + // A deliverable (non-suppressed) workspace-turn wake keeps the agent-bound group out of + // the batch until the batch's send is rejected. + const taskHandleStore = new TaskHandleStore(config); + await taskHandleStore.upsertWorkspaceTurn({ + kind: "workspace_turn", + handleId: "wst_backoff_deliverable", + ownerWorkspaceId: parentId, + workspaceId: parentId, + turnId: "backoff-deliverable", + status: "completed", + reportMarkdown: "turn done", + createdAt: "2026-08-11T00:00:00.000Z", + updatedAt: "2026-08-11T00:00:01.000Z", + createdWorkspace: false, + disposableWorkspace: false, + }); + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_backoff_deliverable", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // First attempt sends the non-workflow batch and is rejected; the re-poked drain lets the + // backed-off batch sit out so the agent-bound group delivers in the same cycle. + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[1]?.[2]).toMatchObject({ agentId: "plan" }); + expect(String(sendMessage.mock.calls[1]?.[1])).toContain(runId); + // The rejected wake stays pending for the sweep-cadence retry, never dropped. + const stillPending = await terminalAttentionStore.listPending(parentId); + expect(stillPending.map((notification) => notification.sourceId)).toEqual([ + "wst_backoff_deliverable", + ]); + const queued = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.get(parentId); + expect(queued?.has(runId) ?? false).toBe(false); + }); + test("settlement writes the stable marker the previous build dedupes recovery on", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index fc72d46842a..bf1f1870dcc 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1166,6 +1166,11 @@ function workflowWakeGroupKey(agent: WorkflowWakeInitiatingAgent): string { return `${agent.agentId}\u0000${pin === undefined ? "walk" : JSON.stringify(pin)}`; } +// Reserved workflowWakeGroupSendBackoffUntilMs key for the non-workflow (sub-agent and +// workspace-turn) send batch. Group keys start with a non-empty agentId and the empty string +// keys the unpinned group, so a leading \u0000 cannot collide. +const NON_WORKFLOW_WAKE_BACKOFF_KEY = "\u0000non-workflow"; + function isTypedWorkspaceEvent(value: unknown, type: string): boolean { return ( typeof value === "object" && @@ -8532,6 +8537,25 @@ export class TaskService implements AgentTaskIntegration { } } + /** + * Back the given terminal-wake send batches off until the sweep cadence retries them and + * re-poke the drain so the remaining batches get their send this cycle instead of starving + * behind the failed one. + */ + private backOffTerminalWakeSends(ownerWorkspaceId: string, keys: readonly string[]): void { + assert(keys.length > 0, "backOffTerminalWakeSends requires keys"); + let ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); + if (ownerBackoff == null) { + ownerBackoff = new Map(); + this.workflowWakeGroupSendBackoffUntilMs.set(ownerWorkspaceId, ownerBackoff); + } + const retryAt = Date.now() + WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS; + for (const key of keys) { + ownerBackoff.set(key, retryAt); + } + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + } + /** * Drain pending terminal notifications for one owner workspace: defer (leave pending) when the * owner is busy/queued/preparing, otherwise send one coalesced synthetic wake-up and mark the @@ -8703,8 +8727,27 @@ export class TaskService implements AgentTaskIntegration { // Suppression revalidation below can only shrink the workspace-turn set, so gating on // pre-suppression candidates over-approximates non-workflow deliverables: the safe // direction, deferring agent-bound groups rather than ever mixing identities in one send. + // A backed-off non-workflow batch (its conversation-identity send was rejected) sits out + // the drain entirely, staying pending for the sweep-cadence retry, so agent-bound groups + // are not starved behind a send that fails the same way on every drain. + const nonWorkflowSendBackoffUntil = this.workflowWakeGroupSendBackoffUntilMs + .get(ownerWorkspaceId) + ?.get(NON_WORKFLOW_WAKE_BACKOFF_KEY); + let nonWorkflowSendBackedOff = false; + if (nonWorkflowSendBackoffUntil != null) { + if (nonWorkflowSendBackoffUntil > Date.now()) { + nonWorkflowSendBackedOff = true; + } else { + const ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); + ownerBackoff?.delete(NON_WORKFLOW_WAKE_BACKOFF_KEY); + if (ownerBackoff?.size === 0) { + this.workflowWakeGroupSendBackoffUntilMs.delete(ownerWorkspaceId); + } + } + } const hasNonWorkflowDeliverables = - deliverableAgentNotificationIds.size > 0 || workspaceTurnCandidates.length > 0; + !nonWorkflowSendBackedOff && + (deliverableAgentNotificationIds.size > 0 || workspaceTurnCandidates.length > 0); // Unselected groups stay queued: the delivered group's wake turn ends with a streamEnded // drain (and the sweep backstops an aborted one), which delivers the next group. const workspaceTurnMuxMetadata = @@ -8740,8 +8783,8 @@ export class TaskService implements AgentTaskIntegration { // later candidate's await; suppressed handles are dropped instead of // waking the owner, and their notifications are marked superseded only // after the delivery decision. The residual window is the batch read → - // sendMessage gap below (no awaits in between besides the group-scoped - // resume-option resolution and delivery itself) — closing it would + // sendMessage gap below (no awaits in between besides workflow group + // selection and delivery itself) — closing it would // require holding settlement locks across delivery, which the drain must // not do; worst case is one redundant wake (fail toward notify, never a // lost wake). @@ -8763,6 +8806,9 @@ export class TaskService implements AgentTaskIntegration { const supersededWorkflowPrompts: typeof deliverableWorkflowPrompts = []; let currentWorkflowPrompts: typeof deliverableWorkflowPrompts = []; let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; + let resumeOptions: + | Awaited> + | undefined; let remainingWorkflowPrompts = hasNonWorkflowDeliverables ? deliverableWorkflowPrompts.filter((candidate) => candidate.initiatingAgent == null) : deliverableWorkflowPrompts; @@ -8798,6 +8844,16 @@ export class TaskService implements AgentTaskIntegration { this.workflowWakeGroupSendBackoffUntilMs.delete(ownerWorkspaceId); } } + // Resolve the send identity before the currentness reread so the reread stays the last + // await before dispatch: a history clear that completes during this history and + // agent-settings read retires the sidecar, and a reread taken before it would go stale + // and inject the pre-clear result into the freshly cleared conversation. + const groupResumeOptions = await this.resolveParentAutoResumeOptions( + ownerWorkspaceId, + entry, + defaultModel, + groupAgent != null ? { agentId: groupAgent.agentId } : undefined + ); const groupCurrentness = await Promise.all( groupCandidates.map((candidate) => this.workspaceService @@ -8817,6 +8873,7 @@ export class TaskService implements AgentTaskIntegration { if (groupCurrent.length > 0) { currentWorkflowPrompts = groupCurrent; workflowInitiatingAgent = groupAgent; + resumeOptions = groupResumeOptions; break; } remainingWorkflowPrompts = remainingWorkflowPrompts.filter( @@ -8824,11 +8881,13 @@ export class TaskService implements AgentTaskIntegration { ); } - const resumeOptions = await this.resolveParentAutoResumeOptions( + // No workflow group was selected: resolve under the conversation's own identity. The + // workspace-turn and sub-agent wakes tolerate this await in the residual window (worst + // case one redundant wake, never a stale workflow injection). + resumeOptions ??= await this.resolveParentAutoResumeOptions( ownerWorkspaceId, entry, - defaultModel, - workflowInitiatingAgent != null ? { agentId: workflowInitiatingAgent.agentId } : undefined + defaultModel ); // Pair the pin with the delivered group: the newest pin-bearing history row can belong to // a different group's wake (each wake persists its own pin), and pinning another agent's @@ -8847,6 +8906,10 @@ export class TaskService implements AgentTaskIntegration { suppressedNotificationIds.push(candidate.notification.id); return; } + if (nonWorkflowSendBackedOff) { + // Backed off: sits out this drain and stays pending for the sweep-cadence retry. + return; + } deliverableWorkspaceTurnNotificationIds.add(candidate.notification.id); publicAwaitIds.push(candidate.publicAwaitId); }); @@ -8873,12 +8936,14 @@ export class TaskService implements AgentTaskIntegration { } promptSections.push(...currentWorkflowPrompts.map((candidate) => candidate.prompt)); const prompt = promptSections.join("\n\n"); - const effectivePending = pending.filter((notification) => { - if (notification.sourceKind === "agent_task") { - return deliverableAgentNotificationIds.has(notification.id); - } - return deliverableWorkspaceTurnNotificationIds.has(notification.id); - }); + const effectivePending = nonWorkflowSendBackedOff + ? [] + : pending.filter((notification) => { + if (notification.sourceKind === "agent_task") { + return deliverableAgentNotificationIds.has(notification.id); + } + return deliverableWorkspaceTurnNotificationIds.has(notification.id); + }); if (effectivePending.length === 0 && currentWorkflowPrompts.length === 0) { await markSuppressedSuperseded(); // Suppression can empty the very batch that excluded agent-bound workflow groups; with @@ -8943,6 +9008,11 @@ export class TaskService implements AgentTaskIntegration { ownerWorkspaceId, error: resumeResult.error, }); + // Same starvation shape as the prompt path: agent-bound groups were excluded by this + // batch, so back it off and re-poke to let them send under their own launch identity. + if (deliverableWorkflowPrompts.some((candidate) => candidate.initiatingAgent != null)) { + this.backOffTerminalWakeSends(ownerWorkspaceId, [NON_WORKFLOW_WAKE_BACKOFF_KEY]); + } return; } if (!resumeResult.data.started) { @@ -9005,22 +9075,26 @@ export class TaskService implements AgentTaskIntegration { } if (!sendResult.success) { - if (currentWorkflowPrompts.length > 0 && !isWorkspaceBusyIdleOnlySend(sendResult.error)) { - // A non-busy rejection is likely group-specific (an unresolvable pinned agent, a model - // or provider gate). Back the selected group off until the sweep cadence retries it + if (!isWorkspaceBusyIdleOnlySend(sendResult.error)) { + // A non-busy rejection is likely batch-specific (an unresolvable pinned agent, a model + // or provider gate). Back the sent batches off until the sweep cadence retries them // and re-poke so the remaining groups get their send this cycle instead of starving - // behind newest-first selection. Bounded: each re-poked drain either delivers or backs - // off one more group, and with every group backed off it selects nothing. - let ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); - if (ownerBackoff == null) { - ownerBackoff = new Map(); - this.workflowWakeGroupSendBackoffUntilMs.set(ownerWorkspaceId, ownerBackoff); + // behind newest-first selection; the non-workflow batch backs off the same way so a + // persistently rejected conversation-identity send cannot exclude agent-bound groups + // on every drain. Bounded: each re-poked drain either delivers or backs off one more + // key, and with every key backed off it selects nothing. + const backoffKeys: string[] = []; + if (currentWorkflowPrompts.length > 0) { + backoffKeys.push( + workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : "" + ); + } + if (effectivePending.length > 0) { + backoffKeys.push(NON_WORKFLOW_WAKE_BACKOFF_KEY); + } + if (backoffKeys.length > 0) { + this.backOffTerminalWakeSends(ownerWorkspaceId, backoffKeys); } - ownerBackoff.set( - workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : "", - Date.now() + WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS - ); - this.scheduleTerminalAttentionDrain(ownerWorkspaceId); } // Busy rejection: the owner started work between the idle check and the send; leave // pending and retry on the next drain trigger. From d2de0cb9d5c5cae3830ff1e1f6582d959e6c9691 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:12:51 +0000 Subject: [PATCH 56/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20honor=20stable=20se?= =?UTF-8?q?ttlement=20markers=20on=20upgrade=20and=20reconcile=20wakes=20o?= =?UTF-8?q?n=20unarchive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 135 ++++++++++++++++++ src/node/services/taskService.ts | 72 +++++++++- .../services/taskWorkspaceSeam.testUtils.ts | 1 + src/node/services/taskWorkspaceSeam.ts | 1 + src/node/services/workspaceService.test.ts | 17 +++ src/node/services/workspaceService.ts | 9 ++ 6 files changed, 231 insertions(+), 4 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 47cbcd87488..c2859df0c12 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6132,6 +6132,141 @@ describe("TaskService", () => { expect(sendMessage).toHaveBeenCalledTimes(2); }); + test("the sweep honors a recent stable marker from the previous build and re-queues past a stale one", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_upgrade_stable_marker"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const sweep = () => + ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention(); + + // The previous build consumed the result (e.g. a kernel-nested task_await) and recorded + // only the stable un-suffixed marker; no generation marker exists. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.recordSettled({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + terminalOutcome: "failed", + status: "delivered", + }); + + // Upgrade sweep: the stable marker postdates the terminal generation, so the wake is + // already consumed and the decision migrates onto this generation's marker. + expect(await sweep()).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + const run = await runStore.getRun(runId); + const migrated = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(migrated?.status).toBe("delivered"); + + // A resume that reaches terminal after the marker was written makes the stable marker + // stale (its restart-time clear is best-effort): the newer generation must re-queue. + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + expect(await sweep()).toBe(1); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + }); + + test("unarchive reconciliation delivers a wake parked by the archived-owner drain", async () => { + const config = await createTestConfig(rootDir); + const { parentId, projectPath } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_unarchive_requeue"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + const setArchivedState = async (field: "archivedAt" | "unarchivedAt") => { + await config.editConfig((cfg) => { + const entry = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === parentId); + assert(entry, "parent workspace must exist"); + entry[field] = new Date().toISOString(); + return cfg; + }); + }; + await setArchivedState("archivedAt"); + + // Terminal lands while archived: the drain parks the wake durably (queue dropped, no + // settlement marker). + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + + // Unarchive-time reconciliation re-queues and delivers without waiting for the interval + // sweep. + await setArchivedState("unarchivedAt"); + await taskService.noteWorkspaceUnarchived(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(String(sendMessage.mock.calls[0]?.[1])).toContain(runId); + }); + test("an unreadable settlement marker skips only that run and never rejects the sweep", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index bf1f1870dcc..7c4b247815d 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7653,15 +7653,17 @@ export class TaskService implements AgentTaskIntegration { * startup and on a fixed sweep interval, so missed terminal callbacks, crashes, and * deferred (transiently unreadable) evaluations always get re-evaluated without any * per-failure retry bookkeeping. Archived workspaces are skipped, which parks their wakes - * unsettled: unarchiving re-queues them on the next sweep instead of dropping them. + * unsettled: the unarchive hook (noteWorkspaceUnarchived) and the next interval sweep + * re-queue them instead of dropping them. */ - private async sweepWorkflowRunTerminalAttention(): Promise { + private async sweepWorkflowRunTerminalAttention(onlyWorkspaceId?: string): Promise { const cfg = this.config.loadConfigOrDefault(); let queuedCount = 0; for (const project of cfg.projects.values()) { for (const workspace of project.workspaces) { if ( workspace.id == null || + (onlyWorkspaceId != null && workspace.id !== onlyWorkspaceId) || isWorkspaceArchived(workspace.archivedAt, workspace.unarchivedAt) ) { continue; @@ -7712,6 +7714,59 @@ export class TaskService implements AgentTaskIntegration { if (marker != null) { continue; } + // Upgrade compatibility: the previous build recorded consumption only under the + // stable un-suffixed id (no generation markers), including history-invisible + // consumption such as a kernel-nested task_await. The recency guard keeps + // generation markers authoritative: the restart-time stable clear is best-effort, + // so a marker older than this terminal generation is stale and must not suppress + // the newer result; unparseable timestamps fail toward notify. + let stableMarker: Awaited>; + try { + stableMarker = await this.terminalAttentionStore.get( + workspace.id, + TerminalAttentionStore.notificationId("workflow_run", run.id) + ); + } catch (error: unknown) { + log.warn("Failed to read stable workflow settlement marker; skipping run", { + workspaceId: workspace.id, + runId: run.id, + error: getErrorMessage(error), + }); + continue; + } + if ( + stableMarker != null && + (stableMarker.status === "delivered" || stableMarker.status === "superseded") + ) { + const stableMarkerAt = Date.parse(stableMarker.createdAt); + const terminalGenerationAt = Date.parse(run.updatedAt); + if ( + Number.isFinite(stableMarkerAt) && + Number.isFinite(terminalGenerationAt) && + stableMarkerAt >= terminalGenerationAt + ) { + try { + // Migrate the decision onto this generation's marker so later sweeps stay + // single-read; the stable marker already proves consumption, so a failed + // migration only re-runs this fallback on the next sweep. + await this.terminalAttentionStore.recordSettled({ + ownerWorkspaceId: workspace.id, + sourceKind: "workflow_run", + sourceId: run.id, + generationId: run.updatedAt, + terminalOutcome: terminalAttentionOutcome(run.status), + status: stableMarker.status, + }); + } catch (error: unknown) { + log.warn("Failed to migrate stable workflow settlement marker", { + workspaceId: workspace.id, + runId: run.id, + error: getErrorMessage(error), + }); + } + continue; + } + } if (this.queueWorkflowRunAttention(workspace.id, run.id)) { queuedCount += 1; } @@ -7949,8 +8004,9 @@ export class TaskService implements AgentTaskIntegration { * the previous build's whole-run dedupe (see markWorkflowRunTerminalAttentionSettled), and a * restarted run invalidates it. Without this delete, downgrading after a resume would leave * the old build refusing to enqueue the run's newer result behind the stale stable marker. - * This build never reads the stable marker (generation markers are the authority), so the - * delete is best-effort and must never fail the status transition. + * This build reads the stable marker only as recency-gated upgrade evidence behind + * generation markers (see sweepWorkflowRunTerminalAttention), so the delete stays + * best-effort and must never fail the status transition. */ async clearWorkflowRunDowngradeSettlement(params: { ownerWorkspaceId: string; @@ -13450,6 +13506,14 @@ export class TaskService implements AgentTaskIntegration { return blocking; } + async noteWorkspaceUnarchived(workspaceId: string): Promise { + assert(workspaceId.length > 0, "noteWorkspaceUnarchived requires workspaceId"); + // Archived owners park workflow terminal wakes unsettled (the drain drops the in-memory + // queue and the sweep skips archived workspaces), so without this unarchive-time + // reconciliation an idle owner would stay silent until the interval sweep. + await this.sweepWorkflowRunTerminalAttention(workspaceId); + } + /** * Whether any top-level workflow runs are durably active for this workspace. The archive * sink rechecks this after arming its admission gate (see archiveUnlocked) so a workflow diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index db2c5428178..5b81af09771 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -74,6 +74,7 @@ export function makeAgentTaskIntegrationFake( markParentWorkspaceInterrupted: () => undefined, latchHardInterruptCascade: () => undefined, terminateAllDescendantAgentTasks: () => Promise.resolve([]), + noteWorkspaceUnarchived: () => Promise.resolve(), ...overrides, }; } diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index d8189cbe829..61ba8f67b77 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -302,6 +302,7 @@ export interface AgentTaskIntegration { workspaceId: string, options?: { workflowRunId?: string } ): Promise; + noteWorkspaceUnarchived(workspaceId: string): Promise; } export function normalizeArchiveUntrackedPaths(paths: readonly string[]): string[] { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 8354cbbfce2..3ef2a61a163 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -17299,6 +17299,23 @@ describe("WorkspaceService setPinned", () => { expect((await workspaceService.setPinned(rootId, true)).success).toBe(true); expect(getEntry(rootId)?.pinnedAt).toBeDefined(); }); + + test("unarchive pokes task-side workflow attention reconciliation", async () => { + const noteWorkspaceUnarchived = mock(() => Promise.resolve()); + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ noteWorkspaceUnarchived }) + ); + expect((await workspaceService.archive(rootId)).success).toBe(true); + expect(noteWorkspaceUnarchived).not.toHaveBeenCalled(); + + expect((await workspaceService.unarchive(rootId)).success).toBe(true); + expect(noteWorkspaceUnarchived).toHaveBeenCalledWith(rootId); + expect(noteWorkspaceUnarchived).toHaveBeenCalledTimes(1); + + // No archived -> unarchived transition: a repeat unarchive must not re-poke. + expect((await workspaceService.unarchive(rootId)).success).toBe(true); + expect(noteWorkspaceUnarchived).toHaveBeenCalledTimes(1); + }); }); describe("WorkspaceService reorderPinned", () => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 1e0f169b912..ada185aacd7 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -9537,6 +9537,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return Ok(undefined); } + // Archived owners park workflow terminal wakes unsettled; reconcile now so an idle + // workspace does not stay silent until the interval sweep. Contained: reconciliation + // failure must not fail the unarchive (the sweep retries on its own cadence). + try { + await this.agentTaskIntegration?.noteWorkspaceUnarchived(workspaceId); + } catch (error: unknown) { + log.warn("Unarchive workflow attention reconciliation failed", { workspaceId, error }); + } + // Emit updated metadata const allMetadata = await this.config.getAllWorkspaceMetadata(); const updatedMetadata = allMetadata.find((m) => m.id === workspaceId); From d6c4f305a092a02f2c2ae52d5521dca53f9a62d2 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:30:38 +0000 Subject: [PATCH 57/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20revalidate=20run=20?= =?UTF-8?q?generation=20after=20settlement=20writes=20to=20protect=20resum?= =?UTF-8?q?ed=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 85 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 33 +++++++++++ 2 files changed, 118 insertions(+) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index c2859df0c12..1b00f4ffbd2 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6267,6 +6267,91 @@ describe("TaskService", () => { expect(String(sendMessage.mock.calls[0]?.[1])).toContain(runId); }); + test("settling a stale generation snapshot does not suppress a newer resumed result", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_mid_settlement_resume"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + // The first wake turn resumes the run in the background and the newer generation reaches + // terminal before the outer drain settles its stale first-generation snapshot. The owner + // stays busy (streaming the wake turn) until that settlement completes, so the callback's + // interim drain defers instead of delivering the newer generation early. + let ownerBusy = false; + let simulateResumeDuringWake: (() => Promise) | undefined; + const sendMessage = mock(async (..._args: unknown[]): Promise> => { + const simulate = simulateResumeDuringWake; + simulateResumeDuringWake = undefined; + await simulate?.(); + return Ok(undefined); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + (workspaceService as unknown as Record).hasPendingQueuedOrPreparingTurn = mock( + () => ownerBusy + ); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + simulateResumeDuringWake = async () => { + ownerBusy = true; + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + }); + }; + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const sweep = () => + ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention(); + + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await drain(parentId); + ownerBusy = false; + await flushTerminalAttentionDrains(taskService); + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + // The stale snapshot's settlement must neither drop the newer generation's queue entry + // nor leave a stable marker that postdates it (which the sweep's upgrade fallback would + // migrate as delivered, permanently suppressing the result). + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(String(sendMessage.mock.calls[1]?.[1])).toContain(runId); + expect(await sweep()).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + }); + test("an unreadable settlement marker skips only that run and never rejects the sweep", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 7c4b247815d..8f71582acfb 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7996,6 +7996,39 @@ export class TaskService implements AgentTaskIntegration { }); return; } + // Post-write revalidation: a wake-turn workflow_resume can restart the run while this + // settlement's snapshot was in flight, so the restart-time stable clear can land BEFORE + // the stable write above re-creates the marker. That marker postdates the newer + // generation's updatedAt, so the sweep's upgrade fallback (and a downgraded build's + // whole-run dedupe) would permanently suppress the newer result, and the by-run-id queue + // delete below would drop its owed wake. Reading the run AFTER the writes closes the + // write-side race: any restart after this read re-clears the stable marker itself and + // its terminal callback re-queues behind this deletion. + let currentRunUpdatedAt: string | null; + try { + const runStore = new WorkflowRunStore({ + sessionDir: this.config.getSessionDir(params.ownerWorkspaceId), + }); + currentRunUpdatedAt = (await runStore.getRun(params.runId)).updatedAt; + } catch { + currentRunUpdatedAt = null; + } + if (currentRunUpdatedAt !== params.runUpdatedAt) { + // The settled snapshot is no longer the run's newest generation (or the run is + // unreadable): the stable whole-run marker must not outlive the snapshot. The + // generation marker stays; it truthfully settles only this snapshot. + await this.clearWorkflowRunDowngradeSettlement({ + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + }); + if (currentRunUpdatedAt != null) { + // The queue entry now represents the newer generation's owed wake: leave it for the + // drain the terminal callback scheduled (the sweep backstops a lost poke). + return; + } + // Unreadable run: fall through to the delete. With no stable marker surviving, the + // sweep re-derives any owed newer generation from durable state. + } this.pendingWorkflowRunAttention.get(params.ownerWorkspaceId)?.delete(params.runId); } From 2ac7f66e146f71d91ccee898fe20488c830d6778 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:39:07 +0000 Subject: [PATCH 58/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20revalidate=20workfl?= =?UTF-8?q?ow=20currentness=20before=20the=20busy=20fallback=20send?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 68 +++++++++++++++++++++++++++ src/node/services/taskService.ts | 23 +++++++++ 2 files changed, 91 insertions(+) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 1b00f4ffbd2..40d7683472a 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6352,6 +6352,74 @@ describe("TaskService", () => { expect(sendMessage).toHaveBeenCalledTimes(2); }); + test("a history clear during the busy fallback settles the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_busy_fallback_clear"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + // The idle-only send loses the busy race while a full clear completes; the fallback path + // must not inject the retained pre-clear prompt without revalidating. + let cleared = false; + const sendMessage = mock((..._args: unknown[]): Promise> => { + const internal = _args[3] as { requireIdle?: boolean } | undefined; + if (internal?.requireIdle === true) { + cleared = true; + return Promise.resolve( + Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }) + ); + } + return Promise.resolve(Ok(undefined)); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve(cleared ? "not_current" : "current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // Only the rejected idle-only attempt: the fallback aborts on the currentness reread and + // the re-poked drain settles the superseded generation instead of delivering it. + expect(sendMessage).toHaveBeenCalledTimes(1); + expect((sendMessage.mock.calls[0]?.[3] as { requireIdle?: boolean })?.requireIdle).toBe(true); + const terminalAttentionStore = new TerminalAttentionStore(config); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("superseded"); + }); + test("an unreadable settlement marker skips only that run and never rejects the sweep", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 8f71582acfb..38a8fc5996b 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -9135,6 +9135,29 @@ export class TaskService implements AgentTaskIntegration { !this.interruptedParentWorkspaceIds.has(ownerWorkspaceId) && !(await this.hasBlockingActiveWorkForTerminalDrain(ownerWorkspaceId, latestTaskIndex)) ) { + // Security: the composed prompt retains the pre-check workflow results, and this + // fallback send omits requireIdle, so its epoch snapshot postdates any clear or reset + // that completed during the awaited checks above and the clear guard would accept the + // stale injection into the fresh context. Reread currentness so the fallback keeps + // the primary path's contract (no awaits between revalidation and delivery); any + // non-current candidate aborts toward a fresh drain that re-derives, and the queue + // entries survive for it. + if (currentWorkflowPrompts.length > 0) { + const fallbackCurrentness = await Promise.all( + currentWorkflowPrompts.map((candidate) => + this.workspaceService + .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) + .catch(() => "indeterminate" as const) + ) + ); + if (fallbackCurrentness.some((currentness) => currentness !== "current")) { + log.debug("Terminal wake busy fallback aborted; workflow candidates went stale", { + ownerWorkspaceId, + }); + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + return; + } + } let fallbackAccepted = false; sendResult = await this.workspaceService.sendMessage( ownerWorkspaceId, From 355ac54d4124f95910fe5b16fd77563b836c2e13 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 05:45:51 +0000 Subject: [PATCH 59/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20refuse=20truncation?= =?UTF-8?q?=20when=20the=20scope=20preflight=20is=20unreadable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/workspaceService.test.ts | 69 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 15 +++-- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 3ef2a61a163..ccdae49ae23 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9304,6 +9304,75 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("an unreadable truncation scope preflight refuses instead of applying full-clear effects", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-preflight-refuse"; + const runId = "wfr_currentness_preflight_refuse"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-preflight-refuse", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // A transiently unreadable preflight must refuse: an unknown scope labeled "all" would + // apply full-clear side effects while a prefix removal can leave rows behind. + const preflightSpy = spyOn(historyService, "classifyTruncationRemoval").mockRejectedValueOnce( + new Error("EIO: history unreadable") + ); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("classify"); + } + } finally { + preflightSpy.mockRestore(); + } + // Lossless refusal: the transcript is intact and the kernel workflow reference survives + // for the retry (no wake was settled superseded by a truncation that never happened). + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a partial prefix truncation retires kernel workflow references", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-prefix-retire"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index ada185aacd7..906fc7346d5 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12857,9 +12857,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // turn's restriction rows without a supersession decision), and a no-op must retire // nothing, or active runs' wakes would settle superseded under an unchanged transcript. // Decide up front; historyService revalidates the dangerous drift directions under the - // history write lock (refuseFullDelete / refuseRowRemoval below). A preflight read failure - // counts as emptying: the guarded path fails safe (references retired first, wakes dropped - // but resumable) even if the truncation itself later fails. + // history write lock (refuseFullDelete / refuseRowRemoval below). const truncationScope = effectivePercentage >= 1.0 ? ("all" as const) @@ -12868,12 +12866,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { : await this.historyService .classifyTruncationRemoval(workspaceId, effectivePercentage) .catch((error: unknown) => { - log.warn("History truncation scope preflight failed; treating as full clear", { + log.warn("History truncation scope preflight failed; refusing truncation", { workspaceId, error, }); - return "all" as const; + return null; }); + if (truncationScope == null) { + // An unknown scope must not choose a side-effect set: labeling it a full clear would + // discard goal/plan/retry state and advance the context epoch while rows may remain, + // and labeling it smaller would skip full-clear guards. Nothing is mutated or retired + // yet, so refusing is lossless and the user can simply retry. + return Err("Failed to read history to classify the truncation scope. Try again."); + } const isFullClear = truncationScope === "all"; // Every row-removing truncation holds the admission guard across its awaits (full clear: // the refine drain/lock below; partial: kernel workflow reference retirement): without From 32ad57027b5aa496454281efc32acffb8ac6acd0 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:03:22 +0000 Subject: [PATCH 60/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20refresh=20the=20sta?= =?UTF-8?q?ble=20settlement=20marker=20with=20generation=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 134 ++++++++++++++++++++ src/node/services/taskService.ts | 51 +++++--- src/node/services/terminalAttentionStore.ts | 20 ++- 3 files changed, 182 insertions(+), 23 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 40d7683472a..16a70b800c1 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6420,6 +6420,140 @@ describe("TaskService", () => { expect(marker?.status).toBe("superseded"); }); + test("a newer generation's settlement refreshes a surviving stale stable marker", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_stable_marker_refresh"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const sweep = () => + ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention(); + + // First generation delivers and records the stable whole-run marker. + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstGeneration = (await runStore.getRun(runId)).updatedAt; + + // The run resumes without the restart-time bookkeeping (its best-effort stable clear + // failed), so the stale first-generation marker survives into the new generation. + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + expect(await sweep()).toBe(1); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + + // The newer generation's settlement must refresh the write-once stable marker: a + // downgraded build reads it as "latest consumed generation", and a record still carrying + // the previous generation would suppress the newer result's wake after a downgrade. + const run = await runStore.getRun(runId); + expect(run.updatedAt).not.toBe(firstGeneration); + const terminalAttentionStore = new TerminalAttentionStore(config); + const stableMarker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId) + ); + expect(stableMarker?.status).toBe("delivered"); + expect(stableMarker?.generationId).toBe(run.updatedAt); + }); + + test("the sweep honors a generation-tagged stable marker across clock corrections", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_stable_marker_clock_skew"; + const runStore = new WorkflowRunStore({ sessionDir: config.getSessionDir(parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 30_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 10_000).toISOString()); + // The clock stepped back after this terminal transition, so the settlement marker below + // carries a createdAt that PRECEDES the generation it consumed. + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.recordSettled( + { + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + generationId: run.updatedAt, + terminalOutcome: "failed", + status: "delivered", + }, + { wholeSourceRefresh: true } + ); + + // Exact generation evidence must win over wall-clock ordering: the marker consumed this + // very generation, so the sweep must not re-queue and re-deliver it. + expect( + await ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention() + ).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + const migrated = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(migrated?.status).toBe("delivered"); + }); + test("an unreadable settlement marker skips only that run and never rejects the sweep", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 38a8fc5996b..59476fca442 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -7716,10 +7716,10 @@ export class TaskService implements AgentTaskIntegration { } // Upgrade compatibility: the previous build recorded consumption only under the // stable un-suffixed id (no generation markers), including history-invisible - // consumption such as a kernel-nested task_await. The recency guard keeps - // generation markers authoritative: the restart-time stable clear is best-effort, - // so a marker older than this terminal generation is stale and must not suppress - // the newer result; unparseable timestamps fail toward notify. + // consumption such as a kernel-nested task_await. The guard keeps generation + // markers authoritative: the restart-time stable clear is best-effort, so a stale + // previous-generation marker must not suppress the newer result; mismatched or + // unparseable evidence fails toward notify. let stableMarker: Awaited>; try { stableMarker = await this.terminalAttentionStore.get( @@ -7738,13 +7738,20 @@ export class TaskService implements AgentTaskIntegration { stableMarker != null && (stableMarker.status === "delivered" || stableMarker.status === "superseded") ) { - const stableMarkerAt = Date.parse(stableMarker.createdAt); - const terminalGenerationAt = Date.parse(run.updatedAt); - if ( - Number.isFinite(stableMarkerAt) && - Number.isFinite(terminalGenerationAt) && - stableMarkerAt >= terminalGenerationAt - ) { + // Prefer exact generation evidence (this build's settlement refresh records the + // consumed generation), which is immune to wall-clock corrections; the createdAt + // recency heuristic remains only for legacy markers from the previous build, + // which recorded no generation. + let consumedByStableMarker = stableMarker.generationId === run.updatedAt; + if (!consumedByStableMarker && stableMarker.generationId == null) { + const stableMarkerAt = Date.parse(stableMarker.createdAt); + const terminalGenerationAt = Date.parse(run.updatedAt); + consumedByStableMarker = + Number.isFinite(stableMarkerAt) && + Number.isFinite(terminalGenerationAt) && + stableMarkerAt >= terminalGenerationAt; + } + if (consumedByStableMarker) { try { // Migrate the decision onto this generation's marker so later sweeps stay // single-read; the stable marker already proves consumption, so a failed @@ -7968,13 +7975,21 @@ export class TaskService implements AgentTaskIntegration { // a downgraded build re-create a pending wake for a result the user already consumed. // Stable-first ordering keeps the generation marker (this build's authority) retryable: // if either write fails, the queue entry survives and the next drain re-settles both. - await this.terminalAttentionStore.recordSettled({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - sourceId: params.runId, - terminalOutcome: terminalAttentionOutcome(params.status), - status: params.settledAs, - }); + // The refresh (not write-once) matters: a stale previous-generation stable marker can + // survive its best-effort restart-time clear, and a downgraded build would read it as + // consumption of THIS generation's result. The recorded generation also gives the + // sweep's upgrade fallback exact evidence immune to wall-clock corrections. + await this.terminalAttentionStore.recordSettled( + { + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + sourceId: params.runId, + generationId: params.runUpdatedAt, + terminalOutcome: terminalAttentionOutcome(params.status), + status: params.settledAs, + }, + { wholeSourceRefresh: true } + ); await this.terminalAttentionStore.recordSettled({ ownerWorkspaceId: params.ownerWorkspaceId, sourceKind: "workflow_run", diff --git a/src/node/services/terminalAttentionStore.ts b/src/node/services/terminalAttentionStore.ts index d038d42d584..99f457cc544 100644 --- a/src/node/services/terminalAttentionStore.ts +++ b/src/node/services/terminalAttentionStore.ts @@ -143,6 +143,12 @@ export class TerminalAttentionStore { * single write (no pending intermediate a concurrent reader could misread as an owed wake). * An existing record for the same id is left untouched. Omitting generationId settles the * stable (un-suffixed) id, which older builds use for whole-source dedupe. + * + * wholeSourceRefresh instead settles the stable id while keeping the generationId FIELD and + * overwrites any existing record: that marker means "latest consumed generation of this + * source", so settlement must refresh a stale previous-generation record that survived its + * best-effort restart-time clear, and the recorded generation gives readers exact evidence + * that is immune to wall-clock corrections. */ async recordSettled( notification: Omit< @@ -150,16 +156,20 @@ export class TerminalAttentionStore { "id" | "status" | "createdAt" | "outputDelivery" > & { status: "delivered" | "superseded"; - } + }, + options?: { wholeSourceRefresh?: boolean } ): Promise { + const wholeSourceRefresh = options?.wholeSourceRefresh === true; const id = TerminalAttentionStore.notificationId( notification.sourceKind, notification.sourceId, - notification.generationId + wholeSourceRefresh ? undefined : notification.generationId ); - const existing = await this.get(notification.ownerWorkspaceId, id); - if (existing != null) { - return; + if (!wholeSourceRefresh) { + const existing = await this.get(notification.ownerWorkspaceId, id); + if (existing != null) { + return; + } } await this.write( { From 0d1c089fcfbccdfc2289e975c5fb96430a7b03af Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 06:45:11 +0000 Subject: [PATCH 61/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20refuse=20a=20full-c?= =?UTF-8?q?lear-classified=20truncation=20that=20would=20leave=20rows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scope preflight runs before the admission guard is acquired, so a turn completing during that await can grow history: the locked recomputation then removes only a prefix while the caller still applies every full-clear-only side effect (context epoch advance, goal/plan/retry discards). Guard the third drift direction symmetrically: requireFullDelete makes historyService refuse, under the history write lock, any all-classified truncation whose recomputed budget would leave messages; a retry re-classifies. --- src/node/services/historyService.test.ts | 21 ++++++ src/node/services/historyService.ts | 17 ++++- src/node/services/workspaceService.test.ts | 78 ++++++++++++++++++++++ src/node/services/workspaceService.ts | 8 ++- 4 files changed, 120 insertions(+), 4 deletions(-) diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 65c4d17baca..a6925641878 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2360,6 +2360,27 @@ describe("HistoryService", () => { expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); }); + it("requireFullDelete refuses a truncation whose recomputed budget leaves messages", async () => { + await appendNumberedMessages(service, wsId, 8); + const chatBefore = await fs.readFile(chatPath(wsId), "utf-8"); + + // The caller classified this request as emptying (and applies full-clear-only side + // effects after the rewrite), but history grew between that unserialized read and the + // locked rewrite so rows would survive. Refuse instead of leaving survivors behind a + // "full clear". + const refused = await service.truncateHistory(wsId, 0.5, { requireFullDelete: true }); + expect(refused.success).toBe(false); + if (!refused.success) { + expect(refused.error).toContain("leave messages"); + } + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + + // A truncation that does empty history stays a success under the same flag. + const emptied = await service.truncateHistory(wsId, 0.99, { requireFullDelete: true }); + expect(emptied.success).toBe(true); + expect(await service.getHistoryFromLatestBoundary(wsId)).toEqual({ success: true, data: [] }); + }); + it("does not reseed usage from before a partial prefix truncation", async () => { await appendNumberedMessages(service, wsId, 8); await service.appendToHistory( diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 82771e16394..472205a7e37 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -3002,7 +3002,11 @@ export class HistoryService { async truncateHistory( workspaceId: string, percentage: number, - options?: { refuseFullDelete?: boolean; refuseRowRemoval?: boolean } + options?: { + refuseFullDelete?: boolean; + refuseRowRemoval?: boolean; + requireFullDelete?: boolean; + } ): Promise> { return this.withRecoveredHistoryWriteResultLock( workspaceId, @@ -3042,6 +3046,17 @@ export class HistoryService { ); } + // Third drift direction: the caller classified this request as emptying (and will + // apply full-clear-only side effects after the rewrite: context epoch advance, + // goal/plan/retry discards), but history grew between that unserialized read and + // this locked rewrite so rows would survive. Refuse instead of leaving survivors + // behind a "full clear"; a retry re-classifies. + if (options?.requireFullDelete === true && removeCount < messages.length) { + return Err( + "Truncation classified as a full clear would leave messages; retry to re-run it." + ); + } + // No-op truncation (percentage 0 or rounding to zero tokens) must not // rewrite anything — collapsing the archive back into chat.jsonl would // undo rotation and put lifetime history back on the hot path. diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ccdae49ae23..3394b674c08 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9373,6 +9373,84 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("a full-clear-classified truncation that would leave rows is refused under the history lock", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-all-drift-refuse"; + const runId = "wfr_currentness_all_drift_refuse"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-all-drift-refuse", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: config.getSessionDir(workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + for (let i = 0; i < 6; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`manual-user-${i}`, "user", `padding message ${i}`, { + timestamp: 1_200 + i, + }) + ); + } + + // Simulate rows appended during the unserialized preflight (e.g. a turn completing + // before the admission guard is acquired): it classified this request as emptying, but + // the locked rewrite's recomputation removes only a prefix. The serialized revalidation + // must refuse rather than apply full-clear side effects (context epoch advance, + // goal/plan/retry discards) while rows remain. + const preflightSpy = spyOn( + historyService, + "classifyTruncationRemoval" + ).mockImplementationOnce(() => Promise.resolve("all" as const)); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("leave messages"); + } + } finally { + preflightSpy.mockRestore(); + } + // The transcript is intact; the reference was already retired before the refused + // rewrite (retirement precedes every row-removing truncation), which is the fail-safe + // direction: a dropped wake, with the result still retrievable via resume. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(6); + } + expect( + existsSync(path.join(config.getSessionDir(workspaceId), "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a partial prefix truncation retires kernel workflow references", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-prefix-retire"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 906fc7346d5..8071114ab77 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -12857,7 +12857,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // turn's restriction rows without a supersession decision), and a no-op must retire // nothing, or active runs' wakes would settle superseded under an unchanged transcript. // Decide up front; historyService revalidates the dangerous drift directions under the - // history write lock (refuseFullDelete / refuseRowRemoval below). + // history write lock (refuseFullDelete / refuseRowRemoval / requireFullDelete below). const truncationScope = effectivePercentage >= 1.0 ? ("all" as const) @@ -12971,13 +12971,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { session?.clearUsageState(); } // historyService revalidates the scope preflight under the history write lock: an - // overlapping truncation can shift this one across a scope boundary in either dangerous + // overlapping mutation can shift this one across a scope boundary in any dangerous // direction (a partial cut becoming a full delete skips the full-clear guards; a no-op - // becoming a real cut skips reference retirement). + // becoming a real cut skips reference retirement; a full clear leaving survivors would + // apply full-clear-only discards while rows remain). const truncate = () => this.historyService.truncateHistory(workspaceId, effectivePercentage, { refuseFullDelete: truncationScope === "partial", refuseRowRemoval: truncationScope === "none", + requireFullDelete: truncationScope === "all", }); const truncateResult = effectivePercentage > 0 From 70b22c3631d1ac004c755289c3664fa3431939c7 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 07:23:00 +0000 Subject: [PATCH 62/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20revalidate=20workfl?= =?UTF-8?q?ow=20wakes=20at=20dispatch=20and=20harden=20truncation/unarchiv?= =?UTF-8?q?e=20windows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 18 review findings, all in the terminal-wake delivery path: - A shared revalidateWorkflowPromptForDispatch helper reruns invocation currentness, the run generation (status + updatedAt vs the derivation snapshot), and the generation settlement marker as the last awaits before both the group send and the busy fallback: a Workflows UI resume/retry or a kernel-nested task_await consumption arrives without history evidence, and currentness alone would deliver a stale or already-consumed result. - truncateHistory acquires the admission guard BEFORE the scope preflight (a turn admitted mid-classification could launch a workflow whose sidecar reference the wholesale retirement then deletes) and rechecks turn activity after the retirement await for both row-removing scopes. - Unarchive workflow-attention reconciliation moved after snapshot restoration and lifecycle startup so its synthetic turn cannot run against a half-restored checkout or a rolled-back unarchive. --- src/node/services/taskService.test.ts | 145 ++++++++++++++ src/node/services/taskService.ts | 84 ++++++-- src/node/services/workspaceService.test.ts | 221 +++++++++++++++++++++ src/node/services/workspaceService.ts | 83 ++++---- 4 files changed, 482 insertions(+), 51 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 1f6003d3333..85e16bb4fa0 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6425,6 +6425,151 @@ describe("TaskService", () => { expect(marker?.status).toBe("superseded"); }); + test("a run generation change after prompt derivation defers the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_dispatch_generation_drift"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "failed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => + Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // A Workflows UI retry flips the run back to running (a NEW generation) after the prompt + // snapshot is taken, without touching history or owner busy-ness: model it inside the + // derivation-time currentness read so the materialized candidate retains the failed + // generation while the run record has already moved on. + let retried = false; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(async () => { + if (!retried) { + retried = true; + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:05.000Z", { + allowFailedCheckpointRetry: true, + }); + } + return "current" as const; + }); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + const pending = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention; + pending.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // The pre-dispatch revalidation sees the changed generation and defers: the retained + // prompt would present the superseded failed result as final. The queue entry survives + // so the resumed run's next terminal transition (or the sweep) re-derives. + expect(sendMessage).not.toHaveBeenCalled(); + expect(pending.get(parentId)?.has(runId)).toBe(true); + }); + + test("a kernel-consumed generation during the busy fallback is not redelivered", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_busy_fallback_consumed"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + // The busy race the idle-only send loses IS a competing owner turn consuming this very + // generation through kernel-nested task_await: that consumption writes only the + // settlement marker (no history evidence, owner idle again afterwards). + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + async (..._args: unknown[]): Promise> => { + const internal = _args[3] as { requireIdle?: boolean } | undefined; + if (internal?.requireIdle === true) { + await terminalAttentionStore.recordSettled({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + generationId: run.updatedAt, + terminalOutcome: "completed", + status: "delivered", + }); + return Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }); + } + return Ok(undefined); + } + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + const pending = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention; + pending.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // Only the rejected idle-only attempt: the fallback's settlement-marker recheck sees the + // consumption and aborts instead of replaying the result without requireIdle. The + // re-poked drain then drops the consumed candidate from the queue. + expect(sendMessage).toHaveBeenCalledTimes(1); + expect((sendMessage.mock.calls[0]?.[3] as { requireIdle?: boolean })?.requireIdle).toBe(true); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("delivered"); + expect(pending.get(parentId)?.has(runId) ?? false).toBe(false); + }); + test("a newer generation's settlement refreshes a surviving stale stable marker", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index c1b065e39f1..a0cc3dfb04b 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -8317,6 +8317,57 @@ export class TaskService implements AgentTaskIntegration { this.pendingTerminalAttentionDrains.add(promise); } + /** + * Last-moment revalidation for an already-materialized workflow prompt candidate. The + * composed prompt retains the run snapshot captured at derivation, and invocation + * currentness alone (conversation evidence) misses two hazards that arrive without + * touching history or owner busy-ness: + * - the run's generation can change (a Workflows UI resume/retry flips it back to running; + * a kernel resume can complete a NEWER generation), so the retained prompt would deliver + * a stale result as if final; + * - a kernel-nested task_await can consume this generation, writing only the settlement + * marker, so resending would replay output the conversation already handled. + * "superseded" settles the candidate; "defer" leaves its queue entry pending so a later + * drain re-derives from the then-current run record and markers. + */ + private async revalidateWorkflowPromptForDispatch( + ownerWorkspaceId: string, + candidate: { runId: string; run: WorkflowRunRecord } + ): Promise<"deliverable" | "superseded" | "defer"> { + const currentness = await this.workspaceService + .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) + .catch(() => "indeterminate" as const); + if (currentness !== "current") { + return currentness === "not_current" ? "superseded" : "defer"; + } + try { + const runStore = new WorkflowRunStore({ + sessionDir: path.join(this.config.sessionsDir, ownerWorkspaceId), + }); + const currentRun = await runStore.getRun(candidate.runId); + if ( + currentRun.status !== candidate.run.status || + currentRun.updatedAt !== candidate.run.updatedAt + ) { + return "defer"; + } + const settledMarker = await this.terminalAttentionStore.get( + ownerWorkspaceId, + TerminalAttentionStore.notificationId( + "workflow_run", + candidate.runId, + candidate.run.updatedAt + ) + ); + if (settledMarker != null) { + return "defer"; + } + } catch { + return "defer"; + } + return "deliverable"; + } + private async buildWorkflowTerminalPrompt( ownerWorkspaceId: string, runId: string @@ -8960,7 +9011,7 @@ export class TaskService implements AgentTaskIntegration { this.workflowWakeGroupSendBackoffUntilMs.delete(ownerWorkspaceId); } } - // Resolve the send identity before the currentness reread so the reread stays the last + // Resolve the send identity before the revalidation reread so the reread stays the last // await before dispatch: a history clear that completes during this history and // agent-settings read retires the sidecar, and a reread taken before it would go stale // and inject the pre-clear result into the freshly cleared conversation. @@ -8970,19 +9021,17 @@ export class TaskService implements AgentTaskIntegration { defaultModel, groupAgent != null ? { agentId: groupAgent.agentId } : undefined ); - const groupCurrentness = await Promise.all( + const groupRevalidation = await Promise.all( groupCandidates.map((candidate) => - this.workspaceService - .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) - .catch(() => "indeterminate" as const) + this.revalidateWorkflowPromptForDispatch(ownerWorkspaceId, candidate) ) ); const groupCurrent: typeof deliverableWorkflowPrompts = []; groupCandidates.forEach((candidate, index) => { - const currentness = groupCurrentness[index]; - if (currentness === "current") { + const verdict = groupRevalidation[index]; + if (verdict === "deliverable") { groupCurrent.push(candidate); - } else if (currentness === "not_current") { + } else if (verdict === "superseded") { supersededWorkflowPrompts.push(candidate); } }); @@ -9165,19 +9214,20 @@ export class TaskService implements AgentTaskIntegration { // Security: the composed prompt retains the pre-check workflow results, and this // fallback send omits requireIdle, so its epoch snapshot postdates any clear or reset // that completed during the awaited checks above and the clear guard would accept the - // stale injection into the fresh context. Reread currentness so the fallback keeps - // the primary path's contract (no awaits between revalidation and delivery); any - // non-current candidate aborts toward a fresh drain that re-derives, and the queue - // entries survive for it. + // stale injection into the fresh context. The busy race the primary send just lost + // may also have been a competing owner turn consuming these very results through + // kernel-nested task_await (settlement marker only, no history evidence) or a + // Workflows UI resume changing the run generation. Revalidate everything so the + // fallback keeps the primary path's contract (no awaits between revalidation and + // delivery); any stale candidate aborts toward a fresh drain that re-derives, and + // the queue entries survive for it. if (currentWorkflowPrompts.length > 0) { - const fallbackCurrentness = await Promise.all( + const fallbackRevalidation = await Promise.all( currentWorkflowPrompts.map((candidate) => - this.workspaceService - .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) - .catch(() => "indeterminate" as const) + this.revalidateWorkflowPromptForDispatch(ownerWorkspaceId, candidate) ) ); - if (fallbackCurrentness.some((currentness) => currentness !== "current")) { + if (fallbackRevalidation.some((verdict) => verdict !== "deliverable")) { log.debug("Terminal wake busy fallback aborted; workflow candidates went stale", { ownerWorkspaceId, }); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index e37533f43c1..838cc828bac 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9474,6 +9474,149 @@ describe("WorkspaceService workflow invocation events", () => { } }); + test("the admission guard is held across the truncation scope preflight", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-preflight-guard"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-preflight-guard", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + for (let i = 0; i < 6; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`manual-user-${i}`, "user", `padding message ${i}`, { + timestamp: 1_200 + i, + }) + ); + } + + // A turn admitted during the preflight could launch a kernel workflow whose sidecar + // reference the wholesale retirement deletes while its rows survive the prefix cut, + // permanently suppressing that run's wake. Admission must therefore already be held + // while the classification snapshot is read: park the preflight and prove a concurrent + // context mutation is refused for the whole window. + let releasePreflight: ((scope: "partial") => void) | undefined; + const preflightGate = new Promise<"partial">((resolve) => { + releasePreflight = resolve; + }); + const preflightSpy = spyOn( + historyService, + "classifyTruncationRemoval" + ).mockImplementationOnce(() => preflightGate); + try { + const first = workspaceService.truncateHistory(workspaceId, 0.5); + const second = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(second.success).toBe(false); + if (!second.success) { + expect(second.error).toContain("already in progress"); + } + releasePreflight?.("partial"); + const firstResult = await first; + expect(firstResult.success).toBe(true); + } finally { + preflightSpy.mockRestore(); + } + // The refused full clear touched nothing: the prefix cut left a suffix behind. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data.length).toBeGreaterThan(0); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a turn becoming active during reference retirement refuses the truncation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retirement-recheck"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retirement-recheck", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + let streaming = false; + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + isStreaming: mock(() => streaming), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + for (let i = 0; i < 6; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`manual-user-${i}`, "user", `padding message ${i}`, { + timestamp: 1_200 + i, + }) + ); + } + + // An in-turn compaction retry bypasses admission gating across a transient idle gap, + // and the retirement await is the last one before the rewrite: a retry that becomes + // active during it must refuse the truncation instead of streaming across it. + const retireSpy = spyOn( + workspaceService as unknown as { + retireKernelWorkflowRunReferences: (id: string) => Promise; + }, + "retireKernelWorkflowRunReferences" + ).mockImplementationOnce(() => { + streaming = true; + return Promise.resolve(); + }); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("turn is active"); + } + } finally { + retireSpy.mockRestore(); + } + // Refused before the rewrite: the transcript is intact. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(6); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + test("a partial prefix truncation retires kernel workflow references", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); const workspaceId = "workflow-currentness-prefix-retire"; @@ -19040,6 +19183,84 @@ describe("WorkspaceService archive snapshots", () => { expect(entry?.worktreeArchiveSnapshot).toBeUndefined(); expect(editConfigSpy).toHaveBeenCalledTimes(0); }); + + test("unarchive reconciles workflow attention only after snapshot restoration", async () => { + const snapshot = { + version: 1 as const, + capturedAt: "2026-03-30T00:00:00.000Z", + stateDirPath: "archive-state", + projects: [ + { + projectPath, + projectName: "proj", + storageKey: "proj", + branchName: "ws-archive-snapshot", + trunkBranch: "main", + baseSha: "base-sha", + headSha: "head-sha", + }, + ], + }; + const order: string[] = []; + const noteWorkspaceUnarchived = mock((_workspaceId: string) => { + order.push("reconcile"); + return Promise.resolve(); + }); + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ noteWorkspaceUnarchived }) + ); + workspaceService.setWorktreeArchiveSnapshotService({ + preflightSnapshotForArchive: mock(() => Promise.resolve(Ok(undefined))), + captureSnapshotForArchive: mock(() => Promise.resolve(Ok(snapshot))), + restoreSnapshotAfterUnarchive: mock(() => { + order.push("restore"); + return Promise.resolve(Ok("skipped" as const)); + }), + getUnsupportedUntrackedPaths: mock(() => Promise.resolve(Ok([]))), + }); + + expect((await workspaceService.archive(workspaceId)).success).toBe(true); + expect((await workspaceService.unarchive(workspaceId)).success).toBe(true); + // The reconciliation drain can admit a synthetic agent turn, which must never run + // against a half-restored checkout. + expect(order).toEqual(["restore", "reconcile"]); + }); + + test("a failed snapshot restoration skips workflow attention reconciliation", async () => { + const snapshot = { + version: 1 as const, + capturedAt: "2026-03-30T00:00:00.000Z", + stateDirPath: "archive-state", + projects: [ + { + projectPath, + projectName: "proj", + storageKey: "proj", + branchName: "ws-archive-snapshot", + trunkBranch: "main", + baseSha: "base-sha", + headSha: "head-sha", + }, + ], + }; + const noteWorkspaceUnarchived = mock((_workspaceId: string) => Promise.resolve()); + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ noteWorkspaceUnarchived }) + ); + workspaceService.setWorktreeArchiveSnapshotService({ + preflightSnapshotForArchive: mock(() => Promise.resolve(Ok(undefined))), + captureSnapshotForArchive: mock(() => Promise.resolve(Ok(snapshot))), + restoreSnapshotAfterUnarchive: mock(() => Promise.resolve(Err("restore failed"))), + getUnsupportedUntrackedPaths: mock(() => Promise.resolve(Ok([]))), + }); + + expect((await workspaceService.archive(workspaceId)).success).toBe(true); + const result = await workspaceService.unarchive(workspaceId); + expect(result.success).toBe(false); + // The failed restoration rolled the unarchive back; reconciling would admit a synthetic + // turn into a workspace that is still archived. + expect(noteWorkspaceUnarchived).not.toHaveBeenCalled(); + }); }); describe("WorkspaceService preflightArchive and acknowledged archive", () => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index a8a7b9eb1e9..bda429c1ae9 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -9548,15 +9548,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return Ok(undefined); } - // Archived owners park workflow terminal wakes unsettled; reconcile now so an idle - // workspace does not stay silent until the interval sweep. Contained: reconciliation - // failure must not fail the unarchive (the sweep retries on its own cadence). - try { - await this.agentTaskIntegration?.noteWorkspaceUnarchived(workspaceId); - } catch (error: unknown) { - log.warn("Unarchive workflow attention reconciliation failed", { workspaceId, error }); - } - // Emit updated metadata const allMetadata = await this.config.getAllWorkspaceMetadata(); const updatedMetadata = allMetadata.find((m) => m.id === workspaceId); @@ -9636,6 +9627,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { subProjectPath: hookMetadata?.subProjectPath, }); + // Archived owners park workflow terminal wakes unsettled; reconcile so an idle + // workspace does not stay silent until the interval sweep. Only AFTER snapshot + // restoration and lifecycle startup above: the drain can admit a synthetic agent turn, + // which must not run against a half-restored checkout or precede a failed restoration's + // config rollback. Contained: reconciliation failure must not fail the unarchive (the + // sweep retries on its own cadence). + try { + await this.agentTaskIntegration?.noteWorkspaceUnarchived(workspaceId); + } catch (error: unknown) { + log.warn("Unarchive workflow attention reconciliation failed", { workspaceId, error }); + } + return Ok(undefined); } catch (error) { const message = getErrorMessage(error); @@ -12861,6 +12864,33 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async truncateHistory(workspaceId: string, percentage?: number): Promise> { const effectivePercentage = percentage ?? 1.0; + // The admission guard is acquired BEFORE the scope preflight and held across every await + // below: a turn admitted during any of them could snapshot the pre-truncation transcript + // and stream across the mutation, and one admitted during the preflight itself could + // launch a kernel workflow whose sidecar reference the wholesale retirement below would + // delete while its launch turn's rows survive the prefix cut, permanently suppressing + // that run's wake. The preflight cannot yet prove scope "none", so every request that + // may remove rows (percentage > 0) pays the guard; percentage <= 0 is a deterministic + // no-op that retires nothing and keeps the plain busy pre-check. + let admissionGuard: Disposable | null = null; + if (effectivePercentage > 0) { + const guardResult = this.acquireContextMutationAdmissionGuard( + workspaceId, + "truncate history" + ); + if (!guardResult.success) { + return Err(guardResult.error); + } + admissionGuard = guardResult.data; + } else if ( + this.sessions.get(workspaceId)?.isBusy() || + this.aiService.isStreaming(workspaceId) + ) { + return Err( + "Cannot truncate history while a turn is active. Press Esc to stop the stream first." + ); + } + using _admissionGuard = admissionGuard; // A token-proportional truncation below 100% can remove nothing (budget rounds to zero), // a proper prefix, or everything (historyService's full-delete fast path), and each scope // carries different obligations: an emptied transcript needs every full-clear guard, a @@ -12891,31 +12921,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return Err("Failed to read history to classify the truncation scope. Try again."); } const isFullClear = truncationScope === "all"; - // Every row-removing truncation holds the admission guard across its awaits (full clear: - // the refine drain/lock below; partial: kernel workflow reference retirement): without - // it, a send admitted during those awaits could snapshot the pre-truncation transcript - // and stream across the mutation, or lose its turn's workflow provenance to the - // retirement. Scope "none" keeps the plain pre-check: it retires nothing, and - // historyService refuses row-removing drift under the write lock. - let admissionGuard: Disposable | null = null; - if (truncationScope !== "none") { - const guardResult = this.acquireContextMutationAdmissionGuard( - workspaceId, - "truncate history" - ); - if (!guardResult.success) { - return Err(guardResult.error); - } - admissionGuard = guardResult.data; - } else if ( - this.sessions.get(workspaceId)?.isBusy() || - this.aiService.isStreaming(workspaceId) - ) { - return Err( - "Cannot truncate history while a turn is active. Press Esc to stop the stream first." - ); - } - using _admissionGuard = admissionGuard; const session = this.sessions.get(workspaceId); // A full clear discards the transcript a streaming refine pass may be @@ -12977,6 +12982,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { `(${getErrorMessage(error)}). Retry once the session storage is writable.` ); } + // In-turn compaction retries bypass admission gating across a transient idle gap (see + // the full-clear recheck above), and this retirement is the last await before the + // rewrite for BOTH row-removing scopes: revalidate here or a retry admitted during it + // would have its history truncated underneath the stream. Refusing after retirement is + // the documented fail-safe direction (dropped wake, retrievable via resume). + if (session?.hasActiveOrPendingTurnWork() || this.aiService.isStreaming(workspaceId)) { + return Err( + "Cannot truncate history while a turn is active. Press Esc to stop the stream first." + ); + } } if (effectivePercentage > 0) { session?.clearUsageState(); From a15a79d0fdd19b1cca21859ffd30084d7a2be7b7 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 31 Aug 2026 07:50:43 +0000 Subject: [PATCH 63/63] =?UTF-8?q?=F0=9F=A4=96=20fix:=20order=20dispatch=20?= =?UTF-8?q?revalidation=20reads=20and=20serialize=20workflow=20settlements?= =?UTF-8?q?=20per=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/node/services/taskService.test.ts | 188 ++++++++++++++++++++++++++ src/node/services/taskService.ts | 46 ++++++- 2 files changed, 228 insertions(+), 6 deletions(-) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 85e16bb4fa0..2580c441338 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -6570,6 +6570,194 @@ describe("TaskService", () => { expect(pending.get(parentId)?.has(runId) ?? false).toBe(false); }); + test("a history mutation during the revalidation reads supersedes the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_dispatch_mutation_during_reads"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => + Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // A history clear retires the run's invocation DURING the revalidation's own run/marker + // reads: model it on the second generation-marker read (the first is derivation's), so + // only a currentness read taken AFTER those reads can observe it. + let cleared = false; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve(cleared ? ("not_current" as const) : ("current" as const))); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + }; + const generationMarkerId = TerminalAttentionStore.notificationId( + "workflow_run", + runId, + run.updatedAt + ); + const realGet = internal.terminalAttentionStore.get.bind(internal.terminalAttentionStore); + let generationMarkerReads = 0; + const getSpy = spyOn(internal.terminalAttentionStore, "get").mockImplementation( + (ownerWorkspaceId, notificationId) => { + if (notificationId === generationMarkerId) { + generationMarkerReads += 1; + if (generationMarkerReads === 2) { + cleared = true; + } + } + return realGet(ownerWorkspaceId, notificationId); + } + ); + + try { + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + internal.pendingWorkflowRunAttention.set(parentId, new Set([runId])); + await internal.drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + } finally { + getSpy.mockRestore(); + } + + // Currentness is the final await before dispatch: it postdates the run/marker reads, so + // the clear is observed and the retained prompt is settled superseded, never sent. + expect(sendMessage).not.toHaveBeenCalled(); + const probeStore = new TerminalAttentionStore(config); + const marker = await probeStore.get(parentId, generationMarkerId); + expect(marker?.status).toBe("superseded"); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has(runId) ?? false).toBe(false); + }); + + test("overlapping settlements preserve the newer generation's stable marker", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_overlapping_settlements"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "failed", "2026-06-19T00:00:03.000Z"); + const oldGeneration = (await runStore.getRun(runId)).updatedAt; + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:05.000Z", { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:07.000Z"); + const newGeneration = (await runStore.getRun(runId)).updatedAt; + + const { taskService } = createTaskServiceHarness(config); + const internal = taskService as unknown as { + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + }; + internal.pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + // Park the older generation's settlement inside its first marker write: the newer + // generation's settlement (started while the older one is parked) can then only + // interleave with the older one's post-write mismatch delete if settlements overlap. + let releaseOldSettlement: () => void = () => undefined; + const oldSettlementParked = new Promise((resolve) => { + releaseOldSettlement = resolve; + }); + let parkedReached: () => void = () => undefined; + const oldSettlementReached = new Promise((resolve) => { + parkedReached = resolve; + }); + const realRecordSettled = internal.terminalAttentionStore.recordSettled.bind( + internal.terminalAttentionStore + ); + let parkedOnce = false; + const settleSpy = spyOn(internal.terminalAttentionStore, "recordSettled").mockImplementation( + async (record, options) => { + if (record.generationId === oldGeneration && !parkedOnce) { + parkedOnce = true; + parkedReached(); + await oldSettlementParked; + } + return realRecordSettled(record, options); + } + ); + + try { + const oldSettlement = taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + runUpdatedAt: oldGeneration, + settledAs: "superseded", + }); + await oldSettlementReached; + const newSettlement = taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: newGeneration, + settledAs: "delivered", + }); + // The newer settlement must queue behind the parked older one instead of interleaving. + await new Promise((resolve) => setTimeout(resolve, 50)); + const midProbeStore = new TerminalAttentionStore(config); + expect( + await midProbeStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, newGeneration) + ) + ).toBeNull(); + releaseOldSettlement(); + await Promise.all([oldSettlement, newSettlement]); + } finally { + settleSpy.mockRestore(); + } + + // The older settlement's mismatch delete ran before the newer settlement's stable + // refresh, so the newer generation's markers survive and the queue entry is consumed. + const probeStore = new TerminalAttentionStore(config); + const stable = await probeStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId) + ); + expect(stable?.generationId).toBe(newGeneration); + const generationMarker = await probeStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, newGeneration) + ); + expect(generationMarker?.status).toBe("delivered"); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has(runId) ?? false).toBe(false); + }); + test("a newer generation's settlement refreshes a surviving stale stable marker", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index a0cc3dfb04b..282d2867c98 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -1663,6 +1663,11 @@ export class TaskService implements AgentTaskIntegration { // tests and shutdown can await them; drains are idempotent and re-triggered on owner idle events. private readonly pendingTerminalAttentionDrainsByOwner = new Map>(); private readonly pendingTerminalAttentionDrains = new Set>(); + // Terminal settlements of the same run must not overlap: settlement is multi-step (stable + // refresh, generation marker, post-write mismatch delete), so an older generation reaching + // its mismatch delete after a newer settlement's stable refresh would remove the newer + // generation's valid marker and let a downgraded build re-deliver a consumed result. + private readonly workflowRunSettlementByRun = new Map>(); // Owed workflow terminal wakes (owner -> runIds believed terminal and not yet settled). An // in-memory work queue over durable state, not a delivery record: entries are (re)derived // from run records + settled markers at startup and on the periodic sweep and added by live @@ -7981,6 +7986,32 @@ export class TaskService implements AgentTaskIntegration { if (!isTerminalWorkflowRunStatus(params.status)) { return; } + const key = `${params.ownerWorkspaceId}\u0000${params.runId}`; + const previous = this.workflowRunSettlementByRun.get(key) ?? Promise.resolve(); + const run = previous + .catch(() => undefined) + .then(() => this.settleWorkflowRunTerminalAttention(params)); + const tracked = run + .then( + () => undefined, + () => undefined + ) + .finally(() => { + if (this.workflowRunSettlementByRun.get(key) === tracked) { + this.workflowRunSettlementByRun.delete(key); + } + }); + this.workflowRunSettlementByRun.set(key, tracked); + return await run; + } + + private async settleWorkflowRunTerminalAttention(params: { + ownerWorkspaceId: string; + runId: string; + status: WorkflowRunStatus; + runUpdatedAt: string; + settledAs: "delivered" | "superseded"; + }): Promise { try { // Downgrade compatibility: the previous build dedupes its startup re-derivation on the // stable un-suffixed workflow_run id, so settling only the generation marker would let @@ -8329,17 +8360,14 @@ export class TaskService implements AgentTaskIntegration { * marker, so resending would replay output the conversation already handled. * "superseded" settles the candidate; "defer" leaves its queue entry pending so a later * drain re-derives from the then-current run record and markers. + * Currentness is deliberately the LAST await: it is the read that observes a destructive + * history mutation (clear/truncation) retiring the run's invocation, and any awaited read + * after it would reopen the stale-injection window this reread exists to close. */ private async revalidateWorkflowPromptForDispatch( ownerWorkspaceId: string, candidate: { runId: string; run: WorkflowRunRecord } ): Promise<"deliverable" | "superseded" | "defer"> { - const currentness = await this.workspaceService - .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) - .catch(() => "indeterminate" as const); - if (currentness !== "current") { - return currentness === "not_current" ? "superseded" : "defer"; - } try { const runStore = new WorkflowRunStore({ sessionDir: path.join(this.config.sessionsDir, ownerWorkspaceId), @@ -8365,6 +8393,12 @@ export class TaskService implements AgentTaskIntegration { } catch { return "defer"; } + const currentness = await this.workspaceService + .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) + .catch(() => "indeterminate" as const); + if (currentness !== "current") { + return currentness === "not_current" ? "superseded" : "defer"; + } return "deliverable"; }