diff --git a/src/browser/utils/chatCommands.test.ts b/src/browser/utils/chatCommands.test.ts index b3bee5b49d..631260e3a3 100644 --- a/src/browser/utils/chatCommands.test.ts +++ b/src/browser/utils/chatCommands.test.ts @@ -1216,6 +1216,32 @@ describe("prepareCompactionMessage", () => { expect(metadata.parsed.followUpContent?.agentId).toBe("exec"); }); + test("compaction recovery keeps the persisted follow-up's restrictions when retry options lack them", () => { + // Retrying a failed compaction passes the already-persisted follow-up together with + // storage-derived send options, which never carry a caller toolPolicy. The preserved + // restrictions must survive that recomposition or the recovered follow-up resumes with + // unrestricted caller tools. + const recoveredFollowUp = { + text: "Keep building", + model: "openai:gpt-4o", + agentId: "code", + toolPolicy: [{ regex_match: "^bash$", action: "disable" as const }], + disableWorkspaceAgents: true, + }; + const { metadata } = prepareCompactionMessage({ + workspaceId: "ws-1", + followUpContent: recoveredFollowUp, + sendMessageOptions: { model: "anthropic:claude-sonnet-4-6", agentId: "exec" }, + }); + + expectCompactionMetadata(metadata); + expect(metadata.parsed.followUpContent?.toolPolicy).toEqual(recoveredFollowUp.toolPolicy); + expect(metadata.parsed.followUpContent?.disableWorkspaceAgents).toBe(true); + // Existing model/agentId still win over the retry-time options. + expect(metadata.parsed.followUpContent?.model).toBe("openai:gpt-4o"); + expect(metadata.parsed.followUpContent?.agentId).toBe("code"); + }); + test("does not create followUpContent when no text or images provided", () => { const sendMessageOptions = createBaseOptions(); const { metadata } = prepareCompactionMessage({ diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 66beb1dfef..128d63f0b0 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -63,6 +63,7 @@ type PreservedSendOptions = Pick< | "providerOptions" | "experiments" | "disableWorkspaceAgents" + | "toolPolicy" | "strictAgentResolution" | "allowAgentSetGoal" | "skipAiSettingsPersistence" @@ -73,21 +74,39 @@ type PreservedSendOptions = Pick< * Use this helper to avoid duplicating the field list when building CompactionFollowUpRequest. */ export function pickPreservedSendOptions(options: SendMessageOptions): PreservedSendOptions { + // Unset fields are OMITTED, not emitted as explicit undefined: compaction recovery spreads + // this pick over an already-persisted follow-up, and an undefined key would clobber the + // original preserved value (e.g. a restricted turn's toolPolicy) instead of leaving it. return { - thinkingLevel: options.thinkingLevel, - reasoningMode: options.reasoningMode, - additionalSystemInstructions: options.additionalSystemInstructions, - providerOptions: options.providerOptions, + ...(options.thinkingLevel !== undefined ? { thinkingLevel: options.thinkingLevel } : {}), + ...(options.reasoningMode !== undefined ? { reasoningMode: options.reasoningMode } : {}), + ...(options.additionalSystemInstructions !== undefined + ? { additionalSystemInstructions: options.additionalSystemInstructions } + : {}), + ...(options.providerOptions !== undefined ? { providerOptions: options.providerOptions } : {}), // Downgrade-compat (see withLegacyPtcExclusiveMirror): preserved options // can persist across restarts and build versions. - experiments: withLegacyPtcExclusiveMirror(options.experiments), - disableWorkspaceAgents: options.disableWorkspaceAgents, + ...(options.experiments !== undefined + ? { experiments: withLegacyPtcExclusiveMirror(options.experiments) } + : {}), + ...(options.disableWorkspaceAgents !== undefined + ? { disableWorkspaceAgents: options.disableWorkspaceAgents } + : {}), + // Security: a restricted turn (including a terminal-wake send restoring the caller's + // policy) that triggers on-send compaction must not redispatch its follow-up allow-all. + ...(options.toolPolicy !== undefined ? { toolPolicy: options.toolPolicy } : {}), // Delegated turns with explicit agent overrides must stay loud across the // compaction replay too — dropping this would let the follow-up silently // fall back to exec if the agent vanished in the meantime. - strictAgentResolution: options.strictAgentResolution, - allowAgentSetGoal: options.allowAgentSetGoal, - skipAiSettingsPersistence: options.skipAiSettingsPersistence, + ...(options.strictAgentResolution !== undefined + ? { strictAgentResolution: options.strictAgentResolution } + : {}), + ...(options.allowAgentSetGoal !== undefined + ? { allowAgentSetGoal: options.allowAgentSetGoal } + : {}), + ...(options.skipAiSettingsPersistence !== undefined + ? { skipAiSettingsPersistence: options.skipAiSettingsPersistence } + : {}), }; } diff --git a/src/common/types/workflow.ts b/src/common/types/workflow.ts index 564ecc1ad9..50f70d9044 100644 --- a/src/common/types/workflow.ts +++ b/src/common/types/workflow.ts @@ -51,6 +51,16 @@ export function isTerminalWorkflowRunStatus(status: WorkflowRunStatus): boolean return status === "completed" || status === "failed" || status === "interrupted"; } +/** + * Terminal statuses that owe a proactive background continuation (terminal wake). An + * interrupted run was stopped deliberately, so neither the terminal callback (unless the + * service opts in) nor the level-triggered attention sweep may notify it. + */ +export const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES: ReadonlySet = new Set([ + "completed", + "failed", +]); + /** * Status of a nested-workflow ("child") run event embedded in a parent run's event stream. * Distinct from {@link WorkflowRunStatus}: an in-progress child event reports "started" rather diff --git a/src/common/utils/tools/tools.ts b/src/common/utils/tools/tools.ts index d8d0a95dbe..807474287e 100644 --- a/src/common/utils/tools/tools.ts +++ b/src/common/utils/tools/tools.ts @@ -2,7 +2,7 @@ import { xai } from "@ai-sdk/xai"; import { type LanguageModel, type Tool } from "ai"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; -import type { ProvidersConfigMap } from "@/common/orpc/types"; +import type { ProvidersConfigMap, SendMessageOptions } from "@/common/orpc/types"; import { isGrokFrontierModel } from "@/common/types/thinking"; import type { BackgroundWorkAttentionPolicy } from "@/common/types/backgroundWorkAttention"; import { cloneToolPreservingDescriptors } from "@/common/utils/tools/cloneToolPreservingDescriptors"; @@ -184,6 +184,10 @@ export interface ToolConfiguration { workspaceSessionDir?: string; /** Workspace ID for tracking background processes and plan storage */ workspaceId?: string; + /** Resolved agent identity of the turn executing the tools (workflow wake provenance). */ + agentId?: string; + /** The turn's strict-agent pin, persisted with workflow run provenance so wakes re-pin the launch agent. */ + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; /** Pre-resolved mux-managed resource scope (global ~/.xum vs project root). */ xumScope?: XumToolScope; /** Memory service for the memory tool (present only when the memory experiment is enabled). */ diff --git a/src/common/utils/workflowRunMessages.ts b/src/common/utils/workflowRunMessages.ts index 9fc393e9f7..66a2b3b5fa 100644 --- a/src/common/utils/workflowRunMessages.ts +++ b/src/common/utils/workflowRunMessages.ts @@ -190,6 +190,41 @@ export function buildWorkflowResultContextMessage(input: { ].join("\n\n"); } +/** + * Recognize this run's result payload inside a coalesced terminal-attention prompt from the + * builder's own output format. The drain's synthetic user row can coalesce several runs into + * one message and carries no workflow-result metadata, so currentness checks must read the + * consumption evidence out of the text: each payload block is parsed back and matched on the + * exact workflow.runId the builder wrote, not on a raw substring, so a run ID merely quoted + * inside another run's report cannot count as consumption. + */ +export function textContainsWorkflowResultPayload(text: string, runId: string): boolean { + assert(runId.length > 0, "textContainsWorkflowResultPayload: runId is required"); + if (!text.includes(WORKFLOW_RESULT_MESSAGE_OPENING_SENTENCE)) { + return false; + } + const blockPattern = new RegExp( + `<${WORKFLOW_RESULT_XML_TAG}>\\n([\\s\\S]*?)\\n`, + "g" + ); + for (const match of text.matchAll(blockPattern)) { + let payload: unknown; + try { + payload = JSON.parse(match[1] ?? ""); + } catch { + continue; + } + if (!isRecordValue(payload)) { + continue; + } + const workflow = payload.workflow; + if (isRecordValue(workflow) && workflow.runId === runId) { + return true; + } + } + return false; +} + export interface WorkflowRunCardInput { scriptPath?: string; scriptSource?: string; diff --git a/src/node/services/agentSession.autoCompaction.test.ts b/src/node/services/agentSession.autoCompaction.test.ts index 72bccaddb5..0161cb9d1e 100644 --- a/src/node/services/agentSession.autoCompaction.test.ts +++ b/src/node/services/agentSession.autoCompaction.test.ts @@ -97,10 +97,12 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { getThreshold: mock(() => 0.85), } as unknown as CompactionMonitor; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; const result = await session.sendMessage("please inspect @foo.ts", { model: "openai:gpt-4o", agentId: "exec", disableWorkspaceAgents: true, + toolPolicy: restrictedPolicy, }); expect(result.success).toBe(true); @@ -122,6 +124,15 @@ describe("AgentSession on-send auto-compaction snapshot deferral", () => { ); expect(persistedCompactionMessage).toBeDefined(); expect(persistedCompactionMessage?.metadata?.disableWorkspaceAgents).toBe(true); + // The durable follow-up must keep the caller's restrictions: the redispatched turn + // reconstructs its options from this persisted request, and dropping the policy there + // would resume the conversation allow-all after compaction. + const followUpContent = + persistedCompactionMessage?.metadata?.muxMetadata?.type === "compaction-request" + ? persistedCompactionMessage.metadata.muxMetadata.parsed.followUpContent + : undefined; + expect(followUpContent?.toolPolicy).toEqual(restrictedPolicy); + expect(followUpContent?.disableWorkspaceAgents).toBe(true); const emittedSnapshot = events.some( (message) => diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index bdb220d713..3df43c6c9e 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -45,6 +45,7 @@ import { SendMessageOptionsSchema, SkillNameSchema, } from "@/common/orpc/schemas"; +import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { buildStreamErrorEventData, @@ -7294,6 +7295,18 @@ export class AgentSession { followUp.muxMetadata ); + // Same raw JSON boundary as below: a persisted follow-up may carry a malformed toolPolicy, + // and restoring it unvalidated would throw during resolution. Invalid values are dropped + // like any corrupt persisted policy (self-healing doctrine); a restricted turn's + // follow-up must otherwise keep its policy instead of redispatching allow-all. + const persistedToolPolicy = + followUp.toolPolicy != null ? ToolPolicySchema.safeParse(followUp.toolPolicy) : undefined; + if (persistedToolPolicy != null && !persistedToolPolicy.success) { + log.warn("Ignoring malformed persisted toolPolicy on compaction follow-up", { + workspaceId: this.workspaceId, + }); + } + // Build options for the follow-up message from the preserved send settings captured // when the compaction handoff was staged. Avoid forwarding internal-only recovery flags. const options: SendMessageOptions & { @@ -7315,6 +7328,7 @@ export class AgentSession { experiments: aliasLegacyPtcExclusive(followUp.experiments), allowAgentSetGoal: followUp.allowAgentSetGoal, disableWorkspaceAgents: followUp.disableWorkspaceAgents, + ...(persistedToolPolicy?.success ? { toolPolicy: persistedToolPolicy.data } : {}), // Explicit-agent turns stay loud on the resumed turn too: the requested agent // may have been removed/hidden/disabled while compaction ran. strictAgentResolution: followUp.strictAgentResolution, diff --git a/src/node/services/agentWorkflowRunReferences.test.ts b/src/node/services/agentWorkflowRunReferences.test.ts index 88a6436139..1939e4f305 100644 --- a/src/node/services/agentWorkflowRunReferences.test.ts +++ b/src/node/services/agentWorkflowRunReferences.test.ts @@ -31,4 +31,271 @@ describe("agent workflow run references", () => { await fs.rm(workspaceSessionDir, { recursive: true, force: true }); } }); + + test("collapses persisted duplicate entries to the newest sane timestamp", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Corrupted files can carry duplicates in either order; order-sensitive consumers must + // never observe a stale duplicate ahead of a legitimate re-record. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_dup", createdAtMs: 1_000 }, + { runId: "wfr_dup", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 2_000 }, + { runId: "wfr_dup_reversed", createdAtMs: 1_000 }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(2); + expect(references).toContainEqual({ runId: "wfr_dup", createdAtMs: 2_000 }); + expect(references).toContainEqual({ runId: "wfr_dup_reversed", createdAtMs: 2_000 }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("roundtrips the initiating agent and drops invalid persisted shapes", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + let references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ + runId: "wfr_agent", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + + // Identity is advisory: a malformed persisted agentId drops the field, not the entry, + // so the run keeps its wake and identity falls back to the history walk. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_agent_number", createdAtMs: 1_000, agentId: 7 }, + { runId: "wfr_agent_empty", createdAtMs: 1_000, agentId: "" }, + // Non-empty but schema-invalid: stream resolution would normalize it to exec, + // silently swapping a restricted agent's wake onto exec's tool surface. + { runId: "wfr_agent_malformed", createdAtMs: 1_000, agentId: "bad id" }, + // Invalid pin shapes degrade to the legacy walk fallback (field dropped); a + // persisted false means verified-unpinned (null), like absence at record time. + { + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: 42 }, + }, + { + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: false, + }, + ], + }) + ); + references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toContainEqual({ runId: "wfr_agent_number", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_empty", createdAtMs: 1_000 }); + expect(references).toContainEqual({ runId: "wfr_agent_malformed", createdAtMs: 1_000 }); + expect(references).toContainEqual({ + runId: "wfr_pin_invalid", + createdAtMs: 1_000, + agentId: "plan", + }); + expect(references).toContainEqual({ + runId: "wfr_pin_false", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: null, + }); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("drops persisted future-dated references and repairs them on the next record", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const futureMs = Date.now() + 86_400_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_corrupt_future", createdAtMs: futureMs }, + { runId: "wfr_sane", createdAtMs: 1_000 }, + ], + }) + ); + + // A per-read clamp would re-evaluate to "now" on every read and outrank every later + // user/reset boundary; the corrupted entry must be dropped instead. + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_sane", createdAtMs: 1_000 }, + ]); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_corrupt_future", + createdAtMs: 2_000, + }); + const repaired = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(repaired).toContainEqual({ runId: "wfr_corrupt_future", createdAtMs: 2_000 }); + const raw = await fs.readFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + "utf-8" + ); + expect(raw).not.toContain(String(futureMs)); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("keeps references within the backward-clock skew tolerance", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A backward clock correction makes a legitimately recorded reference look slightly + // future-dated; dropping it would strand the run's terminal wake. + const slightlyFutureMs = Date.now() + 5 * 60_000; + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ references: [{ runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }] }) + ); + + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([ + { runId: "wfr_clock_skew", createdAtMs: slightlyFutureMs }, + ]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("rejects entries with a present-but-invalid boundary snapshot", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // "" and non-string values are corruption, not legacy records: migrating them into the + // wall-clock fallback could outrank a newer boundary during tolerated clock skew. + await fs.writeFile( + path.join(workspaceSessionDir, "agent-workflow-runs.json"), + JSON.stringify({ + references: [ + { runId: "wfr_empty_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "" }, + { runId: "wfr_numeric_boundary", createdAtMs: 1_000, afterBoundaryMessageId: 42 }, + { runId: "wfr_valid_boundary", createdAtMs: 1_000, afterBoundaryMessageId: "row-1" }, + { runId: "wfr_null_boundary", createdAtMs: 1_000, afterBoundaryMessageId: null }, + ], + }) + ); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(new Set(references.map((reference) => reference.runId))).toEqual( + new Set(["wfr_valid_boundary", "wfr_null_boundary"]) + ); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("propagates non-ENOENT read failures instead of flattening them to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // A directory at the file path fails reads with EISDIR. Callers deciding wake delivery + // must observe the failure rather than "no references". + await fs.mkdir(path.join(workspaceSessionDir, "agent-workflow-runs.json")); + let readError: unknown; + try { + await readAgentWorkflowRunReferences(workspaceSessionDir); + } catch (error: unknown) { + readError = error; + } + expect(String(readError)).toContain("EISDIR"); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("record propagates a sidecar read failure instead of clobbering existing references", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_existing", + createdAtMs: 1_000, + }); + const filePath = path.join(workspaceSessionDir, "agent-workflow-runs.json"); + // Unreadable file, writable directory: the atomic rewrite could replace contents it + // never saw, destroying every other run's only durable provenance. + await fs.chmod(filePath, 0o000); + let recordError: unknown; + try { + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId: "wfr_new", + createdAtMs: 2_000, + }); + } catch (error: unknown) { + recordError = error; + } finally { + await fs.chmod(filePath, 0o600); + } + expect(String(recordError)).toContain("EACCES"); + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references.map((reference) => reference.runId)).toEqual(["wfr_existing"]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("self-heals unparseable file contents to empty", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + // Unlike a failed read, corrupted contents cannot be repaired by rereading. + await fs.writeFile(path.join(workspaceSessionDir, "agent-workflow-runs.json"), "{not json"); + expect(await readAgentWorkflowRunReferences(workspaceSessionDir)).toEqual([]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("clamps future-dated createdAtMs to the current time", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_future"; + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs: Date.now() + 86_400_000, + }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toHaveLength(1); + expect(references[0]?.createdAtMs).toBeLessThanOrEqual(Date.now()); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); + + test("keeps the newest createdAtMs across re-records", async () => { + const workspaceSessionDir = await fs.mkdtemp(path.join(os.tmpdir(), "agent-workflow-runs-")); + try { + const runId = "wfr_re_recorded"; + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 2_000 }); + await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs: 1_000 }); + + const references = await readAgentWorkflowRunReferences(workspaceSessionDir); + expect(references).toEqual([{ runId, createdAtMs: 2_000 }]); + } finally { + await fs.rm(workspaceSessionDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/node/services/agentWorkflowRunReferences.ts b/src/node/services/agentWorkflowRunReferences.ts index b202f4a83d..9322f7dc0e 100644 --- a/src/node/services/agentWorkflowRunReferences.ts +++ b/src/node/services/agentWorkflowRunReferences.ts @@ -3,16 +3,55 @@ import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { SendMessageOptionsSchema } from "@/common/orpc/schemas/stream"; +import type { SendMessageOptions } from "@/common/orpc/types"; +import { AgentIdSchema } from "@/common/schemas/ids"; import assert from "@/common/utils/assert"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; +/** A meaningful strict-agent pin: `false` and absence both mean "not pinned" and persist as null. */ +export type AgentWorkflowRunStrictPin = Exclude< + NonNullable, + false +>; + export interface AgentWorkflowRunReference { runId: string; createdAtMs: number; + /** + * Message ID of the newest invocation-decision row (manual user/reset supersession, consumed + * terminal result for this run, or direct invocation part) at record time; null when history + * had none. Row identity, not wall clock: currentness compares this against the row the + * history walk stops at, so a backward clock correction cannot reorder the comparison. + * Absent on legacy entries, which fail safe to not-current. + */ + afterBoundaryMessageId?: string | null; + /** + * Agent identity of the turn that launched/resumed the run. The terminal wake binds to this + * instead of the newest agent-bearing assistant row, which a later synthetic turn (e.g. a + * heartbeat) can own without superseding the run. Advisory: absent on legacy entries, which + * fall back to the history walk. + */ + agentId?: string; + /** + * The launch turn's strict-agent pin, paired with agentId: a wake that re-binds this agent + * must re-pin the launch turn's provenance, because the newest pin-bearing history row can + * belong to a different group's wake and a mismatched pin makes resolution reject every + * retry. null records a verified-unpinned launch; absent (legacy or invalid persisted + * shape) falls back to the history-walk pin. + */ + strictAgentResolution?: AgentWorkflowRunStrictPin | null; } +const StrictPinSchema = SendMessageOptionsSchema.shape.strictAgentResolution; + const AGENT_WORKFLOW_RUN_REFERENCES_FILE = "agent-workflow-runs.json"; +// Backward clock corrections (e.g. an NTP step after booting with a fast clock) can make a +// legitimately recorded reference look future-dated. Tolerate that bounded skew so the run's +// terminal wake is not dropped; only implausibly future values are treated as corruption. +const MAX_FUTURE_SKEW_MS = 60 * 60_000; + const referenceFileLocks = new MutexMap(); function referencesPath(workspaceSessionDir: string): string { @@ -29,7 +68,8 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { return []; } - const parsed: AgentWorkflowRunReference[] = []; + const parsedByRunId = new Map(); + const now = Date.now(); for (const reference of references) { if (reference == null || typeof reference !== "object") { continue; @@ -41,41 +81,149 @@ function parseReferences(value: unknown): AgentWorkflowRunReference[] { if (typeof record.createdAtMs !== "number" || !Number.isFinite(record.createdAtMs)) { continue; } - parsed.push({ runId: record.runId, createdAtMs: record.createdAtMs }); + // Reject implausibly future-dated references (corruption) instead of clamping at read + // time: a per-read clamp re-evaluates to "now" on every read, so the entry would outrank + // every later user/reset boundary until wall time catches up. Values within + // MAX_FUTURE_SKEW_MS are kept as-is (backward clock correction, not corruption). Rejected + // entries are replaced with a sane timestamp by the next legitimate record. + if (record.createdAtMs > now + MAX_FUTURE_SKEW_MS) { + continue; + } + const hasBoundary = "afterBoundaryMessageId" in record; + const boundaryRaw = record.afterBoundaryMessageId; + // A present-but-invalid snapshot ("" or a non-string) is corruption, not a legacy record: + // demoting it to a boundaryless entry would misclassify a recorded boundary as legacy + // provenance (silently settling its wake as superseded). Reject the entry; absence stays + // reserved for records that genuinely predate the field. + if ( + hasBoundary && + boundaryRaw !== null && + (typeof boundaryRaw !== "string" || boundaryRaw.length === 0) + ) { + continue; + } + const afterBoundaryMessageId = hasBoundary + ? typeof boundaryRaw === "string" + ? boundaryRaw + : null + : undefined; + // Identity is advisory (the wake falls back to the history walk), so an invalid shape + // drops only the field, not the entry. Schema-validate rather than accepting any string: + // stream resolution normalizes an unknown requested agent to exec, so a corrupt persisted + // ID would silently swap a restricted agent's wake onto exec's tool surface. + const agentIdParse = AgentIdSchema.safeParse(record.agentId); + const agentId = agentIdParse.success ? agentIdParse.data : undefined; + // The pin only means anything paired with a surviving identity; false and invalid shapes + // degrade differently (unpinned vs legacy walk fallback), matching the field doc above. + let strictAgentResolution: AgentWorkflowRunStrictPin | null | undefined; + if (agentId !== undefined && "strictAgentResolution" in record) { + const pinRaw = record.strictAgentResolution; + if (pinRaw === null || pinRaw === false) { + strictAgentResolution = null; + } else { + const pinParse = StrictPinSchema.safeParse(pinRaw); + strictAgentResolution = + pinParse.success && pinParse.data != null && pinParse.data !== false + ? pinParse.data + : undefined; + } + } + // Collapse corrupted duplicate entries to the newest sane timestamp so order-sensitive + // consumers cannot pick a stale duplicate and declare a legitimately re-recorded run + // superseded. The chosen record is kept wholesale, including its boundary snapshot. + const existing = parsedByRunId.get(record.runId); + if (existing == null || record.createdAtMs > existing.createdAtMs) { + parsedByRunId.set(record.runId, { + runId: record.runId, + createdAtMs: record.createdAtMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(agentId !== undefined ? { agentId } : {}), + ...(strictAgentResolution !== undefined ? { strictAgentResolution } : {}), + }); + } } - return parsed; + return Array.from(parsedByRunId.values()); } export async function readAgentWorkflowRunReferences( workspaceSessionDir: string ): Promise { + let raw: string; try { - const raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); - return parseReferences(JSON.parse(raw) as unknown); + raw = await fs.readFile(referencesPath(workspaceSessionDir), "utf-8"); } catch (error: unknown) { if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { return []; } + // For kernel-launched runs this file is the only durable invocation evidence, and callers + // deciding wake delivery must distinguish "no reference" from "cannot know right now": + // flattening a transient read failure into [] would let the terminal drain settle the + // run's wake as superseded. Corrupted contents below stay self-healing because rereading + // cannot repair them, while a failed read can succeed later. + throw error; + } + try { + return parseReferences(JSON.parse(raw) as unknown); + } catch { return []; } } +/** + * Retire every reference for this workspace. A full history clear removes all rows without + * appending a reset boundary, which makes a verified-empty (null) boundary snapshot recorded + * before the clear indistinguishable from one recorded after it; retiring the references with + * the transcript keeps pre-clear workflow results out of the fresh conversation. A post-clear + * workflow_resume re-records provenance. + */ +export async function clearAgentWorkflowRunReferences(workspaceSessionDir: string): Promise { + const filePath = referencesPath(workspaceSessionDir); + await referenceFileLocks.withLock(filePath, async () => { + await fs.rm(filePath, { force: true }); + }); +} + export async function recordAgentWorkflowRunReference(input: { workspaceSessionDir: string; runId: string; createdAtMs?: number; + afterBoundaryMessageId?: string | null; + agentId?: string; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; }): Promise { assert(input.runId.length > 0, "agent workflow reference requires runId"); const filePath = referencesPath(input.workspaceSessionDir); await referenceFileLocks.withLock(filePath, async () => { + // A read failure propagates instead of being treated as empty: the atomic rewrite below + // would otherwise replace valid-but-momentarily-unreadable contents with only this run, + // destroying every other active run's sole durable provenance. The failed record is + // retryable (workflow_resume re-records), while parse corruption still self-heals to + // empty inside readAgentWorkflowRunReferences because rereading cannot repair it. const existing = await readAgentWorkflowRunReferences(input.workspaceSessionDir); const byRunId = new Map(existing.map((reference) => [reference.runId, reference])); - const createdAtMs = input.createdAtMs ?? Date.now(); + // Clamp like parseReferences: never persist a future-dated timestamp. + const createdAtMs = Math.min(input.createdAtMs ?? Date.now(), Date.now()); const previous = byRunId.get(input.runId); byRunId.set(input.runId, { runId: input.runId, - createdAtMs: previous ? Math.min(previous.createdAtMs, createdAtMs) : createdAtMs, + // Latest record wins: workflow_resume re-records the reference, and a resume issued after + // a manual user message must re-establish provenance for supersession-timestamp + // comparisons (listAgentReferencedWorkflowRunIds). + createdAtMs: previous ? Math.max(previous.createdAtMs, createdAtMs) : createdAtMs, + // The new record event defines currentness provenance wholesale; a caller without + // boundary knowledge produces a legacy-style entry that fails safe to not-current. + ...(input.afterBoundaryMessageId !== undefined + ? { afterBoundaryMessageId: input.afterBoundaryMessageId } + : {}), + ...(input.agentId != null && input.agentId.length > 0 + ? { + agentId: input.agentId, + ...(input.strictAgentResolution !== undefined + ? { strictAgentResolution: input.strictAgentResolution } + : {}), + } + : {}), }); await fs.mkdir(path.dirname(filePath), { recursive: true }); diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index d967560208..12b419f469 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2321,6 +2321,66 @@ describe("HistoryService", () => { expect(await fs.readFile(archivePath(wsId), "utf-8")).toBe(archiveBefore); }); + it("refuseFullDelete refuses a partial truncation that would remove every message", async () => { + await appendNumberedMessages(service, wsId, 1); + const chatBefore = await fs.readFile(chatPath(wsId), "utf-8"); + + // The caller classified this request as non-emptying; the locked recomputation says it + // empties (an overlapping truncation shrank history in between). Refuse instead of + // taking the full-delete fast path without the caller's full-clear guards. + const refused = await service.truncateHistory(wsId, 0.9, { refuseFullDelete: true }); + expect(refused.success).toBe(false); + if (!refused.success) { + expect(refused.error).toContain("full clear"); + } + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + + // Without the guard the same request takes the fast path and empties history. + const emptied = await service.truncateHistory(wsId, 0.9); + expect(emptied.success).toBe(true); + expect(await service.getHistoryFromLatestBoundary(wsId)).toEqual({ success: true, data: [] }); + }); + + it("refuseRowRemoval refuses a truncation whose recomputed budget removes messages", async () => { + await appendNumberedMessages(service, wsId, 1); + const chatBefore = await fs.readFile(chatPath(wsId), "utf-8"); + + // The caller classified this request as a no-op (and skipped its row-removal guards), + // but the locked recomputation reaches real rows. Refuse instead of removing them. + const refused = await service.truncateHistory(wsId, 0.9, { refuseRowRemoval: true }); + expect(refused.success).toBe(false); + if (!refused.success) { + expect(refused.error).toContain("no-op"); + } + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + + // A genuine no-op stays a silent success under the same flag. + const noop = await service.truncateHistory(wsId, 0.0001, { refuseRowRemoval: true }); + expect(noop.success).toBe(true); + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + }); + + it("requireFullDelete refuses a truncation whose recomputed budget leaves messages", async () => { + await appendNumberedMessages(service, wsId, 8); + const chatBefore = await fs.readFile(chatPath(wsId), "utf-8"); + + // The caller classified this request as emptying (and applies full-clear-only side + // effects after the rewrite), but history grew between that unserialized read and the + // locked rewrite so rows would survive. Refuse instead of leaving survivors behind a + // "full clear". + const refused = await service.truncateHistory(wsId, 0.5, { requireFullDelete: true }); + expect(refused.success).toBe(false); + if (!refused.success) { + expect(refused.error).toContain("leave messages"); + } + expect(await fs.readFile(chatPath(wsId), "utf-8")).toBe(chatBefore); + + // A truncation that does empty history stays a success under the same flag. + const emptied = await service.truncateHistory(wsId, 0.99, { requireFullDelete: true }); + expect(emptied.success).toBe(true); + expect(await service.getHistoryFromLatestBoundary(wsId)).toEqual({ success: true, data: [] }); + }); + it("does not reseed usage from before a partial prefix truncation", async () => { await appendNumberedMessages(service, wsId, 8); await service.appendToHistory( diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index b0bbb3b9f5..bdc9a09973 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -2949,9 +2949,71 @@ export class HistoryService { * @param percentage Percentage to truncate (0.0 to 1.0). 1.0 = delete all * @returns Result containing array of deleted historySequence numbers */ - async truncateHistory( + /** + * Token-proportional prefix length that truncateHistory removes at this percentage. Messages + * are stringified whole for counting; only relative weights matter. + */ + private async computeTruncationRemoveCount( + messages: MuxMessage[], + percentage: number + ): Promise { + const tokenizer = await getTokenizerForModel(KNOWN_MODELS.SONNET.id); + const messageTokens = await Promise.all( + messages.map((msg) => tokenizer.countTokens(safeStringifyForCounting(msg))) + ); + const totalTokens = messageTokens.reduce((sum, tokens) => sum + tokens, 0); + const tokensToRemove = Math.floor(totalTokens * percentage); + let tokensRemoved = 0; + let removeCount = 0; + for (const tokens of messageTokens) { + if (tokensRemoved >= tokensToRemove) { + break; + } + tokensRemoved += tokens; + removeCount++; + } + return removeCount; + } + + /** + * Preflight for truncateHistory: whether this percentage removes no rows, a proper prefix, + * or every message (the full-delete fast path). The requested percentage alone cannot + * distinguish these, and callers that must apply per-scope semantics before the rewrite + * commits (workspaceService retires kernel workflow run references only when rows will + * actually be removed, and applies full-clear guards when everything will) need the answer + * up front. + */ + async classifyTruncationRemoval( workspaceId: string, percentage: number + ): Promise<"none" | "partial" | "all"> { + if (percentage >= 1.0) { + return "all"; + } + if (percentage <= 0) { + return "none"; + } + const archivedMessages = await this.readArchivedHistory(workspaceId); + const chatMessages = await this.readChatHistory(workspaceId); + const messages = [...archivedMessages, ...chatMessages]; + if (messages.length === 0) { + return "none"; + } + const removeCount = await this.computeTruncationRemoveCount(messages, percentage); + if (removeCount === 0) { + return "none"; + } + return removeCount >= messages.length ? "all" : "partial"; + } + + async truncateHistory( + workspaceId: string, + percentage: number, + options?: { + refuseFullDelete?: boolean; + refuseRowRemoval?: boolean; + requireFullDelete?: boolean; + } ): Promise> { return this.withRecoveredHistoryWriteResultLock( workspaceId, @@ -2978,31 +3040,28 @@ export class HistoryService { return Ok([]); // Nothing to truncate } - // Get tokenizer for counting (use a default model) - const tokenizer = await getTokenizerForModel(KNOWN_MODELS.SONNET.id); + const removeCount = await this.computeTruncationRemoveCount(messages, percentage); - // Count tokens for each message - // We stringify the entire message for simplicity - only relative weights matter - const messageTokens: Array<{ message: MuxMessage; tokens: number }> = await Promise.all( - messages.map(async (msg) => { - const tokens = await tokenizer.countTokens(safeStringifyForCounting(msg)); - return { message: msg, tokens }; - }) - ); - - // Calculate total tokens and target to remove - const totalTokens = messageTokens.reduce((sum, mt) => sum + mt.tokens, 0); - const tokensToRemove = Math.floor(totalTokens * percentage); + // Mirror of refuseFullDelete for the opposite drift direction: the caller + // classified this request as a no-op (and so skipped its row-removal guards, e.g. + // kernel workflow reference retirement), but history grew enough between that + // unserialized read and this locked rewrite for the budget to reach real rows. + // Refuse instead of removing them unguarded; a retry re-classifies. + if (options?.refuseRowRemoval === true && removeCount > 0) { + return Err( + "Truncation classified as a no-op would remove messages; retry to re-run it." + ); + } - // Remove messages from beginning until we've removed enough tokens - let tokensRemoved = 0; - let removeCount = 0; - for (const mt of messageTokens) { - if (tokensRemoved >= tokensToRemove) { - break; - } - tokensRemoved += mt.tokens; - removeCount++; + // Third drift direction: the caller classified this request as emptying (and will + // apply full-clear-only side effects after the rewrite: context epoch advance, + // goal/plan/retry discards), but history grew between that unserialized read and + // this locked rewrite so rows would survive. Refuse instead of leaving survivors + // behind a "full clear"; a retry re-classifies. + if (options?.requireFullDelete === true && removeCount < messages.length) { + return Err( + "Truncation classified as a full clear would leave messages; retry to re-run it." + ); } // No-op truncation (percentage 0 or rounding to zero tokens) must not @@ -3014,6 +3073,17 @@ export class HistoryService { // If we're removing all messages, use fast path if (removeCount >= messages.length) { + // Serialized revalidation of the caller's emptiness preflight: an overlapping + // truncation can shrink history between that unserialized read and this locked + // rewrite, turning a partial-classified request into a full delete that skipped + // the caller's full-clear guards (most critically kernel workflow reference + // retirement). Refuse instead of emptying; a retry re-runs the preflight against + // the settled history and routes through the full-clear path. + if (options?.refuseFullDelete === true) { + return Err( + "Truncation would remove every remaining message; retry to run it as a full clear." + ); + } await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index e87e5f1c73..2580c44133 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -2,6 +2,7 @@ import { SecretsStore } from "@/node/config"; import * as path from "path"; import { describe, test, expect, beforeEach, afterEach, mock, spyOn } from "bun:test"; import * as fsPromises from "fs/promises"; +import { existsSync } from "fs"; import * as os from "os"; import { execSync } from "node:child_process"; @@ -528,6 +529,8 @@ type WorkspaceHostMockOverrides = Partial<{ }> & { unarchive?: ReturnType }; function createWorkspaceServiceMocks(overrides: WorkspaceHostMockOverrides = {}) { + const isWorkflowInvocationCurrent = + overrides.isWorkflowInvocationCurrent ?? mock(() => Promise.resolve(true)); const mocks = { sendMessage: overrides.sendMessage ?? mock((): Promise> => Promise.resolve(Ok(undefined))), @@ -555,8 +558,20 @@ function createWorkspaceServiceMocks(overrides: WorkspaceHostMockOverrides = {}) overrides.unarchive ?? overrides.unarchiveWhileTaskTreeLocked ?? mock((): Promise> => Promise.resolve(Ok(undefined))), - isWorkflowInvocationCurrent: - overrides.isWorkflowInvocationCurrent ?? mock(() => Promise.resolve(true)), + isWorkflowInvocationCurrent, + // Derived from the boolean mock so tests that override isWorkflowInvocationCurrent keep + // steering the drain's three-state check. + getWorkflowInvocationCurrentness: + overrides.getWorkflowInvocationCurrentness ?? + mock( + async (workspaceId: string, runId: string) => + ((await isWorkflowInvocationCurrent(workspaceId, runId)) === true + ? "current" + : "not_current") as "current" | "not_current" | "indeterminate" + ), + getWorkflowInvocationBoundaryMessageId: + overrides.getWorkflowInvocationBoundaryMessageId ?? + mock(() => Promise.resolve(null)), create: overrides.create ?? mock( @@ -5653,167 +5668,2894 @@ describe("TaskService", () => { testTaskSettings() ); - const resumeStream = mock( - (): Promise> => - Promise.resolve(Ok({ started: true })) - ); + const resumeStream = mock( + (): Promise> => + Promise.resolve(Ok({ started: true })) + ); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ resumeStream, sendMessage }); + const { historyService, taskService } = createTaskServiceHarness(config, { workspaceService }); + const taskHandleStore = (taskService as unknown as { taskHandleStore: TaskHandleStore }) + .taskHandleStore; + await taskHandleStore.upsertWorkspaceTurn({ + kind: "workspace_turn", + handleId, + ownerWorkspaceId: parentWorkspaceId, + workspaceId: childTaskId, + turnId: "turn-continuation-report", + status: "completed", + createdAt: "2026-08-10T00:00:01.000Z", + updatedAt: "2026-08-10T00:00:02.000Z", + createdWorkspace: false, + disposableWorkspace: false, + reportMarkdown: "Private continuation output", + }); + await historyService.appendToHistory( + parentWorkspaceId, + createMuxMessage( + "continuation-report", + "user", + formatSubagentReportEnvelope({ + taskId: childTaskId, + agentType: "explore", + status: "completed", + title: "Tooling Mapper", + reportMarkdown: "Stable child report", + }), + { + timestamp: Date.parse("2026-08-10T00:00:02.000Z"), + synthetic: true, + uiVisible: true, + } + ) + ); + + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentWorkspaceId, + sourceKind: "agent_task", + sourceId: childTaskId, + }); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentWorkspaceId, + sourceKind: "workspace_turn", + sourceId: handleId, + }); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentWorkspaceId); + + expect(resumeStream).toHaveBeenCalledTimes(1); + expect(sendMessage).not.toHaveBeenCalled(); + expect( + await terminalAttentionStore.get(parentWorkspaceId, `workspace_turn:${handleId}`) + ).toMatchObject({ status: "superseded" }); + expect( + await terminalAttentionStore.get(parentWorkspaceId, `agent_task:${childTaskId}`) + ).toMatchObject({ status: "delivered" }); + }); + + test("persistent child continuation keeps the wake prompt when no current report was delivered", async () => { + const config = await createTestConfig(rootDir); + const projectPath = path.join(rootDir, "repo"); + const parentWorkspaceId = "parent-continuation-fallback"; + const childTaskId = "child-continuation-fallback"; + const handleId = "wst_continuation_fallback"; + await saveWorkspaces( + config, + projectPath, + [ + projectWorkspace(projectPath, "parent", parentWorkspaceId), + projectWorkspace(projectPath, "child", childTaskId, { + parentWorkspaceId, + agentId: "explore", + agentType: "explore", + taskStatus: "reported", + reportedAt: "2026-08-10T00:00:00.000Z", + taskExecutionId: handleId, + taskExecutionStatus: "completed", + }), + ], + testTaskSettings() + ); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { historyService, taskService } = createTaskServiceHarness(config, { workspaceService }); + const taskHandleStore = (taskService as unknown as { taskHandleStore: TaskHandleStore }) + .taskHandleStore; + await taskHandleStore.upsertWorkspaceTurn({ + kind: "workspace_turn", + handleId, + ownerWorkspaceId: parentWorkspaceId, + workspaceId: childTaskId, + turnId: "turn-continuation-fallback", + status: "completed", + createdAt: "2026-08-10T00:00:01.000Z", + updatedAt: "2026-08-10T00:00:02.000Z", + createdWorkspace: false, + disposableWorkspace: false, + reportMarkdown: "Continuation output without a new agent report", + }); + await historyService.appendToHistory( + parentWorkspaceId, + createMuxMessage( + "old-report", + "user", + formatSubagentReportEnvelope({ + taskId: childTaskId, + agentType: "explore", + status: "completed", + title: "Earlier report", + reportMarkdown: "This report predates the continuation.", + }), + { + timestamp: Date.parse("2026-08-10T00:00:00.000Z"), + synthetic: true, + uiVisible: true, + } + ) + ); + + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentWorkspaceId, + sourceKind: "workspace_turn", + sourceId: handleId, + }); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentWorkspaceId); + + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(String(sendMessage.mock.calls[0]?.[1])).toContain(childTaskId); + expect(String(sendMessage.mock.calls[0]?.[1])).toContain("task_await"); + expect( + await terminalAttentionStore.get(parentWorkspaceId, `workspace_turn:${handleId}`) + ).toMatchObject({ status: "delivered" }); + }); + + test("terminal workflow wake-up reconstructs durable result context", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_notify"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const prompt = String(sendMessage.mock.calls[0]?.[1]); + expect(prompt).toContain("mux_workflow_result"); + expect(prompt).toContain("Workflow finished"); + expect(prompt).toContain(runId); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("terminal workflow wake-up defers when history is unreadable", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // History unreadable at drain time: currentness is indeterminate, so the run must stay + // queued for a later drain or sweep instead of being settled as superseded. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + const run = await runStore.getRun(runId); + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ) + ).toBeNull(); + }); + + test("a deferred wake delivers on the next drain trigger", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_defer_retry"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // The first drain sees a transient storage fault: the run stays queued with no timer + // bookkeeping, and any later drain trigger (stream end, sweep) re-evaluates and delivers. + let currentnessCalls = 0; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => { + currentnessCalls += 1; + return Promise.resolve(currentnessCalls === 1 ? "indeterminate" : "current"); + }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + + ( + taskService as unknown as { scheduleTerminalAttentionDrain(id: string): void } + ).scheduleTerminalAttentionDrain(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const run = await runStore.getRun(runId); + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ) + ).toMatchObject({ status: "delivered" }); + }); + + test("drains for a removed workspace drop queued workflow wakes without touching disk", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_terminal_enqueue_removed_owner"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendNextEvent(runId, { + type: "result", + at: "2026-06-19T00:00:02.000Z", + result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + // The owner is removed (config entry gone, session directory deleted) before the drain + // runs; the drain must not recreate the deleted session directory or leave queued state + // for a future workspace reusing the ID. + const cfg = config.loadConfigOrDefault(); + for (const project of cfg.projects.values()) { + project.workspaces = project.workspaces.filter((workspace) => workspace.id !== parentId); + } + await config.editConfig(() => cfg); + const sessionDir = path.join(config.sessionsDir, parentId); + await fsPromises.rm(sessionDir, { recursive: true, force: true }); + + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).not.toHaveBeenCalled(); + expect(existsSync(sessionDir)).toBe(false); + const queued = ( + taskService as unknown as { + pendingWorkflowRunAttention: Map>; + } + ).pendingWorkflowRunAttention; + expect(queued.has(parentId)).toBe(false); + }); + + test("a legacy pending workflow outbox record is deleted by the next drain", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const { taskService } = createTaskServiceHarness(config); + const terminalAttentionStore = new TerminalAttentionStore(config); + const legacy = await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: "wfr_legacy_outbox", + }); + assert(legacy, "legacy workflow attention must enqueue"); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + + // Deleted outright rather than superseded: workflow wakes are re-derived from run + // records now, so a pre-reconciler pending record is dead state that would otherwise + // hold the drain hot forever. + expect(await terminalAttentionStore.get(parentId, legacy.id)).toBeNull(); + }); + + test("the sweep re-queues a resumed run's new terminal generation past the old delivered marker", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_recovery_stale_generation"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const sweep = () => + ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention(); + + // First generation delivers normally, leaving a delivered marker bound to that terminal + // generation; the sweep must not re-queue an already-settled generation. + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(await sweep()).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + + // The resumed run reaches terminal again with a newer updatedAt; the old delivered marker + // belongs to the previous generation, so the sweep re-queues the wake without any reset + // bookkeeping having run. + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + + expect(await sweep()).toBe(1); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + }); + + test("the sweep honors a recent stable marker from the previous build and re-queues past a stale one", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_upgrade_stable_marker"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const sweep = () => + ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention(); + + // The previous build consumed the result (e.g. a kernel-nested task_await) and recorded + // only the stable un-suffixed marker; no generation marker exists. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.recordSettled({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + terminalOutcome: "failed", + status: "delivered", + }); + + // Upgrade sweep: the stable marker postdates the terminal generation, so the wake is + // already consumed and the decision migrates onto this generation's marker. + expect(await sweep()).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + const run = await runStore.getRun(runId); + const migrated = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(migrated?.status).toBe("delivered"); + + // A resume that reaches terminal after the marker was written makes the stable marker + // stale (its restart-time clear is best-effort): the newer generation must re-queue. + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + expect(await sweep()).toBe(1); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + }); + + test("unarchive reconciliation delivers a wake parked by the archived-owner drain", async () => { + const config = await createTestConfig(rootDir); + const { parentId, projectPath } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_unarchive_requeue"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + const setArchivedState = async (field: "archivedAt" | "unarchivedAt") => { + await config.editConfig((cfg) => { + const entry = cfg.projects + .get(projectPath) + ?.workspaces.find((workspace) => workspace.id === parentId); + assert(entry, "parent workspace must exist"); + entry[field] = new Date().toISOString(); + return cfg; + }); + }; + await setArchivedState("archivedAt"); + + // Terminal lands while archived: the drain parks the wake durably (queue dropped, no + // settlement marker). + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + + // Unarchive-time reconciliation re-queues and delivers without waiting for the interval + // sweep. + await setArchivedState("unarchivedAt"); + await taskService.noteWorkspaceUnarchived(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(String(sendMessage.mock.calls[0]?.[1])).toContain(runId); + }); + + test("settling a stale generation snapshot does not suppress a newer resumed result", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_mid_settlement_resume"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + // The first wake turn resumes the run in the background and the newer generation reaches + // terminal before the outer drain settles its stale first-generation snapshot. The owner + // stays busy (streaming the wake turn) until that settlement completes, so the callback's + // interim drain defers instead of delivering the newer generation early. + let ownerBusy = false; + let simulateResumeDuringWake: (() => Promise) | undefined; + const sendMessage = mock(async (..._args: unknown[]): Promise> => { + const simulate = simulateResumeDuringWake; + simulateResumeDuringWake = undefined; + await simulate?.(); + return Ok(undefined); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + (workspaceService as unknown as Record).hasPendingQueuedOrPreparingTurn = mock( + () => ownerBusy + ); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + simulateResumeDuringWake = async () => { + ownerBusy = true; + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + }); + }; + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + const sweep = () => + ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention(); + + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await drain(parentId); + ownerBusy = false; + await flushTerminalAttentionDrains(taskService); + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + // The stale snapshot's settlement must neither drop the newer generation's queue entry + // nor leave a stable marker that postdates it (which the sweep's upgrade fallback would + // migrate as delivered, permanently suppressing the result). + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(String(sendMessage.mock.calls[1]?.[1])).toContain(runId); + expect(await sweep()).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + }); + + test("a history clear during the busy fallback settles the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_busy_fallback_clear"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + // The idle-only send loses the busy race while a full clear completes; the fallback path + // must not inject the retained pre-clear prompt without revalidating. + let cleared = false; + const sendMessage = mock((..._args: unknown[]): Promise> => { + const internal = _args[3] as { requireIdle?: boolean } | undefined; + if (internal?.requireIdle === true) { + cleared = true; + return Promise.resolve( + Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }) + ); + } + return Promise.resolve(Ok(undefined)); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve(cleared ? "not_current" : "current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // Only the rejected idle-only attempt: the fallback aborts on the currentness reread and + // the re-poked drain settles the superseded generation instead of delivering it. + expect(sendMessage).toHaveBeenCalledTimes(1); + expect((sendMessage.mock.calls[0]?.[3] as { requireIdle?: boolean })?.requireIdle).toBe(true); + const terminalAttentionStore = new TerminalAttentionStore(config); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("superseded"); + }); + + test("a run generation change after prompt derivation defers the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_dispatch_generation_drift"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "failed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => + Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // A Workflows UI retry flips the run back to running (a NEW generation) after the prompt + // snapshot is taken, without touching history or owner busy-ness: model it inside the + // derivation-time currentness read so the materialized candidate retains the failed + // generation while the run record has already moved on. + let retried = false; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(async () => { + if (!retried) { + retried = true; + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:05.000Z", { + allowFailedCheckpointRetry: true, + }); + } + return "current" as const; + }); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + const pending = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention; + pending.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // The pre-dispatch revalidation sees the changed generation and defers: the retained + // prompt would present the superseded failed result as final. The queue entry survives + // so the resumed run's next terminal transition (or the sweep) re-derives. + expect(sendMessage).not.toHaveBeenCalled(); + expect(pending.get(parentId)?.has(runId)).toBe(true); + }); + + test("a kernel-consumed generation during the busy fallback is not redelivered", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_busy_fallback_consumed"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + // The busy race the idle-only send loses IS a competing owner turn consuming this very + // generation through kernel-nested task_await: that consumption writes only the + // settlement marker (no history evidence, owner idle again afterwards). + const terminalAttentionStore = new TerminalAttentionStore(config); + const sendMessage = mock( + async (..._args: unknown[]): Promise> => { + const internal = _args[3] as { requireIdle?: boolean } | undefined; + if (internal?.requireIdle === true) { + await terminalAttentionStore.recordSettled({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + generationId: run.updatedAt, + terminalOutcome: "completed", + status: "delivered", + }); + return Err({ type: "unknown", raw: "Workspace is busy; idle-only send was skipped." }); + } + return Ok(undefined); + } + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + const pending = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention; + pending.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // Only the rejected idle-only attempt: the fallback's settlement-marker recheck sees the + // consumption and aborts instead of replaying the result without requireIdle. The + // re-poked drain then drops the consumed candidate from the queue. + expect(sendMessage).toHaveBeenCalledTimes(1); + expect((sendMessage.mock.calls[0]?.[3] as { requireIdle?: boolean })?.requireIdle).toBe(true); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("delivered"); + expect(pending.get(parentId)?.has(runId) ?? false).toBe(false); + }); + + test("a history mutation during the revalidation reads supersedes the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_dispatch_mutation_during_reads"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => + Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // A history clear retires the run's invocation DURING the revalidation's own run/marker + // reads: model it on the second generation-marker read (the first is derivation's), so + // only a currentness read taken AFTER those reads can observe it. + let cleared = false; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve(cleared ? ("not_current" as const) : ("current" as const))); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + }; + const generationMarkerId = TerminalAttentionStore.notificationId( + "workflow_run", + runId, + run.updatedAt + ); + const realGet = internal.terminalAttentionStore.get.bind(internal.terminalAttentionStore); + let generationMarkerReads = 0; + const getSpy = spyOn(internal.terminalAttentionStore, "get").mockImplementation( + (ownerWorkspaceId, notificationId) => { + if (notificationId === generationMarkerId) { + generationMarkerReads += 1; + if (generationMarkerReads === 2) { + cleared = true; + } + } + return realGet(ownerWorkspaceId, notificationId); + } + ); + + try { + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + internal.pendingWorkflowRunAttention.set(parentId, new Set([runId])); + await internal.drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + } finally { + getSpy.mockRestore(); + } + + // Currentness is the final await before dispatch: it postdates the run/marker reads, so + // the clear is observed and the retained prompt is settled superseded, never sent. + expect(sendMessage).not.toHaveBeenCalled(); + const probeStore = new TerminalAttentionStore(config); + const marker = await probeStore.get(parentId, generationMarkerId); + expect(marker?.status).toBe("superseded"); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has(runId) ?? false).toBe(false); + }); + + test("overlapping settlements preserve the newer generation's stable marker", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_overlapping_settlements"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "failed", "2026-06-19T00:00:03.000Z"); + const oldGeneration = (await runStore.getRun(runId)).updatedAt; + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:05.000Z", { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:07.000Z"); + const newGeneration = (await runStore.getRun(runId)).updatedAt; + + const { taskService } = createTaskServiceHarness(config); + const internal = taskService as unknown as { + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + }; + internal.pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + // Park the older generation's settlement inside its first marker write: the newer + // generation's settlement (started while the older one is parked) can then only + // interleave with the older one's post-write mismatch delete if settlements overlap. + let releaseOldSettlement: () => void = () => undefined; + const oldSettlementParked = new Promise((resolve) => { + releaseOldSettlement = resolve; + }); + let parkedReached: () => void = () => undefined; + const oldSettlementReached = new Promise((resolve) => { + parkedReached = resolve; + }); + const realRecordSettled = internal.terminalAttentionStore.recordSettled.bind( + internal.terminalAttentionStore + ); + let parkedOnce = false; + const settleSpy = spyOn(internal.terminalAttentionStore, "recordSettled").mockImplementation( + async (record, options) => { + if (record.generationId === oldGeneration && !parkedOnce) { + parkedOnce = true; + parkedReached(); + await oldSettlementParked; + } + return realRecordSettled(record, options); + } + ); + + try { + const oldSettlement = taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + runUpdatedAt: oldGeneration, + settledAs: "superseded", + }); + await oldSettlementReached; + const newSettlement = taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: newGeneration, + settledAs: "delivered", + }); + // The newer settlement must queue behind the parked older one instead of interleaving. + await new Promise((resolve) => setTimeout(resolve, 50)); + const midProbeStore = new TerminalAttentionStore(config); + expect( + await midProbeStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, newGeneration) + ) + ).toBeNull(); + releaseOldSettlement(); + await Promise.all([oldSettlement, newSettlement]); + } finally { + settleSpy.mockRestore(); + } + + // The older settlement's mismatch delete ran before the newer settlement's stable + // refresh, so the newer generation's markers survive and the queue entry is consumed. + const probeStore = new TerminalAttentionStore(config); + const stable = await probeStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId) + ); + expect(stable?.generationId).toBe(newGeneration); + const generationMarker = await probeStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, newGeneration) + ); + expect(generationMarker?.status).toBe("delivered"); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has(runId) ?? false).toBe(false); + }); + + test("a newer generation's settlement refreshes a surviving stale stable marker", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_stable_marker_refresh"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 120_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 90_000).toISOString()); + await runStore.appendStatus(runId, "failed", new Date(Date.now() - 60_000).toISOString()); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const sweep = () => + ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention(); + + // First generation delivers and records the stable whole-run marker. + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "failed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstGeneration = (await runStore.getRun(runId)).updatedAt; + + // The run resumes without the restart-time bookkeeping (its best-effort stable clear + // failed), so the stale first-generation marker survives into the new generation. + await runStore.appendStatus(runId, "running", new Date(Date.now() + 30_000).toISOString(), { + allowFailedCheckpointRetry: true, + }); + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + expect(await sweep()).toBe(1); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + + // The newer generation's settlement must refresh the write-once stable marker: a + // downgraded build reads it as "latest consumed generation", and a record still carrying + // the previous generation would suppress the newer result's wake after a downgrade. + const run = await runStore.getRun(runId); + expect(run.updatedAt).not.toBe(firstGeneration); + const terminalAttentionStore = new TerminalAttentionStore(config); + const stableMarker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId) + ); + expect(stableMarker?.status).toBe("delivered"); + expect(stableMarker?.generationId).toBe(run.updatedAt); + }); + + test("the sweep honors a generation-tagged stable marker across clock corrections", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_stable_marker_clock_skew"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: new Date(Date.now() - 30_000).toISOString(), + }); + await runStore.appendStatus(runId, "running", new Date(Date.now() - 10_000).toISOString()); + // The clock stepped back after this terminal transition, so the settlement marker below + // carries a createdAt that PRECEDES the generation it consumed. + await runStore.appendStatus(runId, "failed", new Date(Date.now() + 60_000).toISOString()); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.recordSettled( + { + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + generationId: run.updatedAt, + terminalOutcome: "failed", + status: "delivered", + }, + { wholeSourceRefresh: true } + ); + + // Exact generation evidence must win over wall-clock ordering: the marker consumed this + // very generation, so the sweep must not re-queue and re-deliver it. + expect( + await ( + taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + } + ).sweepWorkflowRunTerminalAttention() + ).toBe(0); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + const migrated = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(migrated?.status).toBe("delivered"); + }); + + test("an unreadable settlement marker skips only that run and never rejects the sweep", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + for (const runId of ["wfr_sweep_marker_a", "wfr_sweep_marker_b"]) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + } + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Keep the queue observable: indeterminate currentness defers every drain delivery. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + }; + const realGet = internal.terminalAttentionStore.get.bind(internal.terminalAttentionStore); + const getSpy = spyOn(internal.terminalAttentionStore, "get") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's unhandled-rejection + // detector on this host before the sweep consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("EACCES: marker unreadable"))) + .mockImplementation(realGet); + + try { + // Startup awaits this sweep: one damaged marker must skip its run, not abort the sweep. + expect(await internal.sweepWorkflowRunTerminalAttention()).toBe(1); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.size).toBe(1); + + // The skipped run is re-derived once the marker read recovers. + expect(await internal.sweepWorkflowRunTerminalAttention()).toBe(1); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.size).toBe(2); + } finally { + getSpy.mockRestore(); + } + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + }); + + test("a failed settlement marker write never rejects and keeps the queue entry", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const { taskService } = createTaskServiceHarness(config); + const internal = taskService as unknown as { + terminalAttentionStore: TerminalAttentionStore; + pendingWorkflowRunAttention: Map>; + }; + internal.pendingWorkflowRunAttention.set(parentId, new Set(["wfr_marker_soft_fail"])); + const settleSpy = spyOn(internal.terminalAttentionStore, "recordSettled") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's unhandled-rejection + // detector on this host before the call consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("EACCES: marker dir unwritable"))); + + const settleParams = { + ownerWorkspaceId: parentId, + runId: "wfr_marker_soft_fail", + status: "completed" as const, + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered" as const, + }; + try { + // Marker I/O must stay contained (workflow_resume/task_await return durable results + // through this call), and the queue entry must survive so the next drain re-attempts. + await taskService.markWorkflowRunTerminalAttentionSettled(settleParams); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has("wfr_marker_soft_fail")).toBe( + true + ); + + await taskService.markWorkflowRunTerminalAttentionSettled(settleParams); + expect(internal.pendingWorkflowRunAttention.get(parentId)?.has("wfr_marker_soft_fail")).toBe( + false + ); + } finally { + settleSpy.mockRestore(); + } + }); + + test("a history clear between classification and dispatch settles the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_clear_race"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Classification sees a current invocation; a full clear then retires the sidecar before + // the batch reaches sendMessage, so the last-moment reread must see not_current. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("not_current")).mockImplementationOnce(() => + Promise.resolve("current") + ); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId, + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + // The pre-clear result must not wake the freshly cleared conversation; the run settles + // superseded for this terminal generation and stays retrievable via workflow_resume. + expect(sendMessage).not.toHaveBeenCalled(); + const terminalAttentionStore = new TerminalAttentionStore(config); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("superseded"); + expect( + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(runId) ?? false + ).toBe(false); + }); + + test("a history clear during resume-option resolution settles the wake instead of delivering", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_resolve_clear_race"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const run = await runStore.getRun(runId); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Classification sees a current invocation; the clear completes while the drain resolves + // resume options, so only a currentness reread taken AFTER that resolution observes it. + let cleared = false; + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve(cleared ? "not_current" : "current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const svc = taskService as unknown as { + resolveParentAutoResumeOptions: (...args: unknown[]) => Promise; + }; + const originalResolve = svc.resolveParentAutoResumeOptions.bind(taskService); + svc.resolveParentAutoResumeOptions = async (...args: unknown[]) => { + const resolved = await originalResolve(...args); + cleared = true; + return resolved; + }; + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId, + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + // The pre-clear result must not wake the freshly cleared conversation. + expect(sendMessage).not.toHaveBeenCalled(); + const terminalAttentionStore = new TerminalAttentionStore(config); + const marker = await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ); + expect(marker?.status).toBe("superseded"); + }); + + test("an indeterminate newest group does not stall an older deliverable group", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const oldRunId = "wfr_group_old"; + const newRunId = "wfr_group_new"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + for (const runId of [oldRunId, newRunId]) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + } + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Classification (first call per run) sees both runs current; the last-moment reread then + // fails transiently for the NEWEST group only. The drain must fall through to the older + // group in the same cycle instead of parking every wake on the sweep. + const currentnessCalls = new Map(); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock((_workspaceId: string, runId: string) => { + const count = (currentnessCalls.get(runId) ?? 0) + 1; + currentnessCalls.set(runId, count); + if (count === 1) { + return Promise.resolve("current"); + } + return Promise.resolve(runId === newRunId ? "indeterminate" : "current"); + }); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: oldRunId, + createdAtMs: 1_100, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: newRunId, + createdAtMs: 1_500, + agentId: "plan", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([oldRunId, newRunId])); + + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const prompt = String(sendMessage.mock.calls[0]?.[1]); + expect(prompt).toContain(oldRunId); + expect(prompt).not.toContain(newRunId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + // The unreadable group stays queued for the next drain or sweep, never settled. + expect( + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(newRunId) + ).toBe(true); + }); + + test("a generation settled during the owner's stream is not redelivered by the terminal callback", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_settled_requeue"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + + // A kernel-nested task_await consumed the durable result and settled the generation while + // the owner was still streaming, before WorkflowService reached its terminal callback. + await taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered", + }); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + + // Kernel consumption leaves no history evidence, so only the durable marker can stop the + // re-queued entry from waking the owner with a duplicate result. + expect(sendMessage).not.toHaveBeenCalled(); + expect( + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(runId) ?? false + ).toBe(false); + }); + + test("stuck pending outbox attention is re-poked by the sweep-cadence reconciler", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_stuck", + }); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { historyService, taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + schedulePendingTerminalAttentionOwnerDrains: () => Promise; + }; + + // Transient restriction read failure: the drain fails closed, leaving the durable record + // pending with no later stream or task event to retry it. + const iterateSpy = spyOn(historyService, "iterateFullHistory") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's unhandled-rejection + // detector on this host before the drain consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("EIO: history unreadable"))); + try { + await internal.drainTerminalAttention(parentId); + expect(sendMessage).not.toHaveBeenCalled(); + + expect(await internal.schedulePendingTerminalAttentionOwnerDrains()).toBe(1); + await flushTerminalAttentionDrains(taskService); + } finally { + iterateSpy.mockRestore(); + } + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(String(sendMessage.mock.calls[0]?.[1])).toContain("wst_stuck"); + expect(await terminalAttentionStore.get(parentId, "workspace_turn:wst_stuck")).toMatchObject({ + status: "delivered", + }); + }); + + test("a rejected group send backs off and lets an older group deliver in the same cycle", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const oldRunId = "wfr_backoff_old"; + const newRunId = "wfr_backoff_new"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + for (const runId of [oldRunId, newRunId]) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + } + + // The newest group's send is persistently rejected (its pinned agent cannot resolve); + // the older group's send succeeds. + const sendMessage = mock((..._args: unknown[]): Promise> => { + const options = _args[2] as { agentId?: string } | undefined; + return options?.agentId === "plan" + ? Promise.resolve(Err({ type: "unknown", raw: "agent not resolvable" })) + : Promise.resolve(Ok(undefined)); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: oldRunId, + createdAtMs: 1_100, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: newRunId, + createdAtMs: 1_500, + agentId: "plan", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([oldRunId, newRunId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // First attempt selects the newest group and is rejected; the re-poked drain skips the + // backed-off group and delivers the older one instead of parking it on the sweep. + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[0]?.[2]).toMatchObject({ agentId: "plan" }); + expect(sendMessage.mock.calls[1]?.[2]).toMatchObject({ agentId: "exec" }); + expect(String(sendMessage.mock.calls[1]?.[1])).toContain(oldRunId); + const queued = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.get(parentId); + // The rejected group stays queued for the sweep-cadence retry, never settled. + expect(queued?.has(newRunId)).toBe(true); + expect(queued?.has(oldRunId) ?? false).toBe(false); + }); + + test("a rejected non-workflow send backs off and lets an agent-bound group deliver", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_nonworkflow_backoff"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + // The workspace-turn batch's conversation-identity send is persistently rejected; the + // agent-bound group's own pinned identity can still send. + const sendMessage = mock((..._args: unknown[]): Promise> => { + const options = _args[2] as { agentId?: string } | undefined; + return options?.agentId === "plan" + ? Promise.resolve(Ok(undefined)) + : Promise.resolve(Err({ type: "unknown", raw: "agent not resolvable" })); + }); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId, + agentId: "plan", + }); + + // A deliverable (non-suppressed) workspace-turn wake keeps the agent-bound group out of + // the batch until the batch's send is rejected. + const taskHandleStore = new TaskHandleStore(config); + await taskHandleStore.upsertWorkspaceTurn({ + kind: "workspace_turn", + handleId: "wst_backoff_deliverable", + ownerWorkspaceId: parentId, + workspaceId: parentId, + turnId: "backoff-deliverable", + status: "completed", + reportMarkdown: "turn done", + createdAt: "2026-08-11T00:00:00.000Z", + updatedAt: "2026-08-11T00:00:01.000Z", + createdWorkspace: false, + disposableWorkspace: false, + }); + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_backoff_deliverable", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + await ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention(parentId); + await flushTerminalAttentionDrains(taskService); + + // First attempt sends the non-workflow batch and is rejected; the re-poked drain lets the + // backed-off batch sit out so the agent-bound group delivers in the same cycle. + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[1]?.[2]).toMatchObject({ agentId: "plan" }); + expect(String(sendMessage.mock.calls[1]?.[1])).toContain(runId); + // The rejected wake stays pending for the sweep-cadence retry, never dropped. + const stillPending = await terminalAttentionStore.listPending(parentId); + expect(stillPending.map((notification) => notification.sourceId)).toEqual([ + "wst_backoff_deliverable", + ]); + const queued = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.get(parentId); + expect(queued?.has(runId) ?? false).toBe(false); + }); + + test("settlement writes the stable marker the previous build dedupes recovery on", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_downgrade_stable"; + // A real run record also materializes the owner session dir: settlement markers refuse to + // recreate a removed owner dir by design, so the fixture must exist like in production. + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const { taskService } = createTaskServiceHarness(config); + await taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered", + }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + // The previous build recovers by enqueueIfAbsent on the stable un-suffixed id: an existing + // record must block it from re-creating a pending wake for the consumed result. + expect( + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workflow_run", + sourceId: runId, + }) + ).toBeNull(); + // This build's generation marker is written alongside it. + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, "2026-06-19T00:00:03.000Z") + ) + ).toMatchObject({ status: "delivered" }); + }); + + test("the sweep does not queue an intentionally interrupted run", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + for (const [runId, status] of [ + ["wfr_sweep_interrupted", "interrupted"], + ["wfr_sweep_completed", "completed"], + ] as const) { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, status, "2026-06-19T00:00:03.000Z"); + } + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Keep the queue observable: indeterminate currentness defers every drain delivery. + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("indeterminate")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const internal = taskService as unknown as { + sweepWorkflowRunTerminalAttention(): Promise; + pendingWorkflowRunAttention: Map>; + }; + + // The user stopped the interrupted run: re-deriving a continuation wake for it would undo + // the stop with new agent actions. Only the completed run owes attention. + expect(await internal.sweepWorkflowRunTerminalAttention()).toBe(1); + const queued = internal.pendingWorkflowRunAttention.get(parentId); + expect(queued?.has("wfr_sweep_completed")).toBe(true); + expect(queued?.has("wfr_sweep_interrupted") ?? false).toBe(false); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + }); + + test("a restarted run clears the stable downgrade marker but keeps generation markers", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_restart_compat"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + const { taskService } = createTaskServiceHarness(config); + await taskService.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + runUpdatedAt: "2026-06-19T00:00:03.000Z", + settledAs: "delivered", + }); + + await taskService.clearWorkflowRunDowngradeSettlement({ ownerWorkspaceId: parentId, runId }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + // The previous build re-arms a restarted run by deleting the stable id; after the clear + // its recovery probe can enqueue the run's next result again instead of dropping it. + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId) + ) + ).toBeNull(); + // This build's generation marker is untouched: the old generation stays settled here. + expect( + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, "2026-06-19T00:00:03.000Z") + ) + ).toMatchObject({ status: "delivered" }); + }); + + test("workflow wake restriction recovery stops at a context reset boundary", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_policy_reset_boundary"; + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The pre-reset manual row disabled bash, but the context reset discarded that + // conversation. The workflow launched from a post-reset synthetic turn (heartbeat), so + // its wake must use fresh defaults instead of resurrecting the discarded restriction. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("reset-boundary", "assistant", "Context reset", { + timestamp: 2_000, + contextBoundaryKind: "reset", + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-launch", "user", "[heartbeat] launch the workflow", { + timestamp: 3_000, + synthetic: true, + }) + ); + + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBeUndefined(); + }); + + test("workflow wakes restore the caller tool policy from the newest manual row", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_policy_restore"); + await createRun("wfr_policy_lifted"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + // The launch turn disabled bash; a later synthetic row (an earlier wake) defines no + // policy and must be skipped. Omitting the policy on the wake would let workflow output + // regain the disabled tool at a time the workflow chooses. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("earlier-wake", "user", "results delivered", { + timestamp: 1_100, + synthetic: true, + }) + ); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_restore", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + toolPolicy: restrictedPolicy, + }); + + // A newer manual row without a policy means the caller lifted it: no restoration. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-unrestricted", "user", "carry on", { timestamp: 1_200 }) + ); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_policy_lifted", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + const liftedOptions = sendMessage.mock.calls[1]?.[2] as { toolPolicy?: unknown }; + expect(liftedOptions.toolPolicy).toBeUndefined(); + }); + + test("wake restoration walks a long agent-less tail: caller policy, agent identity, disable flag", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_policy_long_tail"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "exec", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, + }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_000, agentId: "plan" }) + ); + // A tail longer than any bounded history read: the launch turn's restrictions must still + // be found, not silently lifted once enough rows accumulate after the manual turn. + for (let i = 0; i < 60; i++) { + await historyService.appendToHistory( + parentId, + createMuxMessage(`assistant-${i}`, "assistant", `progress ${i}`, { timestamp: 1_001 + i }) + ); + } + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + toolPolicy: restrictedPolicy, + disableWorkspaceAgents: true, + }); + }); + + test("workflow wakes bind to the initiating agent, not a later synthetic turn's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_initiating_agent"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("launch-turn", "assistant", "starting", { + timestamp: 1_001, + agentId: "plan", + }) + ); + // A heartbeat is synthetic, not a manual supersession boundary: the run stays current, but + // its agent-bearing assistant row is now the newest one in history. The wake must use the + // launch turn's agent from the sidecar, not the heartbeat's. + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat", "user", "heartbeat", { timestamp: 1_002, synthetic: true }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("heartbeat-turn", "assistant", "idle check", { + timestamp: 1_003, + agentId: "exec", + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId, + agentId: "plan", + }); + + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + }); + + test("coalesced workflow wakes split by initiating agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_split_plan"); + await createRun("wfr_split_exec"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Two current runs from different initiating agents: one coalesced wake would hand the + // older run's (attacker-influenced) output to the newer agent's tool grants. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: "wfr_split_exec", + createdAtMs: 1_000, + agentId: "exec", + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: "wfr_split_plan", + createdAtMs: 2_000, + agentId: "plan", + }); + + // Seed the in-memory queue directly so ONE drain observes both runs; per-note drains + // would deliver them separately without exercising the coalescing path. + const terminalAttentionStore = new TerminalAttentionStore(config); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set(["wfr_split_plan", "wfr_split_exec"])); + await drain(parentId); + + // The newest launch's group delivers first, alone, under its own agent. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_split_plan"); + expect(firstPrompt).not.toContain("wfr_split_exec"); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + // The deferred group delivers on a later drain under its own agent. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_split_exec"); + expect(secondPrompt).not.toContain("wfr_split_plan"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("coalesced workflow wakes split by strict pin within one agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_split_pinned"); + await createRun("wfr_pin_split_unpinned"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run both audits", { timestamp: 1_000 }) + ); + // Same agentId, different launch pins (an agent definition replaced between synthetic + // launches): one coalesced wake would process the pinned run's output under the newer + // verified-unpinned launch. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: "wfr_pin_split_pinned", + createdAtMs: 1_000, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: "wfr_pin_split_unpinned", + createdAtMs: 2_000, + agentId: "plan", + strictAgentResolution: null, + }); + + const terminalAttentionStore = new TerminalAttentionStore(config); + // Seed the in-memory queue directly so ONE drain observes both runs. + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set( + parentId, + new Set(["wfr_pin_split_pinned", "wfr_pin_split_unpinned"]) + ); + await drain(parentId); + + // The newest launch delivers first, alone, without the other launch's pin. + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wfr_pin_split_unpinned"); + expect(firstPrompt).not.toContain("wfr_pin_split_pinned"); + const firstOptions = sendMessage.mock.calls[0]?.[2] as Record; + expect(firstOptions.agentId).toBe("plan"); + expect(firstOptions.strictAgentResolution).toBeUndefined(); + + // The pinned launch delivers on the retry drain under its own recorded pin. + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain("wfr_pin_split_pinned"); + expect(secondPrompt).not.toContain("wfr_pin_split_unpinned"); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("mixed drains keep workspace-turn attention off the workflow's agent", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_mixed"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + parentId, + createMuxMessage("agent-turn", "assistant", "on it", { timestamp: 1_001, agentId: "plan" }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId, + agentId: "exec", + }); + + // A workspace-turn result resumes under the conversation's own identity; sharing its wake + // with an agent-bound workflow group would process it under the workflow's agent instead. + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_mixed_handle", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + await drain(parentId); + + expect(sendMessage).toHaveBeenCalledTimes(1); + const firstPrompt = String(sendMessage.mock.calls[0]?.[1]); + expect(firstPrompt).toContain("wst_mixed_handle"); + expect(firstPrompt).not.toContain(runId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + }); + + await drain(parentId); + expect(sendMessage).toHaveBeenCalledTimes(2); + const secondPrompt = String(sendMessage.mock.calls[1]?.[1]); + expect(secondPrompt).toContain(runId); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("a fully suppressed batch re-pokes the drain for unselected workflow groups", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_repoke"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + const drain = ( + taskService as unknown as { + drainTerminalAttention: (ownerWorkspaceId: string) => Promise; + } + ).drainTerminalAttention.bind(taskService); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual", "user", "run the audit", { timestamp: 1_000 }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId, + agentId: "exec", + }); + + // A pending workspace-turn notification whose handle already carries an owner-follow-up + // supersede: the pre-suppression batch counts it (excluding the agent-bound workflow + // group from the send), then the last-moment reread drops it, emptying the batch. + const taskHandleStore = new TaskHandleStore(config); + await taskHandleStore.upsertWorkspaceTurn({ + kind: "workspace_turn", + handleId: "wst_repoke_suppressed", + ownerWorkspaceId: parentId, + workspaceId: parentId, + turnId: "repoke-suppressed", + status: "interrupted", + error: + "Workspace turn superseded by follow-up turn wst_repoke_successor from the same owner workspace", + createdAt: "2026-08-11T00:00:00.000Z", + updatedAt: "2026-08-11T00:00:01.000Z", + createdWorkspace: false, + disposableWorkspace: false, + }); + const terminalAttentionStore = new TerminalAttentionStore(config); + await terminalAttentionStore.enqueueIfAbsent({ + ownerWorkspaceId: parentId, + sourceKind: "workspace_turn", + sourceId: "wst_repoke_suppressed", + }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention.set(parentId, new Set([runId])); + + // The empty suppressed batch must re-poke the drain, not park the wake on the sweep. + await drain(parentId); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + const prompt = String(sendMessage.mock.calls[0]?.[1]); + expect(prompt).toContain(runId); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "exec", + }); + expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + }); + + test("wake keeps a synthetic launch row's strict pin without lifting the manual policy", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const restrictedPolicy = [{ regex_match: "^bash$", action: "disable" as const }]; + const runId = "wfr_synthetic_pin"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-restricted", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: restrictedPolicy, + }) + ); + // The kernel workflow launched from a pinned synthetic turn (preserved heartbeat or + // compaction follow-up): its pin must ride the wake without lifting the manual policy. + await historyService.appendToHistory( + parentId, + createMuxMessage("synthetic-launch", "user", "heartbeat", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }, + }) + ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId, + agentId: "plan", + }); + + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(sendMessage.mock.calls[0]?.[2] as Record).toMatchObject({ + agentId: "plan", + toolPolicy: restrictedPolicy, + strictAgentResolution: { expectedScope: "built-in" }, + }); + }); + + test("transient run-store read failures defer the wake instead of dropping it", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); const sendMessage = mock( (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); - const { workspaceService } = createWorkspaceServiceMocks({ resumeStream, sendMessage }); - const { historyService, taskService } = createTaskServiceHarness(config, { workspaceService }); - const taskHandleStore = (taskService as unknown as { taskHandleStore: TaskHandleStore }) - .taskHandleStore; - await taskHandleStore.upsertWorkspaceTurn({ - kind: "workspace_turn", - handleId, - ownerWorkspaceId: parentWorkspaceId, - workspaceId: childTaskId, - turnId: "turn-continuation-report", + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const terminalAttentionStore = new TerminalAttentionStore(config); + const queued = ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention; + + // run.json exists but is unreadable (EISDIR): potentially transient, so the wake must + // stay queued for a later drain or sweep instead of being dropped. + const unreadableRunId = "wfr_unreadable"; + await fsPromises.mkdir( + path.join(config.sessionsDir, parentId, "workflows", unreadableRunId, "run.json"), + { recursive: true } + ); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: unreadableRunId, status: "completed", - createdAt: "2026-08-10T00:00:01.000Z", - updatedAt: "2026-08-10T00:00:02.000Z", - createdWorkspace: false, - disposableWorkspace: false, - reportMarkdown: "Private continuation output", }); - await historyService.appendToHistory( - parentWorkspaceId, - createMuxMessage( - "continuation-report", - "user", - formatSubagentReportEnvelope({ - taskId: childTaskId, - agentType: "explore", - status: "completed", - title: "Tooling Mapper", - reportMarkdown: "Stable child report", - }), - { - timestamp: Date.parse("2026-08-10T00:00:02.000Z"), - synthetic: true, - uiVisible: true, - } - ) - ); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(queued.get(parentId)?.has(unreadableRunId)).toBe(true); - const terminalAttentionStore = new TerminalAttentionStore(config); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentWorkspaceId, - sourceKind: "agent_task", - sourceId: childTaskId, + // A definitively missing run (ENOENT) is dropped from the queue: the sweep re-derives + // owed wakes from run records, so nothing durable is needed to keep it away. + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_missing", + status: "completed", }); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentWorkspaceId, - sourceKind: "workspace_turn", - sourceId: handleId, + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).not.toHaveBeenCalled(); + expect(queued.get(parentId)?.has("wfr_missing")).toBe(false); + expect(queued.get(parentId)?.has(unreadableRunId)).toBe(true); + expect(await terminalAttentionStore.get(parentId, "workflow_run:wfr_missing")).toBeNull(); + }); + + test("wake defers when the launch-identity read fails after currentness succeeds", async () => { + const config = await createTestConfig(rootDir); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runId = "wfr_identity_unreadable"; + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - await ( - taskService as unknown as { - drainTerminalAttention: (ownerWorkspaceId: string) => Promise; - } - ).drainTerminalAttention(parentWorkspaceId); + const sendMessage = mock( + (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) + ); + const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); + // Currentness succeeds without the sidecar (e.g. a direct invocation row)... + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService } = createTaskServiceHarness(config, { workspaceService }); + const terminalAttentionStore = new TerminalAttentionStore(config); - expect(resumeStream).toHaveBeenCalledTimes(1); + // ...but the launch-identity read fails transiently (EISDIR). Delivering without the + // recorded identity would bind the wake to the newest agent-bearing history row, so the + // wake must stay queued for the retry drain. + await fsPromises.mkdir(path.join(config.sessionsDir, parentId, "agent-workflow-runs.json"), { + recursive: true, + }); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId, + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); expect(sendMessage).not.toHaveBeenCalled(); expect( - await terminalAttentionStore.get(parentWorkspaceId, `workspace_turn:${handleId}`) - ).toMatchObject({ status: "superseded" }); + ( + taskService as unknown as { pendingWorkflowRunAttention: Map> } + ).pendingWorkflowRunAttention + .get(parentId) + ?.has(runId) + ).toBe(true); + const run = await runStore.getRun(runId); expect( - await terminalAttentionStore.get(parentWorkspaceId, `agent_task:${childTaskId}`) - ).toMatchObject({ status: "delivered" }); + await terminalAttentionStore.get( + parentId, + TerminalAttentionStore.notificationId("workflow_run", runId, run.updatedAt) + ) + ).toBeNull(); }); - test("persistent child continuation keeps the wake prompt when no current report was delivered", async () => { + test("wake re-pins the selected group's recorded launch pin, not the newest row's", async () => { const config = await createTestConfig(rootDir); - const projectPath = path.join(rootDir, "repo"); - const parentWorkspaceId = "parent-continuation-fallback"; - const childTaskId = "child-continuation-fallback"; - const handleId = "wst_continuation_fallback"; - await saveWorkspaces( - config, - projectPath, - [ - projectWorkspace(projectPath, "parent", parentWorkspaceId), - projectWorkspace(projectPath, "child", childTaskId, { - parentWorkspaceId, - agentId: "explore", - agentType: "explore", - taskStatus: "reported", - reportedAt: "2026-08-10T00:00:00.000Z", - taskExecutionId: handleId, - taskExecutionStatus: "completed", - }), - ], - testTaskSettings() - ); + const { parentId } = await saveLocalParentWorkspace(config, rootDir); + const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); + const createRun = async (runId: string) => { + await runStore.createRun({ + id: runId, + workspaceId: parentId, + workflow: { + name: "research", + description: "Research workflow", + scope: "built-in", + executable: true, + }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-06-19T00:00:00.000Z", + }); + await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); + await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); + }; + await createRun("wfr_pin_unpinned"); + await createRun("wfr_pin_recorded"); const sendMessage = mock( (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - const { historyService, taskService } = createTaskServiceHarness(config, { workspaceService }); - const taskHandleStore = (taskService as unknown as { taskHandleStore: TaskHandleStore }) - .taskHandleStore; - await taskHandleStore.upsertWorkspaceTurn({ - kind: "workspace_turn", - handleId, - ownerWorkspaceId: parentWorkspaceId, - workspaceId: childTaskId, - turnId: "turn-continuation-fallback", - status: "completed", - createdAt: "2026-08-10T00:00:01.000Z", - updatedAt: "2026-08-10T00:00:02.000Z", - createdWorkspace: false, - disposableWorkspace: false, - reportMarkdown: "Continuation output without a new agent report", - }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); + await historyService.appendToHistory( - parentWorkspaceId, - createMuxMessage( - "old-report", - "user", - formatSubagentReportEnvelope({ - taskId: childTaskId, - agentType: "explore", - status: "completed", - title: "Earlier report", - reportMarkdown: "This report predates the continuation.", - }), - { - timestamp: Date.parse("2026-08-10T00:00:00.000Z"), - synthetic: true, - uiVisible: true, - } - ) + parentId, + createMuxMessage("manual", "user", "run the audits", { timestamp: 1_000 }) + ); + // The newest pin-bearing row belongs to a DIFFERENT group's wake: pinning its provenance + // onto this group's agentId would make resolution reject the wake on every retry. + await historyService.appendToHistory( + parentId, + createMuxMessage("other-group-wake", "user", "earlier group results", { + timestamp: 1_100, + synthetic: true, + retrySendOptions: { + model: "openai:gpt-4o", + agentId: "plan", + strictAgentResolution: { expectedScope: "project", expectedSource: "/repo/.xum/agents" }, + }, + }) ); - const terminalAttentionStore = new TerminalAttentionStore(config); - await terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: parentWorkspaceId, - sourceKind: "workspace_turn", - sourceId: handleId, + // A verified-unpinned launch (null) must suppress the walk pin entirely. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: "wfr_pin_unpinned", + agentId: "exec", + strictAgentResolution: null, }); - - await ( - taskService as unknown as { - drainTerminalAttention: (ownerWorkspaceId: string) => Promise; - } - ).drainTerminalAttention(parentWorkspaceId); - + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_unpinned", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); expect(sendMessage).toHaveBeenCalledTimes(1); - expect(String(sendMessage.mock.calls[0]?.[1])).toContain(childTaskId); - expect(String(sendMessage.mock.calls[0]?.[1])).toContain("task_await"); - expect( - await terminalAttentionStore.get(parentWorkspaceId, `workspace_turn:${handleId}`) - ).toMatchObject({ status: "delivered" }); + const unpinnedOptions = sendMessage.mock.calls[0]?.[2] as { + agentId?: string; + strictAgentResolution?: unknown; + }; + expect(unpinnedOptions.agentId).toBe("exec"); + expect(unpinnedOptions.strictAgentResolution).toBeUndefined(); + + // A recorded launch pin overrides the walk pin exactly. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, parentId), + runId: "wfr_pin_recorded", + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: parentId, + runId: "wfr_pin_recorded", + status: "completed", + }); + await flushTerminalAttentionDrains(taskService); + expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage.mock.calls[1]?.[2] as Record).toMatchObject({ + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); }); - test("terminal workflow wake-up reconstructs durable result context", async () => { + test("a malformed persisted toolPolicy cannot block the wake or leak into the send", async () => { const config = await createTestConfig(rootDir); const { parentId } = await saveLocalParentWorkspace(config, rootDir); - const runId = "wfr_terminal_notify"; + const runId = "wfr_policy_corrupt"; const runStore = new WorkflowRunStore({ sessionDir: path.join(config.sessionsDir, parentId) }); await runStore.createRun({ id: runId, @@ -5830,33 +8572,40 @@ describe("TaskService", () => { now: "2026-06-19T00:00:00.000Z", }); await runStore.appendStatus(runId, "running", "2026-06-19T00:00:01.000Z"); - await runStore.appendNextEvent(runId, { - type: "result", - at: "2026-06-19T00:00:02.000Z", - result: { reportMarkdown: "Workflow finished", structuredOutput: { ok: true } }, - }); await runStore.appendStatus(runId, "completed", "2026-06-19T00:00:03.000Z"); - const terminalAttentionStore = new TerminalAttentionStore(config); const sendMessage = mock( (..._args: unknown[]): Promise> => Promise.resolve(Ok(undefined)) ); const { workspaceService } = createWorkspaceServiceMocks({ sendMessage }); - const { taskService } = createTaskServiceHarness(config, { workspaceService }); + (workspaceService as unknown as Record).getWorkflowInvocationCurrentness = + mock(() => Promise.resolve("current")); + const { taskService, historyService } = createTaskServiceHarness(config, { workspaceService }); - await taskService.enqueueWorkflowRunTerminalAttention({ + // Persisted metadata is untrusted disk state: a corrupt toolPolicy shape must be dropped + // (not copied into the send, where it would throw during resolution and permanently block + // the wake), while the intact disable flag on the same row still applies. + await historyService.appendToHistory( + parentId, + createMuxMessage("manual-corrupt", "user", "run the audit", { + timestamp: 1_000, + toolPolicy: { bogus: true }, + disableWorkspaceAgents: true, + } as unknown as Parameters[3]) + ); + taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: parentId, runId, status: "completed", }); await flushTerminalAttentionDrains(taskService); - expect(sendMessage).toHaveBeenCalledTimes(1); - const prompt = String(sendMessage.mock.calls[0]?.[1]); - expect(prompt).toContain("mux_workflow_result"); - expect(prompt).toContain("Workflow finished"); - expect(prompt).toContain(runId); - expect(await terminalAttentionStore.listPending(parentId)).toHaveLength(0); + const options = sendMessage.mock.calls[0]?.[2] as { + toolPolicy?: unknown; + disableWorkspaceAgents?: unknown; + }; + expect(options.toolPolicy).toBeUndefined(); + expect(options.disableWorkspaceAgents).toBe(true); }); test("initialize replays and clears persisted pending task guidance", async () => { @@ -12466,7 +15215,8 @@ describe("TaskService", () => { ); const queuedInitStatusPath = path.join( - path.join(config.sessionsDir, queued.data.taskId), + config.sessionsDir, + queued.data.taskId, "init-status.json" ); await fsPromises.stat(queuedInitStatusPath).then( diff --git a/src/node/services/taskService.ts b/src/node/services/taskService.ts index 3bc6f72bd9..282d2867c9 100644 --- a/src/node/services/taskService.ts +++ b/src/node/services/taskService.ts @@ -110,6 +110,7 @@ import { import type { ProjectRef, WorkspaceMetadata } from "@/common/types/workspace"; import { getRuntimeType } from "@/node/runtime/initHook"; import { AgentIdSchema } from "@/common/orpc/schemas"; +import { SendMessageOptionsSchema, ToolPolicySchema } from "@/common/orpc/schemas/stream"; import type { AgentDefinitionScope } from "@/common/types/agentDefinition"; import { normalizeAgentId, @@ -143,6 +144,8 @@ import type { ErrorEvent, StreamAbortEvent, StreamEndEvent } from "@/common/type import { isActiveWorkflowRunStatus, isTerminalWorkflowRunStatus, + WORKFLOW_BACKGROUND_CONTINUATION_STATUSES, + type WorkflowRunRecord, type WorkflowRunStatus, } from "@/common/types/workflow"; import { isDynamicToolPart, type DynamicToolPart } from "@/common/types/toolParts"; @@ -184,6 +187,7 @@ import { isNonRetryableStreamError } from "@/common/utils/messages/retryEligibil import type { SendMessageError, StreamErrorType } from "@/common/types/errors"; import { hasCompletedAgentReport } from "@/common/utils/agentTaskCompletion"; import { isWorkspaceArchived } from "@/common/utils/archive"; +import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; import { DEFAULT_CODER_ARCHIVE_BEHAVIOR } from "@/common/config/coderArchiveBehavior"; import { isSSHRuntime, isWorktreeRuntime } from "@/common/types/runtime"; @@ -204,6 +208,7 @@ import { type TerminalAttentionOutcome, } from "@/node/services/terminalAttentionStore"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { AgentWorkflowRunStrictPin } from "@/node/services/agentWorkflowRunReferences"; import { isWorkflowRunTaskId } from "@/node/services/tools/taskId"; import { normalizeWorkflowAgentReportPayloadForHostSchema } from "@/common/utils/tools/workflowReportPayload"; import { @@ -858,6 +863,12 @@ function isWorkspaceBusyIdleOnlySend(error: unknown): boolean { const REMOVED_AGENT_TASKS_DIR = "removed-agent-tasks"; const COMPLETED_REPORT_CACHE_MAX_ENTRIES = 128; +// Level-triggered backstop for workflow terminal wakes: the sweep re-derives owed wakes from +// durable state (run records + settled markers), so lost terminal callbacks, deferred +// (transiently unreadable) evaluations, and crashes all recover here without any per-failure +// retry bookkeeping. +const WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS = 5 * 60_000; + /** Maximum consecutive auto-resumes before stopping. Prevents infinite loops when descendants are stuck. */ // Task-recovery paths must stay deterministic and editing-capable even when // workspace/default agent preferences evolve (e.g., auto router defaults). @@ -1144,6 +1155,27 @@ interface ParentAutoResumeHint { agentId?: string; } +/** Launch identity recorded with a workflow run reference; see AgentWorkflowRunReference. */ +interface WorkflowWakeInitiatingAgent { + agentId: string; + createdAtMs: number; + strictAgentResolution?: AgentWorkflowRunStrictPin | null; +} + +// Coalescing key for terminal workflow wakes: the pin is part of the launch identity, so an +// agentId alone must not merge a pinned launch with an unpinned (or differently pinned) one. +// undefined (legacy walk fallback), null (verified unpinned), and each concrete pin are +// distinct groups; over-splitting structurally equal pins is safe, merging them is not. +function workflowWakeGroupKey(agent: WorkflowWakeInitiatingAgent): string { + const pin = agent.strictAgentResolution; + return `${agent.agentId}\u0000${pin === undefined ? "walk" : JSON.stringify(pin)}`; +} + +// Reserved workflowWakeGroupSendBackoffUntilMs key for the non-workflow (sub-agent and +// workspace-turn) send batch. Group keys start with a non-empty agentId and the empty string +// keys the unpinned group, so a leading \u0000 cannot collide. +const NON_WORKFLOW_WAKE_BACKOFF_KEY = "\u0000non-workflow"; + function isTypedWorkspaceEvent(value: unknown, type: string): boolean { return ( typeof value === "object" && @@ -1631,6 +1663,23 @@ export class TaskService implements AgentTaskIntegration { // tests and shutdown can await them; drains are idempotent and re-triggered on owner idle events. private readonly pendingTerminalAttentionDrainsByOwner = new Map>(); private readonly pendingTerminalAttentionDrains = new Set>(); + // Terminal settlements of the same run must not overlap: settlement is multi-step (stable + // refresh, generation marker, post-write mismatch delete), so an older generation reaching + // its mismatch delete after a newer settlement's stable refresh would remove the newer + // generation's valid marker and let a downgraded build re-deliver a consumed result. + private readonly workflowRunSettlementByRun = new Map>(); + // Owed workflow terminal wakes (owner -> runIds believed terminal and not yet settled). An + // in-memory work queue over durable state, not a delivery record: entries are (re)derived + // from run records + settled markers at startup and on the periodic sweep and added by live + // terminal callbacks, so losing the map merely delays a wake until the next sweep. + private readonly pendingWorkflowRunAttention = new Map>(); + // Owner -> wake-group key -> epoch ms before which the drain skips selecting the group. A + // group whose send was rejected for group-specific reasons (for example an unresolvable + // strictly pinned agent) would otherwise be re-selected newest-first by every drain and + // sweep, starving older deliverable groups. In-memory only: a restart retries every group, + // and entries expire on read. + private readonly workflowWakeGroupSendBackoffUntilMs = new Map>(); + private workflowAttentionSweepTimer: ReturnType | null = null; private readonly pendingWaitersByTaskId = new Map(); private readonly pendingStartWaitersByTaskId = new Map(); // Tracks workspaces currently blocked in a foreground wait (e.g. a task tool call awaiting @@ -1783,9 +1832,15 @@ export class TaskService implements AgentTaskIntegration { } const runIds = new Set(); - const references = await readAgentWorkflowRunReferences( - path.join(this.config.sessionsDir, workspaceId) - ); + let references: Awaited> = []; + try { + references = await readAgentWorkflowRunReferences( + path.join(this.config.sessionsDir, workspaceId) + ); + } catch (error: unknown) { + // Rediscovery is non-destructive and re-runs on the next listing; skip this pass. + log.warn("Failed to read agent workflow run references", { workspaceId, error }); + } for (const reference of references) { // If the latest user/reset supersession has no durable timestamp, fail safe: only trust // workflow provenance re-established by current/post-supersession assistant output below. @@ -2400,26 +2455,30 @@ export class TaskService implements AgentTaskIntegration { // Compaction is internal bookkeeping, not an identity for resuming user work. let agentId = hint?.agentId === "compact" ? undefined : hint?.agentId; - // Durable history preserves the parent identity across process restarts. + // Durable history preserves the parent identity across process restarts. The walk is + // unbounded: synthetic rows without an agent identity (drain-appended sub-agent reports, + // heartbeat scaffolding) can push the newest agent-bearing assistant row past any fixed + // tail, and a truncated read would silently recompose terminal-wake sends from the exec + // fallback, lifting a restricted agent's tool policy. if (!agentId) { - try { - const historyResult = await this.historyService.getLastMessages(parentWorkspaceId, 20); - if (historyResult.success) { - for (let i = historyResult.data.length - 1; i >= 0; i--) { - const msg = historyResult.data[i]; - if ( - msg?.role === "assistant" && - msg.metadata?.agentId && - msg.metadata.agentId !== "compact" - ) { - agentId = msg.metadata.agentId; - break; - } + const found: { agentId?: string } = {}; + await this.historyService.iterateFullHistory(parentWorkspaceId, "backward", (messages) => { + for (const msg of messages) { + if ( + msg.role === "assistant" && + typeof msg.metadata?.agentId === "string" && + msg.metadata.agentId.length > 0 && + msg.metadata.agentId !== "compact" + ) { + found.agentId = msg.metadata.agentId; + return false; } } - } catch { - // Best-effort; fall through to defaults - } + return undefined; + }); + // A failed read falls through to defaults (best-effort); the terminal drain separately + // fails closed on unreadable history via resolveTerminalWakeCallerSendRestrictions. + agentId = found.agentId; } // 3) Default @@ -3133,16 +3192,29 @@ export class TaskService implements AgentTaskIntegration { log.error("Startup workflow task archive sweep failed", { error }); } - const recoveredTerminalWorkflowRunNotificationCount = - await this.recoverTerminalWorkflowRunAttentionNotifications(); + let queuedTerminalWorkflowRunAttentionCount = 0; + try { + queuedTerminalWorkflowRunAttentionCount = await this.sweepWorkflowRunTerminalAttention(); + } catch (error: unknown) { + // Startup-time initialization must never crash the app; the interval sweep retries. + log.warn("Startup workflow terminal attention sweep failed", { error }); + } + if (this.workflowAttentionSweepTimer == null) { + this.workflowAttentionSweepTimer = setInterval(() => { + void this.sweepWorkflowRunTerminalAttention().catch((error: unknown) => { + log.warn("Workflow terminal attention sweep failed", { error }); + }); + void this.schedulePendingTerminalAttentionOwnerDrains().catch((error: unknown) => { + log.warn("Pending terminal attention re-poke failed", { error }); + }); + }, WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS); + this.workflowAttentionSweepTimer.unref?.(); + } const recoveredTerminalWorkspaceTurnNotificationCount = await this.recoverTerminalWorkspaceTurnAttentionNotifications(); const terminalAttentionDrainStartedAt = Date.now(); - const pendingTerminalAttentionOwnerWorkspaceIds = - await this.terminalAttentionStore.listPendingOwnerWorkspaceIds(); - for (const ownerWorkspaceId of pendingTerminalAttentionOwnerWorkspaceIds) { - this.scheduleTerminalAttentionDrain(ownerWorkspaceId); - } + const pendingTerminalAttentionOwnerWorkspaceCount = + await this.schedulePendingTerminalAttentionOwnerDrains(); const terminalAttentionDrainMs = Date.now() - terminalAttentionDrainStartedAt; log.info("[startup] TaskService.initialize completed", { @@ -3160,9 +3232,9 @@ export class TaskService implements AgentTaskIntegration { patchGenerationRecoveryMs, bestOfParentRecoveryCount: bestOfParentWorkspaceIds.size, bestOfRecoveryMs, - recoveredTerminalWorkflowRunNotificationCount, + queuedTerminalWorkflowRunAttentionCount, recoveredTerminalWorkspaceTurnNotificationCount, - pendingTerminalAttentionOwnerWorkspaceCount: pendingTerminalAttentionOwnerWorkspaceIds.length, + pendingTerminalAttentionOwnerWorkspaceCount, terminalAttentionDrainMs, cleanupReportedTasksMs, }); @@ -7591,12 +7663,26 @@ export class TaskService implements AgentTaskIntegration { }); } - private async recoverTerminalWorkflowRunAttentionNotifications(): Promise { + /** + * Level-triggered reconciliation scan: re-derive owed workflow terminal wakes from durable + * state (top-level notify_on_terminal run records in a terminal status, minus + * generation-scoped settled markers) into the in-memory queue and poke the drain. Runs at + * startup and on a fixed sweep interval, so missed terminal callbacks, crashes, and + * deferred (transiently unreadable) evaluations always get re-evaluated without any + * per-failure retry bookkeeping. Archived workspaces are skipped, which parks their wakes + * unsettled: the unarchive hook (noteWorkspaceUnarchived) and the next interval sweep + * re-queue them instead of dropping them. + */ + private async sweepWorkflowRunTerminalAttention(onlyWorkspaceId?: string): Promise { const cfg = this.config.loadConfigOrDefault(); - let recoveredCount = 0; + let queuedCount = 0; for (const project of cfg.projects.values()) { for (const workspace of project.workspaces) { - if (workspace.id == null) { + if ( + workspace.id == null || + (onlyWorkspaceId != null && workspace.id !== onlyWorkspaceId) || + isWorkspaceArchived(workspace.archivedAt, workspace.unarchivedAt) + ) { continue; } const runStore = new WorkflowRunStore({ @@ -7606,38 +7692,116 @@ export class TaskService implements AgentTaskIntegration { try { runs = await runStore.listRuns(); } catch (error: unknown) { - log.warn("Failed to recover workflow terminal notifications", { + log.warn("Failed to sweep workflow terminal attention", { workspaceId: workspace.id, error: getErrorMessage(error), }); continue; } + let queuedForWorkspace = false; for (const run of runs) { if ( run.workspaceId !== workspace.id || run.parentWorkflow != null || resolveBackgroundWorkAttentionPolicy(run.attentionPolicy) !== "notify_on_terminal" || - !isTerminalWorkflowRunStatus(run.status) + // Interrupted runs were stopped deliberately: the terminal callback only notifies + // them under an explicit service opt-in, and this re-derivation must not undo the + // user's stop by injecting a continuation prompt. Opt-in callbacks queue directly. + !WORKFLOW_BACKGROUND_CONTINUATION_STATUSES.has(run.status) ) { continue; } - if (!(await this.workspaceService.isWorkflowInvocationCurrent(workspace.id, run.id))) { + let marker: Awaited>; + try { + marker = await this.terminalAttentionStore.get( + workspace.id, + TerminalAttentionStore.notificationId("workflow_run", run.id, run.updatedAt) + ); + } catch (error: unknown) { + // Startup awaits this sweep, so one unreadable marker must not abort it (or app + // init). Skip the run: an unreadable marker cannot prove the wake is owed, and + // the next sweep retries, so delivery is delayed, never crashed or duplicated. + log.warn("Failed to read workflow terminal settlement marker; skipping run", { + workspaceId: workspace.id, + runId: run.id, + error: getErrorMessage(error), + }); continue; } - const created = await this.terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: workspace.id, - sourceKind: "workflow_run", - sourceId: run.id, - terminalOutcome: terminalAttentionOutcome(run.status), - }); - if (created != null) { - this.scheduleTerminalAttentionDrain(workspace.id); - recoveredCount += 1; + if (marker != null) { + continue; } + // Upgrade compatibility: the previous build recorded consumption only under the + // stable un-suffixed id (no generation markers), including history-invisible + // consumption such as a kernel-nested task_await. The guard keeps generation + // markers authoritative: the restart-time stable clear is best-effort, so a stale + // previous-generation marker must not suppress the newer result; mismatched or + // unparseable evidence fails toward notify. + let stableMarker: Awaited>; + try { + stableMarker = await this.terminalAttentionStore.get( + workspace.id, + TerminalAttentionStore.notificationId("workflow_run", run.id) + ); + } catch (error: unknown) { + log.warn("Failed to read stable workflow settlement marker; skipping run", { + workspaceId: workspace.id, + runId: run.id, + error: getErrorMessage(error), + }); + continue; + } + if ( + stableMarker != null && + (stableMarker.status === "delivered" || stableMarker.status === "superseded") + ) { + // Prefer exact generation evidence (this build's settlement refresh records the + // consumed generation), which is immune to wall-clock corrections; the createdAt + // recency heuristic remains only for legacy markers from the previous build, + // which recorded no generation. + let consumedByStableMarker = stableMarker.generationId === run.updatedAt; + if (!consumedByStableMarker && stableMarker.generationId == null) { + const stableMarkerAt = Date.parse(stableMarker.createdAt); + const terminalGenerationAt = Date.parse(run.updatedAt); + consumedByStableMarker = + Number.isFinite(stableMarkerAt) && + Number.isFinite(terminalGenerationAt) && + stableMarkerAt >= terminalGenerationAt; + } + if (consumedByStableMarker) { + try { + // Migrate the decision onto this generation's marker so later sweeps stay + // single-read; the stable marker already proves consumption, so a failed + // migration only re-runs this fallback on the next sweep. + await this.terminalAttentionStore.recordSettled({ + ownerWorkspaceId: workspace.id, + sourceKind: "workflow_run", + sourceId: run.id, + generationId: run.updatedAt, + terminalOutcome: terminalAttentionOutcome(run.status), + status: stableMarker.status, + }); + } catch (error: unknown) { + log.warn("Failed to migrate stable workflow settlement marker", { + workspaceId: workspace.id, + runId: run.id, + error: getErrorMessage(error), + }); + } + continue; + } + } + if (this.queueWorkflowRunAttention(workspace.id, run.id)) { + queuedCount += 1; + } + queuedForWorkspace = true; + } + if (queuedForWorkspace) { + this.scheduleTerminalAttentionDrain(workspace.id); } } } - return recoveredCount; + return queuedCount; } private async recoverTerminalWorkspaceTurnAttentionNotifications(): Promise { @@ -7730,75 +7894,232 @@ export class TaskService implements AgentTaskIntegration { } // ---- Terminal attention notifier ------------------------------------------------------------ - // Deep module for delivering terminal wake-ups for notify_on_terminal work. Settlement paths - // enqueue a persisted notification (outside any settlement lock); the notifier drains pending - // notifications when the owner is idle, sends one coalesced synthetic wake-up, and marks each - // delivered only after an accepted send. Crash/restart safe via the persisted store. + // Deep module for delivering terminal wake-ups for notify_on_terminal work. Sub-agent and + // workspace-turn settlements enqueue a persisted outbox notification (outside any settlement + // lock); workflow wakes are level-triggered instead, re-derived from run records + settled + // markers (see sweepWorkflowRunTerminalAttention). The drain fires when the owner is idle, + // sends one coalesced synthetic wake-up, and records delivery only after an accepted send. /** - * Persist a pending terminal wake-up for the owner workspace and schedule an async drain. - * Idempotent by source kind/id. Must NOT be called while holding settlement/event locks; only - * the persisted enqueue happens synchronously inside callers, the drain is deferred. + * Note a top-level background workflow run's terminal transition and poke the drain. Purely + * an accelerator over the durable state the sweep re-derives (run records + settled + * markers): a lost poke, a removed workspace, or a later resume needs no compensation here, + * so this writes nothing to disk. */ - async enqueueWorkflowRunTerminalAttention(params: { + noteWorkflowRunTerminalAttention(params: { ownerWorkspaceId: string; runId: string; status: WorkflowRunStatus; - }): Promise { + }): void { assert( params.ownerWorkspaceId.length > 0, - "enqueueWorkflowRunTerminalAttention requires ownerWorkspaceId" + "noteWorkflowRunTerminalAttention requires ownerWorkspaceId" ); - assert(params.runId.length > 0, "enqueueWorkflowRunTerminalAttention requires runId"); + assert(params.runId.length > 0, "noteWorkflowRunTerminalAttention requires runId"); if (!isTerminalWorkflowRunStatus(params.status)) { return; } - await this.enqueueTerminalAttention({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - terminalOutcome: terminalAttentionOutcome(params.status), - sourceId: params.runId, - }); + this.queueWorkflowRunAttention(params.ownerWorkspaceId, params.runId); + this.scheduleTerminalAttentionDrain(params.ownerWorkspaceId); + } + + /** + * Level-triggered retry for outbox (sub-agent / workspace-turn) attention: pending records + * are the durable "wake owed" state, but unlike workflow runs they have no periodic + * re-derivation of their own, so a drain that failed transiently (for example an unreadable + * history for caller restrictions) would otherwise leave them stuck until restart. Startup + * and the sweep interval both re-poke their owners; drains are idempotent and no-op when + * nothing is deliverable. + */ + private async schedulePendingTerminalAttentionOwnerDrains(): Promise { + const ownerWorkspaceIds = await this.terminalAttentionStore.listPendingOwnerWorkspaceIds(); + for (const ownerWorkspaceId of ownerWorkspaceIds) { + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + } + return ownerWorkspaceIds.length; + } + + /** Returns true when the run was newly queued for this owner. */ + private queueWorkflowRunAttention(ownerWorkspaceId: string, runId: string): boolean { + let runIds = this.pendingWorkflowRunAttention.get(ownerWorkspaceId); + if (runIds == null) { + runIds = new Set(); + this.pendingWorkflowRunAttention.set(ownerWorkspaceId, runIds); + } + if (runIds.has(runId)) { + return false; + } + runIds.add(runId); + return true; + } + + /** + * Tool-path access to the invocation-boundary snapshot recorded into the + * agent-workflow-runs sidecar (see recordBackgroundWorkflowRunReference). + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + return this.workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); } - async resetWorkflowRunTerminalAttention(params: { + /** + * Durable "this terminal generation needs no further wake" marker, keyed by the run's + * terminal updatedAt: a later resume produces a new generation and thereby re-arms + * attention without any reset bookkeeping. Write-once and best-effort by design: if the + * write fails or never happens, the next drain evaluation re-derives the same answer from + * run + history evidence and merely re-attempts the marker. + */ + async markWorkflowRunTerminalAttentionSettled(params: { ownerWorkspaceId: string; runId: string; + status: WorkflowRunStatus; + runUpdatedAt: string; + settledAs: "delivered" | "superseded"; }): Promise { assert( params.ownerWorkspaceId.length > 0, - "resetWorkflowRunTerminalAttention requires ownerWorkspaceId" - ); - assert(params.runId.length > 0, "resetWorkflowRunTerminalAttention requires runId"); - await this.terminalAttentionStore.delete( - params.ownerWorkspaceId, - TerminalAttentionStore.notificationId("workflow_run", params.runId) + "markWorkflowRunTerminalAttentionSettled requires ownerWorkspaceId" ); + assert(params.runId.length > 0, "markWorkflowRunTerminalAttentionSettled requires runId"); + if (!isTerminalWorkflowRunStatus(params.status)) { + return; + } + const key = `${params.ownerWorkspaceId}\u0000${params.runId}`; + const previous = this.workflowRunSettlementByRun.get(key) ?? Promise.resolve(); + const run = previous + .catch(() => undefined) + .then(() => this.settleWorkflowRunTerminalAttention(params)); + const tracked = run + .then( + () => undefined, + () => undefined + ) + .finally(() => { + if (this.workflowRunSettlementByRun.get(key) === tracked) { + this.workflowRunSettlementByRun.delete(key); + } + }); + this.workflowRunSettlementByRun.set(key, tracked); + return await run; } - async markWorkflowRunTerminalAttentionConsumed(params: { + private async settleWorkflowRunTerminalAttention(params: { ownerWorkspaceId: string; runId: string; status: WorkflowRunStatus; + runUpdatedAt: string; + settledAs: "delivered" | "superseded"; + }): Promise { + try { + // Downgrade compatibility: the previous build dedupes its startup re-derivation on the + // stable un-suffixed workflow_run id, so settling only the generation marker would let + // a downgraded build re-create a pending wake for a result the user already consumed. + // Stable-first ordering keeps the generation marker (this build's authority) retryable: + // if either write fails, the queue entry survives and the next drain re-settles both. + // The refresh (not write-once) matters: a stale previous-generation stable marker can + // survive its best-effort restart-time clear, and a downgraded build would read it as + // consumption of THIS generation's result. The recorded generation also gives the + // sweep's upgrade fallback exact evidence immune to wall-clock corrections. + await this.terminalAttentionStore.recordSettled( + { + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + sourceId: params.runId, + generationId: params.runUpdatedAt, + terminalOutcome: terminalAttentionOutcome(params.status), + status: params.settledAs, + }, + { wholeSourceRefresh: true } + ); + await this.terminalAttentionStore.recordSettled({ + ownerWorkspaceId: params.ownerWorkspaceId, + sourceKind: "workflow_run", + sourceId: params.runId, + generationId: params.runUpdatedAt, + terminalOutcome: terminalAttentionOutcome(params.status), + status: params.settledAs, + }); + } catch (error: unknown) { + // Contain marker I/O here so no caller fails on bookkeeping: workflow_resume and + // task_await must still return the run's durable result, and the drain must move on to + // its other candidates. Keep the queue entry so the next drain re-evaluates from run + + // history evidence and re-attempts the marker; at worst a truthful wake re-delivers. + log.warn("Failed to record workflow terminal settlement marker", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + settledAs: params.settledAs, + error, + }); + return; + } + // Post-write revalidation: a wake-turn workflow_resume can restart the run while this + // settlement's snapshot was in flight, so the restart-time stable clear can land BEFORE + // the stable write above re-creates the marker. That marker postdates the newer + // generation's updatedAt, so the sweep's upgrade fallback (and a downgraded build's + // whole-run dedupe) would permanently suppress the newer result, and the by-run-id queue + // delete below would drop its owed wake. Reading the run AFTER the writes closes the + // write-side race: any restart after this read re-clears the stable marker itself and + // its terminal callback re-queues behind this deletion. + let currentRunUpdatedAt: string | null; + try { + const runStore = new WorkflowRunStore({ + sessionDir: path.join(this.config.sessionsDir, params.ownerWorkspaceId), + }); + currentRunUpdatedAt = (await runStore.getRun(params.runId)).updatedAt; + } catch { + currentRunUpdatedAt = null; + } + if (currentRunUpdatedAt !== params.runUpdatedAt) { + // The settled snapshot is no longer the run's newest generation (or the run is + // unreadable): the stable whole-run marker must not outlive the snapshot. The + // generation marker stays; it truthfully settles only this snapshot. + await this.clearWorkflowRunDowngradeSettlement({ + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + }); + if (currentRunUpdatedAt != null) { + // The queue entry now represents the newer generation's owed wake: leave it for the + // drain the terminal callback scheduled (the sweep backstops a lost poke). + return; + } + // Unreadable run: fall through to the delete. With no stable marker surviving, the + // sweep re-derives any owed newer generation from durable state. + } + this.pendingWorkflowRunAttention.get(params.ownerWorkspaceId)?.delete(params.runId); + } + + /** + * Downgrade-compat bookkeeping only: settlement dual-writes a stable un-suffixed marker for + * the previous build's whole-run dedupe (see markWorkflowRunTerminalAttentionSettled), and a + * restarted run invalidates it. Without this delete, downgrading after a resume would leave + * the old build refusing to enqueue the run's newer result behind the stale stable marker. + * This build reads the stable marker only as recency-gated upgrade evidence behind + * generation markers (see sweepWorkflowRunTerminalAttention), so the delete stays + * best-effort and must never fail the status transition. + */ + async clearWorkflowRunDowngradeSettlement(params: { + ownerWorkspaceId: string; + runId: string; }): Promise { assert( params.ownerWorkspaceId.length > 0, - "markWorkflowRunTerminalAttentionConsumed requires ownerWorkspaceId" + "clearWorkflowRunDowngradeSettlement requires ownerWorkspaceId" ); - assert(params.runId.length > 0, "markWorkflowRunTerminalAttentionConsumed requires runId"); - if (!isTerminalWorkflowRunStatus(params.status)) { - return; + assert(params.runId.length > 0, "clearWorkflowRunDowngradeSettlement requires runId"); + try { + await this.terminalAttentionStore.delete( + params.ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", params.runId) + ); + } catch (error: unknown) { + log.warn("Failed to clear stale workflow downgrade settlement marker", { + ownerWorkspaceId: params.ownerWorkspaceId, + runId: params.runId, + error, + }); } - await this.terminalAttentionStore.enqueueIfAbsent({ - ownerWorkspaceId: params.ownerWorkspaceId, - sourceKind: "workflow_run", - terminalOutcome: terminalAttentionOutcome(params.status), - sourceId: params.runId, - }); - await this.terminalAttentionStore.markDelivered( - params.ownerWorkspaceId, - TerminalAttentionStore.notificationId("workflow_run", params.runId) - ); } async markWorkspaceTurnTerminalAttentionConsumed(params: { @@ -7907,6 +8228,108 @@ export class TaskService implements AgentTaskIntegration { this.pendingTerminalAttentionDrains.add(promise); } + /** + * Caller send restrictions (tool policy, workspace-agent disable flag, strict-agent pin) to + * restore on a terminal-attention wake. The newest manual user row carries the + * conversation's persisted restrictions; synthetic rows without any (earlier wakes, + * heartbeat scaffolding) do not define them and are skipped. The walk is unbounded: a long + * assistant/synthetic tail after the launch turn must not push the defining row out of + * sight and silently lift the restrictions. It stops at the newest context reset boundary, + * since rows from the discarded context must not re-disable tools available to the reset + * context. Throws when history is unreadable so the caller can fail closed instead of + * waking with unrestricted tools. + */ + private async resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId: string): Promise<{ + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; + }> { + // The pin and the policy resolve independently: a synthetic launch row (preserved + // heartbeat, compaction follow-up) can carry only a strict pin, and the wake bound to that + // turn's agent must keep the pin loud without lifting an older manual row's policy. + // Manual rows still define both wholesale (absence means lifted). + const state: { + pin: { strictAgentResolution?: SendMessageOptions["strictAgentResolution"] } | null; + restrictions: { toolPolicy?: ToolPolicy; disableWorkspaceAgents?: boolean } | null; + } = { pin: null, restrictions: null }; + const historyResult = await this.historyService.iterateFullHistory( + ownerWorkspaceId, + "backward", + (messages) => { + for (const message of messages) { + // A context reset discards everything before it: pre-reset rows must not define + // the wake's restrictions or pin. Stopping here leaves undefined fields as fresh + // defaults, matching a manual send in the post-reset context. + if (isResetBoundaryMessage(message)) { + return false; + } + if (message.role !== "user") { + continue; + } + const metadata = message.metadata; + // The strict-agent pin lives in the row's retry snapshot: an explicit agent override + // must stay loud on the wake too, or a vanished/corrupted definition would silently + // recompose the send from the exec fallback (same rule as startup retry and + // compaction follow-ups). Forwarded verbatim per the field's design note: the object + // form pins the validated definition's scope/source/chain provenance, not just + // loudness. Schema-validated like toolPolicy below, since it crosses the same + // persisted-row boundary; invalid shapes are dropped. + const rawStrictPin = metadata?.retrySendOptions?.strictAgentResolution; + const parsedStrictPin = + rawStrictPin != null && rawStrictPin !== false + ? SendMessageOptionsSchema.shape.strictAgentResolution.safeParse(rawStrictPin) + : null; + if (parsedStrictPin != null && !parsedStrictPin.success) { + log.warn("Ignoring malformed persisted strictAgentResolution on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } + const strictAgentResolution = parsedStrictPin?.success ? parsedStrictPin.data : undefined; + if ( + state.pin == null && + (strictAgentResolution != null || metadata?.synthetic !== true) + ) { + state.pin = strictAgentResolution != null ? { strictAgentResolution } : {}; + } + if ( + state.restrictions == null && + (metadata?.toolPolicy != null || + metadata?.disableWorkspaceAgents != null || + metadata?.synthetic !== true) + ) { + // Persisted rows are untrusted disk state: a malformed toolPolicy would throw deep + // inside send resolution and leave the wake permanently blocked on the same corrupt + // row. Sanitize instead of trusting the JSON shape; an unparseable policy restores + // nothing while a valid disable flag still applies (self-healing doctrine). + const parsedPolicy = + metadata?.toolPolicy != null ? ToolPolicySchema.safeParse(metadata.toolPolicy) : null; + if (parsedPolicy != null && !parsedPolicy.success) { + log.warn("Ignoring malformed persisted toolPolicy on terminal wake", { + ownerWorkspaceId, + messageId: message.id, + }); + } + state.restrictions = { + ...(parsedPolicy?.success ? { toolPolicy: parsedPolicy.data } : {}), + ...(typeof metadata?.disableWorkspaceAgents === "boolean" + ? { disableWorkspaceAgents: metadata.disableWorkspaceAgents } + : {}), + }; + } + if (state.pin != null && state.restrictions != null) { + return false; + } + } + return undefined; + } + ); + if (!historyResult.success) { + throw new Error(`history unavailable: ${historyResult.error}`); + } + return { ...(state.restrictions ?? {}), ...(state.pin ?? {}) }; + } + private scheduleTerminalAttentionDrainAfterIdle(ownerWorkspaceId: string): void { const promise = this.workspaceService .waitForIdleAndNoQueuedMessages(ownerWorkspaceId) @@ -7925,10 +8348,77 @@ export class TaskService implements AgentTaskIntegration { this.pendingTerminalAttentionDrains.add(promise); } + /** + * Last-moment revalidation for an already-materialized workflow prompt candidate. The + * composed prompt retains the run snapshot captured at derivation, and invocation + * currentness alone (conversation evidence) misses two hazards that arrive without + * touching history or owner busy-ness: + * - the run's generation can change (a Workflows UI resume/retry flips it back to running; + * a kernel resume can complete a NEWER generation), so the retained prompt would deliver + * a stale result as if final; + * - a kernel-nested task_await can consume this generation, writing only the settlement + * marker, so resending would replay output the conversation already handled. + * "superseded" settles the candidate; "defer" leaves its queue entry pending so a later + * drain re-derives from the then-current run record and markers. + * Currentness is deliberately the LAST await: it is the read that observes a destructive + * history mutation (clear/truncation) retiring the run's invocation, and any awaited read + * after it would reopen the stale-injection window this reread exists to close. + */ + private async revalidateWorkflowPromptForDispatch( + ownerWorkspaceId: string, + candidate: { runId: string; run: WorkflowRunRecord } + ): Promise<"deliverable" | "superseded" | "defer"> { + try { + const runStore = new WorkflowRunStore({ + sessionDir: path.join(this.config.sessionsDir, ownerWorkspaceId), + }); + const currentRun = await runStore.getRun(candidate.runId); + if ( + currentRun.status !== candidate.run.status || + currentRun.updatedAt !== candidate.run.updatedAt + ) { + return "defer"; + } + const settledMarker = await this.terminalAttentionStore.get( + ownerWorkspaceId, + TerminalAttentionStore.notificationId( + "workflow_run", + candidate.runId, + candidate.run.updatedAt + ) + ); + if (settledMarker != null) { + return "defer"; + } + } catch { + return "defer"; + } + const currentness = await this.workspaceService + .getWorkflowInvocationCurrentness(ownerWorkspaceId, candidate.runId) + .catch(() => "indeterminate" as const); + if (currentness !== "current") { + return currentness === "not_current" ? "superseded" : "defer"; + } + return "deliverable"; + } + private async buildWorkflowTerminalPrompt( ownerWorkspaceId: string, runId: string - ): Promise { + ): Promise< + | { + outcome: "deliver"; + prompt: string; + run: WorkflowRunRecord; + initiatingAgent?: WorkflowWakeInitiatingAgent; + } + // settle: superseded or already consumed; record the generation marker so scans stop here. + | { outcome: "settle"; run: WorkflowRunRecord } + // drop: not (or no longer) a wake candidate; just dequeue, nothing durable to mark. + | { outcome: "drop" } + // defer: state transiently unreadable; keep queued for the next drain trigger or sweep. + | { outcome: "defer" } + > { assert(ownerWorkspaceId.length > 0, "buildWorkflowTerminalPrompt requires ownerWorkspaceId"); assert(runId.length > 0, "buildWorkflowTerminalPrompt requires runId"); const runStore = new WorkflowRunStore({ @@ -7938,30 +8428,111 @@ export class TaskService implements AgentTaskIntegration { try { run = await runStore.getRun(runId); } catch (error: unknown) { + // A missing run (ENOENT) or an unparseable record (no fs code; rereading cannot repair + // it) is definitively ineligible. Every other fs failure (EIO, EACCES, EISDIR...) is + // potentially transient, and dropping on it would delay the wake to the next sweep's + // re-derivation for no reason: defer those like indeterminate currentness below. + const code = + error != null && typeof error === "object" && "code" in error + ? (error as { code?: unknown }).code + : undefined; + if (typeof code === "string" && code !== "ENOENT") { + log.warn("Deferring workflow terminal wake-up; run record unreadable", { + ownerWorkspaceId, + runId, + error: getErrorMessage(error), + }); + return { outcome: "defer" }; + } log.warn("Failed to load terminal workflow run for wake-up", { ownerWorkspaceId, runId, error: getErrorMessage(error), }); - return null; + return { outcome: "drop" }; } if ( run.workspaceId !== ownerWorkspaceId || run.parentWorkflow != null || - !isTerminalWorkflowRunStatus(run.status) || - !(await this.workspaceService.isWorkflowInvocationCurrent(ownerWorkspaceId, run.id)) + // A resumed run left terminal state; its next terminal transition re-queues it. + !isTerminalWorkflowRunStatus(run.status) ) { - return null; + return { outcome: "drop" }; + } + // The durable settled marker outranks the in-memory queue entry: a kernel-nested + // task_await or workflow_resume can settle this generation after the terminal callback + // queued it, and that consumption leaves no history evidence for the classification + // below, so skipping this check would deliver a duplicate wake. + let settledMarker: Awaited>; + try { + settledMarker = await this.terminalAttentionStore.get( + ownerWorkspaceId, + TerminalAttentionStore.notificationId("workflow_run", run.id, run.updatedAt) + ); + } catch (error: unknown) { + // An unreadable marker cannot prove the wake is owed; defer like indeterminate currentness. + log.warn("Deferring workflow terminal wake-up; settlement marker unreadable", { + ownerWorkspaceId, + runId, + error: getErrorMessage(error), + }); + return { outcome: "defer" }; + } + if (settledMarker != null) { + return { outcome: "drop" }; + } + const currentness = await this.workspaceService.getWorkflowInvocationCurrentness( + ownerWorkspaceId, + run.id + ); + // Indeterminate means history was unreadable, not that the run was superseded: settling + // now would permanently drop the wake over a transient fault, so defer and retry instead. + if (currentness === "indeterminate") { + return { outcome: "defer" }; + } + if (currentness === "not_current") { + return { outcome: "settle", run }; + } + // Bind the wake to the agent recorded at launch: the newest agent-bearing assistant row + // can belong to an unrelated later synthetic turn (a heartbeat is not a supersession + // boundary), which would pair a different agent's tool surface with the launch turn's + // caller policy. Advisory: legacy references fall back to the history walk. + let initiatingAgent: WorkflowWakeInitiatingAgent | undefined; + try { + const references = await readAgentWorkflowRunReferences( + path.join(this.config.sessionsDir, ownerWorkspaceId) + ); + const reference = references.find((candidate) => candidate.runId === run.id); + if (reference?.agentId != null) { + initiatingAgent = { + agentId: reference.agentId, + createdAtMs: reference.createdAtMs, + ...(reference.strictAgentResolution !== undefined + ? { strictAgentResolution: reference.strictAgentResolution } + : {}), + }; + } + } catch { + // Currentness can succeed (e.g. a direct invocation row) and this identity read still + // fail transiently. Delivering without the recorded identity would bind the wake to the + // newest agent-bearing history row, handing the run's output to an unrelated later + // synthetic turn's agent; defer like an unreadable run record. + return { outcome: "defer" }; } const scriptPath = run.workflow.sourcePath ?? run.workflow.name; - return buildWorkflowResultContextMessage({ - rawCommand: `workflow_run ${scriptPath}`, - name: scriptPath, - runId: run.id, - status: run.status, - result: null, + return { + outcome: "deliver", run, - }); + ...(initiatingAgent != null ? { initiatingAgent } : {}), + prompt: buildWorkflowResultContextMessage({ + rawCommand: `workflow_run ${scriptPath}`, + name: scriptPath, + runId: run.id, + status: run.status, + result: null, + run, + }), + }; } private async ensureAgentTerminalMessages( @@ -8167,14 +8738,45 @@ export class TaskService implements AgentTaskIntegration { } } + /** + * Back the given terminal-wake send batches off until the sweep cadence retries them and + * re-poke the drain so the remaining batches get their send this cycle instead of starving + * behind the failed one. + */ + private backOffTerminalWakeSends(ownerWorkspaceId: string, keys: readonly string[]): void { + assert(keys.length > 0, "backOffTerminalWakeSends requires keys"); + let ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); + if (ownerBackoff == null) { + ownerBackoff = new Map(); + this.workflowWakeGroupSendBackoffUntilMs.set(ownerWorkspaceId, ownerBackoff); + } + const retryAt = Date.now() + WORKFLOW_TERMINAL_ATTENTION_SWEEP_INTERVAL_MS; + for (const key of keys) { + ownerBackoff.set(key, retryAt); + } + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + } + /** * Drain pending terminal notifications for one owner workspace: defer (leave pending) when the * owner is busy/queued/preparing, otherwise send one coalesced synthetic wake-up and mark the * drained notifications delivered. Stale (deleted-workspace) notifications are marked superseded. */ private async drainTerminalAttention(ownerWorkspaceId: string): Promise { - const pending = await this.terminalAttentionStore.listPending(ownerWorkspaceId); - if (pending.length === 0) { + const allPending = await this.terminalAttentionStore.listPending(ownerWorkspaceId); + // Legacy pre-reconciler outbox records for workflow runs (no generation suffix) are dead + // state now that workflow wakes are re-derived from run records + settled markers: delete + // them so they cannot hold the drain hot forever. + for (const notification of allPending) { + if (notification.sourceKind === "workflow_run") { + await this.terminalAttentionStore.delete(ownerWorkspaceId, notification.id); + } + } + const pending = allPending.filter((notification) => notification.sourceKind !== "workflow_run"); + const queuedWorkflowRunIds = Array.from( + this.pendingWorkflowRunAttention.get(ownerWorkspaceId) ?? [] + ); + if (pending.length === 0 && queuedWorkflowRunIds.length === 0) { return; } @@ -8182,6 +8784,8 @@ export class TaskService implements AgentTaskIntegration { const entry = findWorkspaceEntry(cfg, ownerWorkspaceId); if (entry == null) { // Owner workspace no longer exists: the terminal artifacts remain retrievable elsewhere. + // Queue only, no markers: a settled-marker write would recreate the deleted session dir. + this.pendingWorkflowRunAttention.delete(ownerWorkspaceId); for (const notification of pending) { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); } @@ -8189,6 +8793,9 @@ export class TaskService implements AgentTaskIntegration { } if (isWorkspaceArchived(entry.workspace.archivedAt, entry.workspace.unarchivedAt)) { + // Workflow wakes stay unsettled while archived: the sweep skips archived workspaces, so + // dropping the queue parks them until an unarchive-time sweep re-derives the entries. + this.pendingWorkflowRunAttention.delete(ownerWorkspaceId); for (const notification of pending) { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); } @@ -8265,33 +8872,110 @@ export class TaskService implements AgentTaskIntegration { publicAwaitId: isPersistentChildContinuation ? record.workspaceId : notification.sourceId, }); } - const workflowNotifications = pending.filter( - (notification) => notification.sourceKind === "workflow_run" - ); - const deliverableWorkflowNotificationIds = new Set(); + const deliverableWorkflowPrompts: Array<{ + runId: string; + run: WorkflowRunRecord; + prompt: string; + initiatingAgent?: WorkflowWakeInitiatingAgent; + }> = []; - const workflowPromptSections: string[] = []; - for (const notification of workflowNotifications) { - const workflowPrompt = await this.buildWorkflowTerminalPrompt( - ownerWorkspaceId, - notification.sourceId - ); - if (workflowPrompt == null) { - await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, notification.id); + for (const runId of queuedWorkflowRunIds) { + const workflowPrompt = await this.buildWorkflowTerminalPrompt(ownerWorkspaceId, runId); + if (workflowPrompt.outcome === "defer") { + // History, sidecar, or the run record was transiently unreadable: the run stays + // queued and the next drain trigger or sweep re-evaluates it. + log.warn("Deferring workflow terminal attention; state unavailable", { + ownerWorkspaceId, + runId, + }); + continue; + } + if (workflowPrompt.outcome === "drop") { + this.pendingWorkflowRunAttention.get(ownerWorkspaceId)?.delete(runId); + continue; + } + if (workflowPrompt.outcome === "settle") { + // Dropping a notify_on_terminal wake strands the run's owner; keep the drop diagnosable. + log.warn("Settling superseded workflow terminal attention", { + ownerWorkspaceId, + runId, + }); + await this.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId, + runId, + status: workflowPrompt.run.status, + runUpdatedAt: workflowPrompt.run.updatedAt, + settledAs: "superseded", + }); continue; } - deliverableWorkflowNotificationIds.add(notification.id); - workflowPromptSections.push(workflowPrompt); + deliverableWorkflowPrompts.push({ + runId, + run: workflowPrompt.run, + prompt: workflowPrompt.prompt, + ...(workflowPrompt.initiatingAgent != null + ? { initiatingAgent: workflowPrompt.initiatingAgent } + : {}), + }); } - - const resumeOptions = await this.resolveParentAutoResumeOptions( - ownerWorkspaceId, - entry, - defaultModel - ); + // Deliver one launch-identity group per drain, keyed by agentId AND recorded strict pin: + // the whole coalesced prompt is handled under the single agentId/pin passed to + // sendMessage, so batching runs from different launch identities would hand a restricted + // launch's (attacker-influenced) output to another launch's tool grants. Mixed batches + // too: workspace-turn and sub-agent attention resumes under the conversation's own + // (history-walk) identity, so agent-bound workflow groups never share their send. + // Unselected groups stay queued for a later drain; the newest launch goes first. + // Suppression revalidation below can only shrink the workspace-turn set, so gating on + // pre-suppression candidates over-approximates non-workflow deliverables: the safe + // direction, deferring agent-bound groups rather than ever mixing identities in one send. + // A backed-off non-workflow batch (its conversation-identity send was rejected) sits out + // the drain entirely, staying pending for the sweep-cadence retry, so agent-bound groups + // are not starved behind a send that fails the same way on every drain. + const nonWorkflowSendBackoffUntil = this.workflowWakeGroupSendBackoffUntilMs + .get(ownerWorkspaceId) + ?.get(NON_WORKFLOW_WAKE_BACKOFF_KEY); + let nonWorkflowSendBackedOff = false; + if (nonWorkflowSendBackoffUntil != null) { + if (nonWorkflowSendBackoffUntil > Date.now()) { + nonWorkflowSendBackedOff = true; + } else { + const ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); + ownerBackoff?.delete(NON_WORKFLOW_WAKE_BACKOFF_KEY); + if (ownerBackoff?.size === 0) { + this.workflowWakeGroupSendBackoffUntilMs.delete(ownerWorkspaceId); + } + } + } + const hasNonWorkflowDeliverables = + !nonWorkflowSendBackedOff && + (deliverableAgentNotificationIds.size > 0 || workspaceTurnCandidates.length > 0); + // Unselected groups stay queued: the delivered group's wake turn ends with a streamEnded + // drain (and the sweep backstops an aborted one), which delivers the next group. const workspaceTurnMuxMetadata = await this.getActiveWorkspaceTurnMuxMetadataForWorkspace(ownerWorkspaceId); + // Security: restore the conversation's active caller tool policy on the wake. The ordinary + // in-stream workflow continuation carries the live turn's effectiveToolPolicy; this + // synthetic send starts a fresh turn, and omitting the policy would let a workflow wake + // regain tools the caller disabled (with attacker-influenced workflow output choosing the + // timing). The agent-level policy recomposes from agentId at send resolution. + let wakeRestrictions: { + toolPolicy?: ToolPolicy; + disableWorkspaceAgents?: boolean; + strictAgentResolution?: SendMessageOptions["strictAgentResolution"]; + }; + try { + wakeRestrictions = await this.resolveTerminalWakeCallerSendRestrictions(ownerWorkspaceId); + } catch (error: unknown) { + // Fail closed: an unknown policy must not fall back to unrestricted tools. Everything + // stays pending/queued for the next drain trigger or sweep. + log.warn("Deferring terminal wake; caller tool policy unavailable", { + ownerWorkspaceId, + error, + }); + return; + } + // Last-moment suppression revalidation: a quiet owner-follow-up resettle // deletes its notification files, but cannot retract this drain's // already-taken listPending() snapshot, so the handle records are the @@ -8300,15 +8984,118 @@ export class TaskService implements AgentTaskIntegration { // later candidate's await; suppressed handles are dropped instead of // waking the owner, and their notifications are marked superseded only // after the delivery decision. The residual window is the batch read → - // sendMessage gap below (no awaits in between besides delivery itself) — - // closing it would require holding settlement locks across delivery, - // which the drain must not do; worst case is one redundant wake (fail - // toward notify, never a lost wake). + // sendMessage gap below (no awaits in between besides workflow group + // selection and delivery itself) — closing it would + // require holding settlement locks across delivery, which the drain must + // not do; worst case is one redundant wake (fail toward notify, never a + // lost wake). const candidateRecords = await Promise.all( workspaceTurnCandidates.map((candidate) => this.taskHandleStore.getWorkspaceTurn(ownerWorkspaceId, candidate.notification.sourceId) ) ); + // Workflow candidates get the same last-moment treatment, one launch-identity group at a + // time: a full history clear that completes after buildWorkflowTerminalPrompt classified + // these runs retires the sidecar but cannot retract the materialized candidates, and once + // the clear releases its admission guard the owner is idle again, so this requireIdle + // send would inject a pre-clear workflow result into the freshly cleared conversation. + // The clear retires references before truncating, so a reread sees not_current and + // settles; unreadable state stays queued for the next drain or sweep. Groups are tried + // newest-first until one revalidates, so one unreadable group cannot stall independent + // wakes behind the sweep. Bounded: every iteration permanently removes one group from + // this drain's consideration. + const supersededWorkflowPrompts: typeof deliverableWorkflowPrompts = []; + let currentWorkflowPrompts: typeof deliverableWorkflowPrompts = []; + let workflowInitiatingAgent: WorkflowWakeInitiatingAgent | undefined; + let resumeOptions: + | Awaited> + | undefined; + let remainingWorkflowPrompts = hasNonWorkflowDeliverables + ? deliverableWorkflowPrompts.filter((candidate) => candidate.initiatingAgent == null) + : deliverableWorkflowPrompts; + while (remainingWorkflowPrompts.length > 0) { + let groupAgent: WorkflowWakeInitiatingAgent | undefined; + for (const candidate of remainingWorkflowPrompts) { + const agent = candidate.initiatingAgent; + if (agent != null && (groupAgent == null || agent.createdAtMs > groupAgent.createdAtMs)) { + groupAgent = agent; + } + } + const groupKey = groupAgent != null ? workflowWakeGroupKey(groupAgent) : undefined; + const groupCandidates = remainingWorkflowPrompts.filter((candidate) => + groupKey == null + ? candidate.initiatingAgent == null + : candidate.initiatingAgent != null && + workflowWakeGroupKey(candidate.initiatingAgent) === groupKey + ); + // Group keys embed \u0000, so the empty string safely keys the unpinned group. + const backoffKey = groupKey ?? ""; + const ownerBackoff = this.workflowWakeGroupSendBackoffUntilMs.get(ownerWorkspaceId); + const backoffUntil = ownerBackoff?.get(backoffKey); + if (backoffUntil != null) { + if (backoffUntil > Date.now()) { + // Recently rejected send: leave the group queued and give the next group its turn. + remainingWorkflowPrompts = remainingWorkflowPrompts.filter( + (candidate) => !groupCandidates.includes(candidate) + ); + continue; + } + ownerBackoff?.delete(backoffKey); + if (ownerBackoff?.size === 0) { + this.workflowWakeGroupSendBackoffUntilMs.delete(ownerWorkspaceId); + } + } + // Resolve the send identity before the revalidation reread so the reread stays the last + // await before dispatch: a history clear that completes during this history and + // agent-settings read retires the sidecar, and a reread taken before it would go stale + // and inject the pre-clear result into the freshly cleared conversation. + const groupResumeOptions = await this.resolveParentAutoResumeOptions( + ownerWorkspaceId, + entry, + defaultModel, + groupAgent != null ? { agentId: groupAgent.agentId } : undefined + ); + const groupRevalidation = await Promise.all( + groupCandidates.map((candidate) => + this.revalidateWorkflowPromptForDispatch(ownerWorkspaceId, candidate) + ) + ); + const groupCurrent: typeof deliverableWorkflowPrompts = []; + groupCandidates.forEach((candidate, index) => { + const verdict = groupRevalidation[index]; + if (verdict === "deliverable") { + groupCurrent.push(candidate); + } else if (verdict === "superseded") { + supersededWorkflowPrompts.push(candidate); + } + }); + if (groupCurrent.length > 0) { + currentWorkflowPrompts = groupCurrent; + workflowInitiatingAgent = groupAgent; + resumeOptions = groupResumeOptions; + break; + } + remainingWorkflowPrompts = remainingWorkflowPrompts.filter( + (candidate) => !groupCandidates.includes(candidate) + ); + } + + // No workflow group was selected: resolve under the conversation's own identity. The + // workspace-turn and sub-agent wakes tolerate this await in the residual window (worst + // case one redundant wake, never a stale workflow injection). + resumeOptions ??= await this.resolveParentAutoResumeOptions( + ownerWorkspaceId, + entry, + defaultModel + ); + // Pair the pin with the delivered group: the newest pin-bearing history row can belong to + // a different group's wake (each wake persists its own pin), and pinning another agent's + // provenance onto this group's agentId makes resolution reject the wake on every retry. A + // recorded pin (or a verified-unpinned null) overrides the walk; legacy references + // without the field keep the walk pin. + const groupPin = workflowInitiatingAgent?.strictAgentResolution; + const effectiveStrictPin = + groupPin !== undefined ? (groupPin ?? undefined) : wakeRestrictions.strictAgentResolution; const deliverableWorkspaceTurnNotificationIds = new Set(); const publicAwaitIds: string[] = []; const suppressedNotificationIds: string[] = []; @@ -8318,6 +9105,10 @@ export class TaskService implements AgentTaskIntegration { suppressedNotificationIds.push(candidate.notification.id); return; } + if (nonWorkflowSendBackedOff) { + // Backed off: sits out this drain and stays pending for the sweep-cadence retry. + return; + } deliverableWorkspaceTurnNotificationIds.add(candidate.notification.id); publicAwaitIds.push(candidate.publicAwaitId); }); @@ -8325,6 +9116,15 @@ export class TaskService implements AgentTaskIntegration { for (const id of suppressedNotificationIds) { await this.terminalAttentionStore.markSuperseded(ownerWorkspaceId, id); } + for (const candidate of supersededWorkflowPrompts) { + await this.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId, + runId: candidate.runId, + status: candidate.run.status, + runUpdatedAt: candidate.run.updatedAt, + settledAs: "superseded", + }); + } }; // Sub-agent reports and failures are already durable user-context messages. Resume from history @@ -8333,19 +9133,29 @@ export class TaskService implements AgentTaskIntegration { if (publicAwaitIds.length > 0) { promptSections.push(buildCompletedWorkspaceTurnPrompt(publicAwaitIds)); } - promptSections.push(...workflowPromptSections); + promptSections.push(...currentWorkflowPrompts.map((candidate) => candidate.prompt)); const prompt = promptSections.join("\n\n"); - const effectivePending = pending.filter((notification) => { - if (notification.sourceKind === "agent_task") { - return deliverableAgentNotificationIds.has(notification.id); - } - if (notification.sourceKind === "workflow_run") { - return deliverableWorkflowNotificationIds.has(notification.id); - } - return deliverableWorkspaceTurnNotificationIds.has(notification.id); - }); - if (effectivePending.length === 0) { + const effectivePending = nonWorkflowSendBackedOff + ? [] + : pending.filter((notification) => { + if (notification.sourceKind === "agent_task") { + return deliverableAgentNotificationIds.has(notification.id); + } + return deliverableWorkspaceTurnNotificationIds.has(notification.id); + }); + if (effectivePending.length === 0 && currentWorkflowPrompts.length === 0) { await markSuppressedSuperseded(); + // Suppression can empty the very batch that excluded agent-bound workflow groups; with + // nothing sent there is no streamEnded drain, so re-poke instead of parking the queued + // wake on the sweep. No spin: re-poke only when this drain durably changed state (a + // suppressed turn or superseded workflow was just marked), so the re-drain sees a + // different candidate set; an all-indeterminate batch parks for the sweep instead. + if ( + deliverableWorkflowPrompts.length > 0 && + (suppressedNotificationIds.length > 0 || supersededWorkflowPrompts.length > 0) + ) { + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + } return; } @@ -8353,6 +9163,17 @@ export class TaskService implements AgentTaskIntegration { for (const notification of effectivePending) { await this.terminalAttentionStore.markDelivered(ownerWorkspaceId, notification.id); } + for (const candidate of currentWorkflowPrompts) { + // Marker failures are contained inside the settle method; the delivered wake itself is + // durable history evidence, so the next evaluation settles this run as consumed. + await this.markWorkflowRunTerminalAttentionSettled({ + ownerWorkspaceId, + runId: candidate.runId, + status: candidate.run.status, + runUpdatedAt: candidate.run.updatedAt, + settledAs: "delivered", + }); + } }; const markPendingForRetry = async () => { @@ -8366,6 +9187,9 @@ export class TaskService implements AgentTaskIntegration { agentId: resumeOptions.agentId, thinkingLevel: resumeOptions.thinkingLevel, reasoningMode: resumeOptions.reasoningMode, + ...(wakeRestrictions.toolPolicy != null ? { toolPolicy: wakeRestrictions.toolPolicy } : {}), + ...(wakeRestrictions.disableWorkspaceAgents === true ? { disableWorkspaceAgents: true } : {}), + ...(effectiveStrictPin != null ? { strictAgentResolution: effectiveStrictPin } : {}), ...(workspaceTurnMuxMetadata != null ? { muxMetadata: workspaceTurnMuxMetadata } : {}), }; if (prompt.length === 0) { @@ -8383,6 +9207,11 @@ export class TaskService implements AgentTaskIntegration { ownerWorkspaceId, error: resumeResult.error, }); + // Same starvation shape as the prompt path: agent-bound groups were excluded by this + // batch, so back it off and re-poke to let them send under their own launch identity. + if (deliverableWorkflowPrompts.some((candidate) => candidate.initiatingAgent != null)) { + this.backOffTerminalWakeSends(ownerWorkspaceId, [NON_WORKFLOW_WAKE_BACKOFF_KEY]); + } return; } if (!resumeResult.data.started) { @@ -8416,6 +9245,30 @@ export class TaskService implements AgentTaskIntegration { !this.interruptedParentWorkspaceIds.has(ownerWorkspaceId) && !(await this.hasBlockingActiveWorkForTerminalDrain(ownerWorkspaceId, latestTaskIndex)) ) { + // Security: the composed prompt retains the pre-check workflow results, and this + // fallback send omits requireIdle, so its epoch snapshot postdates any clear or reset + // that completed during the awaited checks above and the clear guard would accept the + // stale injection into the fresh context. The busy race the primary send just lost + // may also have been a competing owner turn consuming these very results through + // kernel-nested task_await (settlement marker only, no history evidence) or a + // Workflows UI resume changing the run generation. Revalidate everything so the + // fallback keeps the primary path's contract (no awaits between revalidation and + // delivery); any stale candidate aborts toward a fresh drain that re-derives, and + // the queue entries survive for it. + if (currentWorkflowPrompts.length > 0) { + const fallbackRevalidation = await Promise.all( + currentWorkflowPrompts.map((candidate) => + this.revalidateWorkflowPromptForDispatch(ownerWorkspaceId, candidate) + ) + ); + if (fallbackRevalidation.some((verdict) => verdict !== "deliverable")) { + log.debug("Terminal wake busy fallback aborted; workflow candidates went stale", { + ownerWorkspaceId, + }); + this.scheduleTerminalAttentionDrain(ownerWorkspaceId); + return; + } + } let fallbackAccepted = false; sendResult = await this.workspaceService.sendMessage( ownerWorkspaceId, @@ -8445,7 +9298,29 @@ export class TaskService implements AgentTaskIntegration { } if (!sendResult.success) { - // Owner became busy between the idle check and the send: leave pending and retry next drain. + if (!isWorkspaceBusyIdleOnlySend(sendResult.error)) { + // A non-busy rejection is likely batch-specific (an unresolvable pinned agent, a model + // or provider gate). Back the sent batches off until the sweep cadence retries them + // and re-poke so the remaining groups get their send this cycle instead of starving + // behind newest-first selection; the non-workflow batch backs off the same way so a + // persistently rejected conversation-identity send cannot exclude agent-bound groups + // on every drain. Bounded: each re-poked drain either delivers or backs off one more + // key, and with every key backed off it selects nothing. + const backoffKeys: string[] = []; + if (currentWorkflowPrompts.length > 0) { + backoffKeys.push( + workflowInitiatingAgent != null ? workflowWakeGroupKey(workflowInitiatingAgent) : "" + ); + } + if (effectivePending.length > 0) { + backoffKeys.push(NON_WORKFLOW_WAKE_BACKOFF_KEY); + } + if (backoffKeys.length > 0) { + this.backOffTerminalWakeSends(ownerWorkspaceId, backoffKeys); + } + } + // Busy rejection: the owner started work between the idle check and the send; leave + // pending and retry on the next drain trigger. log.debug("Terminal attention wake-up not accepted; leaving pending", { ownerWorkspaceId, error: sendResult.error, @@ -11948,7 +12823,8 @@ export class TaskService implements AgentTaskIntegration { private removedAgentTaskTombstonePath(ownerWorkspaceId: string, taskId: string): string { return path.join( - path.join(this.config.sessionsDir, ownerWorkspaceId), + this.config.sessionsDir, + ownerWorkspaceId, REMOVED_AGENT_TASKS_DIR, `${encodeURIComponent(taskId)}.json` ); @@ -12798,6 +13674,14 @@ export class TaskService implements AgentTaskIntegration { return blocking; } + async noteWorkspaceUnarchived(workspaceId: string): Promise { + assert(workspaceId.length > 0, "noteWorkspaceUnarchived requires workspaceId"); + // Archived owners park workflow terminal wakes unsettled (the drain drops the in-memory + // queue and the sweep skips archived workspaces), so without this unarchive-time + // reconciliation an idle owner would stay silent until the interval sweep. + await this.sweepWorkflowRunTerminalAttention(workspaceId); + } + /** * Whether any top-level workflow runs are durably active for this workspace. The archive * sink rechecks this after arming its admission gate (see archiveUnlocked) so a workflow diff --git a/src/node/services/taskWorkspaceSeam.testUtils.ts b/src/node/services/taskWorkspaceSeam.testUtils.ts index 8890e60807..5b81af0977 100644 --- a/src/node/services/taskWorkspaceSeam.testUtils.ts +++ b/src/node/services/taskWorkspaceSeam.testUtils.ts @@ -51,6 +51,8 @@ export function makeWorkspaceHostFake(overrides: Partial = {}): W emitChatEvent: () => undefined, isExperimentEnabled: () => false, isWorkflowInvocationCurrent: () => Promise.resolve(true), + getWorkflowInvocationCurrentness: () => Promise.resolve("current" as const), + getWorkflowInvocationBoundaryMessageId: () => Promise.resolve(null), ...overrides, }; } @@ -72,6 +74,7 @@ export function makeAgentTaskIntegrationFake( markParentWorkspaceInterrupted: () => undefined, latchHardInterruptCascade: () => undefined, terminateAllDescendantAgentTasks: () => Promise.resolve([]), + noteWorkspaceUnarchived: () => Promise.resolve(), ...overrides, }; } diff --git a/src/node/services/taskWorkspaceSeam.ts b/src/node/services/taskWorkspaceSeam.ts index 2283005c7e..61ba8f67b7 100644 --- a/src/node/services/taskWorkspaceSeam.ts +++ b/src/node/services/taskWorkspaceSeam.ts @@ -268,6 +268,14 @@ export interface WorkspaceMetadataHost { ): boolean; emit(event: "chat", payload: { workspaceId: string; message: WorkspaceChatMessage }): boolean; emitChatEvent(workspaceId: string, message: WorkspaceChatMessage): void; + getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise; + getWorkflowInvocationCurrentness( + workspaceId: string, + runId: string + ): Promise<"current" | "not_current" | "indeterminate">; isExperimentEnabled(experimentId: ExperimentId): boolean; isWorkflowInvocationCurrent(workspaceId: string, runId: string): Promise; } @@ -294,6 +302,7 @@ export interface AgentTaskIntegration { workspaceId: string, options?: { workflowRunId?: string } ): Promise; + noteWorkspaceUnarchived(workspaceId: string): Promise; } export function normalizeArchiveUntrackedPaths(paths: readonly string[]): string[] { diff --git a/src/node/services/terminalAttentionStore.test.ts b/src/node/services/terminalAttentionStore.test.ts index 8b17667e1d..1543d6f575 100644 --- a/src/node/services/terminalAttentionStore.test.ts +++ b/src/node/services/terminalAttentionStore.test.ts @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import * as fsPromises from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; @@ -53,6 +54,33 @@ describe("TerminalAttentionStore", () => { expect(pending.map((n) => n.sourceId)).toEqual(["wst_abc"]); }); + test("recordSettled never recreates a removed owner session dir", async () => { + const config = makeConfig(rootDir); + const store = new TerminalAttentionStore(config); + const settled = { + sourceKind: "workflow_run" as const, + sourceId: "wfr_removed", + generationId: "2026-06-19T00:00:03.000Z", + terminalOutcome: "completed" as const, + status: "superseded" as const, + }; + + // No session dir: a workspace removal racing an in-flight drain settlement. The marker is + // re-derivable dedupe, so it must be dropped rather than resurrecting orphaned session + // state. + await store.recordSettled({ ...settled, ownerWorkspaceId: "owner-removed" }); + expect(existsSync(path.join(config.sessionsDir, "owner-removed"))).toBe(false); + + // A live owner still gets the terminal-attention subdir created and the marker written. + await fsPromises.mkdir(path.join(config.sessionsDir, "owner-live"), { recursive: true }); + await store.recordSettled({ ...settled, ownerWorkspaceId: "owner-live" }); + const record = await store.get( + "owner-live", + TerminalAttentionStore.notificationId("workflow_run", "wfr_removed", settled.generationId) + ); + expect(record?.status).toBe("superseded"); + }); + test("loads pending notifications written with legacy derived fields", async () => { const config = makeConfig(rootDir); const dir = path.join(config.sessionsDir, "owner-1", TERMINAL_ATTENTION_DIR); diff --git a/src/node/services/terminalAttentionStore.ts b/src/node/services/terminalAttentionStore.ts index c52481caad..ac8388b712 100644 --- a/src/node/services/terminalAttentionStore.ts +++ b/src/node/services/terminalAttentionStore.ts @@ -138,6 +138,59 @@ export class TerminalAttentionStore { return record; } + /** + * Write-once settlement marker: records a notification directly in a terminal status with a + * single write (no pending intermediate a concurrent reader could misread as an owed wake). + * An existing record for the same id is left untouched. Omitting generationId settles the + * stable (un-suffixed) id, which older builds use for whole-source dedupe. + * + * wholeSourceRefresh instead settles the stable id while keeping the generationId FIELD and + * overwrites any existing record: that marker means "latest consumed generation of this + * source", so settlement must refresh a stale previous-generation record that survived its + * best-effort restart-time clear, and the recorded generation gives readers exact evidence + * that is immune to wall-clock corrections. + */ + async recordSettled( + notification: Omit< + TerminalAttentionNotification, + "id" | "status" | "createdAt" | "outputDelivery" + > & { + status: "delivered" | "superseded"; + }, + options?: { wholeSourceRefresh?: boolean } + ): Promise { + const wholeSourceRefresh = options?.wholeSourceRefresh === true; + const id = TerminalAttentionStore.notificationId( + notification.sourceKind, + notification.sourceId, + wholeSourceRefresh ? undefined : notification.generationId + ); + if (!wholeSourceRefresh) { + const existing = await this.get(notification.ownerWorkspaceId, id); + if (existing != null) { + return; + } + } + await this.write( + { + id, + ownerWorkspaceId: notification.ownerWorkspaceId, + sourceKind: notification.sourceKind, + sourceId: notification.sourceId, + ...(notification.generationId != null ? { generationId: notification.generationId } : {}), + outputDelivery: outputDeliveryForSource(notification.sourceKind), + terminalOutcome: notification.terminalOutcome, + status: notification.status, + createdAt: new Date().toISOString(), + ...(notification.status === "delivered" ? { deliveredAt: new Date().toISOString() } : {}), + }, + // Settlement markers are re-derivable dedupe over run + history evidence, and reconcilers + // write them while workspace removal may be deleting the session directory: never mkdir + // the owner dir back into existence for one (orphaned session state); drop it instead. + { createOwnerDir: false } + ); + } + async get(ownerWorkspaceId: string, id: string): Promise { let raw: string; try { @@ -230,9 +283,31 @@ export class TerminalAttentionStore { }); } - private async write(record: TerminalAttentionNotification): Promise { + private async write( + record: TerminalAttentionNotification, + options?: { createOwnerDir?: boolean } + ): Promise { const dir = this.dir(record.ownerWorkspaceId); - await fsPromises.mkdir(dir, { recursive: true }); + if (options?.createOwnerDir === false) { + try { + // Non-recursive: creates only the terminal-attention subdir under an owner session dir + // that still exists; a missing parent means the owner was removed. + await fsPromises.mkdir(dir); + } catch (error) { + if (isErrnoWithCode(error, "ENOENT")) { + log.debug("Dropping terminal attention write for removed owner session dir", { + ownerWorkspaceId: record.ownerWorkspaceId, + id: record.id, + }); + return; + } + if (!isErrnoWithCode(error, "EEXIST")) { + throw error; + } + } + } else { + await fsPromises.mkdir(dir, { recursive: true }); + } await fsPromises.writeFile( this.file(record.ownerWorkspaceId, record.id), JSON.stringify(record, null, 2), diff --git a/src/node/services/tools/task_await.test.ts b/src/node/services/tools/task_await.test.ts index 4863fcd303..ddcc5a35a9 100644 --- a/src/node/services/tools/task_await.test.ts +++ b/src/node/services/tools/task_await.test.ts @@ -1194,11 +1194,11 @@ describe("task_await tool", () => { }, ]); - const markWorkflowRunTerminalAttentionConsumed = mock(() => Promise.resolve()); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); const taskService = { listActiveDescendantAgentTaskIds: mock(() => []), isDescendantAgentTask: mock(() => Promise.resolve(false)), - markWorkflowRunTerminalAttentionConsumed, + markWorkflowRunTerminalAttentionSettled, waitForAgentReport: mock(() => { throw new Error("workflow run IDs should not be treated as agent tasks"); }), @@ -1233,10 +1233,12 @@ describe("task_await tool", () => { workspaceId: "parent-workspace", runId: "wfr_demo", }); - expect(markWorkflowRunTerminalAttentionConsumed).toHaveBeenCalledWith({ + expect(markWorkflowRunTerminalAttentionSettled).toHaveBeenCalledWith({ ownerWorkspaceId: "parent-workspace", status: "completed", runId: "wfr_demo", + runUpdatedAt: "2026-01-01T00:00:05.000Z", + settledAs: "delivered", }); }); diff --git a/src/node/services/tools/task_await.ts b/src/node/services/tools/task_await.ts index 0c8c8b13e0..4ecc2954ce 100644 --- a/src/node/services/tools/task_await.ts +++ b/src/node/services/tools/task_await.ts @@ -411,10 +411,12 @@ export const createTaskAwaitTool: ToolFactory = (config: ToolConfiguration) => { if (!isTerminalWorkflowRunStatus(run.status)) { return; } - await taskService.markWorkflowRunTerminalAttentionConsumed?.({ + await taskService.markWorkflowRunTerminalAttentionSettled?.({ ownerWorkspaceId: workspaceId, status: run.status, runId: run.id, + runUpdatedAt: run.updatedAt, + settledAs: "delivered", }); }; diff --git a/src/node/services/tools/toolUtils.ts b/src/node/services/tools/toolUtils.ts index eb60511d88..562b993b48 100644 --- a/src/node/services/tools/toolUtils.ts +++ b/src/node/services/tools/toolUtils.ts @@ -89,17 +89,75 @@ export async function emitWorkflowRunAttachedEvent(input: { export async function recordBackgroundWorkflowRunReference( config: ToolConfiguration, runId: string, - createdAtMs: number + createdAtMs: number, + options?: { + /** + * Pre-launch records must not fail soft: the sidecar is the kernel invocation's only + * durable provenance, and starting the runner without it lets a fast terminal run have + * its wake permanently marked superseded. Throwing before dispatch leaves the run + * pending and resumable (workflow_resume re-records), so the caller surfaces a loud, + * recoverable launch failure instead. Post-dispatch records stay best-effort because the + * run already started and failing the tool would strand it. + */ + propagateWriteFailure?: boolean; + } ): Promise { const workspaceSessionDir = config.workspaceSessionDir; if (workspaceSessionDir == null || workspaceSessionDir.length === 0) { + if (options?.propagateWriteFailure === true) { + throw new Error( + `Cannot record workflow run provenance without a workspace session dir: ${runId}` + ); + } log.warn("Skipping agent workflow run reference without workspace session dir", { runId }); return; } + // Snapshot which invocation-decision row is newest at launch so the terminal-wake + // currentness check compares row identity instead of wall-clock order, which clock + // corrections can reorder (see WorkspaceService.isWorkflowInvocationCurrent). A history read + // failure must not be persisted as a verified-empty boundary (null): record without the + // field instead, so the run stays rediscoverable (listAgentReferencedWorkflowRunIds) and a + // later workflow_resume re-record can repair provenance, while the unverifiable boundary + // fails safe to a not-current wake instead of guessing from wall-clock order. + let afterBoundaryMessageId: string | null | undefined; + const taskService = config.taskService; + if (config.workspaceId != null && taskService?.getWorkflowInvocationBoundaryMessageId != null) { + try { + afterBoundaryMessageId = await taskService.getWorkflowInvocationBoundaryMessageId( + config.workspaceId, + runId + ); + } catch (error: unknown) { + log.error("Failed to snapshot workflow invocation boundary for run reference", { + runId, + error: getErrorMessage(error), + }); + } + } + try { - await recordAgentWorkflowRunReference({ workspaceSessionDir, runId, createdAtMs }); + await recordAgentWorkflowRunReference({ + workspaceSessionDir, + runId, + createdAtMs, + ...(afterBoundaryMessageId !== undefined ? { afterBoundaryMessageId } : {}), + ...(config.agentId != null && config.agentId.length > 0 + ? { + agentId: config.agentId, + // The pin pairs with the identity: null records a verified-unpinned launch so the + // wake never inherits another row's pin (see AgentWorkflowRunReference). + strictAgentResolution: + config.strictAgentResolution != null && config.strictAgentResolution !== false + ? config.strictAgentResolution + : null, + } + : {}), + }); } catch (error: unknown) { + if (options?.propagateWriteFailure === true) { + throw error; + } log.warn("Failed to record agent workflow run reference", { runId, error: getErrorMessage(error), diff --git a/src/node/services/tools/workflow_resume.test.ts b/src/node/services/tools/workflow_resume.test.ts index 1ae69b9402..c1dcc63b5c 100644 --- a/src/node/services/tools/workflow_resume.test.ts +++ b/src/node/services/tools/workflow_resume.test.ts @@ -6,6 +6,7 @@ import { TestTempDir, createTestToolConfig } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; import { WORKFLOW_CHECKPOINT_RETRY_ERROR_MESSAGE } from "@/common/utils/workflowRetryEligibility"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +import type { TaskService } from "@/node/services/taskService"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; const mockToolCallOptions: ToolExecutionOptions = { @@ -168,7 +169,19 @@ describe("workflow_resume tool", () => { test("resumes in background and records an agent workflow run reference", async () => { using tempDir = new TestTempDir("test-workflow-resume-bg"); - const workflowService = buildWorkflowService(); + // The reference must NOT be durable before the dispatch: the run still sits in its old + // terminal state until the dispatch restarts it, and a pre-dispatch reference would let a + // crash in that window replay the stale failure/interruption as a current wake. + let referenceDurableAtDispatch = false; + const workflowService = buildWorkflowService({ + resumeRunInBackground: mock(async () => { + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableAtDispatch = references.some( + (reference) => reference.runId === "wfr_resume_me" + ); + return { runId: "wfr_resume_me", status: "running" as const, result: null }; + }), + }); const tool = createWorkflowResumeTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: false, @@ -186,6 +199,7 @@ describe("workflow_resume tool", () => { projectTrusted: false, }); expect(workflowService.resumeRun).not.toHaveBeenCalled(); + expect(referenceDurableAtDispatch).toBe(false); const references = await readAgentWorkflowRunReferences(tempDir.path); expect(references.map((reference) => reference.runId)).toContain("wfr_resume_me"); expect(result).toMatchObject({ status: "running", runId: "wfr_resume_me", mode: "resume" }); @@ -229,6 +243,144 @@ describe("workflow_resume tool", () => { }); }); + test("marks terminal attention settled when returning an already-completed run's result", async () => { + using tempDir = new TestTempDir("test-workflow-resume-consumed"); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { + sequence: 2, + type: "result", + at: "2026-05-29T00:00:01.000Z", + result: { reportMarkdown: "already done" }, + }, + { sequence: 3, type: "status", at: "2026-05-29T00:00:01.000Z", status: "completed" }, + ], + }); + const workflowService = buildWorkflowService({ getRun: mock(async () => completedRun) }); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionSettled).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + runUpdatedAt: completedRun.updatedAt, + settledAs: "delivered", + }); + }); + + test("does not mark terminal attention settled for background dispatches", async () => { + using tempDir = new TestTempDir("test-workflow-resume-background-no-consume"); + // The refresh after a background dispatch can still observe the stale pre-dispatch failed + // status; settling it would absorb the retried run's future terminal wake. + const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: true, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionSettled).not.toHaveBeenCalled(); + }); + + test("marks terminal attention settled when a foreground retry finishes terminal", async () => { + using tempDir = new TestTempDir("test-workflow-resume-foreground-consume"); + const failedRun = buildFailedRun(); + const completedRun = buildRun({ + status: "completed", + events: [ + { sequence: 1, type: "status", at: "2026-05-29T00:00:00.000Z", status: "running" }, + { sequence: 2, type: "status", at: "2026-05-29T00:00:02.000Z", status: "completed" }, + ], + }); + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? failedRun : completedRun; + }), + retryRunFromCheckpoint: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "retried" }, + })), + }); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, + }); + + await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: "retry_from_checkpoint" }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionSettled).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_me", + status: "completed", + runUpdatedAt: completedRun.updatedAt, + settledAs: "delivered", + }); + }); + + test("skips the settled marker when the refresh read fails but still returns the result", async () => { + using tempDir = new TestTempDir("test-workflow-resume-refresh-failure-consume"); + // WorkflowService.getRun collapses transient read failures to null. The marker binds to the + // run's terminal generation (updatedAt), which is unknowable without the refreshed record: + // the terminal result still returns to the model, and the wake settles as consumed from + // history evidence on the next scan instead of being marked here. + let getRunCalls = 0; + const workflowService = buildWorkflowService({ + getRun: mock(async () => { + getRunCalls += 1; + return getRunCalls === 1 ? buildRun() : null; + }), + resumeRun: mock(async () => ({ + runId: "wfr_resume_me", + status: "completed" as const, + result: { reportMarkdown: "resumed" }, + })), + }); + const markWorkflowRunTerminalAttentionSettled = mock(() => Promise.resolve()); + const tool = createWorkflowResumeTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + workflowService, + taskService: { markWorkflowRunTerminalAttentionSettled } as unknown as TaskService, + }); + + const result = await tool.execute!( + { run_id: "wfr_resume_me", run_in_background: false, mode: null }, + mockToolCallOptions + ); + + expect(markWorkflowRunTerminalAttentionSettled).not.toHaveBeenCalled(); + expect(result).toMatchObject({ status: "completed", runId: "wfr_resume_me", mode: "resume" }); + }); + test("rejects default resume of a failed run with checkpoint retry guidance", async () => { using tempDir = new TestTempDir("test-workflow-resume-failed"); const workflowService = buildWorkflowService({ getRun: mock(async () => buildFailedRun()) }); diff --git a/src/node/services/tools/workflow_resume.ts b/src/node/services/tools/workflow_resume.ts index a94188569a..1463c36d04 100644 --- a/src/node/services/tools/workflow_resume.ts +++ b/src/node/services/tools/workflow_resume.ts @@ -2,7 +2,7 @@ import { tool } from "ai"; import { getErrorMessage } from "@/common/utils/errors"; import type { ToolConfiguration, ToolFactory } from "@/common/utils/tools/tools"; -import type { WorkflowRunRecord } from "@/common/types/workflow"; +import { isTerminalWorkflowRunStatus, type WorkflowRunRecord } from "@/common/types/workflow"; import { getWorkflowCheckpointRetryEligibility } from "@/common/utils/workflowRetryEligibility"; import { WorkflowRunRecordSchema } from "@/common/orpc/schemas"; import { @@ -154,9 +154,29 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const mode: WorkflowResumeMode = args.mode ?? "resume"; const invocationStartedAtMs = Date.now(); + // A kernel-nested resume (mux.workflow_resume inside code_execution) leaves no top-level + // workflow_resume part in history, so the history-walk consumption predicates cannot see + // that this turn already received the terminal result. Persist consumption durably so the + // terminal-attention drain never re-delivers it. + const markTerminalAttentionSettled = async ( + terminalRun: Pick + ) => { + if (!isTerminalWorkflowRunStatus(terminalRun.status)) { + return; + } + await config.taskService?.markWorkflowRunTerminalAttentionSettled?.({ + ownerWorkspaceId: workspaceId, + runId: terminalRun.id, + status: terminalRun.status, + runUpdatedAt: terminalRun.updatedAt, + settledAs: "delivered", + }); + }; + // Idempotent success: the work is already done, so hand back the durable result instead // of failing the agent's recovery loop (e.g. resuming after a crash that actually finished). if (run.status === "completed" && mode === "resume") { + await markTerminalAttentionSettled(run); return parseToolResult( WorkflowResumeToolResultSchema, { @@ -216,6 +236,12 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) // Background-style resumes outlive this turn; persist provenance so the run is // rediscoverable (task_await/task_list) and its terminal result re-engages the agent. + // Deliberately AFTER the dispatch, unlike workflow_run's onRunCreated record: the + // resumed run sits in an old terminal state until the dispatch durably restarts it, and + // a pre-dispatch reference would let a crash in that window make startup recovery + // deliver the stale failure/interruption as a current wake. Recording late fails safe + // instead (a crash loses this resume's wake); with the process alive, delivery always + // waits for the owner to go idle, by which point this record is durable. const isBackgroundDispatch = args.run_in_background === true || dispatched.status === "backgrounded"; if (isBackgroundDispatch) { @@ -231,6 +257,20 @@ export const createWorkflowResumeTool: ToolFactory = (config: ToolConfiguration) const refreshedRunIsStale = isBackgroundDispatch && refreshedRun != null && refreshedRun.status === run.status; + // Foreground only: a background dispatch can still observe the stale pre-dispatch + // terminal status, and settling it would absorb the retried run's future terminal wake. + // The settled marker binds to the run's terminal generation (updatedAt), so it needs a + // refreshed record that reflects the dispatched terminal status; when the refresh failed + // or lags, skip the marker and the wake settles as consumed from the tool result in + // history on the next scan (worst case one redundant wake for a kernel-nested resume). + if ( + !isBackgroundDispatch && + refreshedRun != null && + refreshedRun.status === dispatched.status + ) { + await markTerminalAttentionSettled(refreshedRun); + } + return parseToolResult( WorkflowResumeToolResultSchema, { diff --git a/src/node/services/tools/workflow_run.test.ts b/src/node/services/tools/workflow_run.test.ts index 0636a54d8b..3e394d5c35 100644 --- a/src/node/services/tools/workflow_run.test.ts +++ b/src/node/services/tools/workflow_run.test.ts @@ -13,9 +13,15 @@ import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { resolveWorkflowScript } from "@/node/services/workflows/workflowScriptResolver"; import { TestTempDir, createTestToolConfig, writeProjectSkill } from "./testHelpers"; import { readAgentWorkflowRunReferences } from "@/node/services/agentWorkflowRunReferences"; +import type { TaskService } from "@/node/services/taskService"; +import type { ToolConfiguration } from "@/common/utils/tools/tools"; import type { WorkflowRunAttachedEvent } from "@/common/types/stream"; import type { WorkflowRunRecord } from "@/common/types/workflow"; +type BackgroundStartInput = Parameters< + NonNullable["startWorkflowInBackground"]> +>[0]; + const mockToolCallOptions: ToolExecutionOptions = { toolCallId: "test-call-id", messages: [], @@ -644,15 +650,31 @@ describe("workflow_run tool", () => { const startWorkflow = mock(async () => { throw new Error("foreground start should not be used"); }); - const startWorkflowInBackground = mock(async () => ({ - runId: "wfr_background", - status: "running" as const, - result: null, - })); + // Faithful to WorkflowService: onRunCreated is awaited at run creation, before the runner + // starts executing. The probe checks the sidecar reference is already durable at that + // point, so a fast run (or a crash) after launch cannot lose its terminal wake. + let referenceDurableBeforeRunnerStart = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_background", + status: "pending", + result: null, + run: null, + }); + const references = await readAgentWorkflowRunReferences(tempDir.path); + referenceDurableBeforeRunnerStart = references.some( + (reference) => reference.runId === "wfr_background" + ); + return { runId: "wfr_background", status: "running" as const, result: null }; + }); const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); const tool = createWorkflowRunTool({ ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), trusted: true, + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, workflowService: { startWorkflow, startWorkflowInBackground, @@ -665,8 +687,23 @@ describe("workflow_run tool", () => { mockToolCallOptions ); + expect(referenceDurableBeforeRunnerStart).toBe(true); + // The reference must persist the invocation-boundary snapshot: currentness compares row + // identity, so a reference recorded without it fails safe and the wake is dropped. const references = await readAgentWorkflowRunReferences(tempDir.path); - expect(references.map((reference) => reference.runId)).toContain("wfr_background"); + expect(references).toHaveLength(1); + expect(references[0]).toMatchObject({ + runId: "wfr_background", + afterBoundaryMessageId: "boundary-row-1", + // The wake binds to the launching agent, so the reference must carry its identity + // and the launch turn's provenance pin. + agentId: "plan", + strictAgentResolution: { expectedScope: "built-in" }, + }); + expect(getWorkflowInvocationBoundaryMessageId).toHaveBeenCalledWith( + "workspace-1", + "wfr_background" + ); expect(startWorkflowInBackground).toHaveBeenCalledWith( expect.objectContaining({ @@ -681,6 +718,83 @@ describe("workflow_run tool", () => { expect(result).toEqual({ status: "running", runId: "wfr_background", result: null }); }); + test("records a rediscovery-only reference when the boundary snapshot fails", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-boundary-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_boundary_error", + status: "pending", + result: null, + run: null, + }); + return { runId: "wfr_boundary_error", status: "running" as const, result: null }; + }); + const getWorkflowInvocationBoundaryMessageId = mock(async () => { + throw new Error("history read failed"); + }); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { + startWorkflow: mock(async () => { + throw new Error("foreground start should not be used"); + }), + startWorkflowInBackground, + getRun: mock(async () => null), + }, + }); + + const result = await tool.execute!( + { script_path: scriptPath, args: { topic: "workflow tools" }, run_in_background: true }, + mockToolCallOptions + ); + expect(result).toEqual({ status: "running", runId: "wfr_boundary_error", result: null }); + + // A read failure must not persist a verified-empty boundary (null): the entry keeps the run + // rediscoverable while a later resume re-record can repair provenance. + const references = await readAgentWorkflowRunReferences(tempDir.path); + expect(references).toHaveLength(1); + expect(references[0]?.runId).toBe("wfr_boundary_error"); + expect(references[0] != null && "afterBoundaryMessageId" in references[0]).toBe(false); + }); + + test("a sidecar write failure aborts the background launch before the runner starts", async () => { + using tempDir = new TestTempDir("test-workflow-run-tool-sidecar-write-error"); + const scriptPath = await writeWorkflowScript(tempDir.path); + // The reference path exists as a directory, so every sidecar write fails. + await fs.mkdir(path.join(tempDir.path, "agent-workflow-runs.json")); + let runnerStarted = false; + const startWorkflowInBackground = mock(async (input: BackgroundStartInput) => { + await input.onRunCreated?.({ + runId: "wfr_sidecar_write_error", + status: "pending", + result: null, + run: null, + }); + runnerStarted = true; + return { runId: "wfr_sidecar_write_error", status: "running" as const, result: null }; + }); + const getRun = mock(async () => null); + const getWorkflowInvocationBoundaryMessageId = mock(async () => "boundary-row-1"); + const tool = createWorkflowRunTool({ + ...createTestToolConfig(tempDir.path, { workspaceId: "workspace-1" }), + trusted: true, + agentId: "plan", + taskService: { getWorkflowInvocationBoundaryMessageId } as unknown as TaskService, + workflowService: { startWorkflowInBackground, getRun }, + }); + + // Starting the runner without durable provenance would let a fast terminal run have its + // wake permanently superseded; the launch must fail loudly instead, leaving the run + // pending and resumable (workflow_resume re-records provenance). + await expect( + tool.execute!({ script_path: scriptPath, run_in_background: true }, mockToolCallOptions) + ).rejects.toThrow(/created durable run wfr_sidecar_write_error/); + expect(runnerStarted).toBe(false); + }); + test("requires the workflow service", async () => { using tempDir = new TestTempDir("test-workflow-run-tool-missing"); const scriptPath = await writeWorkflowScript(tempDir.path); diff --git a/src/node/services/tools/workflow_run.ts b/src/node/services/tools/workflow_run.ts index 425b379eae..300da0f60f 100644 --- a/src/node/services/tools/workflow_run.ts +++ b/src/node/services/tools/workflow_run.ts @@ -262,6 +262,7 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => } } const createdRun: { id: string | null } = { id: null }; + const invocationStartedAtMs = Date.now(); const startInput = { script, workspaceId, @@ -271,6 +272,14 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => createdRun.id = event.runId; // The run record is durable now, so a concurrent duplicate launch will see it. releaseAdmission?.(); + // Provenance must be durable BEFORE the runner starts: a fast background run (or a + // process exit mid-dispatch) can reach terminal state before any post-dispatch + // write, and a terminal wake with no sidecar reference is permanently superseded. + if (args.run_in_background === true) { + await recordBackgroundWorkflowRunReference(config, event.runId, invocationStartedAtMs, { + propagateWriteFailure: true, + }); + } await emitWorkflowRunAttachedEvent({ config, workspaceId, @@ -280,7 +289,6 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => }); }, }; - const invocationStartedAtMs = Date.now(); let result: { runId: string; status: string; result: unknown }; try { result = @@ -335,7 +343,10 @@ export const createWorkflowRunTool: ToolFactory = (config: ToolConfiguration) => releaseAdmission?.(); } - if (isBackgroundWorkflowResult(args, result.status)) { + // Explicit background launches already recorded provenance in onRunCreated; this covers + // a foreground dispatch that backgrounded itself, where the run ID outcome is only + // knowable post-dispatch. + if (args.run_in_background !== true && isBackgroundWorkflowResult(args, result.status)) { await recordBackgroundWorkflowRunReference(config, result.runId, invocationStartedAtMs); } diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 8a2101b4bd..1af75a84e5 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -1696,9 +1696,12 @@ export class TurnRequestBuilder { runStore: new WorkflowRunStore({ sessionDir: path.join(this.dependencies.config.sessionsDir, workspaceId), }), + // This build's settled markers are keyed by the run's terminal generation, so + // a resumed run re-arms attention by itself; only the downgrade-compat stable + // marker needs clearing when the run leaves terminal state. onRunStatusChanged: async (event) => { if (!isTerminalWorkflowRunStatus(event.status)) { - await this.dependencies.bindings.taskService?.resetWorkflowRunTerminalAttention({ + await this.dependencies.bindings.taskService?.clearWorkflowRunDowngradeSettlement({ ownerWorkspaceId: event.workspaceId, runId: event.runId, }); @@ -1746,7 +1749,7 @@ export class TurnRequestBuilder { return; } if (this.dependencies.bindings.taskService != null) { - await this.dependencies.bindings.taskService.enqueueWorkflowRunTerminalAttention({ + this.dependencies.bindings.taskService.noteWorkflowRunTerminalAttention({ ownerWorkspaceId: workspaceId, runId, status, @@ -2012,6 +2015,8 @@ export class TurnRequestBuilder { planFilePath, ancestorPlanFilePaths, workspaceId, + agentId: effectiveAgentId, + strictAgentResolution, xumScope, timelineService: timelineExperimentEnabled ? this.dependencies.bindings.timelineService diff --git a/src/node/services/workflows/WorkflowService.context.test.ts b/src/node/services/workflows/WorkflowService.context.test.ts index 3a33934457..aed7a174df 100644 --- a/src/node/services/workflows/WorkflowService.context.test.ts +++ b/src/node/services/workflows/WorkflowService.context.test.ts @@ -8,7 +8,9 @@ import { QuickJSRuntimeFactory } from "@/node/services/ptc/quickjsRuntime"; import { DisposableTempDir } from "@/node/services/tempDir"; import { WorkflowRunStore } from "./WorkflowRunStore"; import { + listWorkflowRuns, listWorkflowScripts, + resumeWorkflowRun, startWorkflowRun, type WorkflowServiceContext, } from "./WorkflowService"; @@ -94,7 +96,10 @@ describe("WorkflowService request orchestration", () => { })), }, workspaceService, - taskService: {}, + taskService: { + noteWorkflowRunTerminalAttention: mock(() => undefined), + clearWorkflowRunDowngradeSettlement: mock(async () => undefined), + }, experimentsService: { isExperimentEnabled: mock(() => options.enabled ?? true), }, @@ -233,6 +238,78 @@ describe("WorkflowService request orchestration", () => { } }); + test("crash-resumed background runs note terminal attention on settle", async () => { + const runStore = new WorkflowRunStore({ + sessionDir: path.join(config.sessionsDir, "workspace-1"), + }); + await runStore.createRun({ + id: "wfr_crash_wake", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + // Orphaned by a crash: durable status says running, but no live runner. + await runStore.appendStatus("wfr_crash_wake", "running", "2026-05-29T00:00:01.000Z"); + + const noteWorkflowRunTerminalAttention = mock(() => undefined); + const { context } = createContext(); + (context as unknown as Record).taskService = { + noteWorkflowRunTerminalAttention, + }; + + // A read path triggers crash recovery; the resumed run's settle must poke the drain + // instead of waiting for the next sweep. + await listWorkflowRuns(context, "workspace-1"); + const deadline = Date.now() + 5_000; + while (noteWorkflowRunTerminalAttention.mock.calls.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(noteWorkflowRunTerminalAttention).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_crash_wake", + status: "completed", + }); + }); + + test("resuming a run clears the stale downgrade settlement marker", async () => { + const runStore = new WorkflowRunStore({ + sessionDir: path.join(config.sessionsDir, "workspace-1"), + }); + await runStore.createRun({ + id: "wfr_resume_compat", + workspaceId: "workspace-1", + workflow: { name: "demo", description: "Demo", scope: "built-in", executable: true }, + source: "export default function workflow() { return { reportMarkdown: 'done' }; }\n", + args: {}, + attentionPolicy: "notify_on_terminal", + now: "2026-05-29T00:00:00.000Z", + }); + await runStore.appendStatus("wfr_resume_compat", "running", "2026-05-29T00:00:01.000Z"); + await runStore.appendStatus("wfr_resume_compat", "interrupted", "2026-05-29T00:00:02.000Z"); + + const clearWorkflowRunDowngradeSettlement = mock(async () => undefined); + const { context } = createContext(); + (context as unknown as Record).taskService = { + noteWorkflowRunTerminalAttention: mock(() => undefined), + clearWorkflowRunDowngradeSettlement, + }; + + // Leaving terminal state invalidates the stable downgrade marker written at settlement; + // without the clear, a downgraded build would refuse to enqueue the resumed result. + await resumeWorkflowRun(context, { workspaceId: "workspace-1", runId: "wfr_resume_compat" }); + const deadline = Date.now() + 5_000; + while (clearWorkflowRunDowngradeSettlement.mock.calls.length === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(clearWorkflowRunDowngradeSettlement).toHaveBeenCalledWith({ + ownerWorkspaceId: "workspace-1", + runId: "wfr_resume_compat", + }); + }); + test("rejects disabled dynamic workflows before workspace initialization", async () => { const { context, waitForInit } = createContext({ enabled: false }); try { diff --git a/src/node/services/workflows/WorkflowService.ts b/src/node/services/workflows/WorkflowService.ts index 2ed151b162..affe845de4 100644 --- a/src/node/services/workflows/WorkflowService.ts +++ b/src/node/services/workflows/WorkflowService.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import { isTerminalWorkflowRunStatus, + WORKFLOW_BACKGROUND_CONTINUATION_STATUSES, type WorkflowScriptDescriptor, type WorkflowRunRecord, type WorkflowRunStatus, @@ -134,11 +135,6 @@ export interface StartNamedWorkflowResult { result: unknown; } -const WORKFLOW_BACKGROUND_CONTINUATION_STATUSES = new Set([ - "completed", - "failed", -]); - // oRPC creates a WorkflowService per request, so workflow lifecycle state that spans requests // needs process-wide registries. const pendingCrashResumeTimers = new Map>(); @@ -1080,10 +1076,36 @@ export async function resolveWorkflowContext( includeAgentPlugins, skillStorageContext, }), - onRunStatusChanged: (event) => context.workspaceService.emitWorkflowRunActivity(event), - ...(options.onBackgroundRunTerminal != null - ? { onBackgroundRunTerminal: options.onBackgroundRunTerminal } - : {}), + // Settled markers are keyed by the run's terminal generation, so a resumed run's next + // terminal transition re-arms attention by itself; only the downgrade-compat stable + // marker needs clearing when the run leaves terminal state (see + // clearWorkflowRunDowngradeSettlement). + onRunStatusChanged: async (event) => { + if (!isTerminalWorkflowRunStatus(event.status)) { + await context.taskService.clearWorkflowRunDowngradeSettlement({ + ownerWorkspaceId: event.workspaceId, + runId: event.runId, + }); + } + await context.workspaceService.emitWorkflowRunActivity(event); + }, + // Read paths (listRuns / stream subscribe) create services purely to observe runs, but + // crash recovery can resume an orphaned background run on them: without a terminal + // callback the settled run would owe its wake to the next sweep. Explicit callbacks + // (slash-command continuations, retry) keep their custom behavior. + onBackgroundRunTerminal: + options.onBackgroundRunTerminal ?? + ((event) => { + // Nested runs surface through their parent workflow, not their own wake. + if (event.run.parentWorkflow != null) { + return; + } + context.taskService.noteWorkflowRunTerminalAttention({ + ownerWorkspaceId: workspaceId, + runId: event.runId, + status: event.status, + }); + }), getCurrentProjectTrusted: resolveWorkflowProjectTrusted, runnerId: "workflow-runner:" + workspaceId, }), diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 36d5ec2623..838cc828ba 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -3,6 +3,7 @@ import { WorkspaceService, generateForkBranchName, generateForkTitle } from "./w import { registerInProcessWorkflowRun } from "@/node/services/workflows/workflowArchiveAdmission"; import type { IdleCompactionOutcome } from "./idleCompactionService"; import type { AgentSession } from "./agentSession"; +import { CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE } from "./agentSession"; import { createAgentSessionHarness, createStartedTurnHandle, @@ -58,9 +59,12 @@ import type { SendMessageOptions, WorkspaceChatMessage } from "@/common/orpc/typ import { createMuxMessage } from "@/common/types/message"; import { buildStagedAttachmentNotice } from "@/browser/features/ChatInput/stagedAttachments"; import { + WORKFLOW_RESULT_METADATA_TYPE, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, + buildWorkflowResultContextMessage, } from "@/common/utils/workflowRunMessages"; +import { recordAgentWorkflowRunReference } from "@/node/services/agentWorkflowRunReferences"; import { getPlanFilePath } from "@/common/utils/planStorage"; import * as todoStorageModule from "@/node/services/todos/todoStorage"; import * as runtimeFactory from "@/node/runtime/runtimeFactory"; @@ -889,7 +893,8 @@ describe("WorkspaceService bash monitor wakes", () => { // durably (otherwise the staged-clear grace scan would eventually roll the // staging back and resurrect the retired wakes). const tombPath = path.join( - path.join(config.sessionsDir, workspaceId), + config.sessionsDir, + workspaceId, "bash-monitor-wakes", "cleared-at" ); @@ -1051,7 +1056,8 @@ describe("WorkspaceService bash monitor wakes", () => { // The refused clear touched nothing: no retirement, no staged tombstone. expect((await wakeStore.get(workspaceId, "proc-removal-race"))?.status).toBe("pending"); const tombPath = path.join( - path.join(config.sessionsDir, workspaceId), + config.sessionsDir, + workspaceId, "bash-monitor-wakes", "cleared-at" ); @@ -2306,7 +2312,8 @@ describe("WorkspaceService bash monitor wakes", () => { }); const gen2Start = Date.now() - 1_000; const recordFile = path.join( - path.join(config.sessionsDir, workspaceId), + config.sessionsDir, + workspaceId, "bash-monitor-wakes", "proc-gen.json" ); @@ -2388,7 +2395,8 @@ describe("WorkspaceService bash monitor wakes", () => { terminal: { status: "exited", exitCode: 1 }, }); const recordFile = path.join( - path.join(config.sessionsDir, workspaceId), + config.sessionsDir, + workspaceId, "bash-monitor-wakes", "proc-nan.json" ); @@ -8551,17 +8559,1482 @@ describe("WorkspaceService activity list scoping", () => { await cleanup(); } }); -}); +}); + +describe("WorkspaceService workflow invocation events", () => { + test("emits workflow slash invocation rows through the active session chat stream", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-live-events"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-live-events", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + const session = workspaceService.getOrCreateSession(workspaceId); + const events: WorkspaceChatMessage[] = []; + const unsubscribe = session.onChatEvent(({ message }) => { + events.push(message); + }); + + try { + const persisted = await workspaceService.appendWorkflowRunInvocation({ + workspaceId, + rawCommand: "/demo investigate live events", + scriptPath: "./workflows/demo.js", + args: { input: "investigate live events" }, + runId: "wfr_live_events", + status: "running", + result: null, + }); + + expect(persisted).toBe(true); + expect(events).toHaveLength(2); + const triggerMessage = events[0]; + const cardMessage = events[1]; + if (triggerMessage?.type !== "message" || cardMessage?.type !== "message") { + throw new Error("Expected workflow invocation to emit message events"); + } + expect(triggerMessage).toMatchObject({ role: "user", type: "message" }); + expect(triggerMessage.metadata?.muxMetadata).toEqual( + expect.objectContaining({ type: WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE }) + ); + expect(cardMessage).toMatchObject({ role: "assistant", type: "message" }); + expect(cardMessage.metadata?.muxMetadata).toEqual( + expect.objectContaining({ type: WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE }) + ); + } finally { + unsubscribe(); + workspaceService.disposeSession(workspaceId); + } + } finally { + await cleanup(); + } + }); + + test("keeps workflow invocations current across synthetic user continuations", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness"; + const runId = "wfr_currentness"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-workflow-run", "assistant", "", { timestamp: 1_000 }, [ + { + type: "dynamic-tool", + toolCallId: "workflow-call-1", + toolName: "workflow_run", + state: "output-available", + input: { script_path: "./workflows/demo.js", args: {}, run_in_background: true }, + output: { status: "running", runId, result: null }, + }, + ]) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("synthetic-await", "user", "Call task_await", { + timestamp: 1_100, + synthetic: true, + }) + ); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "Never mind, answer something else", { + timestamp: 1_200, + }) + ); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("counts workflow_resume output as the current invocation after manual supersession", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-resume"; + const runId = "wfr_currentness_resume"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-resume", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-workflow-run", "assistant", "", { timestamp: 1_000 }, [ + { + type: "dynamic-tool", + toolCallId: "workflow-call-1", + toolName: "workflow_run", + state: "output-available", + input: { script_path: "./workflows/demo.js", args: {}, run_in_background: true }, + output: { status: "running", runId, result: null }, + }, + ]) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "Never mind, answer something else", { + timestamp: 1_100, + }) + ); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // An unrelated tool output mentioning the run does not re-establish the invocation. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-unrelated-tool", "assistant", "", { timestamp: 1_200 }, [ + { + type: "dynamic-tool", + toolCallId: "task-list-1", + toolName: "task_list", + state: "output-available", + input: {}, + output: { status: "running", runId, result: null }, + }, + ]) + ); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // workflow_resume re-attaches the agent to the run, so the invocation counts as current + // again and the terminal continuation would be delivered. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-workflow-resume", "assistant", "", { timestamp: 1_300 }, [ + { + type: "dynamic-tool", + toolCallId: "workflow-resume-1", + toolName: "workflow_resume", + state: "output-available", + input: { run_id: runId, mode: "resume", run_in_background: true }, + output: { status: "running", runId, result: null }, + }, + ]) + ); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("counts a kernel-launched run recorded in the sidecar as the current invocation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-kernel"; + const runId = "wfr_currentness_kernel"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-kernel", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // mux.workflow_run inside code_execution leaves no workflow_run tool part in history; the + // agent-workflow-runs sidecar reference is the only durable invocation evidence. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("assistant-kernel-launch", "assistant", "", { timestamp: 1_100 }, [ + { + type: "dynamic-tool", + toolCallId: "code-exec-1", + toolName: "code_execution", + state: "output-available", + input: { code: "return xum.workflow_run({ script_path: './workflows/demo.js' })" }, + output: { success: true, result: { status: "running", runId } }, + }, + ]) + ); + + // The nested runId in the code_execution output alone is not invocation evidence. + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A newer manual user message supersedes the sidecar reference. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel workflow_resume re-records the reference after the supersession and + // re-establishes provenance (latest record wins). + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_250, + afterBoundaryMessageId: "manual-user-2", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Once the terminal result was delivered, the sidecar must not resurrect the invocation. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("workflow-result", "user", "The workflow below has finished.", { + timestamp: 1_300, + synthetic: true, + muxMetadata: { + type: WORKFLOW_RESULT_METADATA_TYPE, + rawCommand: "workflow_run ./workflows/demo.js", + runId, + }, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A kernel background resume issued after the delivered result re-records the reference, + // so the retried run's next terminal wake must count as current again. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_350, + afterBoundaryMessageId: "workflow-result", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("does not treat sidecar references as current after a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-cleared"; + const runId = "wfr_currentness_cleared"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-cleared", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A full clear (truncateHistory) removes every row without appending a reset boundary + // and leaves the sidecar intact; the surviving reference must not inject a workflow + // result into the freshly cleared conversation. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("delivers kernel launches recorded against a decision-free history", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-empty"; + const runId = "wfr_currentness_empty"; + const legacyRunId = "wfr_currentness_empty_legacy"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-empty", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // A kernel launch from a synthetic turn in a new (or fully cleared) workspace records a + // verified-empty snapshot (null). History still having no decision row means the launch + // context is unchanged, so the wake must deliver. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // A reference without a verified snapshot cannot claim the empty history as its launch + // context; it may merely have survived a full clear. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId: legacyRunId, + createdAtMs: 1_150, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, legacyRunId)).toBe( + false + ); + + // A decision row appearing after the launch supersedes the verified-empty snapshot. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("retires kernel workflow run references on a full history clear", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire"; + const runId = "wfr_currentness_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + // Launched from a decision-free history: the verified-empty snapshot delivers. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + + // A full clear returns history to decision-free, making the pre-clear null snapshot + // indistinguishable from a fresh empty-history launch; the clear must retire the + // reference so the stale result cannot inject into the fresh conversation. + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(true); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("retires kernel workflow run references even when a later post-clear step fails", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-early"; + const runId = "wfr_currentness_retire_early"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-early", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + + // The truncation commits, then a later post-clear step fails. Retirement must already + // have happened, or the stale null-snapshot reference survives the committed clear and + // reads current against the emptied history. + const sessionAccessor = workspaceService as unknown as { + getOrCreateSession(id: string): { clearPostCompactionState(): Promise }; + }; + const session = sessionAccessor.getOrCreateSession(workspaceId); + const carryoverSpy = spyOn(session, "clearPostCompactionState").mockImplementationOnce(() => + Promise.reject(new Error("carryover discard failed")) + ); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + } finally { + carryoverSpy.mockRestore(); + } + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a failed reference retirement aborts the clear before truncation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retire-abort"; + const runId = "wfr_currentness_retire_abort"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retire-abort", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + + // Retirement failing after a committed truncation would leave the null-snapshot + // reference reading current against the emptied history (pre-clear output injected + // into the fresh conversation), so the clear must abort with the transcript intact. + const truncateSpy = spyOn(historyService, "truncateHistory"); + const internal = workspaceService as unknown as { + retireKernelWorkflowRunReferences(id: string): Promise; + }; + const retireSpy = spyOn(internal, "retireKernelWorkflowRunReferences") + // Lazy rejection: an eager mockRejectedValueOnce promise trips bun's + // unhandled-rejection detector on this host before the clear consumes it. + .mockImplementationOnce(() => Promise.reject(new Error("read-only session storage"))); + try { + const clearResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(clearResult.success).toBe(false); + if (!clearResult.success) { + expect(clearResult.error).toContain("could not be retired"); + } + expect(truncateSpy).not.toHaveBeenCalled(); + } finally { + retireSpy.mockRestore(); + truncateSpy.mockRestore(); + } + + // A retry once storage recovers clears normally and retires the sidecar. + const retryResult = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(retryResult.success).toBe(true); + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a partial truncation that empties history retires kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-partial-empty"; + const runId = "wfr_currentness_partial_empty"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-partial-empty", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // Half of a single-message transcript crosses the whole-history removal budget, so the + // token-proportional truncation takes historyService's full-delete fast path. The + // emptied transcript must retire the null-snapshot reference exactly like an explicit + // clear, or the pre-truncation workflow result would read current against the emptied + // decision-free history. + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(truncateResult.success).toBe(true); + expect(await historyService.getHistoryFromLatestBoundary(workspaceId)).toEqual(Ok([])); + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("an overlapping truncation that would empty history is refused, not silently cleared", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-preflight-race"; + const runId = "wfr_currentness_preflight_race"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-preflight-race", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // Simulate the preflight racing an overlapping truncation: it classifies this request + // as non-emptying, but the locked rewrite's own recomputation would empty history. The + // serialized revalidation must refuse rather than skip the full-clear guards. + const preflightSpy = spyOn( + historyService, + "classifyTruncationRemoval" + ).mockImplementationOnce(() => Promise.resolve("partial" as const)); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("full clear"); + } + } finally { + preflightSpy.mockRestore(); + } + // The transcript is intact; the reference was already retired before the refused + // rewrite (retirement precedes every row-removing truncation), which is the fail-safe + // direction: a dropped wake, with the result still retrievable via resume. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("an unreadable truncation scope preflight refuses instead of applying full-clear effects", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-preflight-refuse"; + const runId = "wfr_currentness_preflight_refuse"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-preflight-refuse", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // A transiently unreadable preflight must refuse: an unknown scope labeled "all" would + // apply full-clear side effects while a prefix removal can leave rows behind. + const preflightSpy = spyOn(historyService, "classifyTruncationRemoval").mockRejectedValueOnce( + new Error("EIO: history unreadable") + ); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("classify"); + } + } finally { + preflightSpy.mockRestore(); + } + // Lossless refusal: the transcript is intact and the kernel workflow reference survives + // for the retry (no wake was settled superseded by a truncation that never happened). + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a full-clear-classified truncation that would leave rows is refused under the history lock", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-all-drift-refuse"; + const runId = "wfr_currentness_all_drift_refuse"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-all-drift-refuse", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + for (let i = 0; i < 6; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`manual-user-${i}`, "user", `padding message ${i}`, { + timestamp: 1_200 + i, + }) + ); + } + + // Simulate rows appended during the unserialized preflight (e.g. a turn completing + // before the admission guard is acquired): it classified this request as emptying, but + // the locked rewrite's recomputation removes only a prefix. The serialized revalidation + // must refuse rather than apply full-clear side effects (context epoch advance, + // goal/plan/retry discards) while rows remain. + const preflightSpy = spyOn( + historyService, + "classifyTruncationRemoval" + ).mockImplementationOnce(() => Promise.resolve("all" as const)); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("leave messages"); + } + } finally { + preflightSpy.mockRestore(); + } + // The transcript is intact; the reference was already retired before the refused + // rewrite (retirement precedes every row-removing truncation), which is the fail-safe + // direction: a dropped wake, with the result still retrievable via resume. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(6); + } + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("the admission guard is held across the truncation scope preflight", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-preflight-guard"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-preflight-guard", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + for (let i = 0; i < 6; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`manual-user-${i}`, "user", `padding message ${i}`, { + timestamp: 1_200 + i, + }) + ); + } + + // A turn admitted during the preflight could launch a kernel workflow whose sidecar + // reference the wholesale retirement deletes while its rows survive the prefix cut, + // permanently suppressing that run's wake. Admission must therefore already be held + // while the classification snapshot is read: park the preflight and prove a concurrent + // context mutation is refused for the whole window. + let releasePreflight: ((scope: "partial") => void) | undefined; + const preflightGate = new Promise<"partial">((resolve) => { + releasePreflight = resolve; + }); + const preflightSpy = spyOn( + historyService, + "classifyTruncationRemoval" + ).mockImplementationOnce(() => preflightGate); + try { + const first = workspaceService.truncateHistory(workspaceId, 0.5); + const second = await workspaceService.truncateHistory(workspaceId, 1.0); + expect(second.success).toBe(false); + if (!second.success) { + expect(second.error).toContain("already in progress"); + } + releasePreflight?.("partial"); + const firstResult = await first; + expect(firstResult.success).toBe(true); + } finally { + preflightSpy.mockRestore(); + } + // The refused full clear touched nothing: the prefix cut left a suffix behind. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data.length).toBeGreaterThan(0); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a turn becoming active during reference retirement refuses the truncation", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-retirement-recheck"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-retirement-recheck", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + let streaming = false; + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + isStreaming: mock(() => streaming), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + for (let i = 0; i < 6; i++) { + await historyService.appendToHistory( + workspaceId, + createMuxMessage(`manual-user-${i}`, "user", `padding message ${i}`, { + timestamp: 1_200 + i, + }) + ); + } + + // An in-turn compaction retry bypasses admission gating across a transient idle gap, + // and the retirement await is the last one before the rewrite: a retry that becomes + // active during it must refuse the truncation instead of streaming across it. + const retireSpy = spyOn( + workspaceService as unknown as { + retireKernelWorkflowRunReferences: (id: string) => Promise; + }, + "retireKernelWorkflowRunReferences" + ).mockImplementationOnce(() => { + streaming = true; + return Promise.resolve(); + }); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("turn is active"); + } + } finally { + retireSpy.mockRestore(); + } + // Refused before the rewrite: the transcript is intact. + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(6); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a partial prefix truncation retires kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-prefix-retire"; + const runId = "wfr_currentness_prefix_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-prefix-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "before truncation", { timestamp: 1_200 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-b", "user", "still here after", { timestamp: 1_300 }) + ); + + // A genuinely partial prefix cut can delete the launch turn's restriction-bearing rows + // without adding a supersession decision, so the reference must not survive to + // recompose the wake from unrestricted defaults; the run stays retrievable via resume. + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(truncateResult.success).toBe(true); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a partial truncation blocks send admission across reference retirement", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-partial-admission"; + const runId = "wfr_currentness_partial_admission"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-partial-admission", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "before truncation", { timestamp: 1_200 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-b", "user", "still here after", { timestamp: 1_300 }) + ); + + // A send racing the partial truncation during the retirement await must be refused: + // admitted, it would snapshot the pre-truncation transcript and lose its turn's + // workflow provenance to the reference retirement. + const internal = workspaceService as unknown as { + retireKernelWorkflowRunReferences: (id: string) => Promise; + }; + const originalRetire = internal.retireKernelWorkflowRunReferences.bind(workspaceService); + let raceSendOutcome: string | null = null; + const retireSpy = spyOn(internal, "retireKernelWorkflowRunReferences").mockImplementationOnce( + async (id: string) => { + const sendResult = await workspaceService.sendMessage(workspaceId, "race the cut", { + model: "openai:gpt-4o", + agentId: "exec", + }); + raceSendOutcome = sendResult.success ? "accepted" : JSON.stringify(sendResult.error); + await originalRetire(id); + } + ); + try { + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.5); + expect(truncateResult.success).toBe(true); + } finally { + retireSpy.mockRestore(); + } + expect(raceSendOutcome ?? "").toContain(CONTEXT_MUTATION_SEND_BLOCKED_MESSAGE); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a destructive history replacement retires kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-replace-retire"; + const runId = "wfr_currentness_replace_retire"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-replace-retire", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "before replacement", { timestamp: 1_200 }) + ); + + // A destructive non-compaction replacement leaves a decision-free transcript that a + // null-boundary reference would read as current, injecting the pre-replacement result. + const replaceResult = await workspaceService.replaceHistory( + workspaceId, + createMuxMessage("replacement-summary", "assistant", "Replacement summary", {}) + ); + expect(replaceResult.success).toBe(true); + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a truncation that removes no rows preserves kernel workflow references", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-noop-preserve"; + const runId = "wfr_currentness_noop_preserve"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-noop-preserve", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "first message", { timestamp: 1_200 }) + ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-b", "user", "second message", { timestamp: 1_300 }) + ); + + // A tiny percentage rounds to a zero removal budget: the transcript is unchanged, so + // the run's reference must survive or its terminal wake would settle superseded under + // a conversation that never lost a row. + const truncateResult = await workspaceService.truncateHistory(workspaceId, 0.0001); + expect(truncateResult.success).toBe(true); + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(2); + } + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a no-op-classified truncation that would remove rows is refused with references intact", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-noop-race"; + const runId = "wfr_currentness_noop_race"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-noop-race", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "single short message", { timestamp: 1_200 }) + ); + + // Simulate the scope preflight racing history growth: classified a no-op (so reference + // retirement was skipped), but the locked recomputation reaches real rows. The + // serialized guard must refuse rather than remove rows with live references. + const preflightSpy = spyOn( + historyService, + "classifyTruncationRemoval" + ).mockImplementationOnce(() => Promise.resolve("none" as const)); + try { + const result = await workspaceService.truncateHistory(workspaceId, 0.9); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error).toContain("no-op"); + } + } finally { + preflightSpy.mockRestore(); + } + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (history.success) { + expect(history.data).toHaveLength(1); + } + expect( + existsSync(path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json")) + ).toBe(true); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); + + test("a delivered coalesced workflow result consumes the kernel run's currentness", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-coalesced"; + const runId = "wfr_currentness_coalesced"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-coalesced", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); + + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: null, + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // Another run's payload quoting nothing about this run must not count as consumption. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-other", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run other.js", + name: "other.js", + runId: "wfr_currentness_other", + status: "completed", + result: { reportMarkdown: "other done" }, + run: null, + }), + { timestamp: 1_250, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The drain's synthetic coalesced prompt carries no workflow-result metadata. After a + // crash between durable acceptance and the settled-marker write, this row is the only + // evidence the result already reached history; it must read as consumption or the next + // sweep injects the same terminal result again. + await historyService.appendToHistory( + workspaceId, + createMuxMessage( + "coalesced-result", + "user", + buildWorkflowResultContextMessage({ + rawCommand: "workflow_run research.js", + name: "research.js", + runId, + status: "completed", + result: { reportMarkdown: "done" }, + run: null, + }), + { timestamp: 1_300, synthetic: true } + ) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); -describe("WorkspaceService workflow invocation events", () => { - test("emits workflow slash invocation rows through the active session chat stream", async () => { + test("decides sidecar currentness by boundary identity, not wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-live-events"; + const workspaceId = "workflow-currentness-clock"; + const runId = "wfr_currentness_clock"; const projectPath = path.join(config.rootDir, "project"); try { await config.addWorkspace(projectPath, { id: workspaceId, - name: "workflow-live-events", + name: "workflow-currentness-clock", projectName: "project", projectPath, runtimeConfig: { type: "local" }, @@ -8580,56 +10053,46 @@ describe("WorkspaceService workflow invocation events", () => { off: mock(() => undefined as unknown as InitStateManager), } as unknown as InitStateManager, }); - const session = workspaceService.getOrCreateSession(workspaceId); - const events: WorkspaceChatMessage[] = []; - const unsubscribe = session.onChatEvent(({ message }) => { - events.push(message); - }); - try { - const persisted = await workspaceService.appendWorkflowRunInvocation({ - workspaceId, - rawCommand: "/demo investigate live events", - scriptPath: "./workflows/demo.js", - args: { input: "investigate live events" }, - runId: "wfr_live_events", - status: "running", - result: null, - }); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) + ); + // A backward clock correction after recording makes the reference timestamp future-dated + // relative to every later history row; identity comparison must still deliver the wake. + const skewedCreatedAtMs = Date.now() + 30 * 60_000; + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: skewedCreatedAtMs, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - expect(persisted).toBe(true); - expect(events).toHaveLength(2); - const triggerMessage = events[0]; - const cardMessage = events[1]; - if (triggerMessage?.type !== "message" || cardMessage?.type !== "message") { - throw new Error("Expected workflow invocation to emit message events"); - } - expect(triggerMessage).toMatchObject({ role: "user", type: "message" }); - expect(triggerMessage.metadata?.muxMetadata).toEqual( - expect.objectContaining({ type: WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE }) - ); - expect(cardMessage).toMatchObject({ role: "assistant", type: "message" }); - expect(cardMessage.metadata?.muxMetadata).toEqual( - expect.objectContaining({ type: WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE }) - ); - } finally { - unsubscribe(); - workspaceService.disposeSession(workspaceId); - } + // A user message written after the correction has a smaller timestamp than the reference; + // wall-clock ordering would keep the stale reference current, identity must not. + await historyService.appendToHistory( + workspaceId, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_200, + }) + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + workspaceService.disposeSession(workspaceId); } finally { await cleanup(); } }); - test("keeps workflow invocations current across synthetic user continuations", async () => { + test("fails boundaryless sidecar references quiet instead of trusting wall-clock order", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-currentness"; - const runId = "wfr_currentness"; + const workspaceId = "workflow-currentness-legacy"; + const runId = "wfr_currentness_legacy"; const projectPath = path.join(config.rootDir, "project"); try { await config.addWorkspace(projectPath, { id: workspaceId, - name: "workflow-currentness", + name: "workflow-currentness-legacy", projectName: "project", projectPath, runtimeConfig: { type: "local" }, @@ -8651,50 +10114,48 @@ describe("WorkspaceService workflow invocation events", () => { await historyService.appendToHistory( workspaceId, - createMuxMessage("assistant-workflow-run", "assistant", "", { timestamp: 1_000 }, [ - { - type: "dynamic-tool", - toolCallId: "workflow-call-1", - toolName: "workflow_run", - state: "output-available", - input: { script_path: "./workflows/demo.js", args: {}, run_in_background: true }, - output: { status: "running", runId, result: null }, - }, - ]) + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); - await historyService.appendToHistory( - workspaceId, - createMuxMessage("synthetic-await", "user", "Call task_await", { - timestamp: 1_100, - synthetic: true, - }) + // A reference without a boundary snapshot (pre-upgrade entry or record-time history read + // failure) cannot be ordered against the decision row by identity: the wake fails quiet + // (not_current) rather than delivering or deferring forever. + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + }); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "not_current" ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - + // A backward clock correction gives the newer superseding turn an OLDER timestamp than + // the reference. Wall-clock ordering would resurrect the superseded reference as current + // and deliver its output under the newer turn's tool policy; it must stay quiet. await historyService.appendToHistory( workspaceId, - createMuxMessage("manual-user", "user", "Never mind, answer something else", { - timestamp: 1_200, + createMuxMessage("manual-user-2", "user", "never mind, answer something else", { + timestamp: 1_100, }) ); - - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "not_current" + ); workspaceService.disposeSession(workspaceId); } finally { await cleanup(); } }); - test("counts workflow_resume output as the current invocation after manual supersession", async () => { + test("treats an unreadable history as indeterminate, not superseded", async () => { const { config, historyService, cleanup } = await createTestHistoryService(); - const workspaceId = "workflow-currentness-resume"; - const runId = "wfr_currentness_resume"; + const workspaceId = "workflow-currentness-io-error"; + const runId = "wfr_currentness_io_error"; const projectPath = path.join(config.rootDir, "project"); try { await config.addWorkspace(projectPath, { id: workspaceId, - name: "workflow-currentness-resume", + name: "workflow-currentness-io-error", projectName: "project", projectPath, runtimeConfig: { type: "local" }, @@ -8716,60 +10177,111 @@ describe("WorkspaceService workflow invocation events", () => { await historyService.appendToHistory( workspaceId, - createMuxMessage("assistant-workflow-run", "assistant", "", { timestamp: 1_000 }, [ - { - type: "dynamic-tool", - toolCallId: "workflow-call-1", - toolName: "workflow_run", - state: "output-available", - input: { script_path: "./workflows/demo.js", args: {}, run_in_background: true }, - output: { status: "running", runId, result: null }, - }, - ]) - ); - await historyService.appendToHistory( - workspaceId, - createMuxMessage("manual-user", "user", "Never mind, answer something else", { - timestamp: 1_100, - }) + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); - - // An unrelated tool output mentioning the run does not re-establish the invocation. - await historyService.appendToHistory( - workspaceId, - createMuxMessage("assistant-unrelated-tool", "assistant", "", { timestamp: 1_200 }, [ - { - type: "dynamic-tool", - toolCallId: "task-list-1", - toolName: "task_list", - state: "output-available", - input: {}, - output: { status: "running", runId, result: null }, - }, - ]) + const readSpy = spyOn(historyService, "iterateFullHistory").mockResolvedValue( + Err("disk read failed") + ); + try { + // The drain distinguishes a read failure (retain and retry) from supersession + // (settle as superseded); the boolean view stays fail-safe false for non-destructive + // callers. + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + // The record path must fail loudly instead of persisting a verified-empty boundary that + // would permanently strand the run's wake after storage recovers. + let boundaryError: unknown; + try { + await workspaceService.getWorkflowInvocationBoundaryMessageId(workspaceId, runId); + } catch (error: unknown) { + boundaryError = error; + } + expect(String(boundaryError)).toContain("boundary unavailable"); + } finally { + readSpy.mockRestore(); + } + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" ); + workspaceService.disposeSession(workspaceId); + } finally { + await cleanup(); + } + }); - expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + test("treats an unreadable sidecar as indeterminate, not superseded", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const workspaceId = "workflow-currentness-sidecar-error"; + const runId = "wfr_currentness_sidecar_error"; + const projectPath = path.join(config.rootDir, "project"); + try { + await config.addWorkspace(projectPath, { + id: workspaceId, + name: "workflow-currentness-sidecar-error", + projectName: "project", + projectPath, + runtimeConfig: { type: "local" }, + }); + const workspaceService = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + stopStream: mock(() => Promise.resolve(Ok(undefined))), + }), + extensionMetadata: new ExtensionMetadataService( + path.join(config.rootDir, "extensionMetadata.json") + ), + initStateManager: { + ...mockInitStateManager, + off: mock(() => undefined as unknown as InitStateManager), + } as unknown as InitStateManager, + }); - // workflow_resume re-attaches the agent to the run, so the invocation counts as current - // again and the terminal continuation would be delivered. await historyService.appendToHistory( workspaceId, - createMuxMessage("assistant-workflow-resume", "assistant", "", { timestamp: 1_300 }, [ - { - type: "dynamic-tool", - toolCallId: "workflow-resume-1", - toolName: "workflow_resume", - state: "output-available", - input: { run_id: runId, mode: "resume", run_in_background: true }, - output: { status: "running", runId, result: null }, - }, - ]) + createMuxMessage("manual-user", "user", "run the audit workflow", { timestamp: 1_000 }) ); - + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(true); + + // The sidecar is the only invocation evidence for kernel-launched runs: an unreadable + // file must read as "cannot know right now", not "no reference", or the drain would + // settle the wake as superseded on a transient storage fault. + const sidecarPath = path.join(config.sessionsDir, workspaceId, "agent-workflow-runs.json"); + await fsPromises.rm(sidecarPath); + await fsPromises.mkdir(sidecarPath); + try { + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "indeterminate" + ); + expect(await workspaceService.isWorkflowInvocationCurrent(workspaceId, runId)).toBe(false); + } finally { + await fsPromises.rmdir(sidecarPath); + } + await recordAgentWorkflowRunReference({ + workspaceSessionDir: path.join(config.sessionsDir, workspaceId), + runId, + createdAtMs: 1_150, + afterBoundaryMessageId: "manual-user", + }); + expect(await workspaceService.getWorkflowInvocationCurrentness(workspaceId, runId)).toBe( + "current" + ); workspaceService.disposeSession(workspaceId); } finally { await cleanup(); @@ -9981,10 +11493,7 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { // Deterministic unlink failure: a DIRECTORY at the pending-state path // fails unlink with EISDIR (read errors are swallowed at load, so this // models exactly the stale-undeletable-file case). - const pendingStatePath = path.join( - path.join(config.sessionsDir, workspaceId), - "post-compaction.json" - ); + const pendingStatePath = path.join(config.sessionsDir, workspaceId, "post-compaction.json"); await fsPromises.mkdir(pendingStatePath, { recursive: true }); const result = await workspaceService.resetContext(workspaceId); @@ -10064,10 +11573,16 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { projectPath: "/tmp/full-clear-sandbox-project", runtimeConfig: { type: "local" }, }); + // Two similar-size messages: 50% removes only the first, keeping the truncation genuinely + // partial (a one-message 50% empties history and routes as a full clear). await historyService.appendToHistory( workspaceId, createMuxMessage("pre-clear-user", "user", "before clear", {}) ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("pre-clear-user-b", "user", "still here", {}) + ); const discardSpy = spyOn(sandboxHostService, "discardScope").mockImplementation(() => Promise.resolve() ); @@ -10118,10 +11633,16 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { projectPath: "/tmp/clear-drains-refine-project", runtimeConfig: { type: "local" }, }); + // Two similar-size messages keep the 50% truncation genuinely partial (see the sandbox + // discard test above). await historyService.appendToHistory( workspaceId, createMuxMessage("pre-clear-user", "user", "before clear", {}) ); + await historyService.appendToHistory( + workspaceId, + createMuxMessage("pre-clear-user-b", "user", "still here", {}) + ); const drained: string[] = []; workspaceService.setRefinePassCanceller({ cancelInFlightRefinePass: (id) => { @@ -16097,6 +17618,23 @@ describe("WorkspaceService setPinned", () => { expect((await workspaceService.setPinned(rootId, true)).success).toBe(true); expect(getEntry(rootId)?.pinnedAt).toBeDefined(); }); + + test("unarchive pokes task-side workflow attention reconciliation", async () => { + const noteWorkspaceUnarchived = mock(() => Promise.resolve()); + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ noteWorkspaceUnarchived }) + ); + expect((await workspaceService.archive(rootId)).success).toBe(true); + expect(noteWorkspaceUnarchived).not.toHaveBeenCalled(); + + expect((await workspaceService.unarchive(rootId)).success).toBe(true); + expect(noteWorkspaceUnarchived).toHaveBeenCalledWith(rootId); + expect(noteWorkspaceUnarchived).toHaveBeenCalledTimes(1); + + // No archived -> unarchived transition: a repeat unarchive must not re-poke. + expect((await workspaceService.unarchive(rootId)).success).toBe(true); + expect(noteWorkspaceUnarchived).toHaveBeenCalledTimes(1); + }); }); describe("WorkspaceService reorderPinned", () => { @@ -17645,6 +19183,84 @@ describe("WorkspaceService archive snapshots", () => { expect(entry?.worktreeArchiveSnapshot).toBeUndefined(); expect(editConfigSpy).toHaveBeenCalledTimes(0); }); + + test("unarchive reconciles workflow attention only after snapshot restoration", async () => { + const snapshot = { + version: 1 as const, + capturedAt: "2026-03-30T00:00:00.000Z", + stateDirPath: "archive-state", + projects: [ + { + projectPath, + projectName: "proj", + storageKey: "proj", + branchName: "ws-archive-snapshot", + trunkBranch: "main", + baseSha: "base-sha", + headSha: "head-sha", + }, + ], + }; + const order: string[] = []; + const noteWorkspaceUnarchived = mock((_workspaceId: string) => { + order.push("reconcile"); + return Promise.resolve(); + }); + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ noteWorkspaceUnarchived }) + ); + workspaceService.setWorktreeArchiveSnapshotService({ + preflightSnapshotForArchive: mock(() => Promise.resolve(Ok(undefined))), + captureSnapshotForArchive: mock(() => Promise.resolve(Ok(snapshot))), + restoreSnapshotAfterUnarchive: mock(() => { + order.push("restore"); + return Promise.resolve(Ok("skipped" as const)); + }), + getUnsupportedUntrackedPaths: mock(() => Promise.resolve(Ok([]))), + }); + + expect((await workspaceService.archive(workspaceId)).success).toBe(true); + expect((await workspaceService.unarchive(workspaceId)).success).toBe(true); + // The reconciliation drain can admit a synthetic agent turn, which must never run + // against a half-restored checkout. + expect(order).toEqual(["restore", "reconcile"]); + }); + + test("a failed snapshot restoration skips workflow attention reconciliation", async () => { + const snapshot = { + version: 1 as const, + capturedAt: "2026-03-30T00:00:00.000Z", + stateDirPath: "archive-state", + projects: [ + { + projectPath, + projectName: "proj", + storageKey: "proj", + branchName: "ws-archive-snapshot", + trunkBranch: "main", + baseSha: "base-sha", + headSha: "head-sha", + }, + ], + }; + const noteWorkspaceUnarchived = mock((_workspaceId: string) => Promise.resolve()); + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ noteWorkspaceUnarchived }) + ); + workspaceService.setWorktreeArchiveSnapshotService({ + preflightSnapshotForArchive: mock(() => Promise.resolve(Ok(undefined))), + captureSnapshotForArchive: mock(() => Promise.resolve(Ok(snapshot))), + restoreSnapshotAfterUnarchive: mock(() => Promise.resolve(Err("restore failed"))), + getUnsupportedUntrackedPaths: mock(() => Promise.resolve(Ok([]))), + }); + + expect((await workspaceService.archive(workspaceId)).success).toBe(true); + const result = await workspaceService.unarchive(workspaceId); + expect(result.success).toBe(false); + // The failed restoration rolled the unarchive back; reconciling would admit a synthetic + // turn into a workspace that is still archived. + expect(noteWorkspaceUnarchived).not.toHaveBeenCalled(); + }); }); describe("WorkspaceService preflightArchive and acknowledged archive", () => { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2bb77c7c77..bda429c1ae 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -3,6 +3,11 @@ import { TASK_TERMINATION_STOP_STREAM_TIMEOUT_MS } from "@/constants/termination import { raceWithAbortAndTimeout } from "@/node/utils/concurrency/withTimeout"; import { EventEmitter } from "events"; import { acquireCrossProcessLock } from "@/node/utils/main/crossProcessLock"; +import { + clearAgentWorkflowRunReferences, + readAgentWorkflowRunReferences, + type AgentWorkflowRunReference, +} from "@/node/services/agentWorkflowRunReferences"; import * as fsPromises from "fs/promises"; import assert from "@/common/utils/assert"; import { DEFAULT_WORKTREE_ARCHIVE_BEHAVIOR } from "@/common/config/worktreeArchiveBehavior"; @@ -200,6 +205,7 @@ import { } from "@/node/services/workflows/workflowArchiveAdmission"; import { WORKFLOW_RESULT_METADATA_TYPE, + textContainsWorkflowResultPayload, WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, WORKFLOW_TRIGGER_DISPLAY_METADATA_TYPE, buildWorkflowRunCardMessage, @@ -479,6 +485,23 @@ function isWorkflowResultContinuationMessage(message: MuxMessage, runId: string) ); } +/** + * The terminal-attention drain delivers workflow results as one synthetic user prompt that may + * coalesce several runs, so it carries no per-run workflow-result metadata. If a crash lands + * between the send's durable acceptance and the settled-marker write, the next sweep re-queues + * the run; recognizing the accepted row as consumption is what settles it without a re-send. + * Only synthetic rows qualify: a manual user message is a supersession boundary and is + * classified before this check runs. + */ +function isCoalescedWorkflowResultMessage(message: MuxMessage, runId: string): boolean { + if (message.role !== "user" || message.metadata?.synthetic !== true) { + return false; + } + return message.parts.some( + (part) => part.type === "text" && textContainsWorkflowResultPayload(part.text, runId) + ); +} + function isResetBoundaryMessage(message: MuxMessage): boolean { return message.metadata?.contextBoundaryKind === CONTEXT_BOUNDARY_KINDS.RESET; } @@ -4912,7 +4935,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { private async getPersistedPostCompactionDiffPaths(workspaceId: string): Promise { const postCompactionPath = path.join( - path.join(this.config.sessionsDir, workspaceId), + this.config.sessionsDir, + workspaceId, "post-compaction.json" ); @@ -5036,10 +5060,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * Returns empty exclusions if file doesn't exist. */ public async getPostCompactionExclusions(workspaceId: string): Promise { - const exclusionsPath = path.join( - path.join(this.config.sessionsDir, workspaceId), - "exclusions.json" - ); + const exclusionsPath = path.join(this.config.sessionsDir, workspaceId, "exclusions.json"); try { const data = await fsPromises.readFile(exclusionsPath, "utf-8"); return JSON.parse(data) as PostCompactionExclusions; @@ -9606,6 +9627,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { subProjectPath: hookMetadata?.subProjectPath, }); + // Archived owners park workflow terminal wakes unsettled; reconcile so an idle + // workspace does not stay silent until the interval sweep. Only AFTER snapshot + // restoration and lifecycle startup above: the drain can admit a synthetic agent turn, + // which must not run against a half-restored checkout or precede a failed restoration's + // config rollback. Contained: reconciliation failure must not fail the unarchive (the + // sweep retries on its own cadence). + try { + await this.agentTaskIntegration?.noteWorkspaceUnarchived(workspaceId); + } catch (error: unknown) { + log.warn("Unarchive workflow attention reconciliation failed", { workspaceId, error }); + } + return Ok(undefined); } catch (error) { const message = getErrorMessage(error); @@ -10997,38 +11030,137 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } async isWorkflowInvocationCurrent(workspaceId: string, runId: string): Promise { - assert(workspaceId.length > 0, "isWorkflowInvocationCurrent requires workspaceId"); - assert(runId.length > 0, "isWorkflowInvocationCurrent requires runId"); + return (await this.getWorkflowInvocationCurrentness(workspaceId, runId)) === "current"; + } - let current = false; - let foundDecision = false; + /** + * Three-state currentness: "indeterminate" means history/provenance could not be read or + * ordered, so the answer is unknown rather than no. Callers that would permanently settle a + * terminal wake on a negative answer (the terminal-attention drain records a superseded + * settlement marker) must retain and retry on "indeterminate" instead; boolean callers treat + * it as not-current, + * the pre-existing fail-safe for non-destructive decisions. + */ + async getWorkflowInvocationCurrentness( + workspaceId: string, + runId: string + ): Promise<"current" | "not_current" | "indeterminate"> { + assert(workspaceId.length > 0, "getWorkflowInvocationCurrentness requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationCurrentness requires runId"); + + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + if (decision.status === "error") { + return "indeterminate"; + } + if (decision.status === "found" && decision.outcome === "invocation") { + return "current"; + } + + // Kernel-launched runs (mux.workflow_run / mux.workflow_resume inside code_execution) leave + // no recognizable invocation part in history, so the backward walk above stops at the prior + // real user message (or, after a delivered result, at that consumed terminal message) and + // would wrongly drop the run's notify_on_terminal wake. Their durable provenance is the + // agent-workflow-runs sidecar, which snapshots the ID of the decision row that was newest + // at record time: the run is current exactly when that row is still the newest decision row. + // Row identity, not wall-clock ordering, so a backward clock correction can neither strand + // a legitimate wake nor let a pre-supersession reference outrank a newer boundary. For a + // consumed boundary, equality means a background resume/retry was recorded after the prior + // result was delivered. References without a boundary snapshot (pre-upgrade entries, + // record-time read failures) cannot be ordered against the decision row at all and defer + // as indeterminate below. + let references: AgentWorkflowRunReference[]; + try { + references = await readAgentWorkflowRunReferences( + path.join(this.config.sessionsDir, workspaceId) + ); + } catch (error: unknown) { + // The sidecar is the only invocation evidence a kernel-launched run has, so an + // unreadable file is "cannot know right now", not "no reference": defer wake decisions + // exactly like an unreadable history. + log.warn("Could not read workflow run references for currentness", { + workspaceId, + runId, + error, + }); + return "indeterminate"; + } + const reference = references.find((candidate) => candidate.runId === runId); + if (decision.status === "none") { + // A decision-free history is current only for a reference whose snapshot verified an + // empty history at record time (null): kernel launches from a new or fully cleared + // workspace (e.g. a heartbeat turn) have no decision row before or after, and their wake + // must still deliver. Every other surviving reference fails safe, because a full clear + // (truncateHistory) removes every row WITHOUT appending a reset boundary while leaving + // the sidecar intact, and a reference pointing at a cleared row, or one without a + // verified snapshot, must not inject a workflow result into the freshly cleared + // conversation. + return reference?.afterBoundaryMessageId === null ? "current" : "not_current"; + } + if (reference == null) { + return "not_current"; + } + if (reference.afterBoundaryMessageId === undefined) { + // No boundary snapshot (pre-upgrade entry or record-time history read failure): row + // identity cannot be verified, and wall-clock ordering is the exact hole the identity + // path exists to close (a backward clock correction would let a pre-supersession + // reference outrank a newer manual turn and deliver its output under that turn's tool + // policy). Fail quiet rather than deliver or defer forever: this is a deliberately + // accepted narrow window (downgrade-stripped or snapshot-failed launches), and the + // run's result stays retrievable via an explicit workflow_resume, which re-records the + // reference with a fresh boundary. + return "not_current"; + } + if (reference.afterBoundaryMessageId === null) { + // Verified-empty snapshot: a decision row now exists, so it appeared after the record. + return "not_current"; + } + return reference.afterBoundaryMessageId === decision.messageId ? "current" : "not_current"; + } + + /** + * The newest invocation-decision row for this run: a manual user/reset supersession, a + * consumed terminal result for the run, or a direct invocation part. Shared by + * isWorkflowInvocationCurrent and the sidecar record path so both sides of the identity + * comparison classify rows identically. + */ + private async findWorkflowInvocationDecisionRow( + workspaceId: string, + runId: string + ): Promise< + | { + status: "found"; + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + } + | { status: "none" } + | { status: "error" } + > { + const state: { + found: { + outcome: "invocation" | "consumed" | "superseded"; + messageId: string; + } | null; + } = { found: null }; const historyResult = await this.historyService.iterateFullHistory( workspaceId, "backward", (messages) => { for (const message of messages) { - if (isManualUserSupersessionMessage(message)) { - current = false; - foundDecision = true; - return false; - } - if (isResetBoundaryMessage(message)) { - current = false; - foundDecision = true; + if (isManualUserSupersessionMessage(message) || isResetBoundaryMessage(message)) { + state.found = { outcome: "superseded", messageId: message.id }; return false; } if ( isWorkflowResultContinuationMessage(message, runId) || + isCoalescedWorkflowResultMessage(message, runId) || isTerminalWorkflowTaskAwaitResultMessage(message, runId) || isTerminalWorkflowToolResultMessage(message, runId) ) { - current = false; - foundDecision = true; + state.found = { outcome: "consumed", messageId: message.id }; return false; } if (isWorkflowInvocationMessage(message, runId)) { - current = true; - foundDecision = true; + state.found = { outcome: "invocation", messageId: message.id }; return false; } } @@ -11041,10 +11173,38 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { runId, error: historyResult.error, }); - return false; + return { status: "error" }; } + return state.found != null + ? { status: "found", outcome: state.found.outcome, messageId: state.found.messageId } + : { status: "none" }; + } - return foundDecision && current; + /** Testable seam for the pre-truncation retirement in truncateHistory. */ + private async retireKernelWorkflowRunReferences(workspaceId: string): Promise { + await clearAgentWorkflowRunReferences(path.join(this.config.sessionsDir, workspaceId)); + } + + /** + * Boundary snapshot for the agent-workflow-runs sidecar: the message ID of the newest + * invocation-decision row for this run, or null when history has none. Recorded at + * background launch/resume so isWorkflowInvocationCurrent can compare row identity instead + * of wall-clock timestamps, which clock corrections can reorder. + */ + async getWorkflowInvocationBoundaryMessageId( + workspaceId: string, + runId: string + ): Promise { + assert(workspaceId.length > 0, "getWorkflowInvocationBoundaryMessageId requires workspaceId"); + assert(runId.length > 0, "getWorkflowInvocationBoundaryMessageId requires runId"); + const decision = await this.findWorkflowInvocationDecisionRow(workspaceId, runId); + // A read failure must not masquerade as a verified-empty history: persisting null would + // permanently fail the run's currentness check even after storage recovers. Throw so the + // record path can distinguish and record a rediscovery-only reference instead. + if (decision.status === "error") { + throw new Error("workflow invocation boundary unavailable: history read failed"); + } + return decision.status === "found" ? decision.messageId : null; } /** @@ -12704,14 +12864,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async truncateHistory(workspaceId: string, percentage?: number): Promise> { const effectivePercentage = percentage ?? 1.0; - const isFullClear = effectivePercentage >= 1.0; - // A full clear holds the admission guard across the refine drain/lock - // awaits below: without it, a send admitted during those awaits could - // snapshot the pre-clear transcript and stream across the truncation, - // repopulating the cleared context. Partial truncation keeps the plain - // pre-check — no awaits sit between it and the truncation. + // The admission guard is acquired BEFORE the scope preflight and held across every await + // below: a turn admitted during any of them could snapshot the pre-truncation transcript + // and stream across the mutation, and one admitted during the preflight itself could + // launch a kernel workflow whose sidecar reference the wholesale retirement below would + // delete while its launch turn's rows survive the prefix cut, permanently suppressing + // that run's wake. The preflight cannot yet prove scope "none", so every request that + // may remove rows (percentage > 0) pays the guard; percentage <= 0 is a deterministic + // no-op that retires nothing and keeps the plain busy pre-check. let admissionGuard: Disposable | null = null; - if (isFullClear) { + if (effectivePercentage > 0) { const guardResult = this.acquireContextMutationAdmissionGuard( workspaceId, "truncate history" @@ -12729,6 +12891,36 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } using _admissionGuard = admissionGuard; + // A token-proportional truncation below 100% can remove nothing (budget rounds to zero), + // a proper prefix, or everything (historyService's full-delete fast path), and each scope + // carries different obligations: an emptied transcript needs every full-clear guard, a + // prefix cut still needs kernel workflow reference retirement (it can delete the launch + // turn's restriction rows without a supersession decision), and a no-op must retire + // nothing, or active runs' wakes would settle superseded under an unchanged transcript. + // Decide up front; historyService revalidates the dangerous drift directions under the + // history write lock (refuseFullDelete / refuseRowRemoval / requireFullDelete below). + const truncationScope = + effectivePercentage >= 1.0 + ? ("all" as const) + : effectivePercentage <= 0 + ? ("none" as const) + : await this.historyService + .classifyTruncationRemoval(workspaceId, effectivePercentage) + .catch((error: unknown) => { + log.warn("History truncation scope preflight failed; refusing truncation", { + workspaceId, + error, + }); + return null; + }); + if (truncationScope == null) { + // An unknown scope must not choose a side-effect set: labeling it a full clear would + // discard goal/plan/retry state and advance the context epoch while rows may remain, + // and labeling it smaller would skip full-clear guards. Nothing is mutated or retired + // yet, so refusing is lossless and the user can simply retry. + return Err("Failed to read history to classify the truncation scope. Try again."); + } + const isFullClear = truncationScope === "all"; const session = this.sessions.get(workspaceId); // A full clear discards the transcript a streaming refine pass may be @@ -12772,10 +12964,49 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + // Kernel workflow run references belong to the conversation this truncation mutates: a + // full clear leaves a verified-empty (null) boundary snapshot reading the fresh + // conversation as current, and even a prefix truncation can delete the launch turn's + // restriction-bearing rows without appending any supersession decision, letting the wake + // recompose from unrestricted defaults. Retire the references on every row-removing + // truncation, BEFORE it commits, so both fault directions fail safe: a failed retirement + // aborts with the conversation intact, and a failed or refused truncation leaves + // reference-less runs settling superseded (dropped wake, still retrievable via resume, + // which re-records provenance under the surviving context). + if (truncationScope !== "none") { + try { + await this.retireKernelWorkflowRunReferences(workspaceId); + } catch (error) { + return Err( + `Cannot clear history: stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). Retry once the session storage is writable.` + ); + } + // In-turn compaction retries bypass admission gating across a transient idle gap (see + // the full-clear recheck above), and this retirement is the last await before the + // rewrite for BOTH row-removing scopes: revalidate here or a retry admitted during it + // would have its history truncated underneath the stream. Refusing after retirement is + // the documented fail-safe direction (dropped wake, retrievable via resume). + if (session?.hasActiveOrPendingTurnWork() || this.aiService.isStreaming(workspaceId)) { + return Err( + "Cannot truncate history while a turn is active. Press Esc to stop the stream first." + ); + } + } if (effectivePercentage > 0) { session?.clearUsageState(); } - const truncate = () => this.historyService.truncateHistory(workspaceId, effectivePercentage); + // historyService revalidates the scope preflight under the history write lock: an + // overlapping mutation can shift this one across a scope boundary in any dangerous + // direction (a partial cut becoming a full delete skips the full-clear guards; a no-op + // becoming a real cut skips reference retirement; a full clear leaving survivors would + // apply full-clear-only discards while rows remain). + const truncate = () => + this.historyService.truncateHistory(workspaceId, effectivePercentage, { + refuseFullDelete: truncationScope === "partial", + refuseRowRemoval: truncationScope === "none", + requireFullDelete: truncationScope === "all", + }); const truncateResult = effectivePercentage > 0 ? await this.clearHistoryWithRetiredBashMonitorWakes(workspaceId, truncate, { @@ -13213,6 +13444,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); } } + // A destructive non-compaction replacement discards the conversation the kernel + // workflow references belong to, exactly like a full clear: a verified-empty (null) + // boundary snapshot reads the decision-free replacement history as current and would + // inject a pre-replacement workflow result into it. Same ordering and failure posture + // as truncateHistory: retire before the clear commits, abort when retirement fails. + // Compaction replaces preserve conversation identity, so their references stay live. + if (!isCompaction) { + try { + await this.retireKernelWorkflowRunReferences(workspaceId); + } catch (error) { + return Err( + `Cannot replace history: stale workflow run references could not be retired ` + + `(${getErrorMessage(error)}). Retry once the session storage is writable.` + ); + } + } this.sessions.get(workspaceId)?.clearUsageState(); const clearResult = await this.clearHistoryWithRetiredBashMonitorWakes( workspaceId,