From 52ef03a27dac0426a335e803f0dcb96e82b33b13 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:27:05 +0000 Subject: [PATCH 1/5] feat(server): deliver messages by class, never into a compaction A message arriving while the provider is compacting now waits for the compaction to finish instead of aborting it. Senders set a delivery class: interrupt, steer_at_tool_boundary, next_turn, when_idle, or digest. Digests never start a turn and ride with the next one. An undelivered message with the same coalesce key replaces the older one, and one past its expiry is dropped and logged. Alerts default to digest. t3_thread_send accepts the class, coalesce key and expiry, and t3_queue shows them. The pre-commit lint and knip hooks fail in this worktree because Node cannot load the repo's TypeScript oxlint plugin and knip schemas. Both fail the same way on the untouched base commit. Co-Authored-By: Claude Fable 5.2 --- apps/server/src/mcp/OrchestratorMcpService.ts | 26 ++ .../src/mcp/toolkits/orchestrator/tools.ts | 2 +- .../src/mcp/toolkits/thread/handlers.ts | 12 + apps/server/src/mcp/toolkits/thread/tools.ts | 6 +- .../src/orchestration-v2/CommandPolicy.ts | 17 +- .../src/orchestration-v2/IdleCompaction.ts | 5 +- .../orchestration-v2/IdleCompactionService.ts | 6 +- .../src/orchestration-v2/ModelAlertReactor.ts | 42 ++- .../src/orchestration-v2/Orchestrator.ts | 343 ++++++++++++++---- .../src/orchestration-v2/ProjectionStore.ts | 12 +- .../ProviderTurnStartService.ts | 16 +- .../SteeringCompletion.integration.test.ts | 272 ++++++++++++++ .../ThreadManagementService.ts | 10 +- packages/contracts/src/orchestrationV2.ts | 48 +++ packages/contracts/src/orchestratorMcp.ts | 9 + packages/shared/package.json | 4 + packages/shared/src/messageDigest.ts | 31 ++ .../shared/src/orchestrationV2ThreadError.ts | 7 +- 18 files changed, 782 insertions(+), 86 deletions(-) create mode 100644 packages/shared/src/messageDigest.ts diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 8ff41b8650f6d..79c8e389ef6ab 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -3551,6 +3551,30 @@ const make = Effect.gen(function* () { yield* assertTargetWithinLimits(limits, target.thread); const mode = input.mode ?? "auto"; + const deliveryClass = + input.deliveryClass ?? (input.digestCategory === undefined ? undefined : "digest"); + if ( + deliveryClass === undefined && + (input.coalesceKey !== undefined || input.expiresAt !== undefined) + ) { + return yield* failure("invalid_request", "coalesceKey and expiresAt need a deliveryClass."); + } + const expiresAt = + input.expiresAt === undefined + ? undefined + : Option.getOrUndefined(DateTime.make(input.expiresAt)); + if (input.expiresAt !== undefined && expiresAt === undefined) { + return yield* failure("invalid_request", "expiresAt must be an ISO date-time."); + } + const delivery = + deliveryClass === undefined + ? undefined + : { + class: deliveryClass, + ...(input.coalesceKey === undefined ? {} : { coalesceKey: input.coalesceKey }), + ...(expiresAt === undefined ? {} : { expiresAt: DateTime.toUtc(expiresAt) }), + ...(input.digestCategory === undefined ? {} : { category: input.digestCategory }), + }; const key = yield* requestKey(input.clientRequestId); const messageId = stableOperationMessageId({ scope, @@ -3576,6 +3600,7 @@ const make = Effect.gen(function* () { attachments: [], mode, ...(input.freshContext ? { freshContext: true } : {}), + ...(delivery === undefined ? {} : { delivery }), createdBy: "agent", creationSource: "mcp", }) @@ -3618,6 +3643,7 @@ const make = Effect.gen(function* () { ...(result.run.delegatedTaskId === undefined ? {} : { taskId: result.run.delegatedTaskId }), status: result.run.status, delivery: result.delivery, + ...(deliveryClass === undefined ? {} : { deliveryClass }), } satisfies OrchestratorMcpThreadSendResult; }), waitForThread: Effect.fn("OrchestratorMcpService.waitForThread")(function* (scope, input) { diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index ffcb93de70244..2d9ce98bbbe8f 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -285,7 +285,7 @@ const ThreadUpdateTool = Tool.make("t3_thread_update", { const ThreadSendTool = Tool.make("t3_thread_send", { description: - "Send a message to any thread. The target cannot run with broader modes than yours. `mode:'auto'` starts an idle thread, steers a running turn, or queues. `queue`: new turn. `steer`: mid-turn update. `restart`: interrupt and restart. A send to a finished child thread starts a fresh task and returns its `taskId`. In your reply, name it as its `link` verbatim.", + "Send a message to any thread. The target cannot run with broader modes than yours. `mode:'auto'` starts an idle thread, steers a running turn, or queues. `queue`: new turn. `steer`: mid-turn update. `restart`: interrupt and restart. `deliveryClass` overrides `mode`: `interrupt`, `steer_at_tool_boundary` (after the next tool result), `next_turn`, `when_idle` (once nothing runs), or `digest` (never starts a turn; rides with the next one, grouped by `digestCategory`). `coalesceKey` replaces an undelivered message with the same key; past `expiresAt` an undelivered one is dropped. A send to a finished child thread starts a fresh task and returns its `taskId`. In your reply, name it as its `link` verbatim.", parameters: OrchestratorMcpThreadSendInput, success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/mcp/toolkits/thread/handlers.ts b/apps/server/src/mcp/toolkits/thread/handlers.ts index 4a4545b3d5e84..c3131813a9a6d 100644 --- a/apps/server/src/mcp/toolkits/thread/handlers.ts +++ b/apps/server/src/mcp/toolkits/thread/handlers.ts @@ -7,6 +7,7 @@ import { OrchestratorMcpFailure, type OrchestrationV2Command, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import { modelSelectionCommandType } from "@t3tools/shared/model"; @@ -42,6 +43,17 @@ function queueEntry( queuedRunId: run.id, text: characters.slice(0, limit).join(""), truncated: characters.length > limit, + ...(run.delivery === undefined + ? {} + : { + deliveryClass: run.delivery.class, + ...(run.delivery.coalesceKey === undefined + ? {} + : { coalesceKey: run.delivery.coalesceKey }), + ...(run.delivery.expiresAt === undefined + ? {} + : { expiresAt: DateTime.formatIso(run.delivery.expiresAt) }), + }), }; } const dispatch = Effect.fn("mcp.dispatchThreadCommand")(function* ( diff --git a/apps/server/src/mcp/toolkits/thread/tools.ts b/apps/server/src/mcp/toolkits/thread/tools.ts index d6c9a34565ddb..30184102a3a21 100644 --- a/apps/server/src/mcp/toolkits/thread/tools.ts +++ b/apps/server/src/mcp/toolkits/thread/tools.ts @@ -15,6 +15,7 @@ import { IsoDateTime, OrchestratorMcpFailure, OrchestrationV2DispatchCommandResult, + OrchestrationV2DeliveryClass, ThreadId, RunId, NonNegativeInt, @@ -79,12 +80,15 @@ const queueEntry = Schema.Struct({ queuedRunId: RunId, text: Schema.String, truncated: Schema.Boolean, + deliveryClass: Schema.optional(OrchestrationV2DeliveryClass), + coalesceKey: Schema.optional(Schema.String), + expiresAt: Schema.optional(Schema.String), }); const QUEUE_READ_ACTIONS = ["list", "read"] as const; const QueueTool = Tool.make("t3_queue", { ...commandTool, description: - "Manage a thread's queued messages. Omit `threadId` for this thread. list: send order, paged by `cursor` and `limit`. read: up to 16,000 chars of one. edit: replace its `text`; attachments stay. cancel. reorder: before `beforeRunId`, or to the end with null. promote_to_steer: send it now as steering to active `targetRunId`. Writes fail once the run is no longer queued.", + "Manage a thread's queued messages. Omit `threadId` for this thread. list: send order, paged by `cursor` and `limit`. read: up to 16,000 chars of one. edit: replace its `text`; attachments stay. cancel. reorder: before `beforeRunId`, or to the end with null. promote_to_steer: send it now as steering to active `targetRunId`. Entries show their `deliveryClass`, `coalesceKey` and `expiresAt`; a `digest` entry never starts a turn and goes out with the next one. Writes fail once the run is no longer queued.", parameters: Schema.Struct({ action: Schema.Literals([ ...QUEUE_READ_ACTIONS, diff --git a/apps/server/src/orchestration-v2/CommandPolicy.ts b/apps/server/src/orchestration-v2/CommandPolicy.ts index cd56460369bd6..6adecf7af71b9 100644 --- a/apps/server/src/orchestration-v2/CommandPolicy.ts +++ b/apps/server/src/orchestration-v2/CommandPolicy.ts @@ -135,8 +135,23 @@ export function followUpOwnerRun( ); } +/** Steering a provider mid-compaction aborts the compaction, so steers into it wait. */ +export function compactionHoldsSteer( + projection: Pick< + OrchestrationV2ThreadProjection, + "turnItems" | "providerThreads" | "providerSessions" + >, + run: OrchestrationV2Run, +): boolean { + return ( + projection.turnItems.some( + (item) => item.runId === run.id && item.type === "compaction" && item.status === "running", + ) && providerTurnCapabilities(projection, run)?.activeSteeringDuringCompaction !== true + ); +} + function providerTurnCapabilities( - projection: OrchestrationV2ThreadProjection, + projection: Pick, run: OrchestrationV2Run, ) { const providerThread = projection.providerThreads.find( diff --git a/apps/server/src/orchestration-v2/IdleCompaction.ts b/apps/server/src/orchestration-v2/IdleCompaction.ts index c4d586817551f..ecbcdaf693348 100644 --- a/apps/server/src/orchestration-v2/IdleCompaction.ts +++ b/apps/server/src/orchestration-v2/IdleCompaction.ts @@ -1,6 +1,7 @@ import { isListedThreadLineage, type OrchestrationV2ThreadProjection } from "@t3tools/contracts"; import { COMPACTION_SUGGESTION_MIN_TOKENS } from "@t3tools/shared/claudeCompaction"; import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; +import { isWaitingDigestRun } from "@t3tools/shared/orchestrationV2ThreadError"; import * as DateTime from "effect/DateTime"; // Claude Code's idleCompaction defaults to 200K and won't go below 100K. Below @@ -53,7 +54,9 @@ export function idleCompactionWindow( if (!isListedThreadLineage(thread.lineage)) return null; // A removed or steered queued message leaves a cancelled run that never started. const latestRun = runs - .filter((run) => run.status !== "cancelled" || run.startedAt !== null) + .filter( + (run) => (run.status !== "cancelled" || run.startedAt !== null) && !isWaitingDigestRun(run), + ) .reduce<(typeof runs)[number] | null>( (latest, run) => (latest === null || run.ordinal > latest.ordinal ? run : latest), null, diff --git a/apps/server/src/orchestration-v2/IdleCompactionService.ts b/apps/server/src/orchestration-v2/IdleCompactionService.ts index c7035de4ec584..b0be53b7aa513 100644 --- a/apps/server/src/orchestration-v2/IdleCompactionService.ts +++ b/apps/server/src/orchestration-v2/IdleCompactionService.ts @@ -7,6 +7,7 @@ import { type RunId, type ThreadId, } from "@t3tools/contracts"; +import { isWaitingDigestRun } from "@t3tools/shared/orchestrationV2ThreadError"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -89,7 +90,10 @@ export const make = Effect.gen(function* () { ) return; const latestRun = records.runs - .filter((run) => run.status !== "cancelled" || run.startedAt !== null) + .filter( + (run) => + (run.status !== "cancelled" || run.startedAt !== null) && !isWaitingDigestRun(run), + ) .reduce<(typeof records.runs)[number] | undefined>( (latest, run) => (latest === undefined || run.ordinal > latest.ordinal ? run : latest), undefined, diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index 3f1633580d835..d15ed5a86fb04 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -4,6 +4,7 @@ import { isListedThreadLineage, MessageId, type ModelAlertWatcher, + type OrchestrationV2DigestCategory, type OrchestrationV2DomainEvent, type OrchestrationV2Run, type OrchestrationV2ThreadShell, @@ -736,6 +737,33 @@ export const watcherAlertMessage = (alert: WatcherAlert) => { } }; +const ALERT_DIGEST_TTL_MS = 24 * 60 * 60 * 1000; + +/** Alerts ride in the watcher's next digest; a newer alert on the same subject replaces an older one. */ +function alertDelivery(alert: WatcherAlert): { + readonly coalesceKey: string; + readonly category: OrchestrationV2DigestCategory; +} { + switch (alert.type) { + case "peer-link-renewal": + return { coalesceKey: `${alert.type}:${alert.linkedEnvironmentId}`, category: "alert" }; + case "usage-limit-incident": + return { coalesceKey: `${alert.type}:${alert.groupKey}`, category: "alert" }; + case "model-degraded": + return { coalesceKey: `${alert.type}:${alert.threadId}`, category: "alert" }; + case "thread-event": + return { + coalesceKey: `${alert.type}:${alert.kind}:${alert.threadId}`, + category: + alert.kind === "lane-ended" + ? "lane_result" + : alert.kind === "needs-person" + ? "decision_needed" + : "alert", + }; + } +} + /** * Puts an alert in the watcher thread, here or on the linked machine that owns it. The retry key is * one per alert and watcher, so a retry after a lost answer sends once. @@ -754,6 +782,8 @@ export const deliverAlertToWatcher = Effect.fnUntraced(function* ( // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; const key = `${alert.type === "model-degraded" ? "model-alert" : alert.type}:${yield* deps.sha256Hex(identity)}`; + const { coalesceKey, category } = alertDelivery(alert); + const expiresAt = DateTime.add(yield* DateTime.now, { milliseconds: ALERT_DIGEST_TTL_MS }); if (watcher.environmentId === here) { yield* deps.dispatch({ type: "message.dispatch", @@ -763,6 +793,7 @@ export const deliverAlertToWatcher = Effect.fnUntraced(function* ( text: message, attachments: [], dispatchMode: { type: "queue_after_active" }, + delivery: { class: "digest", coalesceKey, expiresAt, category }, createdBy: "agent", creationSource: "server", }); @@ -771,7 +802,16 @@ export const deliverAlertToWatcher = Effect.fnUntraced(function* ( yield* deps.call({ environmentId: watcher.environmentId, tool: "t3_thread_send", - arguments: { threadId: watcher.threadId, message, mode: "queue", clientRequestId: key }, + arguments: { + threadId: watcher.threadId, + message, + mode: "queue", + clientRequestId: key, + deliveryClass: "digest", + coalesceKey, + expiresAt: DateTime.formatIso(expiresAt), + digestCategory: category, + }, limits: { runtimeMode: "full-access", interactionMode: "default" }, success: OrchestratorMcpThreadSendResult, }); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index ad3cce997779b..f4ce1460dc55f 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -56,6 +56,7 @@ import { type OrchestrationV2ProviderTurn, type OrchestrationV2RemoteTaskChild, type OrchestrationV2ModelSwitch, + type OrchestrationV2DigestEntry, type OrchestrationV2Run, type OrchestrationV2RunAttempt, type OrchestrationV2ThreadShell, @@ -114,6 +115,7 @@ import { import { CheckpointServiceV2 } from "./CheckpointService.ts"; import { CommandPolicyV2, + compactionHoldsSteer, followUpOwnerRun, goalCommandStopsActiveRun, isActiveRun, @@ -1022,12 +1024,101 @@ function worktreeSetupPending( ); } +const TERMINAL_TURN_ITEM_STATUSES: ReadonlySet = new Set([ + "completed", + "failed", + "cancelled", + "interrupted", +]); +const TOOL_TURN_ITEM_TYPES: ReadonlySet = new Set([ + "command_execution", + "file_change", + "file_search", + "web_search", + "dynamic_tool", + "subagent", +]); + +/** Digests never start a turn, and when_idle messages wait behind every other queued one. */ function nextQueuedRun( projection: Pick, ): OrchestrationV2Run | undefined { - return queuedRunsInDeliveryOrder(projection).find((run) => run.queueHeld !== true); + const startable = queuedRunsInDeliveryOrder(projection).filter( + (run) => run.queueHeld !== true && run.delivery?.class !== "digest", + ); + return startable.find((run) => run.delivery?.class !== "when_idle") ?? startable[0]; +} + +function deliveryExpired(run: OrchestrationV2Run, now: DateTime.Utc): boolean { + const expiresAt = run.delivery?.expiresAt; + return expiresAt !== undefined && DateTime.isLessThanOrEqualTo(expiresAt, now); +} + +/** The waiting digests a starting turn carries; expired ones are dropped instead. */ +function digestFold( + projection: Pick, + now: DateTime.Utc, +) { + const folded: Array = []; + const expired: Array = []; + const digest: Array = []; + for (const run of queuedRunsInDeliveryOrder(projection)) { + if (run.delivery?.class !== "digest") continue; + if (deliveryExpired(run, now)) { + expired.push(run); + continue; + } + const message = projection.messages.find((candidate) => candidate.id === run.userMessageId); + if (message === undefined) continue; + folded.push(run); + digest.push({ + messageId: message.id, + category: run.delivery.category ?? "fyi", + text: message.text, + }); + } + return { folded, expired, digest }; } +/** Plain turns carry the digest; maintenance, goal, idle-compaction and continuation turns do not. */ +function carriesDigest( + run: Pick< + OrchestrationV2Run, + | "delivery" + | "restartContinuationOfRunId" + | "manualContinuationOfRunId" + | "usageLimitContinuationOfRunId" + >, + message: { + readonly text: string; + readonly attachments: ReadonlyArray; + readonly idleCompaction?: unknown; + }, +): boolean { + return ( + run.delivery?.class !== "digest" && + run.restartContinuationOfRunId === undefined && + run.manualContinuationOfRunId === undefined && + run.usageLimitContinuationOfRunId === undefined && + message.idleCompaction === undefined && + !isNativeMaintenanceCommand(message) && + parseGoalCommand(message) === null + ); +} + +const logExpiredDelivery = (run: OrchestrationV2Run) => + Effect.logInfo("Dropped an expired message", { + threadId: run.threadId, + runId: run.id, + messageId: run.userMessageId, + deliveryClass: run.delivery?.class, + coalesceKey: run.delivery?.coalesceKey, + expiresAt: + run.delivery?.expiresAt === undefined + ? undefined + : DateTime.formatIso(run.delivery.expiresAt), + }); + // Delegated completions, monitor notifications and unpinned scheduled deliveries never choose a // model, so a queued one runs on the thread's current pick rather than the one frozen at queue time. function followsCurrentConversation( @@ -2459,6 +2550,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (yield* cancelStaleQuestionWakes(projection)) { projection = yield* readCommandProjection(threadId); } + const checkedAt = yield* DateTime.now; + const expired = projection.runs.filter( + (run) => run.status === "queued" && deliveryExpired(run, checkedAt), + ); + if (expired.length > 0) { + yield* writeSystemEvents( + expired.flatMap((run) => queuedRunCancellationEvents(projection, run, checkedAt)), + ); + for (const run of expired) yield* logExpiredDelivery(run); + projection = yield* readCommandProjection(threadId); + } let storedQueuedRun = options?.queuedRunId === undefined ? nextQueuedRun(projection) @@ -2996,6 +3098,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio )) ? "explicit-queue" : "automatic-queue"; + const fold = carriesDigest(queuedRun, queuedMessage) + ? digestFold(projection, now) + : undefined; const startingRun: OrchestrationV2Run = { ...queuedRun, providerThreadId: queuedProviderThread.id, @@ -3004,6 +3109,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio queuePosition: null, startedAt: null, contextHandoffId: activeHandoff?.id ?? null, + ...(fold === undefined || fold.digest.length === 0 ? {} : { digest: fold.digest }), ...wakeWorkStartedAt( projection, { @@ -3146,8 +3252,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const detachReason = freshContext ? FRESH_CONTEXT_DETACH_REASON : "Provider or model selection changed."; + for (const expiredRun of fold?.expired ?? []) yield* logExpiredDelivery(expiredRun); yield* writeSystemEvents( [ + ...(fold === undefined ? [] : [...fold.folded, ...fold.expired]).flatMap((spent) => + queuedRunCancellationEvents(projection, spent, now), + ), ...(selectionChanged ? [ { @@ -7164,10 +7274,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection = applyToProjection(projection, event); } let modelSelection = command.modelSelection ?? projection.thread.modelSelection; + const deliveryClass = command.delivery?.class; + const steersByClass = + deliveryClass === "interrupt" || deliveryClass === "steer_at_tool_boundary"; let dispatchMode = resolveMessageDispatchIntent( projection, - command.dispatchMode, - command.deliveryIntent, + deliveryClass === undefined || steersByClass + ? command.dispatchMode + : { type: "queue_after_active" }, + deliveryClass === undefined + ? command.deliveryIntent + : deliveryClass === "interrupt" + ? "restart" + : deliveryClass === "steer_at_tool_boundary" + ? "steer" + : undefined, ); if (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") { const targetRunId = dispatchMode.targetRunId; @@ -7313,6 +7434,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio taskIds: delivery.taskIds, }; } + // Steering or restarting into a running compaction aborts it, so the message + // waits for it to end; steer_at_tool_boundary waits for the next tool result. + // Command reads omit compaction items, so read the target run's own. + const compactionHolds = (run: OrchestrationV2Run) => + loadProjectionForCommand(command, ["turnItems"], { + turnItemTypes: ["compaction"], + turnItemRunIds: [run.id], + }).pipe(Effect.map(({ turnItems }) => compactionHoldsSteer({ ...projection, turnItems }, run))); + let deferSteer = false; + if (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") { + const targetRunId = dispatchMode.targetRunId; + const target = projection.runs.find((run) => run.id === targetRunId); + if ( + target?.status === "running" && + (deliveryClass === "steer_at_tool_boundary" || (yield* compactionHolds(target))) + ) { + deferSteer = !isNativeMaintenanceCommand(command) && goalCommand === null; + dispatchMode = deferSteer + ? { type: "steer_active", targetRunId } + : { type: "queue_after_active" }; + } + } // A provider that can stop a turn but not restart it with new input runs a // restart as the next turn: first in the queue, with the running turn stopped. if (dispatchMode.type === "restart_active" && !isNativeMaintenanceCommand(command)) { @@ -7373,7 +7516,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio activeTurn !== undefined && providerThread?.providerSessionId != null && (!commandNotice || providerThread.goal?.status === "active") && - (activeMessage === undefined || !isNativeMaintenanceCommand(activeMessage)) + (activeMessage === undefined || !isNativeMaintenanceCommand(activeMessage)) && + !(yield* compactionHolds(active)) ) { const session = yield* providerSessions .get(providerThread.providerSessionId) @@ -7460,9 +7604,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (target?.status === "preparing" || target?.status === "starting" || (target?.status === "running" && - !projection.providerTurns.some( - (turn) => turn.runAttemptId === target.activeAttemptId && turn.status === "running", - ))) && + (deferSteer || + !projection.providerTurns.some( + (turn) => turn.runAttemptId === target.activeAttemptId && turn.status === "running", + )))) && !(yield* stopReachedRun(command, command.threadId, target.id)) ) { const adapter = yield* providerAdapters @@ -7533,7 +7678,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection.thread.worktreePath === null ? latestRun : undefined; - const queueBehindRun = activeRun ?? worktreeMissingRun; + // A digest waits on an idle thread too; only a thread that never ran starts one. + const queueBehindRun = + activeRun ?? worktreeMissingRun ?? (deliveryClass === "digest" ? (latestRun ?? undefined) : undefined); const pendingMergeBackTransfers = pendingMergeBackTransfersForThread(projection); const startsOrQueuesRun = dispatchMode.type === "defer_start" || @@ -7581,6 +7728,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: { ...held, queueHeld: false }, }); } + const coalesceKey = command.delivery?.coalesceKey; + if (coalesceKey !== undefined) { + for (const replaced of projection.runs.filter( + (run) => run.status === "queued" && run.delivery?.coalesceKey === coalesceKey, + )) { + for (const event of queuedRunCancellationEvents(projection, replaced, releasedAt)) { + projection = applyToProjection(projection, yield* emit(events, command)(event)); + } + } + } if (shouldQueue) { if (pendingMergeBackTransfers.length > 0) { return yield* new OrchestratorDispatchError({ @@ -7766,6 +7923,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }), ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }), ...(freshContext ? { freshContext: true } : {}), + ...(command.delivery === undefined ? {} : { delivery: command.delivery }), }; const attempt: OrchestrationV2RunAttempt = { id: attemptId, @@ -8153,6 +8311,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), mapDispatchError(command), ); + const fold = carriesDigest(command, command) ? digestFold(projection, now) : undefined; const run: OrchestrationV2Run = { id: runId, threadId: command.threadId, @@ -8186,6 +8345,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }), ...wakeWorkStartedAt(projection, command), ...(freshContext ? { freshContext: true } : {}), + ...(command.delivery === undefined ? {} : { delivery: command.delivery }), + ...(fold === undefined || fold.digest.length === 0 ? {} : { digest: fold.digest }), }; const attempt: OrchestrationV2RunAttempt = { id: attemptId, @@ -8321,6 +8482,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: legacyImportHandoff, }); } + for (const spent of fold === undefined ? [] : [...fold.folded, ...fold.expired]) { + for (const event of queuedRunCancellationEvents(projection, spent, now)) { + yield* emitEvent(event); + } + } + for (const expired of fold?.expired ?? []) yield* logExpiredDelivery(expired); yield* emitEvent({ type: "run.created", threadId: command.threadId, @@ -11365,8 +11532,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio "runtimeRequests", "subagents", ], - // User messages measure a restart's model switch. Steering reads root and live nodes only. - { turnItemTypes: ["user_message"], messageRoles: ["user"], omitSettledLeafNodes: true }, + // User messages measure a restart's model switch; compactions hold the steer. + // Steering reads root and live nodes only. + { + turnItemTypes: ["user_message", "compaction"], + messageRoles: ["user"], + omitSettledLeafNodes: true, + }, ); if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) { return yield* new OrchestratorDispatchError({ @@ -11504,9 +11676,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (target?.status === "preparing" || target?.status === "starting" || (target?.status === "running" && - !projection.providerTurns.some( - (turn) => turn.runAttemptId === target.activeAttemptId && turn.status === "running", - ))) && + (compactionHoldsSteer(projection, target) || + !projection.providerTurns.some( + (turn) => turn.runAttemptId === target.activeAttemptId && turn.status === "running", + )))) && !(yield* stopReachedRun(command, command.threadId, target.id)) ) { yield* ensureSteeringMessage(command, queuedMessage); @@ -15932,65 +16105,99 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio Effect.forkScoped, ); - yield* eventSink - .stream({ afterSequence: terminalEventsAfterSequence, eventType: "provider-turn.updated" }) - .pipe( - Stream.filter( - (stored) => - stored.event.type === "provider-turn.updated" && - stored.event.payload.status === "running", - ), - Stream.runForEach( - Effect.fn("Orchestrator.deliverDeferredSteers")( - function* (stored) { - if (stored.event.type !== "provider-turn.updated") return; - const threadId = stored.event.threadId; - const turn = stored.event.payload; - const projection = yield* projectionStore.getThreadRecords(threadId, ["runs"], { - runStatuses: ["queued", "running"], - }); - const run = projection.runs.find( - (candidate) => - candidate.activeAttemptId === turn.runAttemptId && candidate.status === "running", + // Deferred steers go in once the provider turn runs, a compaction ends, or, for + // steer_at_tool_boundary, a tool item settles. + yield* Stream.merge( + eventSink.stream({ + afterSequence: terminalEventsAfterSequence, + eventType: "provider-turn.updated", + }), + eventSink.stream({ + afterSequence: terminalEventsAfterSequence, + eventType: "turn-item.updated", + }), + ).pipe( + Stream.filter( + (stored) => + (stored.event.type === "provider-turn.updated" && + stored.event.payload.status === "running") || + (stored.event.type === "turn-item.updated" && + (stored.event.payload.type === "compaction" + ? TERMINAL_TURN_ITEM_STATUSES.has(stored.event.payload.status) + : TOOL_TURN_ITEM_TYPES.has(stored.event.payload.type) && + (stored.event.payload.status === "completed" || + stored.event.payload.status === "failed"))), + ), + Stream.runForEach( + Effect.fn("Orchestrator.deliverDeferredSteers")( + function* (stored) { + const { event } = stored; + if (event.type !== "provider-turn.updated" && event.type !== "turn-item.updated") return; + const threadId = event.threadId; + const projection = yield* projectionStore.getThreadRecords(threadId, ["runs"], { + runStatuses: ["queued", "running"], + }); + const run = projection.runs.find( + (candidate) => + candidate.status === "running" && + (event.type === "provider-turn.updated" + ? candidate.activeAttemptId === event.payload.runAttemptId + : candidate.id === event.payload.runId), + ); + if (run === undefined) return; + const atToolBoundary = + event.type === "turn-item.updated" && event.payload.type !== "compaction"; + const trigger = + event.type === "provider-turn.updated" + ? event.payload.runAttemptId + : `${event.payload.id}:${event.payload.status}`; + const pending = projection.runs + .filter( + (queued) => + queued.status === "queued" && + queued.steerWhenReadyRunId === run.id && + (atToolBoundary || queued.delivery?.class !== "steer_at_tool_boundary"), + ) + .toSorted( + (left, right) => + (left.queuePosition ?? left.ordinal) - (right.queuePosition ?? right.ordinal), ); - if (run === undefined) return; - const pending = projection.runs - .filter( - (queued) => queued.status === "queued" && queued.steerWhenReadyRunId === run.id, - ) - .toSorted( - (left, right) => - (left.queuePosition ?? left.ordinal) - (right.queuePosition ?? right.ordinal), - ); - for (const queued of pending) { - yield* dispatchWithReceiptEffect({ - type: "queued-message.promote-to-steer", - commandId: CommandId.make( - `command:deferred-steer:${queued.id}:${run.id}:${turn.runAttemptId}`, - ), - threadId, - queuedRunId: queued.id, - targetRunId: run.id, - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Failed to deliver a deferred steer", { - threadId, - runId: queued.id, - cause, - }), + if (pending.length === 0) return; + // Tool items settle often, so only a thread with a deferred steer takes the lock. + yield* threadDispatch.withLock( + threadId, + Effect.forEach( + pending, + (queued) => + dispatchWithReceiptEffect({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make( + `command:deferred-steer:${queued.id}:${run.id}:${trigger}`, + ), + threadId, + queuedRunId: queued.id, + targetRunId: run.id, + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to deliver a deferred steer", { + threadId, + runId: queued.id, + cause, + }), + ), ), - ); - } - }, - (effect, stored) => threadDispatch.withLock(stored.event.threadId, effect), - Effect.catchCause((cause) => - Effect.logWarning("Failed to deliver deferred steering", { cause }), - ), - (effect, stored) => threadReactions.run(stored.event.threadId, effect), + { discard: true }, + ), + ); + }, + Effect.catchCause((cause) => + Effect.logWarning("Failed to deliver deferred steering", { cause }), ), + (effect, stored) => threadReactions.run(stored.event.threadId, effect), ), - Effect.forkScoped, - ); + ), + Effect.forkScoped, + ); // A child whose run ends while its background tasks or its own subagents still run has no // result yet. When that work settles there is no run update to finalize it, so these events diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 8402be3e50058..13977df7eb05c 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1968,6 +1968,7 @@ function isSilentRunForShell( run: OrchestrationV2ThreadProjection["runs"][number], projection: Pick, ): boolean { + if (run.delivery?.class === "digest" && run.startedAt == null) return true; return ( run.status !== "failed" && projection.messages.some( @@ -4949,6 +4950,7 @@ export const layer: Layer.Layer = WHERE status = 'queued' AND CASE WHEN json_valid(payload_json) THEN json_extract(payload_json, '$.queueHeld') IS NOT 1 + AND json_extract(payload_json, '$.delivery.class') IS NOT 'digest' ELSE 1 END UNION SELECT thread_id FROM orchestration_v2_projection_runtime_requests @@ -6269,7 +6271,8 @@ export const layer: Layer.Layer = WHERE thread.thread_id = ${threadId} AND thread.archived_at IS NULL AND thread.deleted_at IS NULL AND EXISTS (SELECT 1 FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} AND status = 'queued' - AND json_extract(payload_json, '$.queueHeld') IS NOT 1) + AND json_extract(payload_json, '$.queueHeld') IS NOT 1 + AND json_extract(payload_json, '$.delivery.class') IS NOT 'digest') AND NOT EXISTS (SELECT 1 FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} AND status IN ('preparing', 'starting', 'running', 'waiting'))`; @@ -7926,7 +7929,10 @@ export const layer: Layer.Layer = // Mirrors isSilentRunForShell. const isSilentShellRun = (alias: string) => { const run = sql.literal(alias); - return sql`( + return sql`(( + json_extract(${run}.payload_json, '$.delivery.class') IS 'digest' + AND json_extract(${run}.payload_json, '$.startedAt') IS NULL + ) OR ( ${run}.status <> 'failed' AND EXISTS ( SELECT 1 FROM orchestration_v2_projection_messages prompt @@ -7938,7 +7944,7 @@ export const layer: Layer.Layer = AND compaction.type = 'compaction' AND compaction.status = 'failed' ) - )`; + ))`; }; // The newest run not held or outlived before it started, matching latestUnheldRun. const presentedShellRunId = (threadId: Statement.Fragment) => sql`( diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index afbe74a9a1784..7271f5b0daeea 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1,5 +1,6 @@ import { modelSelectionsEqual } from "@t3tools/shared/model"; import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; +import { formatDigestForProvider } from "@t3tools/shared/messageDigest"; import { formatSeenMarkerForProvider } from "@t3tools/shared/seenMarker"; import { CommandId, @@ -911,12 +912,15 @@ export const layer: Layer.Layer< }); let effectiveHandoffs = handoffs; let resumeFellBack = false; - const userText = formatSeenMarkerForProvider( - projectComposerContextForProvider({ - text: providerMessage.text, - records: providerMessage.context?.records ?? [], - }), - providerMessage.seenMarker, + const userText = formatDigestForProvider( + formatSeenMarkerForProvider( + projectComposerContextForProvider({ + text: providerMessage.text, + records: providerMessage.context?.records ?? [], + }), + providerMessage.seenMarker, + ), + run.digest, ); // Native goal commands go verbatim; handoff history waits a turn. const goalDriver = diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 99d544ec2c646..180573316a762 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -17,12 +17,15 @@ import { TurnItemId, type ModelSelection, type OrchestrationV2DomainEvent, + type OrchestrationV2MessageDelivery, } from "@t3tools/contracts"; +import { formatDigestForProvider } from "@t3tools/shared/messageDigest"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; @@ -515,6 +518,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* turn: ProviderAdapter.ProviderAdapterV2SteerInput, ) => Effect.Effect = () => Effect.void, extraAdapters: ReadonlyArray = [], + steersDuringCompaction = true, ) { const driver = ProviderDriverKind.make(nativeGoals === undefined ? "codex" : "opencode"); const cwd = yield* checkpointWorkspace(name); @@ -530,6 +534,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* supportsActiveSteering: canSteer, supportsSteeringByInterruptRestart: canRestart, activeSteeringAcceptsAttachments: steeringAcceptsAttachments, + activeSteeringDuringCompaction: steersDuringCompaction, ...(nativeGoals === undefined ? {} : { supportsNativeGoals: nativeGoals }), }, }; @@ -1709,6 +1714,273 @@ it.effect("queues a steer until the turn ends when the provider cannot steer it" }), ); +const deliveryHarness = Effect.fn("deliveryHarness")(function* ( + name: string, + steersDuringCompaction = true, +) { + const steeredOnce = yield* Deferred.make(); + const harness = yield* nextTurnSelectionHarness( + name, + false, + true, + Effect.void, + true, + undefined, + () => Deferred.succeed(steeredOnce, undefined).pipe(Effect.asVoid), + [], + steersDuringCompaction, + ); + const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + return yield* orchestrator.streamDomainEvents.pipe( + Stream.filter(predicate), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + }); + const finishTurn = (turnInput: ProviderAdapter.ProviderAdapterV2TurnInput) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const now = yield* DateTime.now; + // The harness's running update for this turn may still be queued, so rebuild it. + const turn = { + id: ProviderTurnId.make(`provider-turn:${turnInput.attemptId}`), + providerThreadId: turnInput.providerThread.id, + nodeId: turnInput.rootNodeId, + runAttemptId: turnInput.attemptId, + nativeTurnRef: { driver, nativeId: `native:${turnInput.attemptId}`, strength: "strong" }, + ordinal: turnInput.providerTurnOrdinal, + status: "completed", + startedAt: now, + completedAt: now, + } as const; + const settled = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === turnInput.runId && + event.payload.status === "waiting", + ); + yield* Queue.offer(harness.events, { type: "provider_turn.updated", driver, providerTurn: turn }); + yield* Queue.offer(harness.events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: turnInput.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(settled); + yield* worker.drain(); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + }); + const send = ( + threadId: ThreadId, + id: string, + delivery?: OrchestrationV2MessageDelivery, + ) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`command:${id}`), + threadId, + messageId: MessageId.make(`message:${id}`), + text: id, + attachments: [], + dispatchMode: { type: "start_immediately" }, + ...(delivery === undefined + ? { createdBy: "user", creationSource: "web" } + : { delivery, createdBy: "agent", creationSource: "server" }), + }); + yield* worker.drain(); + return MessageId.make(`message:${id}`); + }); + return { ...harness, steeredOnce, watch, finishTurn, send }; +}); + +it.effect("holds a steer sent during compaction until the compaction completes", () => + Effect.gen(function* () { + const { events, started, steered, interrupted, layer, startFirstTurn, steeredOnce, watch } = + yield* deliveryHarness("compaction-holds-steer", false); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const turn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!; + const now = yield* DateTime.now; + const compaction = (status: "running" | "completed") => + ({ + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("compaction"), + threadId, + runId: first.runId, + nodeId: first.rootNodeId, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status, + title: null, + startedAt: now, + completedAt: status === "running" ? null : now, + updatedAt: now, + type: "compaction", + driver, + }, + }) satisfies ProviderAdapter.ProviderAdapterV2Event; + const compacting = yield* watch( + (event) => event.type === "turn-item.updated" && event.payload.type === "compaction", + ); + yield* Queue.offer(events, compaction("running")); + yield* Fiber.join(compacting); + const dbg = yield* orchestrator.getThreadProjection(threadId); + + const messageId = MessageId.make("message:steer"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("steer"), + threadId, + messageId, + text: "follow up", + attachments: [], + dispatchMode: { type: "steer_active", targetRunId: first.runId }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + const held = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual(steered, []); + assert.deepEqual(interrupted, []); + const queued = held.runs.find((run) => run.userMessageId === messageId); + assert.equal(queued?.status, "queued"); + assert.equal(queued?.steerWhenReadyRunId, first.runId); + + const promoted = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === queued?.id && + event.payload.status !== "queued", + ); + yield* Queue.offer(events, compaction("completed")); + yield* Fiber.join(promoted); + yield* worker.drain(); + yield* Deferred.await(steeredOnce); + const after = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual(steered, ["follow up"]); + assert.deepEqual(interrupted, []); + assert.equal(after.turnItems.find((item) => item.type === "compaction")?.status, "completed"); + assert.equal(after.runs.find((run) => run.id === first.runId)?.status, "running"); + }).pipe(Effect.provide(layer)); + }), +); + +it.effect("starts a when_idle message only after the active run and the next turn end", () => + Effect.gen(function* () { + const { started, layer, startFirstTurn, finishTurn, send } = + yield* deliveryHarness("when-idle"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = yield* startFirstTurn; + const idle = yield* send(threadId, "idle", { class: "when_idle" }); + const next = yield* send(threadId, "next", { class: "next_turn" }); + const runOf = (messageId: MessageId) => + orchestrator + .getThreadProjection(threadId) + .pipe(Effect.map(({ runs }) => runs.find((run) => run.userMessageId === messageId))); + assert.equal(started.length, 1); + assert.equal((yield* runOf(idle))?.status, "queued"); + + yield* finishTurn(started[0]!); + assert.equal(started.length, 2); + assert.equal(started[1]?.message.text, "next"); + assert.equal((yield* runOf(next))?.status, "running"); + assert.equal((yield* runOf(idle))?.status, "queued"); + + yield* finishTurn(started[1]!); + assert.equal(started.length, 3); + assert.equal(started[2]?.message.text, "idle"); + }).pipe(Effect.provide(layer)); + }), +); + +it.effect("coalesces alerts into one digest that rides along with the next turn", () => + Effect.gen(function* () { + const { started, layer, startFirstTurn, finishTurn, send } = + yield* deliveryHarness("coalesced-digest"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = yield* startFirstTurn; + const alert = { class: "digest", coalesceKey: "lane:a", category: "alert" } as const; + const older = yield* send(threadId, "lane a stuck", alert); + const newer = yield* send(threadId, "lane a stuck again", alert); + const fyi = yield* send(threadId, "lane b merged", { class: "digest", category: "fyi" }); + + yield* finishTurn(started[0]!); + assert.equal(started.length, 1, "a digest never starts a turn"); + yield* send(threadId, "what now?"); + + const { runs } = yield* orchestrator.getThreadProjection(threadId); + const statusOf = (messageId: MessageId) => + runs.find((run) => run.userMessageId === messageId)?.status; + assert.equal(started.length, 2); + assert.deepEqual( + started[1]?.message.text, + formatDigestForProvider("what now?", [ + { messageId: newer, category: "alert", text: "lane a stuck again" }, + { messageId: fyi, category: "fyi", text: "lane b merged" }, + ]), + ); + assert.deepEqual( + [statusOf(older), statusOf(newer), statusOf(fyi)], + ["cancelled", "cancelled", "cancelled"], + ); + }).pipe(Effect.provide(layer)); + }), +); + +it.effect("drops and logs a digest that expires before a turn picks it up", () => + Effect.gen(function* () { + const { started, layer, startFirstTurn, finishTurn, send } = + yield* deliveryHarness("expired-digest"); + const logs: Array = []; + const logger = Logger.make(({ message }) => logs.push(message)); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = yield* startFirstTurn; + const expiresAt = DateTime.add(yield* DateTime.now, { minutes: 1 }); + const expiring = yield* send(threadId, "stale alert", { + class: "digest", + coalesceKey: "lane:a", + expiresAt, + category: "alert", + }); + yield* finishTurn(started[0]!); + yield* TestClock.adjust("2 minutes"); + yield* send(threadId, "what now?"); + + const { runs } = yield* orchestrator.getThreadProjection(threadId); + assert.equal(started[1]?.message.text, "what now?"); + assert.equal(runs.find((run) => run.userMessageId === expiring)?.status, "cancelled"); + assert.isTrue( + logs.some((message) => JSON.stringify(message).includes("Dropped an expired message")), + ); + }).pipe( + Effect.provide(Layer.mergeAll(layer, Logger.layer([logger], { mergeWithExisting: true }))), + ); + }), +); + // regression: #33454 a steer that restarts a completion wake run left its cohort claimed forever const restartCompletionWake = Effect.fn("restartCompletionWake")(function* ( harness: Pick< diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 0b57f9e18a988..53e69c1c29e0f 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -28,6 +28,7 @@ import { type OrchestrationV2ConversationMessage, type OrchestrationV2CreationSource, type OrchestrationV2LinkedSender, + type OrchestrationV2MessageDelivery, type OrchestrationV2MessageSentBy, type OrchestrationV2Run, type OrchestrationV2ThreadShellSnapshot, @@ -194,6 +195,8 @@ export interface ThreadManagementSendInput { readonly mode: ThreadManagementSendMode; /** Start a new provider session; the orchestrator queues it instead of steering. */ readonly freshContext?: boolean; + /** Overrides `mode`: the orchestrator places the message by its class. */ + readonly delivery?: OrchestrationV2MessageDelivery; readonly createdBy: OrchestrationV2Actor; readonly creationSource: OrchestrationV2CreationSource; } @@ -1034,7 +1037,9 @@ const make = Effect.gen(function* () { OrchestrationV2Command, { readonly type: "message.dispatch" } >["dispatchMode"]; - if (mode === "steer" || mode === "restart") { + if (input.delivery !== undefined) { + dispatchMode = { type: "queue_after_active" }; + } else if (mode === "steer" || mode === "restart") { if (steerableRun === undefined) { return yield* new ThreadManagementNoSteerableRunError({ threadId: input.threadId, @@ -1086,7 +1091,8 @@ const make = Effect.gen(function* () { ...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }), ...(input.freshContext ? { freshContext: true } : {}), dispatchMode, - ...(mode === "auto" ? { deliveryIntent: "auto" } : {}), + ...(mode === "auto" && input.delivery === undefined ? { deliveryIntent: "auto" } : {}), + ...(input.delivery === undefined ? {} : { delivery: input.delivery }), createdBy: input.createdBy, creationSource: input.creationSource, }); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index c216e6444cb7e..3ffdd93d4ff88 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -749,6 +749,44 @@ export type OrchestrationV2ThreadLaunchWorkspaceStrategy = export const ORCHESTRATION_V2_WORKSPACE_PREPARATION_FAILURE_CODE = "workspace_preparation_failed"; +/** + * How a message reaches a thread. `digest` never starts a turn: it rides along with the next + * turn another message starts. `when_idle` starts only after every other queued message. + */ +export const OrchestrationV2DeliveryClass = Schema.Literals([ + "interrupt", + "steer_at_tool_boundary", + "next_turn", + "when_idle", + "digest", +]); +export type OrchestrationV2DeliveryClass = typeof OrchestrationV2DeliveryClass.Type; + +export const OrchestrationV2DigestCategory = Schema.Literals([ + "alert", + "lane_result", + "fyi", + "decision_needed", +]); +export type OrchestrationV2DigestCategory = typeof OrchestrationV2DigestCategory.Type; + +export const OrchestrationV2MessageDelivery = Schema.Struct({ + class: OrchestrationV2DeliveryClass, + /** An undelivered message with the same key is replaced by the newer one. */ + coalesceKey: Schema.optional(TrimmedNonEmptyString), + /** Undelivered at this time, the message is dropped. */ + expiresAt: Schema.optional(Schema.DateTimeUtc), + category: Schema.optional(OrchestrationV2DigestCategory), +}); +export type OrchestrationV2MessageDelivery = typeof OrchestrationV2MessageDelivery.Type; + +export const OrchestrationV2DigestEntry = Schema.Struct({ + messageId: MessageId, + category: OrchestrationV2DigestCategory, + text: Schema.String, +}); +export type OrchestrationV2DigestEntry = typeof OrchestrationV2DigestEntry.Type; + export const OrchestrationV2Run = Schema.Struct({ id: RunId, threadId: ThreadId, @@ -786,6 +824,9 @@ export const OrchestrationV2Run = Schema.Struct({ workspacePreparation: Schema.optional(OrchestrationV2ThreadLaunchWorkspaceStrategy), /** Starts a new provider conversation; earlier runs stay visible but never reach the provider. */ freshContext: Schema.optional(Schema.Boolean), + delivery: Schema.optional(OrchestrationV2MessageDelivery), + /** Digest messages folded into this run's turn, sent to the provider ahead of its message. */ + digest: Schema.optional(Schema.Array(OrchestrationV2DigestEntry)), }); export type OrchestrationV2Run = typeof OrchestrationV2Run.Type; @@ -2555,6 +2596,12 @@ export const OrchestrationV2RunJson = OrchestrationV2Run.mapFields((fields) => ( startedAt: Schema.NullOr(Schema.DateTimeUtcFromString), completedAt: Schema.NullOr(Schema.DateTimeUtcFromString), workStartedAt: Schema.optional(Schema.DateTimeUtcFromString), + delivery: Schema.optional( + OrchestrationV2MessageDelivery.mapFields((delivery) => ({ + ...delivery, + expiresAt: Schema.optional(Schema.DateTimeUtcFromString), + })), + ), })); export type OrchestrationV2RunJson = typeof OrchestrationV2RunJson.Type; @@ -3483,6 +3530,7 @@ export const OrchestrationV2Command = Schema.Union([ steerFollowUpAfterRunOrdinal: Schema.optional(NonNegativeInt), usageLimitRecoveryRequestId: Schema.optional(CommandId), deliveryIntent: Schema.optional(Schema.Literals(["auto", "steer", "restart"])), + delivery: Schema.optional(OrchestrationV2MessageDelivery), delegatedCompletion: Schema.optional( Schema.Struct({ parentRunId: RunId, diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 2b91f933341ca..1d802c2f21b34 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -41,6 +41,8 @@ import { OrchestrationV2Actor, OrchestrationV2CreationSource, OrchestrationV2DelegatedFrom, + OrchestrationV2DeliveryClass, + OrchestrationV2DigestCategory, OrchestrationV2RunStatus, OrchestrationV2TurnItemStatus, OrchestrationV2WorkerAlert, @@ -590,6 +592,12 @@ export const OrchestratorMcpThreadSendInput = Schema.Struct({ }), ), clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), + deliveryClass: Schema.optional(OrchestrationV2DeliveryClass), + coalesceKey: Schema.optional(TrimmedNonEmptyString.check(Schema.isMaxLength(256))), + expiresAt: Schema.optional( + IsoDateTime.annotate({ description: "ISO time; an undelivered message is dropped then." }), + ), + digestCategory: Schema.optional(OrchestrationV2DigestCategory), }); export type OrchestratorMcpThreadSendInput = typeof OrchestratorMcpThreadSendInput.Type; @@ -601,6 +609,7 @@ export const OrchestratorMcpThreadSendResult = Schema.Struct({ taskId: Schema.optional(NodeId), status: OrchestrationV2RunStatus, delivery: Schema.Literals(["started", "queued", "steered", "restarted"]), + deliveryClass: Schema.optional(OrchestrationV2DeliveryClass), }); export type OrchestratorMcpThreadSendResult = typeof OrchestratorMcpThreadSendResult.Type; diff --git a/packages/shared/package.json b/packages/shared/package.json index 601e520bd24af..b2c70f36d1865 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -7,6 +7,10 @@ "types": "./src/seenMarker.ts", "import": "./src/seenMarker.ts" }, + "./messageDigest": { + "types": "./src/messageDigest.ts", + "import": "./src/messageDigest.ts" + }, "./png": { "types": "./src/png.ts", "import": "./src/png.ts" diff --git a/packages/shared/src/messageDigest.ts b/packages/shared/src/messageDigest.ts new file mode 100644 index 0000000000000..93073cc6ac131 --- /dev/null +++ b/packages/shared/src/messageDigest.ts @@ -0,0 +1,31 @@ +import type { OrchestrationV2DigestCategory, OrchestrationV2DigestEntry } from "@t3tools/contracts"; + +const SECTIONS: ReadonlyArray = [ + ["decision_needed", "Decisions needed"], + ["alert", "Alerts"], + ["lane_result", "Lane results"], + ["fyi", "FYI"], +]; + +/** Prepends the digest messages a turn carries, grouped by category. */ +export function formatDigestForProvider( + text: string, + digest: ReadonlyArray | undefined, +) { + if (digest === undefined || digest.length === 0 || text.trim().startsWith("/")) return text; + const sections = SECTIONS.flatMap(([category, title]) => { + const entries = digest.filter((entry) => entry.category === category); + return entries.length === 0 + ? [] + : [`## ${title}\n${entries.map((entry) => `- ${entry.text.trim()}`).join("\n")}`]; + }); + const noun = digest.length === 1 ? "message" : "messages"; + return [ + ``, + `${digest.length} ${noun} arrived without starting a turn:`, + ...sections, + "", + "", + text, + ].join("\n"); +} diff --git a/packages/shared/src/orchestrationV2ThreadError.ts b/packages/shared/src/orchestrationV2ThreadError.ts index 141d73f402540..4641da98b1d4c 100644 --- a/packages/shared/src/orchestrationV2ThreadError.ts +++ b/packages/shared/src/orchestrationV2ThreadError.ts @@ -116,12 +116,17 @@ export function usageLimitRunPresentedAsLatest( return blocked !== null && runs.some((run) => run.ordinal > blocked.ordinal) ? blocked : null; } +/** A digest waiting for the next turn; it never starts one, so the thread is not busy with it. */ +export function isWaitingDigestRun(run: Pick): boolean { + return run.status === "queued" && run.delivery?.class === "digest"; +} + export function latestUnheldRun( runs: ReadonlyArray, ): OrchestrationV2Run | null { let latest: OrchestrationV2Run | null = null; for (const run of runs) { - if (run.status === "queued" && run.queueHeld === true) continue; + if (run.status === "queued" && (run.queueHeld === true || isWaitingDigestRun(run))) continue; if (outlivedBeforeStart(run, runs)) continue; if (latest === null || run.ordinal > latest.ordinal) latest = run; } From 60ee8d94e1f5a22adcc1d1375df14c88e87b7d2d Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:46:31 +0000 Subject: [PATCH 2/5] fix(server): a digest never starts a turn and is never dropped silently Review of the delivery classes found six holes. A slash-prefixed message now carries its digest instead of discarding it. A digest sent to a thread that has never run waits instead of starting a provider turn. An agent send that names no class rides in the digest. A steer waiting for the next tool result releases when that tool is interrupted or cancelled, not only when it completes or fails. An expired digest is dropped even when it is the only queued message. A turn that starts through a fork, merge-back or provider switch carries the digest too. Co-Authored-By: Claude Fable 5.2 --- apps/server/src/mcp/OrchestratorMcpService.ts | 4 +- .../src/orchestration-v2/Orchestrator.ts | 43 ++++++++++------ .../SteeringCompletion.integration.test.ts | 51 +++++++++++++++++++ packages/shared/src/messageDigest.ts | 2 +- 4 files changed, 83 insertions(+), 17 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 79c8e389ef6ab..02820a3e6aead 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -3551,8 +3551,10 @@ const make = Effect.gen(function* () { yield* assertTargetWithinLimits(limits, target.thread); const mode = input.mode ?? "auto"; + // An agent that names no class rides in the next digest instead of steering the thread. const deliveryClass = - input.deliveryClass ?? (input.digestCategory === undefined ? undefined : "digest"); + input.deliveryClass ?? + (input.digestCategory !== undefined || parent !== undefined ? "digest" : undefined); if ( deliveryClass === undefined && (input.coalesceKey !== undefined || input.expiresAt !== undefined) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index f4ce1460dc55f..cb5029f374799 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -2537,7 +2537,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly userStarted?: boolean; }, ) { - if (!(yield* projectionStore.canStartQueuedRun(threadId))) return; let projection = yield* readCommandProjection(threadId); if ( projection.thread.archivedAt !== null || @@ -2550,6 +2549,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (yield* cancelStaleQuestionWakes(projection)) { projection = yield* readCommandProjection(threadId); } + // A digest never starts a turn, so an expired one is dropped even when it is the only queue. const checkedAt = yield* DateTime.now; const expired = projection.runs.filter( (run) => run.status === "queued" && deliveryExpired(run, checkedAt), @@ -2561,6 +2561,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio for (const run of expired) yield* logExpiredDelivery(run); projection = yield* readCommandProjection(threadId); } + if (!(yield* projectionStore.canStartQueuedRun(threadId))) return; let storedQueuedRun = options?.queuedRunId === undefined ? nextQueuedRun(projection) @@ -7678,15 +7679,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection.thread.worktreePath === null ? latestRun : undefined; - // A digest waits on an idle thread too; only a thread that never ran starts one. - const queueBehindRun = - activeRun ?? worktreeMissingRun ?? (deliveryClass === "digest" ? (latestRun ?? undefined) : undefined); + const queueBehindRun = activeRun ?? worktreeMissingRun; const pendingMergeBackTransfers = pendingMergeBackTransfersForThread(projection); const startsOrQueuesRun = dispatchMode.type === "defer_start" || dispatchMode.type === "start_immediately" || dispatchMode.type === "queue_after_active"; - const shouldQueue = queueBehindRun !== undefined && startsOrQueuesRun; + // A digest never starts a turn, so on a thread with nothing running it waits for one. + const holdDigest = deliveryClass === "digest" && queueBehindRun === undefined && startsOrQueuesRun; + const shouldQueue = (queueBehindRun !== undefined && startsOrQueuesRun) || holdDigest; // An explicit pick queued ahead reaches the thread only when its run starts. A goal or // restart command that stops the active run goes first, so nothing queued applies before it. // Buffered provider wakes drain output their own provider already produced. @@ -7748,14 +7749,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } const queueProviderThread = activeProviderThread ?? - projection.providerThreads.find( - (candidate) => candidate.id === queueBehindRun.providerThreadId, - ); - if (queueProviderThread === undefined) { + (queueBehindRun === undefined + ? undefined + : projection.providerThreads.find( + (candidate) => candidate.id === queueBehindRun.providerThreadId, + )); + if (queueProviderThread === undefined && !holdDigest) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, commandType: command.type, - cause: `Active run ${queueBehindRun.id} has no provider thread for queued dispatch.`, + cause: `Active run ${queueBehindRun?.id} has no provider thread for queued dispatch.`, }); } const now = yield* DateTime.now; @@ -7763,7 +7766,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const runId = idAllocator.derive.run({ threadId: command.threadId, ordinal }); const targetProviderThread = freshContext ? undefined - : modelSelection.instanceId === queueProviderThread.providerInstanceId + : queueProviderThread !== undefined && + modelSelection.instanceId === queueProviderThread.providerInstanceId ? queueProviderThread : rootProviderThreadsForProvider(projection, modelSelection.instanceId)[0]; const queuedAdapter = yield* providerAdapters @@ -7787,7 +7791,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio capabilities: queuedCapabilities, }), ); - const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? { + const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? + queueProviderThread ?? { id: idAllocator.derive.providerThread({ driver: queuedAdapter.driver, nativeThreadId: `pending:${runId}`, @@ -7810,7 +7815,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const attemptId = idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }); const rootNodeId = idAllocator.derive.rootNode({ runId }); const checkpointScope = - queueBehindRun.status === "preparing" || worktreeMissingRun !== undefined + holdDigest || queueBehindRun?.status === "preparing" || worktreeMissingRun !== undefined ? null : yield* runtimePolicy.resolve({ thread: projection.thread, modelSelection }).pipe( Effect.flatMap((resolvedRuntimePolicy) => @@ -9095,6 +9100,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }), ), ); + const forkFold = carriesDigest(command, command) ? digestFold(projection, now) : undefined; const checkpointScope = yield* checkpointService .prepareRootRunScope({ threadId: command.threadId, @@ -9152,6 +9158,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }), ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }), ...wakeWorkStartedAt(projection, command), + ...(command.delivery === undefined ? {} : { delivery: command.delivery }), + ...(forkFold === undefined || forkFold.digest.length === 0 ? {} : { digest: forkFold.digest }), }; const attempt: OrchestrationV2RunAttempt = { id: attemptId, @@ -9500,6 +9508,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, }); } + for (const spent of forkFold === undefined ? [] : [...forkFold.folded, ...forkFold.expired]) { + for (const event of queuedRunCancellationEvents(projection, spent, now)) { + yield* emitEvent(event); + } + } + for (const expired of forkFold?.expired ?? []) yield* logExpiredDelivery(expired); yield* emitEvent({ type: "run.created", threadId: command.threadId, @@ -16125,8 +16139,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (stored.event.payload.type === "compaction" ? TERMINAL_TURN_ITEM_STATUSES.has(stored.event.payload.status) : TOOL_TURN_ITEM_TYPES.has(stored.event.payload.type) && - (stored.event.payload.status === "completed" || - stored.event.payload.status === "failed"))), + TERMINAL_TURN_ITEM_STATUSES.has(stored.event.payload.status))), ), Stream.runForEach( Effect.fn("Orchestrator.deliverDeferredSteers")( diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 180573316a762..9b77d368609ed 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -1981,6 +1981,57 @@ it.effect("drops and logs a digest that expires before a turn picks it up", () = }), ); +it.effect("carries a digest along with a slash-prefixed message", () => + Effect.gen(function* () { + const { started, layer, startFirstTurn, finishTurn, send } = + yield* deliveryHarness("slash-carries-digest"); + yield* Effect.gen(function* () { + const threadId = yield* startFirstTurn; + const alert = yield* send(threadId, "lane a stuck", { + class: "digest", + category: "alert", + }); + yield* finishTurn(started[0]!); + yield* send(threadId, "/review the PR"); + assert.equal(started.length, 2); + assert.deepEqual( + started[1]?.message.text, + formatDigestForProvider("/review the PR", [ + { messageId: alert, category: "alert", text: "lane a stuck" }, + ]), + ); + }).pipe(Effect.provide(layer)); + }), +); + +it.effect("queues a digest on a thread that has never run", () => + Effect.gen(function* () { + const { started, layer, send } = yield* deliveryHarness("digest-before-first-run"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = ThreadId.make("thread:digest-before-first-run"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create:digest-before-first-run"), + threadId, + projectId: ProjectId.make("project:digest-before-first-run"), + title: "delivery", + modelSelection: runSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + const alert = yield* send(threadId, "lane a stuck", { class: "digest", category: "alert" }); + assert.deepEqual(started, []); + const { runs } = yield* orchestrator.getThreadProjection(threadId); + assert.equal(runs.find((run) => run.userMessageId === alert)?.status, "queued"); + }).pipe(Effect.provide(layer)); + }), +); + // regression: #33454 a steer that restarts a completion wake run left its cohort claimed forever const restartCompletionWake = Effect.fn("restartCompletionWake")(function* ( harness: Pick< diff --git a/packages/shared/src/messageDigest.ts b/packages/shared/src/messageDigest.ts index 93073cc6ac131..d96ae77d8ffa3 100644 --- a/packages/shared/src/messageDigest.ts +++ b/packages/shared/src/messageDigest.ts @@ -12,7 +12,7 @@ export function formatDigestForProvider( text: string, digest: ReadonlyArray | undefined, ) { - if (digest === undefined || digest.length === 0 || text.trim().startsWith("/")) return text; + if (digest === undefined || digest.length === 0) return text; const sections = SECTIONS.flatMap(([category, title]) => { const entries = digest.filter((entry) => entry.category === category); return entries.length === 0 From 1fba6c34e7ee38904aea93ea5c62c9c821837c4d Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:48:31 +0000 Subject: [PATCH 3/5] fix(server): a message to the sender's own thread keeps its mode The agent digest default keyed off the caller having any thread, so an agent sending its next instruction to its own thread had it held as a digest that never started a turn. The default now applies only when the sender and the target are different threads. Co-Authored-By: Claude Fable 5.2 --- apps/server/src/mcp/OrchestratorMcpService.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 02820a3e6aead..4c635cd716b41 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -3551,10 +3551,14 @@ const make = Effect.gen(function* () { yield* assertTargetWithinLimits(limits, target.thread); const mode = input.mode ?? "auto"; - // An agent that names no class rides in the next digest instead of steering the thread. + // A message to another thread rides in that thread's next digest unless the sender + // names a class. A message to the sender's own thread keeps the mode it asked for. const deliveryClass = input.deliveryClass ?? - (input.digestCategory !== undefined || parent !== undefined ? "digest" : undefined); + (input.digestCategory !== undefined || + (parent !== undefined && parent.thread.id !== input.threadId) + ? "digest" + : undefined); if ( deliveryClass === undefined && (input.coalesceKey !== undefined || input.expiresAt !== undefined) From b424eaed5771c631c4ff31fac973611b3785f689 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Fri, 20 Nov 2026 20:59:00 +0000 Subject: [PATCH 4/5] fix(server): hold an alert behind a queued digest and retry with its expiry A flush read the watcher once and delivered every due alert together, and a queued digest is hidden from the shell, so the next alert never saw the turn the first one queued and queued behind it instead of waiting. Deliveries now go one at a time, and a watcher whose only turn is a waiting digest counts as busy. A retry recomputed the digest expiry from the clock, so it disagreed with the failed send and was not a replay. The expiry is fixed when the alert is queued. Co-Authored-By: Claude Code --- .../src/orchestration-v2/ModelAlertReactor.ts | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index d15ed5a86fb04..ca74703d4f38a 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -125,6 +125,11 @@ const Delivery = Schema.Struct({ attempts: Schema.Int, nextAttemptAt: Schema.Number, createdAt: Schema.Number, + /** + * When the digest expires, fixed when the alert is queued. A retry must carry the same time, + * or the first send and the retry disagree and the retry is not a replay. + */ + digestExpiresAt: Schema.optional(Schema.Number), }); type Delivery = typeof Delivery.Type; @@ -307,6 +312,7 @@ function enqueueAlert( attempts: 0, nextAttemptAt: nowMs, createdAt: nowMs, + digestExpiresAt: nowMs + ALERT_DIGEST_TTL_MS, })); return { outbox: [...pruneOutbox(outbox, nowMs), ...added].slice(-MAX_OUTBOX), @@ -578,6 +584,8 @@ const BUSY_WATCHER_STATUSES = new Set([ /** Matches the flush interval, so a held alert is looked at again on the next flush. */ export const BUSY_WATCHER_RECHECK_MS = 30_000; export const BUSY_WATCHER_MAX_HOLD_MS = 10 * 60 * 1000; +/** How long an undelivered alert stays in the watcher's digest before it is dropped. */ +const ALERT_DIGEST_TTL_MS = 24 * 60 * 60 * 1000; /** * While the watcher is mid-turn, a sent alert only queues behind that turn and is judged stale or @@ -683,6 +691,7 @@ export function recordUsageLimit( attempts: 0, nextAttemptAt: notBefore, createdAt: nowMs, + digestExpiresAt: nowMs + ALERT_DIGEST_TTL_MS, }, ].slice(-MAX_OUTBOX); } @@ -737,7 +746,6 @@ export const watcherAlertMessage = (alert: WatcherAlert) => { } }; -const ALERT_DIGEST_TTL_MS = 24 * 60 * 60 * 1000; /** Alerts ride in the watcher's next digest; a newer alert on the same subject replaces an older one. */ function alertDelivery(alert: WatcherAlert): { @@ -777,13 +785,16 @@ export const deliverAlertToWatcher = Effect.fnUntraced(function* ( alert: WatcherAlert, watcher: ModelAlertWatcher, here: EnvironmentId, + expiresAtMs?: number, ) { const message = watcherAlertMessage(alert); // Run ids can be long; a retry key must stay under the 256 characters the peer accepts. const identity = `${alert.id}\n${watcher.environmentId}\n${watcher.threadId}`; const key = `${alert.type === "model-degraded" ? "model-alert" : alert.type}:${yield* deps.sha256Hex(identity)}`; const { coalesceKey, category } = alertDelivery(alert); - const expiresAt = DateTime.add(yield* DateTime.now, { milliseconds: ALERT_DIGEST_TTL_MS }); + const expiresAt = DateTime.makeUnsafe( + expiresAtMs ?? DateTime.toEpochMillis(yield* DateTime.now) + ALERT_DIGEST_TTL_MS, + ); if (watcher.environmentId === here) { yield* deps.dispatch({ type: "message.dispatch", @@ -891,12 +902,18 @@ export const make = Effect.gen(function* () { crypto .digest(Digest.sha256, new TextEncoder().encode(identity)) .pipe(Effect.map(Hex.encode), Effect.orDie); - const deliverToWatcher = (alert: WatcherAlert, watcher: ModelAlertWatcher, here: EnvironmentId) => + const deliverToWatcher = ( + alert: WatcherAlert, + watcher: ModelAlertWatcher, + here: EnvironmentId, + expiresAtMs: number, + ) => deliverAlertToWatcher( { dispatch: engine.dispatch, call: peers.call, sha256Hex }, alert, watcher, here, + expiresAtMs, ); type WatcherView = { @@ -1033,8 +1050,15 @@ export const make = Effect.gen(function* () { if (watcher.environmentId === here) { const shell = yield* projections.getThreadShell(watcher.threadId); if (shell === null) return null; + // A queued digest never starts a turn, so the shell hides it and reads idle. It is still a + // turn waiting to be judged, and another alert queueing behind it would be judged stale. + const digestWaiting = + shell.status === "idle" && + (yield* projections.getThreadRecords(watcher.threadId, ["runs"])).runs.some( + (run) => run.status === "queued" && run.delivery?.class === "digest", + ); return { - status: shell.status as string, + status: (digestWaiting ? "queued" : shell.status) as string, providerInstanceId: shell.providerInstanceId as string | null, usageLimited: shell.lastErrorClass === "usage_limit", resetAt: shell.usageLimitResetAt ?? null, @@ -1128,7 +1152,12 @@ export const make = Effect.gen(function* () { holdUntil(view, alert, nowMs) ?? busyWatcherHold(view?.status ?? null, entry.createdAt, nowMs); if (until !== null) return { _tag: "held", until } satisfies DeliveryResult; - yield* deliverToWatcher(alert, target, here); + yield* deliverToWatcher( + alert, + target, + here, + entry.digestExpiresAt ?? nowMs + ALERT_DIGEST_TTL_MS, + ); return { _tag: "sent" } satisfies DeliveryResult; }); @@ -1144,6 +1173,8 @@ export const make = Effect.gen(function* () { return yield* readWatcher(watcher, yield* environment.getEnvironmentId); }).pipe(Effect.catchCause(() => Effect.succeed(null))), ); + // One at a time: a delivery queues a turn on the watcher, and the next alert in this + // flush has to see that turn to hold instead of queueing behind it. yield* Effect.forEach( due, (entry) => @@ -1171,7 +1202,7 @@ export const make = Effect.gen(function* () { : afterDeliveryAttempt(current, entry, true, attemptedAt), ]); }), - { concurrency: 4, discard: true }, + { concurrency: 1, discard: true }, ); }); From bb64beed29e5a09e79205741fac0e9943e2ee7d4 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Sat, 21 Nov 2026 02:05:30 +0000 Subject: [PATCH 5/5] fix(web): keep a timed-out draft test from breaking the next one The ARC voice job's first fake-timer test hit the 15s timeout, then the next test failed: that run was still going, so its cleanup uninstalled the clock or its late upload showed up in the next assertion. Import the modules on real timers, leave setImmediate and performance real so the clock can yield, and ignore a superseded test's cleanup. Model: grok-4.7 Task: fix Agent: Builder Lane: grok Harness: t3 --- .../src/components/chat/chatViewSend.test.ts | 45 +++++++++++++++---- .../chat/draftAttachmentUploads.test.ts | 44 ++++++++++++++---- .../components/chat/draftPlacement.test.ts | 43 ++++++++++++++---- .../chat/draftUploadReconcile.test.ts | 42 ++++++++++++++--- 4 files changed, 142 insertions(+), 32 deletions(-) diff --git a/apps/web/src/components/chat/chatViewSend.test.ts b/apps/web/src/components/chat/chatViewSend.test.ts index 83de6940aa184..61560a259b09f 100644 --- a/apps/web/src/components/chat/chatViewSend.test.ts +++ b/apps/web/src/components/chat/chatViewSend.test.ts @@ -46,14 +46,39 @@ const modelSelection = { model: "claude-sonnet-4-6", }; +// Only one test may own the fake clock. A timed-out test keeps running, and its +// finally used to uninstall the next test's timers and module graph. +let timerScope = 0; + +function useScopedFakeTimers() { + // setImmediate and performance stay real. The loader and fake-timer clock + // both yield through them; faking them before import leaves the import pending + // until the 15s test timeout. + vi.useFakeTimers({ + toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], + }); +} + +async function importFresh(load: () => Promise): Promise { + const scope = timerScope; + const restore = vi.isFakeTimers(); + if (restore) vi.useRealTimers(); + vi.resetModules(); + try { + return await load(); + } finally { + if (restore && scope === timerScope) useScopedFakeTimers(); + } +} + async function openTab() { - return { + return importFresh(async () => ({ store: await import("../../composerDraftStore"), send: await import("./chatViewSend"), logic: await import("./composerSend.logic"), view: await import("../ChatView.logic"), uploads: await import("../../lib/attachmentUploadQueue"), - }; + })); } type Tab = Awaited>; @@ -85,19 +110,21 @@ function faultyStorage() { } async function withReloadableStorage(run: () => Promise) { + const scope = ++timerScope; vi.stubGlobal("localStorage", faultyStorage()); vi.stubGlobal("window", new EventTarget()); vi.stubGlobal("FileReader", BlobFileReader); - vi.useFakeTimers(); - vi.resetModules(); + useScopedFakeTimers(); try { await run(); } finally { - gate = null; - resetStorageFaults(); - vi.useRealTimers(); - vi.unstubAllGlobals(); - vi.resetModules(); + if (scope === timerScope) { + gate = null; + resetStorageFaults(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } } } diff --git a/apps/web/src/components/chat/draftAttachmentUploads.test.ts b/apps/web/src/components/chat/draftAttachmentUploads.test.ts index bb1ad6cab1765..1d6b882908ebc 100644 --- a/apps/web/src/components/chat/draftAttachmentUploads.test.ts +++ b/apps/web/src/components/chat/draftAttachmentUploads.test.ts @@ -152,16 +152,43 @@ const request = { } as const; const storageFaults = { drops: false }; +// Only one test may own the fake clock. A timed-out test keeps running, and its +// finally used to uninstall the next test's timers (and its late upload landed +// in the next test's hoisted doubles). +let timerScope = 0; + +function useScopedFakeTimers() { + // setImmediate and performance stay real. The loader and fake-timer clock + // both yield through them; faking them before import leaves the import pending + // until the 15s test timeout. + vi.useFakeTimers({ + toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], + }); +} + +async function importFresh(load: () => Promise): Promise { + const scope = timerScope; + const restore = vi.isFakeTimers(); + if (restore) vi.useRealTimers(); + vi.resetModules(); + try { + return await load(); + } finally { + if (restore && scope === timerScope) useScopedFakeTimers(); + } +} + async function openTab() { - return { + return importFresh(async () => ({ store: await import("../../composerDraftStore"), placement: await import("./draftPlacement"), queue: await import("../../lib/attachmentUploadQueue"), - }; + })); } type Tab = Awaited>; async function withStorage(run: () => Promise) { + const scope = ++timerScope; const memory = createMemoryStorage(); const storage = { getItem: (name: string) => memory.getItem(name), @@ -173,8 +200,7 @@ async function withStorage(run: () => Promise) { vi.stubGlobal("localStorage", storage); // The image readback reads `window.localStorage`, as in a browser. vi.stubGlobal("window", Object.assign(new EventTarget(), { localStorage: storage })); - vi.useFakeTimers(); - vi.resetModules(); + useScopedFakeTimers(); uploads.to.length = 0; uploads.sequence.length = 0; originServer.bound.length = 0; @@ -182,10 +208,12 @@ async function withStorage(run: () => Promise) { try { await run(); } finally { - storageFaults.drops = false; - vi.useRealTimers(); - vi.unstubAllGlobals(); - vi.resetModules(); + if (scope === timerScope) { + storageFaults.drops = false; + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } } } diff --git a/apps/web/src/components/chat/draftPlacement.test.ts b/apps/web/src/components/chat/draftPlacement.test.ts index ec2091426e378..7af310fd973ca 100644 --- a/apps/web/src/components/chat/draftPlacement.test.ts +++ b/apps/web/src/components/chat/draftPlacement.test.ts @@ -30,15 +30,41 @@ const KEY = "t3code:composer-drafts:v1"; const storageFaults = { drops: false }; +// Only one test may own the fake clock. A timed-out test keeps running, and its +// finally used to uninstall the next test's timers. +let timerScope = 0; + +function useScopedFakeTimers() { + // setImmediate and performance stay real. The loader and fake-timer clock + // both yield through them; faking them before import leaves the import pending + // until the 15s test timeout. + vi.useFakeTimers({ + toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], + }); +} + +async function importFresh(load: () => Promise): Promise { + const scope = timerScope; + const restore = vi.isFakeTimers(); + if (restore) vi.useRealTimers(); + vi.resetModules(); + try { + return await load(); + } finally { + if (restore && scope === timerScope) useScopedFakeTimers(); + } +} + async function openTab() { - return { + return importFresh(async () => ({ store: await import("../../composerDraftStore"), placement: await import("./draftPlacement"), - }; + })); } type Tab = Awaited>; async function withStorage(run: (memory: ReturnType) => Promise) { + const scope = ++timerScope; const memory = createMemoryStorage(); vi.stubGlobal("localStorage", { getItem: (name: string) => memory.getItem(name), @@ -48,15 +74,16 @@ async function withStorage(run: (memory: ReturnType) }, }); vi.stubGlobal("window", new EventTarget()); - vi.useFakeTimers(); - vi.resetModules(); + useScopedFakeTimers(); try { await run(memory); } finally { - storageFaults.drops = false; - vi.useRealTimers(); - vi.unstubAllGlobals(); - vi.resetModules(); + if (scope === timerScope) { + storageFaults.drops = false; + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } } } diff --git a/apps/web/src/components/chat/draftUploadReconcile.test.ts b/apps/web/src/components/chat/draftUploadReconcile.test.ts index 704342138540f..0d3f89fc484fb 100644 --- a/apps/web/src/components/chat/draftUploadReconcile.test.ts +++ b/apps/web/src/components/chat/draftUploadReconcile.test.ts @@ -117,18 +117,45 @@ const request = { multipleModels: false, } as const; +// Only one test may own the fake clock. A timed-out test keeps running, and its +// finally used to uninstall the next test's timers (and its late upload landed +// in the next test's hoisted doubles). +let timerScope = 0; + +function useScopedFakeTimers() { + // setImmediate and performance stay real. The loader and fake-timer clock + // both yield through them; faking them before import leaves the import pending + // until the 15s test timeout. + vi.useFakeTimers({ + toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], + }); +} + +async function importFresh(load: () => Promise): Promise { + const scope = timerScope; + const restore = vi.isFakeTimers(); + if (restore) vi.useRealTimers(); + vi.resetModules(); + try { + return await load(); + } finally { + if (restore && scope === timerScope) useScopedFakeTimers(); + } +} + async function openPage() { - return { + return importFresh(async () => ({ store: await import("../../composerDraftStore"), placement: await import("./draftPlacement"), queue: await import("../../lib/attachmentUploadQueue"), registry: (await import("../../rpc/atomRegistry")).appAtomRegistry, catalog: (await import("../../connection/catalog")).environmentCatalog, - }; + })); } type Page = Awaited>; async function withPage(run: (page: Page) => Promise) { + const scope = ++timerScope; const memory = createMemoryStorage(); const storage = { getItem: (name: string) => memory.getItem(name), @@ -137,8 +164,7 @@ async function withPage(run: (page: Page) => Promise) { }; vi.stubGlobal("localStorage", storage); vi.stubGlobal("window", Object.assign(new EventTarget(), { localStorage: storage })); - vi.useFakeTimers(); - vi.resetModules(); + useScopedFakeTimers(); net.origin.answer = "bound"; net.admissions.length = 0; net.cycles.length = 0; @@ -151,9 +177,11 @@ async function withPage(run: (page: Page) => Promise) { try { await run(await openPage()); } finally { - vi.useRealTimers(); - vi.unstubAllGlobals(); - vi.resetModules(); + if (scope === timerScope) { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.resetModules(); + } } }