diff --git a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts index 742b5969b64c..e5f87411a165 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.activity.test.ts @@ -178,6 +178,7 @@ it("readThread preserves caller access errors and prefers activity-run status ov makeRun({ id: cancelledRunId, ordinal: 2, status: "cancelled" }), ], visibleTurnItems: [], + turnItems: [], providerTurns: [], runtimeRequests: [], messages: [], diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index eb394388218d..169913bdac44 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1785,12 +1785,12 @@ const make = Effect.gen(function* () { : runStatus === "queued" || runStatus === "starting" ? runStatus : "running"; - const derivedResult = - task.result !== null - ? task.result - : taskResultRun !== undefined && isTerminalTaskStatus(status) - ? subagentResultForRun(childProjection, taskResultRun).text - : null; + const derived = + task.result === null && taskResultRun !== undefined && isTerminalTaskStatus(status) + ? subagentResultForRun(childProjection, taskResultRun) + : undefined; + const derivedResult = task.result ?? derived?.text ?? null; + const workerAlert = task.result === null ? derived?.workerAlert : task.workerAlert; const resultTransfers = childControls.contextTransfers.filter( (transfer) => transfer.type === "subagent_result" && @@ -1839,6 +1839,7 @@ const make = Effect.gen(function* () { providerInstanceId: modelSelection.instanceId, model: modelSelection.model, executedModel: runExecutedModel(childControls, executionRun), + ...(workerAlert === undefined ? {} : { workerAlert }), options: modelSelection.options ?? [], summary: derivedResult, resultContextTransferId, @@ -2034,6 +2035,7 @@ const make = Effect.gen(function* () { providerInstanceId: task.providerInstanceId, model: task.model, executedModel: task.executedModel ?? null, + ...(task.workerAlert === undefined ? {} : { workerAlert: task.workerAlert }), summary: task.result, resultContextTransferId: null, latestTerminalRunId: null, @@ -3451,10 +3453,34 @@ const make = Effect.gen(function* () { input.beforePosition === undefined && input.itemId === undefined, ); - const recentRuns = target.runs + const runs = target.runs .toSorted((left, right) => right.ordinal - left.ordinal) - .slice(0, input.runLimit ?? DEFAULT_THREAD_RUN_LIMIT) - .map((run) => threadRun(target, run)); + .slice(0, input.runLimit ?? DEFAULT_THREAD_RUN_LIMIT); + // Only an agent-created thread's latest run is judged, so a read loads one run's replies and notices. + const latestRun = runs[0]; + const latestAlert = + latestRun === undefined || + target.thread.createdBy !== "agent" || + !ThreadManagementService.isTerminalRunStatus(latestRun.status) + ? undefined + : subagentResultForRun( + { + ...target, + ...(yield* threadManagement + .getThreadRecords(input.threadId, ["messages", "turnItems"], { + messageRoles: ["assistant"], + messageRunIds: [latestRun.id], + turnItemRunId: latestRun.id, + turnItemTypes: ["assistant_message", "error", "system_notice"], + }) + .pipe(Effect.mapError(threadManagementFailure))), + }, + latestRun, + ).workerAlert; + const recentRuns = runs.map((run) => ({ + ...threadRun(target, run), + ...(run === latestRun && latestAlert !== undefined ? { workerAlert: latestAlert } : {}), + })); const { items, start } = timelineItemsWithinBudget( { thread, recentRuns, nextPosition: null, hasMore: false, earlierPosition: null }, timeline.items.map((row) => ({ diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index cf533de93e4d..c07ab96746eb 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -10249,7 +10249,8 @@ describe("ClaudeAdapterV2 background wake turns", () => { { name: "hides a passive summary", level: "notice", visible: false }, { name: "shows a blocking notice", level: "notice", preventContinuation: true, visible: true }, { name: "shows a warning", level: "warning", visible: true }, - ])("Claude Stop-hook notices: $name", ({ level, preventContinuation, visible }) => + { name: "marks a safeguard fallback", fallbackModel: "claude-opus-4-8", visible: true }, + ])("Claude system notices: $name", ({ level, preventContinuation, fallbackModel, visible }) => Effect.gen(function* () { const harness = yield* makeWakeHarness; yield* harness.runtime.startTurn( @@ -10267,9 +10268,17 @@ describe("ClaudeAdapterV2 background wake turns", () => { harness.sdkMessages, claudeSdkFrame({ type: "system", - subtype: "informational", + ...(fallbackModel === undefined + ? { subtype: "informational", level } + : { + subtype: "model_refusal_fallback", + trigger: "refusal", + direction: "retry", + original_model: "claude-opus-5-5", + fallback_model: fallbackModel, + request_id: null, + }), content, - level, ...(preventContinuation === undefined ? {} : { prevent_continuation: preventContinuation }), @@ -10285,10 +10294,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.deepEqual( harness.events.flatMap((event) => event.type === "turn_item.updated" && event.turnItem.type === "system_notice" - ? [event.turnItem.message] + ? [[event.turnItem.message, event.turnItem.fallbackModel]] : [], ), - visible ? [content] : [], + visible ? [[content, fallbackModel]] : [], ); assert.isFalse( harness.events.some( diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index fa8d3684f0f2..25d9d616cb93 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -10983,6 +10983,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( status: "completed", title: message.content, message: message.content, + ...(message.subtype === "model_refusal_fallback" + ? { fallbackModel: message.fallback_model } + : {}), startedAt: now, completedAt: now, updatedAt: now, diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index e6e29a1b34a7..b3771fc668da 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -4005,6 +4005,10 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( ], }); yield* orchestrator.recoverDelegatedTasks; + const limited = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (task) => task.id === child.taskId, + ); + assert.deepEqual(limited?.workerAlert?.reasons, ["usage_limit"]); const childShell = Effect.map(orchestrator.getShellSnapshot(), (snapshot) => snapshot.threads.find((thread) => thread.id === child.childThreadId), ); @@ -4740,6 +4744,132 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { }), ); + it.effect("tells the parent when a safeguard swapped its child's model and the child held", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:worker-alert-parent"); + const runId = RunId.make("run:worker-alert-parent"); + const rootNodeId = NodeId.make("node:worker-alert-parent-root"); + yield* seedParentWithTerminalTask({ + threadId, + projectId: ProjectId.make("project:worker-alert-parent"), + runId, + rootNodeId, + taskId: NodeId.make("node:worker-alert-parent-settled"), + deliveryState: "delivered", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId: ProjectId.make("project:worker-alert-parent"), + parentRunId: runId, + rootNodeId, + name: "worker-alert-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + const item = { + threadId: child.childThreadId, + runId: child.childRunId, + nodeId: rootNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + status: "completed" as const, + startedAt: now, + completedAt: now, + updatedAt: now, + }; + const scope = { + threadId: child.childThreadId, + runId: child.childRunId, + nodeId: rootNodeId, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + }; + const reply = "Holding until Kameron gives direct confirmation."; + yield* eventSink.write({ + commandId: CommandId.make("command:worker-alert-child:turn"), + events: [ + { + ...scope, + id: EventId.make("event:worker-alert-child:fallback"), + type: "turn-item.updated", + payload: { + ...item, + id: TurnItemId.make("item:worker-alert-child:fallback"), + type: "system_notice", + ordinal: 1, + title: "Switched models", + message: "Switched to claude-opus-4-8 after a safeguard.", + fallbackModel: "claude-opus-4-8", + }, + }, + { + ...scope, + id: EventId.make("event:worker-alert-child:reply"), + type: "message.updated", + payload: { + id: MessageId.make("message:worker-alert-child:reply"), + threadId: child.childThreadId, + runId: child.childRunId, + nodeId: null, + role: "assistant", + text: reply, + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, + }, + ], + }); + yield* orchestrator.recoverDelegatedTask(child.childThreadId); + const parent = yield* orchestrator.getThreadProjection(threadId); + const task = parent.subagents.find((row) => row.id === child.taskId); + const notice = + "Worker alert: a provider safeguard switched it to claude-opus-4-8; its final message withholds the task. Replace the worker if it cannot do the task."; + assert.deepEqual(task?.workerAlert, { + reasons: ["provider_fallback", "withheld_task"], + notice, + }); + assert.equal(task?.result, `${notice}\n\n${reply}`); + + const delivery = parent.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery; + assert.deepEqual(delivery?.taskIds, [child.taskId]); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "server", + commandId: CommandId.make("command:worker-alert-wake"), + threadId, + messageId: delivery!.messageId, + text: "Delegated task completed", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + delegatedCompletion: { + parentRunId: runId, + generation: delivery!.generation, + taskIds: delivery!.taskIds, + }, + }); + const wake = (yield* orchestrator.getThreadProjection(threadId)).messages.find( + (message) => message.id === delivery!.messageId, + ); + assert.equal( + wake?.text, + `Delegated task ${child.taskId} [completed] (child ):\n${notice}\n\n${reply}`, + ); + }), + ); + // Running does not prove prompt consumption in Claude; re-offer cut deliveries even when resuming the turn. it.effect("re-offers a cut delivery even when a continuation resumes its turn", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 495764336e6e..7defde600c68 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -87,6 +87,7 @@ import { modelSwitchSince } from "./modelSwitch.ts"; import { isPlaceholderThreadTitle } from "./placeholderThreadTitle.ts"; import { combinePullRequestWatchWakes, type PullRequestWatchWake } from "./pullRequestWatch.ts"; import { recordThreadOpen } from "@t3tools/shared/paletteSearch"; +import { formatThreadLink } from "@t3tools/shared/threadLinks"; import { backgroundWorkHoldsCompletion, derivePendingBackgroundWork, @@ -931,7 +932,17 @@ function delegatedCompletionWakeDetail( return taskIds .map((taskId) => { const task = tasks.find((candidate) => candidate.id === taskId); - const label = `Delegated task ${taskId}${task?.title == null ? "" : ` (${task.title})`}${task === undefined ? " reached a terminal state" : ` [${task.status}]`}`; + const child = + task?.childThreadId != null + ? { environmentId: null, threadId: task.childThreadId } + : task?.remoteChild; + const alert = + task?.workerAlert === undefined + ? "" + : child === undefined + ? "" + : ` (child ${formatThreadLink(child, task.title ?? undefined)})`; + const label = `Delegated task ${taskId}${task?.title == null ? "" : ` (${task.title})`}${task === undefined ? " reached a terminal state" : ` [${task.status}]`}${alert}`; if (task?.result != null) { const detail = `${label}:\n${task.result}`; const bytes = Buffer.byteLength(detail); @@ -940,7 +951,7 @@ function delegatedCompletionWakeDetail( return detail; } } - return `${label}. Use task_status with taskId ${taskId} to read the result.`; + return `${label}. ${task?.workerAlert === undefined ? "" : `${task.workerAlert.notice} `}Use task_status with taskId ${taskId} to read the result.`; }) .join("\n\n"); } @@ -10103,6 +10114,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio status: command.status, result: command.result, ...(command.executedModel === undefined ? {} : { executedModel: command.executedModel }), + ...(command.workerAlert === undefined ? {} : { workerAlert: command.workerAlert }), completedAt: now, updatedAt: now, }; @@ -14513,11 +14525,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const parentTurnItem = parentTask.turnItems.find( (candidate) => candidate.type === "subagent" && candidate.subagentId === task.id, ); + const { workerAlert: _previousAlert, ...taskWithoutAlert } = task; const updatedTask: OrchestrationV2Subagent = { - ...task, + ...taskWithoutAlert, providerThreadId: childRun.providerThreadId, status: terminalStatus, result: result.text, + ...(result.workerAlert === undefined ? {} : { workerAlert: result.workerAlert }), completedAt: now, updatedAt: now, // The cancelling parent gets no wake, and the task records the run it settled from. diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index 00ff1049c6e2..c9f217fd1551 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -31,6 +31,8 @@ import { endOrphanedNativeSubagent, endRunlessRootTurns, PROVIDER_STOP_REASON_SIGNAL, + withholdsTask, + workerAlert, } from "@t3tools/provider-core/server/subagentProjection"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; @@ -528,7 +530,85 @@ it("orders a task's assistant messages by turn item, not by timestamps", () => { { ...projection, messages: [report.message], providerTurns: [reported as never] }, run, ).text, - "Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\nAll steps passed.", + "Worker alert: it ran gpt-5.4-mini instead of gpt-5.4. Replace the worker if it cannot do the task.\n\nAll steps passed.", + ); +}); + +it("alerts the parent when a safeguard swapped the model and the worker held the task", () => { + const { projection, run } = taskFixture(); + const at = DateTime.makeUnsafe("2026-07-24T10:05:00.000Z"); + const reply = makeSubagentConversationArtifacts({ + messageId: MessageId.make("held"), + turnItemId: TurnItemId.make("held"), + threadId: parentThreadId, + rootNodeId: NodeId.make("root"), + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + role: "assistant", + text: "I'm holding until Kameron gives direct confirmation.", + ordinal: 2, + now: at, + }); + const notice = (runId: RunId, id: string) => + ({ + ...reply.turnItem, + id: TurnItemId.make(id), + runId, + ordinal: 1, + type: "system_notice", + message: "Switched to claude-opus-4-8 after a safeguard.", + fallbackModel: "claude-opus-4-8", + }) as OrchestrationV2TurnItem; + const result = subagentResultForRun( + { + ...projection, + messages: [{ ...reply.message, runId: run.id }], + turnItems: [ + { ...reply.turnItem, runId: run.id }, + notice(run.id, "fallback"), + notice(RunId.make("run:other"), "other-fallback"), + ], + providerTurns: [ + { runAttemptId: run.activeAttemptId, ordinal: 1, model: "claude-opus-4-8" } as never, + ], + }, + { ...run, modelSelection: { ...run.modelSelection, model: "claude-opus-5-5" } }, + ); + const alert = + "Worker alert: it ran claude-opus-4-8 instead of claude-opus-5-5; a provider safeguard switched it to claude-opus-4-8; its final message withholds the task. Replace the worker if it cannot do the task."; + assert.deepEqual(result.workerAlert, { + reasons: ["model_substituted", "provider_fallback", "withheld_task"], + notice: alert, + }); + assert.equal(result.text, `${alert}\n\nI'm holding until Kameron gives direct confirmation.`); +}); + +it.each([ + ["I'm holding until you confirm.", true], + ["Holding off on the merge until Lead relays approval.", true], + ["I need direct confirmation from Kameron before I start.", true], + ["Please confirm directly that you want this.", true], + ["I need your explicit approval to continue.", true], + ["I can\u2019t help with this request.", true], + ["I won't proceed without a human sign-off.", true], + ["Merged after Kameron's approval; all checks pass.", false], + ["The PR is open and awaiting review.", false], + ["Added a test for the holding-pattern parser.", false], + ["Fixed the bug where the queue kept holding stale entries.", false], + ["The review quotes the old message: holding until the cache warms.", false], + [`${"Implemented and tested the change. ".repeat(30)}I need your approval to merge.`, false], +])("withholdsTask(%j) is %s", (text, expected) => { + assert.equal(withholdsTask(text), expected); +}); + +it("ignores a context-window suffix when comparing models", () => { + assert.isUndefined( + workerAlert({ + requestedModel: "claude-opus-5-5[1m]", + executedModel: "claude-opus-5-5", + finalText: "Done.", + }), ); }); diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index ac63ce91cdd4..bebde781edaf 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -1957,13 +1957,14 @@ it.effect("a task's result is its own run's whole last reply, not a later run's" yield* Deferred.succeed(thread.finished, undefined); const task = yield* taskResult(a, taskId); expect(task?.status).toBe("completed"); - expect(task?.result).toBe( - `Requested model gpt-5.4; the provider reported running gpt-5.4-mini.\n\n${long}`, - ); + const notice = + "Worker alert: it ran gpt-5.4-mini instead of gpt-5.4. Replace the worker if it cannot do the task."; + expect(task?.result).toBe(`${notice}\n\n${long}`); const status = yield* a.call("task_status", { taskId }); expect(status.structuredContent).toMatchObject({ status: "completed", executedModel: "gpt-5.4-mini", + workerAlert: { reasons: ["model_substituted"], notice }, }); // Every wait was for the run the launch started. const waits = yield* Ref.get(thread.waitedFor); diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 132fc6a9badf..ed4ea992a974 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -15,6 +15,7 @@ import { type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, + type OrchestrationV2WorkerAlert, type PeerLink, OrchestrationV2ThreadPlacementAdmitError, type ProviderInteractionMode, @@ -44,7 +45,7 @@ import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as Tool from "effect/ai/Tool"; -import { withExecutedModelNote } from "@t3tools/provider-core/server/subagentProjection"; +import { workerAlert } from "@t3tools/provider-core/server/subagentProjection"; import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import { ProjectToolkit } from "../mcp/toolkits/project/tools.ts"; @@ -1119,6 +1120,7 @@ const make = Effect.gen(function* () { stoppedThere: "ran-out" | "stopped" | undefined, stopCount?: number, executedModel?: string, + alert?: OrchestrationV2WorkerAlert, ) { const attempt = yield* random.uuidV4(); yield* threads @@ -1132,6 +1134,7 @@ const make = Effect.gen(function* () { ...(stoppedThere === undefined ? {} : { stoppedThere }), ...(stopCount === undefined ? {} : { stopCount }), ...(executedModel === undefined ? {} : { executedModel }), + ...(alert === undefined ? {} : { workerAlert: alert }), }) .pipe( Effect.catch((error) => @@ -1187,7 +1190,7 @@ const make = Effect.gen(function* () { * The last reply `runId` gave on the task's thread there, which is its * result, read whole however long it is. A failed run prefers its error * from the activity view over an earlier reply, without taking another run's. - * Led by a note when its provider reported running another model than it asked for. + * Led by a worker alert when the run ran another model or withheld the task. */ const lastReply = Effect.fn("RemoteDelegation.lastReply")(function* ( scope: ThreadScope, @@ -1224,7 +1227,14 @@ const make = Effect.gen(function* () { } reply = error ?? reply; const executedModel = run?.executedModel ?? undefined; - if (reply === undefined) return { text: undefined, executedModel }; + // Peers that predate `workerAlert` report only the model, so judge the run here. + const alertFor = (finalText: string | undefined) => + run === undefined + ? undefined + : (run.workerAlert ?? + workerAlert({ requestedModel: run.model, executedModel: run.executedModel, finalText })); + if (reply === undefined) + return { text: undefined, executedModel, workerAlert: alertFor(undefined) }; let text = reply.text ?? ""; let textOffset = reply.textTruncated ? (reply.nextTextOffset ?? null) : null; while (textOffset !== null) { @@ -1239,9 +1249,11 @@ const make = Effect.gen(function* () { text += part.text ?? ""; textOffset = part.textTruncated ? (part.nextTextOffset ?? null) : null; } + const alert = alertFor(error === undefined ? text : undefined); return { - text: run === undefined ? text : withExecutedModelNote(text, run.model, run.executedModel), + text: alert === undefined ? text : `${alert.notice}\n\n${text}`, executedModel, + workerAlert: alert, }; }); @@ -1449,6 +1461,7 @@ const make = Effect.gen(function* () { "ran-out", undefined, result.executedModel, + result.workerAlert, ); return yield* settled(parentThreadId, taskId); }); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 2a91d8a10eda..3c4238e09d3f 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -920,6 +920,20 @@ export const OrchestrationV2RemoteTaskChild = Schema.Struct({ }); export type OrchestrationV2RemoteTaskChild = typeof OrchestrationV2RemoteTaskChild.Type; +/** Why a delegated worker's run ended unfit for its task, so its parent can replace it. */ +export const OrchestrationV2WorkerAlertReason = Schema.Literals([ + "model_substituted", + "provider_fallback", + "usage_limit", + "withheld_task", +]); +export type OrchestrationV2WorkerAlertReason = typeof OrchestrationV2WorkerAlertReason.Type; +export const OrchestrationV2WorkerAlert = Schema.Struct({ + reasons: Schema.NonEmptyArray(OrchestrationV2WorkerAlertReason), + notice: Schema.String, +}); +export type OrchestrationV2WorkerAlert = typeof OrchestrationV2WorkerAlert.Type; + export const OrchestrationV2Subagent = Schema.Struct({ id: NodeId, threadId: ThreadId, @@ -938,6 +952,8 @@ export const OrchestrationV2Subagent = Schema.Struct({ model: Schema.NullOr(Schema.String), modelSelection: Schema.optional(ModelSelection), executedModel: Schema.optional(Schema.String), + /** Set from the run that produced `result`; informs the parent and takes no action. */ + workerAlert: Schema.optional(OrchestrationV2WorkerAlert), completionWake: Schema.optional(Schema.Literals(["always", "settled_only"])), completionDelivery: Schema.optional(OrchestrationV2DelegatedCompletionTaskDelivery), // The run whose unfinished task was cancelled, committed with its Stop. @@ -1841,6 +1857,8 @@ export const OrchestrationV2TurnItem = Schema.Union([ type: Schema.Literal("system_notice"), message: Schema.String, level: Schema.optional(OrchestrationV2SystemNoticeLevel), + /** A provider safeguard moved the turn to this model. */ + fallbackModel: Schema.optional(Schema.String), }), Schema.Struct({ ...OrchestrationV2TurnItemBaseFields, @@ -2723,6 +2741,8 @@ export const OrchestrationV2TurnItemJson = Schema.Union([ type: Schema.Literal("system_notice"), message: Schema.String, level: Schema.optional(OrchestrationV2SystemNoticeLevel), + /** A provider safeguard moved the turn to this model. */ + fallbackModel: Schema.optional(Schema.String), }), Schema.Struct({ ...OrchestrationV2TurnItemJsonBaseFields, @@ -3700,6 +3720,7 @@ const OrchestrationV2InternalCommand = Schema.Union([ status: Schema.Literals(["completed", "failed", "cancelled", "interrupted"]), result: Schema.String, executedModel: Schema.optional(Schema.String), + workerAlert: Schema.optional(OrchestrationV2WorkerAlert), /** * `ran-out`: the run there ended by itself; `stopped`: a stop from here reached it. Either * sets `remoteChild.stoppedThere` on a task this report completes. On a task already diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index afd1ab215cd6..4f454f9ad758 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -42,6 +42,7 @@ import { OrchestrationV2DelegatedFrom, OrchestrationV2RunStatus, OrchestrationV2TurnItemStatus, + OrchestrationV2WorkerAlert, } from "./orchestrationV2.ts"; import { ProviderOptionSelection, ProviderOptionSelectionValue } from "./model.ts"; import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts"; @@ -218,6 +219,11 @@ const OrchestratorMcpExecutedModel = Schema.NullOr(Schema.String).annotate({ "Model the provider reported running, or null when unavailable. Can differ from the requested `model`.", }); +const OrchestratorMcpWorkerAlert = OrchestrationV2WorkerAlert.annotate({ + description: + "Present when the run ran another model, hit a provider safeguard fallback or usage limit, or withheld the task. Consider replacing the worker; nothing acts on it.", +}); + export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ taskId: NodeId, /** Null for a task that runs in a linked environment; see `remoteChild`. */ @@ -242,6 +248,7 @@ export const OrchestratorMcpDelegateTaskResult = Schema.Struct({ providerInstanceId: ProviderInstanceId, model: Schema.NullOr(Schema.String), executedModel: Schema.optional(OrchestratorMcpExecutedModel), + workerAlert: Schema.optional(OrchestratorMcpWorkerAlert), options: Schema.optional(Schema.Array(ProviderOptionSelection)), summary: Schema.NullOr(Schema.String), resultContextTransferId: Schema.NullOr(ContextTransferId), @@ -522,6 +529,7 @@ export const OrchestratorMcpThreadRun = Schema.Struct({ providerInstanceId: ProviderInstanceId, model: Schema.String, executedModel: Schema.optional(OrchestratorMcpExecutedModel), + workerAlert: Schema.optional(OrchestratorMcpWorkerAlert), requestedAt: IsoDateTime, startedAt: Schema.NullOr(IsoDateTime), completedAt: Schema.NullOr(IsoDateTime), diff --git a/packages/provider-core/src/server/notification.test.ts b/packages/provider-core/src/server/notification.test.ts index f8d0b1d113d4..782cae0e0943 100644 --- a/packages/provider-core/src/server/notification.test.ts +++ b/packages/provider-core/src/server/notification.test.ts @@ -160,4 +160,19 @@ describe("notificationTurnItem", () => { }, }); }); + + it("flags delegated tasks whose worker raised an alert", () => { + const alerted = tasks.map((entry): OrchestrationV2Subagent => + entry.id === NodeId.make("a") + ? { ...entry, workerAlert: { reasons: ["model_substituted"], notice: "Worker alert." } } + : entry, + ); + assert.deepInclude(notificationTurnItem(userMessage, delivery(["a"]), alerted), { + summary: 'Delegated task "Review src/math.ts" finished with a worker alert', + }); + assert.deepInclude(notificationTurnItem(userMessage, delivery(["a", "b"]), alerted), { + summary: + "2 of 3 delegated tasks finished, 1 with a worker alert: Review src/math.ts, Write tests", + }); + }); }); diff --git a/packages/provider-core/src/server/notification.ts b/packages/provider-core/src/server/notification.ts index 9ccb425faf2d..cb4824364273 100644 --- a/packages/provider-core/src/server/notification.ts +++ b/packages/provider-core/src/server/notification.ts @@ -201,6 +201,9 @@ function delegatedCompletionNotification( ).length; const outcome = combinedOutcome(reports.map((report) => report.outcome)); const verb = outcome === "failed" ? "failed" : outcome === "cancelled" ? "stopped" : "finished"; + const alerts = taskIds.filter( + (taskId) => tasks.find((task) => task.id === taskId)?.workerAlert !== undefined, + ).length; const [only] = reports; if (reports.length === 1 && only !== undefined) { const label = reportLabel(only.label); @@ -211,7 +214,7 @@ function delegatedCompletionNotification( ...(only.childThreadId === undefined ? {} : { childThreadId: only.childThreadId }), }, outcome, - summary: `Delegated task${label === undefined ? "" : ` "${label}"`} ${verb}`, + summary: `Delegated task${label === undefined ? "" : ` "${label}"`} ${verb}${alerts === 0 ? "" : " with a worker alert"}`, }; } const labels = reports.flatMap((report) => reportLabel(report.label) ?? []); @@ -222,7 +225,7 @@ function delegatedCompletionNotification( return { source: { kind: "delegated_task", taskIds }, outcome, - summary: `${count} delegated tasks ${verb}${labels.length === 0 ? "" : `: ${labels.join(", ")}`}`, + summary: `${count} delegated tasks ${verb}${alerts === 0 ? "" : `, ${alerts} with a worker alert`}${labels.length === 0 ? "" : `: ${labels.join(", ")}`}`, }; } diff --git a/packages/provider-core/src/server/subagentProjection.ts b/packages/provider-core/src/server/subagentProjection.ts index 74808a3555e0..9df8fa8273fa 100644 --- a/packages/provider-core/src/server/subagentProjection.ts +++ b/packages/provider-core/src/server/subagentProjection.ts @@ -12,6 +12,8 @@ import type { OrchestrationV2Run, OrchestrationV2ThreadProjection, OrchestrationV2TurnItem, + OrchestrationV2WorkerAlert, + OrchestrationV2WorkerAlertReason, ProviderInstanceId, ProviderThreadId, ProviderTurnId, @@ -190,15 +192,70 @@ export function makeSubagentConversationArtifacts(input: { return { message, turnItem }; } -/** A delegated result, led by a note when the provider reported running another model than requested. */ -export function withExecutedModelNote( - result: string, - requestedModel: string, - executedModel: string | null | undefined, -): string { - return executedModel == null || executedModel === requestedModel - ? result - : `Requested model ${requestedModel}; the provider reported running ${executedModel}.\n\n${result}`; +// Small on purpose: a match only flags the worker for its parent, which decides what to do. +const WITHHELD_TASK_PATTERNS: ReadonlyArray = [ + /\b(?:i am|i'm|i will keep|i'll keep|i'm keeping|keeping)\s+(?:this\s+|it\s+|the task\s+)?(?:on hold|holding)\b/i, + /\bholding\s+(?:off|here)\b/i, + /\bdirect(?:ly)?\s+(?:confirmation|approval|go-ahead)\b/i, + /\bconfirm\s+(?:directly|with me)\b/i, + /\b(?:i need|i'd need|i would need|i require)\s+(?:(?:your|explicit|written)\s+)*(?:confirmation|approval|go-ahead|authori[sz]ation)\b/i, + /\bi\s+(?:won't|will not|can't|cannot|can not)\s+(?:proceed|continue|help with|carry out|do this)\b/i, +]; + +// A held or refused task ends in a short reply; a long report that asks for a decision did the work. +const WITHHELD_TASK_MAX_LENGTH = 800; + +/** True when a worker's short final message holds, refuses, or asks for confirmation instead of reporting work. */ +export function withholdsTask(text: string): boolean { + const reply = text.trim(); + return ( + reply.length <= WITHHELD_TASK_MAX_LENGTH && + WITHHELD_TASK_PATTERNS.some((pattern) => pattern.test(reply.replaceAll("\u2019", "'"))) + ); +} + +// Context-window and similar suffixes, such as `[1m]`, name the same model. +const baseModel = (model: string) => model.replace(/\[[^\]]*\]$/, ""); + +/** What made a worker's run unfit for its task, or undefined when nothing did. */ +export function workerAlert(input: { + readonly requestedModel: string | null | undefined; + readonly executedModel: string | null | undefined; + readonly fallbackModels?: ReadonlyArray | undefined; + readonly usageLimited?: boolean | undefined; + readonly finalText: string | null | undefined; +}): OrchestrationV2WorkerAlert | undefined { + const reasons: Array = []; + const parts: Array = []; + const { requestedModel, executedModel } = input; + if ( + requestedModel != null && + executedModel != null && + baseModel(executedModel) !== baseModel(requestedModel) + ) { + reasons.push("model_substituted"); + parts.push(`it ran ${executedModel} instead of ${requestedModel}`); + } + const fallbacks = [...new Set(input.fallbackModels)]; + if (fallbacks.length > 0) { + reasons.push("provider_fallback"); + parts.push(`a provider safeguard switched it to ${fallbacks.join(", ")}`); + } + if (input.usageLimited === true) { + reasons.push("usage_limit"); + parts.push("it hit a provider usage limit"); + } + if (input.finalText != null && withholdsTask(input.finalText)) { + reasons.push("withheld_task"); + parts.push("its final message withholds the task"); + } + const [first, ...rest] = reasons; + return first === undefined + ? undefined + : { + reasons: [first, ...rest], + notice: `Worker alert: ${parts.join("; ")}. Replace the worker if it cannot do the task.`, + }; } /** Run signal of the notice that carries a provider's stop reason on a completed run. */ @@ -221,6 +278,7 @@ export function subagentResultForRun( readonly text: string; readonly messageId: OrchestrationV2ConversationMessage["id"] | null; readonly turnItemId: OrchestrationV2TurnItem["id"] | null; + readonly workerAlert: OrchestrationV2WorkerAlert | undefined; } { const turnItem = projection.turnItems @@ -273,13 +331,22 @@ export function subagentResultForRun( projection.providerTurns, run.activeAttemptId, )?.model; + const alert = workerAlert({ + requestedModel: executedModel === undefined ? undefined : run.modelSelection.model, + executedModel, + fallbackModels: projection.turnItems.flatMap((item) => + item.runId === run.id && item.type === "system_notice" && item.fallbackModel !== undefined + ? [item.fallbackModel] + : [], + ), + usageLimited: failure?.type === "error" && failure.failure.class === "usage_limit", + finalText: message?.text ?? turnItem?.text, + }); return { - text: - executedModel === undefined - ? text - : withExecutedModelNote(text, run.modelSelection.model, executedModel), + text: alert === undefined ? text : `${alert.notice}\n\n${text}`, messageId: failure === undefined ? (message?.id ?? turnItem?.messageId ?? null) : null, turnItemId: failure?.id ?? turnItem?.id ?? null, + workerAlert: alert, }; }