Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/lazurio-fork-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -440,11 +440,18 @@ jobs:
apps/web/src/components/ThreadNotificationCoordinator.tsx
apps/web/src/state/entities.ts
apps/web/src/hooks/useActiveProjectTarget.ts
# A thread waiting on a question wakes whoever waits on it (row 39): t3_thread_wait's
# untilQuestion in the contract and ThreadManagementService (listed above), and the
# remote delegation follower that wakes the parent here, which the orchestrator keeps.
packages/contracts/src/orchestratorMcp.ts
apps/server/src/peer/RemoteDelegation.ts
apps/server/src/peer/RemoteDelegation.test.ts
# Where a message came from, for a linked environment's thread too (row 40): the
# receiving server records the link's label with the sender (OrchestratorMcpService.ts
# and orchestrationV2.ts, listed above), and the tests that see it.
apps/server/src/mcp/OrchestratorMcpService.test.ts
apps/server/src/peer/PeerForwarding.test.ts
# The tool list grows by the two optional parameters and the wait result's field.
apps/server/src/provider/agentContextBudget.json
)
unexpected="$(git diff --name-only "$UPSTREAM_SHA..HEAD" |
Expand Down
2 changes: 2 additions & 0 deletions apps/server/src/mcp/OrchestratorMcpService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3045,6 +3045,8 @@ describe("OrchestratorMcpService provider resolution", () => {
return { taskId: parent.subagents[0]!.id };
}).pipe(Effect.orDie),
runStatus: () => Effect.succeed(null),
pendingQuestionIds: () => Effect.succeed([]),
subscribeQuestionChanges: Effect.succeed(Stream.never),
});
}),
).pipe(Layer.provide(layerWatchedThreads));
Expand Down
72 changes: 50 additions & 22 deletions apps/server/src/mcp/OrchestratorMcpService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2032,8 +2032,9 @@ const make = Effect.gen(function* () {
: null,
latestTerminalSummary: terminal ? task.result : null,
latestTerminalResultContextTransferId: null,
// Its questions and approvals are answered in the environment it runs in.
hasPendingQuestion: false,
// Answered in the environment it runs in; its approvals are not seen from here.
hasPendingQuestion:
!terminal && (yield* remoteDelegation.pendingQuestionIds(task.id)).length > 0,
pendingApproval: null,
waitTimedOut,
} satisfies OrchestratorMcpDelegateTaskResult;
Expand Down Expand Up @@ -2061,36 +2062,56 @@ const make = Effect.gen(function* () {
childThreadId === null
? TASK_WAKE_EVENTS.filter(({ thread }) => thread === "parent")
: TASK_WAKE_EVENTS;
// A remote task's questions are seen by its follower, not by any event here; one it saw
// before this subscribed is read below.
const remoteQuestions: Array<Stream.Stream<unknown, OrchestratorMcpFailure>> = [];
if (childThreadId === null) {
const questionChanges = yield* remoteDelegation.subscribeQuestionChanges;
if ((yield* remoteDelegation.pendingQuestionIds(taskId)).length > 0) {
const asked = yield* readTask(scope, taskId, false, true);
if (endsTaskWait(asked)) return Option.some(asked);
}
remoteQuestions.push(
questionChanges.pipe(Stream.filter((changedTaskId) => changedTaskId === taskId)),
);
}
return yield* Stream.mergeAll(
wakeEvents.map(({ thread, eventType }) =>
threadManagement
.streamStoredEventsFrom({
threadId:
thread === "parent" || childThreadId === null
? scope.thread.threadId
: childThreadId,
afterSequence,
eventType,
})
.pipe(
Stream.filter(
({ event }) =>
thread === "child" ||
event.type !== "subagent.updated" ||
event.payload.id === taskId,
[
...wakeEvents.map(({ thread, eventType }) =>
threadManagement
.streamStoredEventsFrom({
threadId:
thread === "parent" || childThreadId === null
? scope.thread.threadId
: childThreadId,
afterSequence,
eventType,
})
.pipe(
Stream.filter(
({ event }) =>
thread === "child" ||
event.type !== "subagent.updated" ||
event.payload.id === taskId,
),
Stream.mapError(streamError),
),
),
),
),
...remoteQuestions,
],
{ concurrency: "unbounded" },
).pipe(
Stream.mapError(streamError),
Stream.mapEffect(() => readTask(scope, taskId, false, true)),
Stream.filter(endsTaskWait),
Stream.runHead,
);
},
(effect, _scope, _taskId, timeoutMs) =>
effect.pipe(Effect.timeoutOption(Duration.millis(timeoutMs)), Effect.map(Option.flatten)),
effect.pipe(
Effect.scoped,
Effect.timeoutOption(Duration.millis(timeoutMs)),
Effect.map(Option.flatten),
),
);

const refExists = (cwd: string, revision: string) =>
Expand Down Expand Up @@ -3472,6 +3493,9 @@ const make = Effect.gen(function* () {
Math.max(1, input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS),
),
...(input.untilDelegatedWorkSettled === true ? { untilDelegatedWorkSettled: true } : {}),
...(input.untilQuestion === true
? { untilQuestion: true, knownQuestionIds: input.knownQuestionIds ?? [] }
: {}),
})
.pipe(Effect.mapError(threadManagementFailure));
const status = result.run === null ? "idle" : runOutcome(result.run);
Expand Down Expand Up @@ -3508,6 +3532,10 @@ const make = Effect.gen(function* () {
: {
resultRun: { runId: result.resultRun.id, status: runOutcome(result.resultRun) },
}),
...(result.pendingQuestionIds === undefined
? {}
: { pendingQuestionIds: result.pendingQuestionIds }),
...(result.returnedFor === undefined ? {} : { returnedFor: result.returnedFor }),
} satisfies OrchestratorMcpThreadWaitResult;
}),
interruptThread: Effect.fn("OrchestratorMcpService.interruptThread")(function* (scope, input) {
Expand Down
8 changes: 3 additions & 5 deletions apps/server/src/orchestration-v2/Orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2340,11 +2340,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
// An unknown task gives no evidence the question is obsolete.
if (task === undefined) return false;
const childThreadId = task.childThreadId;
if (
childThreadId === null ||
["completed", "failed", "cancelled", "interrupted"].includes(task.status)
)
return true;
if (["completed", "failed", "cancelled", "interrupted"].includes(task.status)) return true;
// Only its follower sees a remote task's questions there; it drops their wakes itself.
if (childThreadId === null) return task.remoteChild === undefined;
const child = yield* projectionStore
.getThreadRecords(childThreadId, ["runtimeRequests"])
.pipe(
Expand Down
106 changes: 106 additions & 0 deletions apps/server/src/orchestration-v2/ThreadManagementService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
ProjectId,
ProviderInstanceId,
RunId,
RuntimeRequestId,
ThreadId,
} from "@t3tools/contracts";
import * as Deferred from "effect/Deferred";
Expand Down Expand Up @@ -553,6 +554,111 @@ it.effect("waitForThread can wait past an ended run for the thread's delegated w
}),
);

it.effect(
"waitForThread with untilQuestion returns once the thread asks, and once it is answered",
() =>
Effect.gen(function* () {
const projectId = ProjectId.make("project:thread-management:wait-question");
const threadId = ThreadId.make("thread:thread-management:wait-question");
const runId = RunId.make("run:thread-management:wait-question");
const questionId = RuntimeRequestId.make("request:thread-management:wait-question");
const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
const subscribed = yield* Queue.unbounded<void>();
const runs = [{ id: runId, ordinal: 1, status: "running" }];
const question = (status: "pending" | "resolved", responseCapability = "live") => ({
id: questionId,
kind: "user_input",
status,
responseCapability: { type: responseCapability },
createdAt: DateTime.makeUnsafe(1_000),
});
// An approval and a question no answer can reach are not questions to return for.
let runtimeRequests: Array<object> = [
{ ...question("pending"), id: "request:approval", kind: "command" },
{ ...question("pending", "not_resumable"), id: "request:stale" },
];
const stored = (sequence: number, event: object) =>
({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent;
const layerTest = ThreadManagementService.layer.pipe(
Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
Layer.provide(
Layer.mock(Orchestrator.OrchestratorV2)({
getThreadEventSequence: () => Effect.succeed(0),
restartContinuationPending: () => Effect.succeed(false),
streamStoredEventsFrom: (input) =>
input?.eventType === "runtime-request.updated"
? Stream.fromEffect(Queue.offer(subscribed, undefined)).pipe(
Stream.drain,
Stream.concat(Stream.fromQueue(events)),
)
: Stream.never,
getThreadRecords: () =>
Effect.sync(
() =>
({
thread: { id: threadId, projectId, deletedAt: null },
runs,
runtimeRequests,
}) as unknown as OrchestrationV2ThreadProjection,
),
}),
),
Layer.provide(ServerSettings.layerTest()),
Layer.provide(layerReceiptsTest),
);
const service = yield* ThreadManagementService.ThreadManagementService.pipe(
Effect.provide(layerTest),
);
const wait = { projectId, threadId, runId, timeoutMs: 60_000 };

// Without the option a question changes nothing: the wait runs out as before.
runtimeRequests = [...runtimeRequests, question("pending")];
const plain = yield* service.waitForThread(wait).pipe(Effect.forkChild);
yield* TestClock.adjust(Duration.minutes(1));
const plainResult = yield* Fiber.join(plain);
expect(plainResult).toMatchObject({ timedOut: true, run: { id: runId, status: "running" } });
expect(plainResult).not.toHaveProperty("pendingQuestionIds");
runtimeRequests = runtimeRequests.slice(0, 2);

const asked = yield* service
.waitForThread({ ...wait, untilQuestion: true })
.pipe(Effect.forkChild);
// A wait that returns early fails below instead of hanging here.
yield* Effect.raceFirst(Queue.take(subscribed), Fiber.await(asked));
runtimeRequests = [...runtimeRequests, question("pending")];
yield* Queue.offer(events, stored(1, { type: "runtime-request.updated", payload: {} }));
expect(yield* Fiber.join(asked)).toMatchObject({
timedOut: false,
run: { id: runId, status: "running" },
pendingQuestionIds: [questionId],
returnedFor: "question",
});

// A caller that knows of the question waits until it is answered.
const answered = yield* service
.waitForThread({ ...wait, untilQuestion: true, knownQuestionIds: [questionId] })
.pipe(Effect.forkChild);
yield* Effect.raceFirst(Queue.take(subscribed), Fiber.await(answered));
runtimeRequests = [...runtimeRequests.slice(0, 2), question("resolved")];
yield* Queue.offer(events, stored(2, { type: "runtime-request.updated", payload: {} }));
expect(yield* Fiber.join(answered)).toMatchObject({
timedOut: false,
run: { id: runId, status: "running" },
pendingQuestionIds: [],
returnedFor: "question",
});

// With no question, the wait runs out and says so.
const quiet = yield* service
.waitForThread({ ...wait, untilQuestion: true })
.pipe(Effect.forkChild);
yield* TestClock.adjust(Duration.minutes(1));
const quietResult = yield* Fiber.join(quiet);
expect(quietResult).toMatchObject({ timedOut: true, pendingQuestionIds: [] });
expect(quietResult).not.toHaveProperty("returnedFor");
}),
);

it.effect("waitForThread settles delegated work on a thread rolled back past its only run", () =>
Effect.gen(function* () {
const projectId = ProjectId.make("project:thread-management:wait-rolled-back");
Expand Down
Loading
Loading