diff --git a/FORK_CHANGES.md b/FORK_CHANGES.md index 502ee7e2aef98..aee12a1eb8c14 100644 --- a/FORK_CHANGES.md +++ b/FORK_CHANGES.md @@ -23,7 +23,7 @@ setting is missing here or a setting named here does not exist. | The pacing gate on `delegate_task`: advise only, or move, lower the effort of, or hold a task on an account that is ahead of pace. | Delegated work follows the pace without the agent having to check. | `server.usagePacingGate`, `server.usagePacingLowerEffort` | off | [#73](https://github.com/scuttletoby/t3code/pull/73) | | Agent slots: `delegate_task` is refused above a number of running agents per machine or account, or below a memory floor. | Too many agents at once can overload a small machine. | `server.agentSlots` | off; rule only observes | [#84](https://github.com/scuttletoby/t3code/pull/84), [#89](https://github.com/scuttletoby/t3code/pull/89) | | Waits: agents can wait for a pull request, a file, a system unit, a named check command or a free agent slot (`wait_for`, `wait_list`, `wait_cancel`), and are woken when it happens. | An agent no longer polls or sleeps while it waits. | `server.waitFor`, `server.waitChecks`, `projectOverride.waitChecks` | off | [#84](https://github.com/scuttletoby/t3code/pull/84), [#90](https://github.com/scuttletoby/t3code/pull/90), [#91](https://github.com/scuttletoby/t3code/pull/91), [#103](https://github.com/scuttletoby/t3code/pull/103), [#106](https://github.com/scuttletoby/t3code/pull/106), [#109](https://github.com/scuttletoby/t3code/pull/109), [#110](https://github.com/scuttletoby/t3code/pull/110), [#116](https://github.com/scuttletoby/t3code/pull/116), [#119](https://github.com/scuttletoby/t3code/pull/119) | -| Delegated results: `task_list` lists a thread's delegated tasks whose result it has not read, and its running children per place from live status; a parent whose finished child's result never arrived gets one reminder. | A finished child's result can no longer sit unseen. | `server.waitFor` (the reminder) | off; `task_list` always (read only) | [#622](https://github.com/scuttletoby/t3code/pull/622) | +| Delegated results: `task_list` lists a thread's delegated tasks whose result it has not read, and its running children per place from live status; a parent whose finished child's result never arrived gets one reminder. | A finished child's result can no longer sit unseen. | `server.waitFor` (the reminder) | off; `task_list` always (read only) | [#622](https://github.com/scuttletoby/t3code/pull/622), [#641](https://github.com/scuttletoby/t3code/pull/641) | | The Settings page **Fork** (`/settings/fork`): every fork setting in one place, with one line per feature at the top saying whether it is on: waits, wait checks, agent slots, usage pacing (per account by name), model policy, review rules, automatic updates, handoff and playtest. MCP profiles are not on it yet. | Change every fork setting without editing a file. | `server.waitFor`, `server.waitChecks`, `server.agentSlots`, `server.usagePacing`, `server.usagePacingGate`, `server.usagePacingLowerEffort`, `server.reviewRules`, `server.modelPolicy`, `server.automaticUpdates`, `server.automaticUpdatesQuietCommand`, `server.handoff`, `server.playtest` | off | [#104](https://github.com/scuttletoby/t3code/pull/104), [#543](https://github.com/scuttletoby/t3code/pull/543), [#544](https://github.com/scuttletoby/t3code/pull/544) | | The fork's checks on `delegate_task` (pacing, slots) sit behind one fork module. | Fewer fork lines in upstream's code, so syncs conflict less. No change in behaviour. | none | always | [#174](https://github.com/scuttletoby/t3code/pull/174) | | The fork's service layers in the server are wired from one fork file (`apps/server/src/fork/forkLayers.ts`). | Fewer fork lines in upstream's code, so syncs conflict less. No change in behaviour. | none | always | [#384](https://github.com/scuttletoby/t3code/pull/384) | diff --git a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts new file mode 100644 index 0000000000000..a841ef9ca656b --- /dev/null +++ b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts @@ -0,0 +1,638 @@ +/** + * Fork (issue 625): the startup orphan scan. State: reruns, restarts, late machines and a moved + * window name each orphan once; a stop half-way; the first scan's format; environments that do + * not answer (fully); busy, stopped or gone parents; threads made after start; a held start. + */ +import { describe, expect, it } from "@effect/vitest"; +import { + type CommandId, + EnvironmentId, + MessageId, + type OrchestrationV2ThreadShell, + ThreadId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; +import * as Option from "effect/Option"; +import { TestClock } from "effect/testing"; + +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import * as McpToolAccessTestkit from "../../mcp/McpToolAccess.testkit.ts"; +import * as CommandReceiptStore from "../../orchestration-v2/CommandReceiptStore.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as PeerForwarding from "../../peer/PeerForwarding.ts"; +import { ServerSettingsService } from "../../serverSettings.ts"; +import * as WorkAdmission from "../../workAdmission.ts"; +import { legacyOrphanSetId, orphanBatchIds } from "./delegatedResults.ts"; +import { at, NOW, PARENT } from "./delegatedResults.testkit.ts"; +import * as DelegatedOrphanScan from "./DelegatedOrphanScan.ts"; +import type { RemoteChildKey } from "./DelegatedResultsStore.ts"; +import * as DelegatedResultsStore from "./DelegatedResultsStore.ts"; + +const HERE = EnvironmentId.make("env-here"); +const ENV_B = EnvironmentId.make("env-b"); +const ENV_C = EnvironmentId.make("env-c"); + +const OPTIONS: DelegatedOrphanScan.OrphanScanOptions = { + delay: Duration.minutes(3), + retryDelay: Duration.minutes(10), + attempts: 3, + maxPages: 3, + pageTimeout: Duration.seconds(10), +}; + +const idleShell = ( + id: ThreadId, + overrides: Partial = {}, +): OrchestrationV2ThreadShell => ({ + ...McpToolAccessTestkit.liveThreadShell(id, { activeRunId: null }), + status: "idle", + ...overrides, +}); + +interface ThreadThere { + readonly threadId: string; + readonly from?: { readonly environmentId: EnvironmentId; readonly threadId: ThreadId }; + /** Before the start by default. */ + readonly createdAt?: number; + /** A minute ago by default. */ + readonly updatedAt?: number; +} + +const iso = (ms: number) => DateTime.formatIso(at(ms)); + +/** + * Both sides as the servers keep them, in memory. `sent` and `receipts` are shared across + * services, as the database is across a restart. + */ +const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) => { + const shells = new Map([[PARENT, idleShell(PARENT)]]); + const followed = new Map | null>(); + const there = new Map>([ + [ENV_B, [{ threadId: "lost-1" }, { threadId: "known-1" }]], + ]); + followed.set(PARENT, [{ environmentId: ENV_B, threadId: ThreadId.make("known-1") }]); + const silent = new Set(); + /** Answers its first page only. */ + const cutShort = new Set(); + const linksOff = { value: false }; + const pageReads: Array<{ environmentId: EnvironmentId; cursor: number }> = []; + const sent: Array<{ threadId: ThreadId; messageId: string; text: string }> = []; + const receipts = new Map(); + const page = (environmentId: EnvironmentId, cursor: number) => { + const all = there.get(environmentId) ?? []; + // Two threads a page, newest first, as `t3_thread_list` orders them. + const threads = all.slice(cursor, cursor + 2).map((thread, index) => ({ + threadId: ThreadId.make(thread.threadId), + title: `title ${thread.threadId}`, + delegatedFrom: { ...(thread.from ?? { environmentId: HERE, threadId: PARENT }), title: "p" }, + createdAt: iso(thread.createdAt ?? NOW - 60 * 60_000), + updatedAt: iso(thread.updatedAt ?? NOW - (cursor + index) * 60_000), + })); + return { + projectId: null, + currentThreadId: null, + threads, + nextCursor: cursor + 2 < all.length ? cursor + 2 : null, + total: all.length, + }; + }; + return { + shells, + followed, + there, + silent, + cutShort, + linksOff, + pageReads, + sent, + receipts, + layer: Layer.mergeAll( + ServerSettingsService.layerTest({ waitFor }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(HERE), + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + getThreadShell: (threadId) => Effect.succeed(shells.get(threadId) ?? null), + getMessagesById: (threadId, ids) => + Effect.succeed( + new Map( + sent + .filter((message) => message.threadId === threadId) + .filter((message) => ids.includes(MessageId.make(message.messageId))) + .map((message) => [MessageId.make(message.messageId), {} as never]), + ), + ), + sendToThread: (input) => + Effect.sync(() => { + sent.push({ threadId: input.threadId, messageId: input.messageId!, text: input.text }); + receipts.set(input.commandId, input.threadId); + return {} as never; + }), + }), + Layer.mock(CommandReceiptStore.CommandReceiptStoreV2)({ + getByCommandId: (commandId: CommandId) => + Effect.succeed( + receipts.has(commandId) + ? Option.some({ status: "accepted", threadId: receipts.get(commandId)! } as never) + : Option.none(), + ), + }), + Layer.mock(DelegatedResultsStore.DelegatedResultsStore)({ + remoteChildrenOf: (parent) => + Effect.succeed(followed.has(parent) ? followed.get(parent)! : []), + // As the real read: the thread's message ids and its accepted receipts' command ids. + sentIds: (threadId, prefix) => + Effect.succeed( + [ + ...sent + .filter((message) => message.threadId === threadId) + .map((message) => message.messageId), + ...[...receipts.entries()] + .filter(([, thread]) => thread === threadId) + .map(([commandId]) => commandId), + ].filter((id) => id.startsWith(prefix)), + ), + }), + Layer.mock(PeerForwarding.PeerForwarding)({ + links: () => + linksOff.value + ? Effect.fail(PeerForwarding.linkedEnvironmentsOff) + : Effect.succeed({ + environmentId: HERE, + links: [ + ...[...there.keys()].map((environmentId) => ({ + environmentId, + label: environmentId, + status: "reachable" as const, + access: {} as never, + expiresAt: "2027-01-01T00:00:00.000Z", + lastError: null, + })), + // An expired link is never asked. + { + environmentId: EnvironmentId.make("env-expired"), + label: "expired", + status: "expired" as const, + access: {} as never, + expiresAt: "2026-01-01T00:00:00.000Z", + lastError: null, + }, + ], + notListed: "", + }), + call: (( + _scope: unknown, + _tool: unknown, + environmentId: EnvironmentId, + params: { readonly cursor: number }, + ) => + Effect.suspend(() => { + pageReads.push({ environmentId, cursor: params.cursor }); + return silent.has(environmentId) || (cutShort.has(environmentId) && params.cursor > 0) + ? Effect.never + : Effect.succeed(page(environmentId, params.cursor)); + })) as never, + }), + ), + }; +}; + +const service = (world: ReturnType, options: Partial = {}) => + DelegatedOrphanScan.make({ ...OPTIONS, ...options }).pipe(Effect.provide(world.layer)); + +const setClock = TestClock.setTime(NOW); + +/** Lets forked fibers reach their next sleep (or finish) before the clock moves. */ +const settle = Effect.repeat(Effect.yieldNow, { times: 30 }); + +/** A scan, while the clock moves past each page timeout of an environment that does not answer. */ +const scanned = (scan: Effect.Effect) => + Effect.gen(function* () { + const fiber = yield* Effect.forkChild(scan); + for (let step = 0; step < 3; step++) { + yield* settle; + yield* TestClock.adjust(OPTIONS.pageTimeout); + } + return yield* Fiber.join(fiber); + }); + +describe("DelegatedOrphanScan.scan", () => { + it.effect("reminds a parent once of a thread there that no task follows", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + const scan = yield* service(world); + const outcome = yield* scanned(scan.scan()); + expect(outcome.parents.get(PARENT)).toEqual({ type: "sent", threadIds: ["lost-1"] }); + expect(world.sent).toHaveLength(1); + expect(world.sent[0]!.text).toContain("lost-1"); + expect(world.sent[0]!.text).toContain(`in ${ENV_B}`); + expect(world.sent[0]!.text).not.toContain("known-1"); + // Runs again: the same orphans, the same message, nothing more. + yield* scanned(scan.scan()); + expect(world.sent).toHaveLength(1); + // The next start: a fresh service finds the message it sent. + yield* scanned((yield* service(world)).scan()); + expect(world.sent).toHaveLength(1); + // An expired link is never asked. + expect(world.pageReads.map((read) => read.environmentId)).not.toContain("env-expired"); + }), + ); + + it.effect("reads every page, and one reminder names the orphans of all environments", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.there.set(ENV_B, [ + { threadId: "known-1" }, + { threadId: "lost-1" }, + { threadId: "lost-2" }, + { threadId: "lost-3" }, + ]); + world.there.set(ENV_C, [{ threadId: "lost-c" }]); + const outcome = yield* scanned((yield* service(world)).scan()); + expect(outcome.partial).toEqual([]); + expect(outcome.parents.get(PARENT)).toMatchObject({ type: "sent" }); + expect(world.sent).toHaveLength(1); + for (const id of ["lost-1", "lost-2", "lost-3", "lost-c"]) { + expect(world.sent[0]!.text).toContain(id); + } + }), + ); + + it.effect("names an environment with more threads than the page limit as partial", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.there.set( + ENV_B, + Array.from({ length: 10 }, (_, index) => ({ threadId: `lost-${index}` })), + ); + const outcome = yield* scanned((yield* service(world)).scan()); + expect(outcome.partial).toEqual([ENV_B]); + expect(world.pageReads).toHaveLength(OPTIONS.maxPages); + // What it read is still reminded. + expect(world.sent).toHaveLength(1); + }), + ); + + it.effect("stops paging at threads last updated before the window", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + // Ten pages, but the clock is a day and more past the second one. + world.there.set( + ENV_B, + Array.from({ length: 20 }, (_, index) => ({ threadId: `t-${index}` })), + ); + yield* TestClock.setTime(NOW + 24 * 60 * 60_000 + 60_000); + const outcome = yield* scanned((yield* service(world)).scan()); + expect(outcome.partial).toEqual([]); + expect(world.pageReads).toHaveLength(1); + }), + ); + + it.effect("never names a thread made after the start, or one from another environment", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.there.set(ENV_B, [ + // Launched after this start (beyond the clock tolerance): its record may still be written. + { threadId: "new-1", createdAt: NOW + DelegatedOrphanScan.CLOCK_SKEW_MS + 1 }, + { threadId: "new-2", createdAt: NOW + DelegatedOrphanScan.CLOCK_SKEW_MS }, + // Launched by a thread of the same id in another environment. + { threadId: "foreign", from: { environmentId: ENV_C, threadId: PARENT } }, + { threadId: "known-1" }, + ]); + const scan = yield* service(world); + yield* TestClock.adjust(Duration.minutes(5)); + const outcome = yield* scanned(scan.scan()); + expect(outcome.parents.size).toBe(1); + expect(outcome.parents.get(PARENT)).toEqual({ type: "none" }); + expect(world.sent).toHaveLength(0); + }), + ); + + it.effect("names a thread there whose clock runs a little ahead of this one", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + // Made just before this start, stamped 30 s after it by a clock ahead of this one. + world.there.set(ENV_B, [{ threadId: "skewed", createdAt: NOW + 30_000 }]); + const outcome = yield* scanned((yield* service(world)).scan()); + expect(outcome.parents.get(PARENT)).toEqual({ type: "sent", threadIds: ["skewed"] }); + }), + ); + + it.effect("queues the message for a busy parent, one waiting on a question too", () => + Effect.gen(function* () { + yield* setClock; + for (const busy of [ + { status: "running", activityRunStatus: "running" }, + { pendingRuntimeRequest: {} as never }, + ] as const) { + const world = makeWorld(); + world.shells.set(PARENT, idleShell(PARENT, busy)); + const outcome = yield* scanned((yield* service(world)).scan()); + // Sent now: it waits in the queue until the parent's run is over. + expect(outcome.parents.get(PARENT)).toEqual({ type: "sent", threadIds: ["lost-1"] }); + expect(world.sent).toHaveLength(1); + } + }), + ); + + it.effect("never writes to a stopped, archived or deleted parent", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + const scan = yield* service(world); + world.shells.set(PARENT, idleShell(PARENT, { status: "interrupted" })); + expect((yield* scanned(scan.scan())).parents.get(PARENT)).toEqual({ + type: "held", + detail: "stopped", + }); + world.shells.set(PARENT, idleShell(PARENT, { archivedAt: at(NOW) })); + expect((yield* scanned(scan.scan())).parents.get(PARENT)).toEqual({ type: "gone" }); + world.shells.delete(PARENT); + expect((yield* scanned(scan.scan())).parents.get(PARENT)).toEqual({ type: "gone" }); + expect(world.sent).toHaveLength(0); + }), + ); + + it.effect("sends nothing when the parent's tasks cannot be read", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.followed.set(PARENT, null); + const outcome = yield* scanned((yield* service(world)).scan()); + expect(outcome.parents.get(PARENT)).toEqual({ type: "unreadable" }); + expect(world.sent).toHaveLength(0); + }), + ); + + it.effect("names only the new orphans after a restart, when the day window has moved", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.there.set(ENV_B, [{ threadId: "lost-1" }, { threadId: "lost-2" }]); + yield* scanned((yield* service(world)).scan()); + expect(world.sent).toHaveLength(1); + // A day later: lost-2 changed since (still in the window), lost-1 fell out of it (paging + // stops before it), and a new one appeared. The next start names only the new one. + const later = NOW + 24 * 60 * 60_000 + 60 * 60_000; + yield* TestClock.setTime(later); + world.followed.set(PARENT, [ + { environmentId: ENV_B, threadId: ThreadId.make("known-1") }, + { environmentId: ENV_B, threadId: ThreadId.make("known-2") }, + ]); + world.there.set(ENV_B, [ + { threadId: "lost-3", updatedAt: later - 60_000 }, + { threadId: "lost-2", updatedAt: later - 2 * 60_000 }, + { threadId: "known-1", updatedAt: NOW - 60_000 }, + { threadId: "known-2", updatedAt: NOW - 2 * 60_000 }, + { threadId: "lost-1", updatedAt: NOW - 3 * 60_000 }, + ]); + const reads = world.pageReads.length; + const outcome = yield* scanned((yield* service(world)).scan()); + expect(outcome.parents.get(PARENT)).toEqual({ type: "sent", threadIds: ["lost-3"] }); + expect(world.sent).toHaveLength(2); + expect(world.sent[1]!.text).toContain("lost-3"); + expect(world.sent[1]!.text).not.toContain("lost-2"); + // lost-1's page was never read. + expect(world.pageReads.slice(reads).map((read) => read.cursor)).toEqual([0, 2]); + }), + ); + + it.effect("after a stop half-way through its messages, sends only the ones not sent", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + const orphans = Array.from({ length: 12 }, (_, index) => ({ + threadId: `lost-${String(index).padStart(2, "0")}`, + })); + world.there.set(ENV_B, orphans); + world.followed.set(PARENT, []); + // The last start sent the first message (ten orphans) and stopped before the second. + const first = orphanBatchIds( + PARENT, + orphans.slice(0, 10).map((orphan) => ({ + environmentId: ENV_B, + threadId: ThreadId.make(orphan.threadId), + })), + ); + world.sent.push({ threadId: PARENT, messageId: first.messageId, text: "" }); + world.receipts.set(first.commandId, PARENT); + // Two threads a page: every page read. + const allPages = { maxPages: 10 }; + const outcome = yield* scanned((yield* service(world, allPages)).scan()); + expect(outcome.parents.get(PARENT)).toEqual({ + type: "sent", + threadIds: ["lost-10", "lost-11"], + }); + expect(world.sent).toHaveLength(2); + // Only the receipt survived (no message row): it still counts as sent. + world.sent.splice(0, 1); + expect( + (yield* scanned((yield* service(world, allPages)).scan())).parents.get(PARENT), + ).toEqual({ type: "none" }); + expect(world.sent).toHaveLength(1); + }), + ); + + it.effect("takes the first scan's message for the same orphans as sent", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + const old = legacyOrphanSetId(PARENT, [ + { environmentId: ENV_B, threadId: ThreadId.make("lost-1") }, + ]); + world.sent.push({ threadId: PARENT, messageId: old, text: "" }); + world.receipts.set(`${old}:0`, PARENT); + const outcome = yield* scanned((yield* service(world)).scan()); + expect(outcome.parents.get(PARENT)).toEqual({ type: "none" }); + expect(world.sent).toHaveLength(1); + }), + ); + + it.effect("is off while waits are off or Linked environments is off", () => + Effect.gen(function* () { + yield* setClock; + const off = makeWorld({ enabled: false }); + expect((yield* scanned((yield* service(off)).scan())).type).toBe("off"); + expect(off.pageReads).toHaveLength(0); + const unlinked = makeWorld(); + unlinked.linksOff.value = true; + expect((yield* scanned((yield* service(unlinked)).scan())).type).toBe("off"); + expect(unlinked.sent).toHaveLength(0); + }), + ); +}); + +describe("DelegatedOrphanScan.run", () => { + it.effect("scans after the delay, and again while an environment does not answer", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.there.set(ENV_C, [{ threadId: "lost-c" }]); + world.silent.add(ENV_C); + const scan = yield* service(world); + const fiber = yield* scan.run().pipe(Effect.forkChild); + yield* settle; + expect(world.pageReads).toHaveLength(0); + yield* TestClock.adjust(OPTIONS.delay); + yield* settle; + yield* TestClock.adjust(OPTIONS.pageTimeout); + yield* settle; + // The first scan reminded what env-b showed; env-c did not answer. + expect(world.sent).toHaveLength(1); + expect(world.sent[0]!.text).not.toContain("lost-c"); + world.silent.delete(ENV_C); + yield* TestClock.adjust(OPTIONS.retryDelay); + yield* settle; + // The next scan read env-c: one new message names only what was not named yet. + expect(world.sent).toHaveLength(2); + expect(world.sent[1]!.text).toContain("lost-c"); + expect(world.sent[1]!.text).not.toContain("lost-1"); + // Nothing left to wait for: no third scan. + yield* Fiber.join(fiber); + const reads = world.pageReads.length; + yield* TestClock.adjust(OPTIONS.retryDelay); + yield* settle; + expect(world.pageReads).toHaveLength(reads); + }), + ); + + it.effect("needs no second scan for a busy parent with no orphan", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.followed.set(PARENT, [ + { environmentId: ENV_B, threadId: ThreadId.make("known-1") }, + { environmentId: ENV_B, threadId: ThreadId.make("lost-1") }, + ]); + world.shells.set( + PARENT, + idleShell(PARENT, { status: "running", activityRunStatus: "running" }), + ); + const fiber = yield* (yield* service(world)).run().pipe(Effect.forkChild); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + yield* settle; + yield* Fiber.join(fiber); + expect(world.pageReads).toHaveLength(1); + }), + ); + + it.effect("scans again while a page or a parent is unread, then logs what is left", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + // env-c answers its first page, never its second. Another parent's tasks cannot be read. + world.there.set(ENV_C, [{ threadId: "c-1" }, { threadId: "c-2" }, { threadId: "c-3" }]); + world.cutShort.add(ENV_C); + const other = ThreadId.make("thread-other"); + world.there.get(ENV_B)!.push({ + threadId: "other-1", + from: { environmentId: HERE, threadId: other }, + }); + world.followed.set(other, null); + const warnings: Array = []; + const logger = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(JSON.stringify(message)); + }); + const scan = yield* service(world); + const fiber = yield* scan + .run() + .pipe( + Effect.provide(Logger.layer([logger], { mergeWithExisting: false })), + Effect.forkChild, + ); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + for (let attempt = 1; attempt <= OPTIONS.attempts; attempt += 1) { + yield* settle; + yield* TestClock.adjust(OPTIONS.pageTimeout); + yield* settle; + if (attempt < OPTIONS.attempts) yield* TestClock.adjust(OPTIONS.retryDelay); + } + yield* Fiber.join(fiber); + // Each scan read env-c's first page again, and its second did not answer. + expect( + world.pageReads.filter((read) => read.environmentId === ENV_C && read.cursor === 0), + ).toHaveLength(OPTIONS.attempts); + const left = warnings.filter((warning) => warning.includes("work unchecked")); + expect(left).toHaveLength(1); + expect(left[0]).toContain(ENV_C); + expect(left[0]).toContain(other); + // A page that did not answer is not called the page limit. + expect(warnings.some((warning) => warning.includes("page limit"))).toBe(false); + // What env-c's first page showed was named once, over all the scans. + expect(world.sent).toHaveLength(1); + expect(world.sent[0]!.text).toContain("c-1"); + }), + ); + + it.effect("scans again while an environment answers only its first page", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.there.set(ENV_C, [{ threadId: "c-1" }, { threadId: "c-2" }, { threadId: "c-3" }]); + world.cutShort.add(ENV_C); + const fiber = yield* (yield* service(world)).run().pipe(Effect.forkChild); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + yield* settle; + yield* TestClock.adjust(OPTIONS.pageTimeout); + yield* settle; + // env-c answers in full now: the next scan reads its second page and names c-3 alone. + world.cutShort.delete(ENV_C); + yield* TestClock.adjust(OPTIONS.retryDelay); + yield* settle; + yield* Fiber.join(fiber); + expect(world.pageReads.filter((read) => read.environmentId === ENV_C)).toHaveLength(4); + expect(world.sent).toHaveLength(2); + expect(world.sent[1]!.text).toContain("c-3"); + expect(world.sent[1]!.text).not.toContain("c-1"); + }), + ); +}); + +describe("DelegatedOrphanScan.layerWith", () => { + const started = (heldFromBoot: boolean) => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + yield* Effect.scoped( + Effect.gen(function* () { + yield* Layer.build( + DelegatedOrphanScan.layerWith(OPTIONS).pipe( + Layer.provide( + Layer.merge( + world.layer, + Layer.succeed(WorkAdmission.WorkAdmission, { heldFromBoot } as never), + ), + ), + ), + ); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + yield* settle; + }), + ); + return world; + }); + + it.effect("scans after a normal start, never after a held one", () => + Effect.gen(function* () { + expect((yield* started(false)).sent).toHaveLength(1); + const held = yield* started(true); + expect(held.pageReads).toHaveLength(0); + expect(held.sent).toHaveLength(0); + }), + ); +}); diff --git a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts new file mode 100644 index 0000000000000..0a1fe5c18f15d --- /dev/null +++ b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts @@ -0,0 +1,298 @@ +/** + * Fork (issue 625, part of 386): names each thread there that no task here follows, made before the + * start, to its parent once (ids carry one key per orphan, so reruns and restarts never repeat one). + * Upstream's fix (record before launch) is not merged; slim or drop this scan once a sync brings it. + */ +import type { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import type { McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; +import { OrchestratorToolkit } from "../../mcp/toolkits/orchestrator/tools.ts"; +import * as CommandReceiptStore from "../../orchestration-v2/CommandReceiptStore.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as PeerForwarding from "../../peer/PeerForwarding.ts"; +import { forkParked } from "../../serverActivation.ts"; +import { ServerSettingsService } from "../../serverSettings.ts"; +import * as WaitDelivery from "../../waits/WaitDelivery.ts"; +import * as WorkAdmission from "../../workAdmission.ts"; +import { + type LinkedThread, + ORPHAN_ID_PREFIX, + orphanBatches, + orphanBatchIds, + orphanReminderText, + orphansOf, + parentGone, + REMINDER_MAX_AGE_MS, +} from "./delegatedResults.ts"; +import * as DelegatedResultsStore from "./DelegatedResultsStore.ts"; +import { LINKED_PAGE_SIZE, listLinkedThreads } from "./linkedThreads.ts"; + +export interface OrphanScanOptions { + /** From the start to the first scan: links come back and followers settle first. */ + readonly delay: Duration.Input; + /** Between scans while an environment did not answer (fully) or a parent could not be read. */ + readonly retryDelay: Duration.Input; + /** Scans in all, the first one included. */ + readonly attempts: number; + /** Pages of `t3_thread_list` read per environment and scan. */ + readonly maxPages: number; + /** The longest one page may take before the environment counts as not answering. */ + readonly pageTimeout: Duration.Input; +} + +/** + * A thread there counts as made before this start up to this long after it: the other machine's + * clock may run ahead of this one. A launch in that minute was recorded long before the first + * scan (`delay`), so it is never raced; only a launch whose record failed (the agent saw an error + * then) may be named too, and the message's wording covers that. + */ +export const CLOCK_SKEW_MS = 60_000; + +const DEFAULT_OPTIONS: OrphanScanOptions = { + delay: Duration.minutes(3), + retryDelay: Duration.minutes(10), + attempts: 6, + maxPages: 20, + pageTimeout: Duration.seconds(10), +}; + +/** What one parent's check did; tests read it. */ +export type ParentOutcome = + /** No orphan, or every one was named before. */ + | { readonly type: "none" } + | { readonly type: "gone" } + | { readonly type: "unreadable" } + /** Named now; a busy parent reads it after its current run (an open question included). */ + | { readonly type: "sent"; readonly threadIds: ReadonlyArray } + | { readonly type: "held"; readonly detail: string }; + +/** What one scan did; tests read it. */ +export interface ScanOutcome { + readonly type: "off" | "scanned"; + /** Environments whose first page did not answer: scanned again. */ + readonly unreachable: ReadonlyArray; + /** Environments whose later page did not answer: scanned again. */ + readonly cutShort: ReadonlyArray; + /** Environments with more matching threads than the page limit reads (a rerun reads the same). */ + readonly partial: ReadonlyArray; + readonly parents: ReadonlyMap; +} + +const OFF: ScanOutcome = { + type: "off", + unreachable: [], + cutShort: [], + partial: [], + parents: new Map(), +}; + +export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( + options: OrphanScanOptions = DEFAULT_OPTIONS, +) { + const threads = yield* ThreadManagement.ThreadManagementService; + const delivery = yield* WaitDelivery.make(); + const settings = yield* ServerSettingsService; + const store = yield* DelegatedResultsStore.DelegatedResultsStore; + const forwarding = yield* PeerForwarding.PeerForwarding; + const hereId = yield* ServerEnvironment.ServerEnvironment.pipe( + Effect.flatMap((environment) => environment.getEnvironmentId), + ); + // Only a launch before this start can have lost its record: a later one is recorded by this + // server, or found by the scan after the next start. + const startedAtMs = yield* Clock.currentTimeMillis; + // The server itself asks, not an agent: no thread, no client. + const scope: McpInvocationScope = { + environmentId: hereId, + requestNamespace: "delegated-orphan-scan", + thread: undefined, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 0, + }; + + const checkParent = Effect.fn("DelegatedOrphanScan.checkParent")(function* ( + parentThreadId: ThreadId, + linked: ReadonlyArray, + ) { + // The orphans first: a parent with none never needs its shell, nor another scan. + const followed = yield* store.remoteChildrenOf(parentThreadId); + if (followed === null) return { type: "unreadable" } satisfies ParentOutcome; + const orphans = orphansOf(linked, hereId, parentThreadId, followed); + if (orphans.length === 0) return { type: "none" } satisfies ParentOutcome; + const sentIds = yield* store.sentIds(parentThreadId, ORPHAN_ID_PREFIX); + if (sentIds === null) return { type: "unreadable" } satisfies ParentOutcome; + const batches = orphanBatches(parentThreadId, orphans, sentIds); + if (batches.length === 0) return { type: "none" } satisfies ParentOutcome; + const shell = yield* threads + .getThreadShell(parentThreadId) + .pipe(Effect.orElseSucceed(() => undefined)); + if (shell === undefined) return { type: "unreadable" } satisfies ParentOutcome; + if (parentGone(shell)) return { type: "gone" } satisfies ParentOutcome; + // A busy parent is not held: its run's end changes nothing about an orphan, and the message + // waits in its queue until that run (a question to a person included) is over. + const sent: Array = []; + let retry = false; + let refused: string | undefined; + for (const batch of batches) { + const ids = orphanBatchIds(parentThreadId, batch); + const result = yield* delivery.deliver({ + threadId: parentThreadId, + messageId: ids.messageId, + commandId: ids.commandId, + text: orphanReminderText(batch), + }); + switch (result.type) { + case "delivered": + sent.push(...batch.map((orphan) => orphan.threadId)); + break; + case "gone": + return { type: "gone" } satisfies ParentOutcome; + case "stopped": + // A person stopped it: never restart it from here. + return { type: "held", detail: "stopped" } satisfies ParentOutcome; + case "rejected": + yield* Effect.logWarning("delegated orphan reminder refused", { + threadId: parentThreadId, + detail: result.detail, + }); + refused = result.detail; + break; + case "retry": + retry = true; + break; + } + } + if (retry) return { type: "unreadable" } satisfies ParentOutcome; + if (refused !== undefined) return { type: "held", detail: refused } satisfies ParentOutcome; + return { type: "sent", threadIds: sent } satisfies ParentOutcome; + }); + + /** One pass over every linked environment; a parent is checked once, over all of them. */ + const scan = Effect.fn("DelegatedOrphanScan.scan")(function* () { + const enabled = yield* settings.getSettings.pipe( + Effect.map((current) => current.waitFor.enabled), + Effect.orElseSucceed(() => false), + ); + if (!enabled) return OFF; + // Fails while Linked environments is off here. + const links = yield* forwarding.links(scope).pipe(Effect.orElseSucceed(() => null)); + if (links === null) return OFF; + const sinceMs = startedAtMs - REMINDER_MAX_AGE_MS; + const unreachable: Array = []; + const cutShort: Array = []; + const partial: Array = []; + const byParent = new Map>(); + yield* Effect.forEach( + links.links.filter((link) => link.status !== "expired"), + (link) => + listLinkedThreads( + (cursor) => + forwarding.call(scope, OrchestratorToolkit.tools.t3_thread_list, link.environmentId, { + allProjects: true, + limit: LINKED_PAGE_SIZE, + cursor, + }), + link.environmentId, + { + maxPages: options.maxPages, + timeout: options.pageTimeout, + sinceMs, + createdBeforeMs: startedAtMs + CLOCK_SKEW_MS, + }, + ).pipe( + Effect.map((read) => { + if (read.state === "unreachable") unreachable.push(link.environmentId); + if (read.state === "cut-short") cutShort.push(link.environmentId); + if (read.state === "limit") partial.push(link.environmentId); + for (const thread of read.threads) { + if (thread.delegatedFrom.environmentId !== hereId) continue; + const parent = thread.delegatedFrom.threadId; + byParent.set(parent, [...(byParent.get(parent) ?? []), thread]); + } + }), + ), + { concurrency: 4, discard: true }, + ); + const parents = new Map(); + for (const [parentThreadId, linked] of byParent) { + parents.set(parentThreadId, yield* checkParent(parentThreadId, linked)); + } + if (partial.length > 0) { + yield* Effect.logWarning( + "delegated orphan scan read only the newest threads: the page limit was reached", + { environmentIds: partial, threads: options.maxPages * LINKED_PAGE_SIZE }, + ); + } + return { type: "scanned", unreachable, cutShort, partial, parents } satisfies ScanOutcome; + }); + + const unreadableParents = (outcome: ScanOutcome) => + [...outcome.parents.entries()] + .filter(([, parent]) => parent.type === "unreadable") + .map(([threadId]) => threadId); + + /** Another pass while an environment did not answer (fully) or a parent could not be read. */ + const needsAnother = (outcome: ScanOutcome): boolean => + outcome.type === "scanned" && + (outcome.unreachable.length > 0 || + outcome.cutShort.length > 0 || + unreadableParents(outcome).length > 0); + + /** + * Scans after the delay, and again while `needsAnother`, at most `attempts` times. What is still + * unchecked then is logged by name; the next start scans it again (within its one-day window). + */ + const run = Effect.fn("DelegatedOrphanScan.run")( + function* () { + yield* Effect.sleep(options.delay); + for (let attempt = 1; attempt <= options.attempts; attempt += 1) { + const outcome = yield* scan(); + if (!needsAnother(outcome)) return; + if (attempt < options.attempts) { + yield* Effect.sleep(options.retryDelay); + continue; + } + yield* Effect.logWarning( + "delegated orphan scan stopped with work unchecked until the next start", + { + scans: options.attempts, + unansweredEnvironmentIds: outcome.unreachable, + cutShortEnvironmentIds: outcome.cutShort, + unreadableParentThreadIds: unreadableParents(outcome), + }, + ); + } + }, + (effect) => + effect.pipe( + Effect.catchCause((cause) => Effect.logWarning("delegated orphan scan failed", { cause })), + ), + ); + + return { scan, run, checkParent }; +}); + +/** A held server (`serve --hold-work`) neither follows nor reminds; its next start does. */ +export const layerWith = (options: OrphanScanOptions = DEFAULT_OPTIONS) => + Layer.effectDiscard( + Effect.gen(function* () { + if ((yield* WorkAdmission.WorkAdmission).heldFromBoot) return; + const service = yield* make(options); + yield* forkParked(service.run()); + }), + ); + +/** + * In the routes beside upstream's follower of linked tasks (fork/forkLayers.ts), which has the + * links. + */ +export const layer = layerWith().pipe( + Layer.provide( + Layer.merge(CommandReceiptStore.layerFromApplicationReceipts, DelegatedResultsStore.layer), + ), +); diff --git a/apps/server/src/fork/delegatedResults/DelegatedResultReminder.test.ts b/apps/server/src/fork/delegatedResults/DelegatedResultReminder.test.ts index 4c924267aeba2..a1e4d7dfe07d3 100644 --- a/apps/server/src/fork/delegatedResults/DelegatedResultReminder.test.ts +++ b/apps/server/src/fork/delegatedResults/DelegatedResultReminder.test.ts @@ -1,7 +1,8 @@ /** - * Fork (issue 386): the reminder of delegated results that never reached their parent. The state - * question: it runs again (no second message), the server restarted (a fresh service finds the - * message it sent), the parent is busy or was stopped, and old rows from before delivery records. + * Fork (issues 386, 625): the reminder of delegated results that never reached their parent. The + * state question: it runs again (no second message), the server restarted (a fresh service finds + * the message it sent, and the table finds a parent with no wake reserved), the parent is busy or + * was stopped, and rows with no delivery state (old ones, or a wait that never acknowledged). */ import { describe, expect, it } from "@effect/vitest"; import { @@ -11,6 +12,7 @@ import { type OrchestrationV2ThreadShell, ThreadId, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -28,6 +30,7 @@ import { ServerSettingsService } from "../../serverSettings.ts"; import { REMINDER_MAX_AGE_MS } from "./delegatedResults.ts"; import { at, delegatedTask, NOW, PARENT } from "./delegatedResults.testkit.ts"; import * as DelegatedResultReminder from "./DelegatedResultReminder.ts"; +import * as DelegatedResultsStore from "./DelegatedResultsStore.ts"; const OPTIONS: DelegatedResultReminder.ReminderOptions = { delay: Duration.minutes(2), @@ -61,10 +64,15 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = const sent: Array = []; const receipts = new Map(); let failSends = 0; + // Parents whose wake upstream reserved (null: every parent with tasks), and those the table names. + const recovery: { ids: ReadonlyArray | null } = { ids: null }; + const unread: { ids: ReadonlyArray; since: Array } = { ids: [], since: [] }; return { shells, tasks, sent, + recovery, + unread, failNextSends: (count: number) => { failSends = count; }, @@ -113,7 +121,14 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = ), }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ - getRecoveryThreadIds: () => Effect.succeed([...tasks.keys()]), + getRecoveryThreadIds: () => Effect.succeed(recovery.ids ?? [...tasks.keys()]), + }), + Layer.mock(DelegatedResultsStore.DelegatedResultsStore)({ + parentsWithUnreadResults: (sinceIso) => + Effect.sync(() => { + unread.since.push(sinceIso); + return unread.ids; + }), }), ), }; @@ -212,7 +227,10 @@ describe("DelegatedResultReminder.check", () => { yield* setClock; const world = makeWorld(); world.tasks.set(PARENT, [ - delegatedTask({ completionDelivery: undefined }), + delegatedTask({ + completionDelivery: undefined, + completedAt: at(NOW - REMINDER_MAX_AGE_MS - 1), + }), delegatedTask({ id: "task-2", completedAt: at(NOW - REMINDER_MAX_AGE_MS - 1) }), delegatedTask({ id: "task-3", origin: "provider_native" }), ]); @@ -221,6 +239,38 @@ describe("DelegatedResultReminder.check", () => { }), ); + it.effect("reminds a wait task whose acknowledgement failed (no delivery state)", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + const task = delegatedTask({ completionDelivery: undefined }); + world.tasks.set(PARENT, [task]); + const reminder = yield* service(world); + expect(reminder.triggers({ type: "subagent.updated", threadId: PARENT, payload: task })).toBe( + true, + ); + expect(yield* reminder.check(PARENT)).toEqual({ type: "sent", taskIds: ["task-1"] }); + // The same task read later with an acknowledgement is settled: nothing more. + world.tasks.set(PARENT, [ + delegatedTask({ completionDelivery: { state: "acknowledged", observedByRunId: null } }), + ]); + expect(yield* reminder.check(PARENT)).toEqual({ type: "none" }); + expect(world.sent).toHaveLength(1); + }), + ); + + it.effect("reminds a parent that left background work running (no run end may come)", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.shells.set( + PARENT, + idleShell(PARENT, { pendingBackgroundTasks: [{ id: "monitor" } as never] }), + ); + expect(yield* (yield* service(world)).check(PARENT)).toMatchObject({ type: "sent" }); + }), + ); + it.effect("waits while the parent works, and checks again when its run ends", () => Effect.gen(function* () { yield* setClock; @@ -243,6 +293,15 @@ describe("DelegatedResultReminder.check", () => { world.shells.set(PARENT, idleShell(PARENT)); expect(yield* reminder.check(PARENT)).toMatchObject({ type: "sent" }); expect(reminder.watched.has(PARENT)).toBe(false); + // A run update that is no end is no trigger, even for a watched parent. + reminder.watched.add(PARENT); + for (const status of ["queued", "running", "waiting"]) { + expect( + reminder.triggers({ type: "run.updated", threadId: PARENT, payload: { status } }), + ).toBe(false); + } + expect(reminder.triggers({ type: "run.updated", threadId: PARENT, payload: {} })).toBe(false); + reminder.watched.delete(PARENT); // A run end of a parent with nothing owed is no trigger. expect( reminder.triggers({ @@ -261,6 +320,15 @@ describe("DelegatedResultReminder.check", () => { world.shells.set(PARENT, idleShell(PARENT, { status: "interrupted" })); const reminder = yield* service(world); expect(yield* reminder.check(PARENT)).toEqual({ type: "held", detail: "stopped" }); + // Not restarted, but its next run end (a person sends it on) checks again. + expect(reminder.watched.has(PARENT)).toBe(true); + expect( + reminder.triggers({ + type: "run.updated", + threadId: PARENT, + payload: { status: "completed" }, + }), + ).toBe(true); world.shells.set(PARENT, idleShell(PARENT, { archivedAt: at(NOW) })); expect(yield* reminder.check(PARENT)).toEqual({ type: "gone" }); world.shells.delete(PARENT); @@ -337,6 +405,57 @@ describe("DelegatedResultReminder.start", () => { }).pipe(Effect.scoped), ); + it.effect("after a restart, finds a parent whose result had no wake reserved", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.recovery.ids = []; + world.unread.ids = [PARENT]; + const reminder = yield* service(world); + const fiber = yield* reminder.start().pipe(Effect.andThen(Effect.never), Effect.forkScoped); + yield* settle; + // It asks for results from the last day only. + expect(world.unread.since).toEqual([DateTime.formatIso(at(NOW - REMINDER_MAX_AGE_MS))]); + yield* TestClock.adjust(OPTIONS.delay); + yield* settle; + expect(world.sent).toHaveLength(1); + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped), + ); + + it.effect("the startup scan leaves a parent to the check an event already moved later", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + const pubsub = yield* PubSub.unbounded(); + const reminder = yield* service(world, Stream.fromPubSub(pubsub)); + const fiber = yield* reminder.start().pipe(Effect.andThen(Effect.never), Effect.forkScoped); + yield* settle; + // The parent works again just before the scan's delay ends: its run end comes later. + yield* TestClock.adjust(Duration.minutes(1)); + yield* PubSub.publish(pubsub, { + type: "subagent.updated", + threadId: PARENT, + payload: delegatedTask(), + }); + yield* settle; + world.shells.set( + PARENT, + idleShell(PARENT, { status: "running", activityRunStatus: "running" }), + ); + yield* TestClock.adjust(Duration.minutes(1)); + yield* settle; + // The scan skipped it: the parent was not marked busy (watched) by a scan of its own. + expect(world.sent).toHaveLength(0); + expect(reminder.watched.has(PARENT)).toBe(false); + world.shells.set(PARENT, idleShell(PARENT)); + yield* TestClock.adjust(Duration.minutes(1)); + yield* settle; + expect(world.sent).toHaveLength(1); + yield* Fiber.interrupt(fiber); + }).pipe(Effect.scoped), + ); + it.effect("ignores events that are no trigger", () => { const world = makeWorld(); return Effect.gen(function* () { diff --git a/apps/server/src/fork/delegatedResults/DelegatedResultReminder.ts b/apps/server/src/fork/delegatedResults/DelegatedResultReminder.ts index 13025caa28eb1..b7bd1ac38343c 100644 --- a/apps/server/src/fork/delegatedResults/DelegatedResultReminder.ts +++ b/apps/server/src/fork/delegatedResults/DelegatedResultReminder.ts @@ -1,10 +1,12 @@ /** - * Fork (issue 386): backstop for upstream's completion wake. A parent's own events (a task ending - * owed, a run ending) start one delayed check; an idle parent with results still owed gets one queued - * message, ids from the task ids, so reruns and restarts never repeat it. Off with `server.waitFor`. + * Fork (issues 386, 625): backstop for upstream's completion wake. A parent's own events (a task + * ending unread, a run ending) start one delayed check; an idle parent with results still unread + * gets one queued message, ids from the task ids, so reruns and restarts never repeat it. At start, + * every parent with a result unread from the last day is checked once. Off with `server.waitFor`. */ import type { OrchestrationV2Subagent, ThreadId } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FiberMap from "effect/FiberMap"; @@ -17,11 +19,13 @@ import * as ThreadManagement from "../../orchestration-v2/ThreadManagementServic import { forkParked } from "../../serverActivation.ts"; import { ServerSettingsService } from "../../serverSettings.ts"; import * as WaitDelivery from "../../waits/WaitDelivery.ts"; +import * as DelegatedResultsStore from "./DelegatedResultsStore.ts"; import { isDelegatedTask, isEndedTask, isRemindable, parentBusy, + REMINDER_MAX_AGE_MS, parentGone, reminderIds, reminderText, @@ -62,6 +66,7 @@ export const make = Effect.fn("DelegatedResultReminder.make")(function* ( const delivery = yield* WaitDelivery.make(); const projections = yield* ProjectionStore.ProjectionStoreV2; const settings = yield* ServerSettingsService; + const store = yield* DelegatedResultsStore.DelegatedResultsStore; // Parents that were busy (or could not be reached) while a result was owed: their next run end // checks again. In memory; a restart rebuilds it from the startup scan. const watched = new Set(); @@ -149,7 +154,10 @@ export const make = Effect.fn("DelegatedResultReminder.make")(function* ( ), ); - /** A task that ended with its result owed, or a run end of a watched parent. */ + /** + * A task that ended with its result owed or with no delivery state (a wait that may not + * acknowledge it), or a run end of a watched parent. + */ const triggers = (event: { readonly type: string; readonly threadId: ThreadId; @@ -157,7 +165,10 @@ export const make = Effect.fn("DelegatedResultReminder.make")(function* ( }): boolean => { if (event.type === "subagent.updated") { const task = event.payload as OrchestrationV2Subagent; - return isDelegatedTask(task) && isEndedTask(task) && resultDelivery(task) === "owed"; + const delivery = resultDelivery(task); + return ( + isDelegatedTask(task) && isEndedTask(task) && (delivery === "owed" || delivery === "none") + ); } if (event.type === "run.updated" && watched.has(event.threadId)) { const run = event.payload as { readonly status?: string }; @@ -176,15 +187,20 @@ export const make = Effect.fn("DelegatedResultReminder.make")(function* ( : Effect.void, ), ); - // Results owed when the server stopped: upstream offers their wakes again at start; a parent - // that still has one owed after the delay gets its reminder. One parent at a time. + // Results unread when the server stopped: upstream offers the wakes it had reserved again at + // start; the table also names parents whose result had no wake reserved yet. A parent that + // still has one unread after the delay gets its reminder. One parent at a time. yield* forkParked( Effect.gen(function* () { - const parents = yield* projections + const recovering = yield* projections .getRecoveryThreadIds("delegated-completions") .pipe(Effect.orElseSucceed((): ReadonlyArray => [])); + const sinceMs = (yield* Clock.currentTimeMillis) - REMINDER_MAX_AGE_MS; + const unread = yield* store.parentsWithUnreadResults( + DateTime.formatIso(DateTime.makeUnsafe(sinceMs)), + ); yield* Effect.sleep(options.delay); - for (const parentThreadId of new Set(parents)) { + for (const parentThreadId of new Set([...recovering, ...unread])) { // An event already moved this parent's check later. if (yield* FiberMap.has(timers, parentThreadId)) continue; yield* check(parentThreadId).pipe( @@ -208,6 +224,10 @@ export const layer = Layer.effectDiscard( make().pipe(Effect.flatMap((service) => service.start())), ).pipe( Layer.provide( - Layer.merge(ProjectionStore.layer, CommandReceiptStore.layerFromApplicationReceipts), + Layer.mergeAll( + ProjectionStore.layer, + CommandReceiptStore.layerFromApplicationReceipts, + DelegatedResultsStore.layer, + ), ), ); diff --git a/apps/server/src/fork/delegatedResults/DelegatedResultsStore.test.ts b/apps/server/src/fork/delegatedResults/DelegatedResultsStore.test.ts new file mode 100644 index 0000000000000..e712668465935 --- /dev/null +++ b/apps/server/src/fork/delegatedResults/DelegatedResultsStore.test.ts @@ -0,0 +1,152 @@ +/** + * Fork (issue 625): the reminder's own reads over the real schema. The state question: rows from + * before delivery states (no state), settled and running rows, rows older than the window, a + * provider's own subagent, an empty table, and a send whose message row is missing but whose + * receipt was kept (or the other way round). + */ +import { assert, it } from "@effect/vitest"; +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/sql/SqlClient"; + +import * as SqlitePersistence from "../../persistence/Sqlite.ts"; +import * as DelegatedResultsStore from "./DelegatedResultsStore.ts"; + +const layer = DelegatedResultsStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)); + +const SINCE = "2026-11-19T12:00:00.000Z"; +const RECENT = "2026-11-20T11:59:00.000Z"; +const OLD = "2026-11-19T11:59:59.999Z"; + +const insert = (row: { + readonly id: string; + readonly thread: string; + readonly status?: string; + readonly origin?: string; + readonly delivery?: string | null; + readonly completedAt?: string | null; + readonly updatedAt?: string; + readonly childThreadId?: string | null; + readonly remote?: { readonly environmentId: string; readonly threadId: string }; +}) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const payload = row.remote === undefined ? {} : { remoteChild: { ...row.remote, label: "b" } }; + yield* sql` + INSERT INTO orchestration_v2_projection_subagents ( + subagent_id, thread_id, run_id, parent_node_id, provider, driver, provider_instance_id, + provider_thread_id, child_thread_id, origin, status, started_at, completed_at, updated_at, + completion_delivery_state, payload_json + ) VALUES ( + ${row.id}, ${row.thread}, NULL, ${`node-${row.id}`}, 'codex', 'codex', 'codex', NULL, + ${row.childThreadId === undefined ? `child-${row.id}` : row.childThreadId}, + ${row.origin ?? "app_owned"}, ${row.status ?? "completed"}, ${RECENT}, + ${row.completedAt === undefined ? RECENT : row.completedAt}, + ${row.updatedAt ?? RECENT}, + ${row.delivery === undefined ? "claimed" : row.delivery}, + ${JSON.stringify(payload)} + ) + `; + }); + +it.layer(layer)("DelegatedResultsStore", (it) => { + it.effect("an empty table names no parent and no linked child", () => + Effect.gen(function* () { + const store = yield* DelegatedResultsStore.DelegatedResultsStore; + assert.deepStrictEqual(yield* store.parentsWithUnreadResults(SINCE), []); + assert.deepStrictEqual(yield* store.remoteChildrenOf(ThreadId.make("p-none")), []); + assert.deepStrictEqual(yield* store.sentIds(ThreadId.make("p-none"), "delegated-orphan"), []); + }), + ); + + it.effect("names each parent once with a recent unread or stateless result", () => + Effect.gen(function* () { + yield* insert({ id: "owed-1", thread: "p-owed" }); + yield* insert({ id: "owed-2", thread: "p-owed", delivery: "pending" }); + // No delivery state: an old row, or a wait that never acknowledged it. + yield* insert({ id: "none-1", thread: "p-none", delivery: null }); + // No end time recorded: its last update counts. + yield* insert({ id: "no-end", thread: "p-no-end", completedAt: null }); + yield* insert({ id: "read", thread: "p-read", delivery: "acknowledged" }); + yield* insert({ id: "sent", thread: "p-read", delivery: "delivered" }); + yield* insert({ id: "running", thread: "p-running", status: "running", delivery: null }); + yield* insert({ id: "old", thread: "p-old", completedAt: OLD, updatedAt: OLD }); + yield* insert({ id: "native", thread: "p-native", origin: "provider_native" }); + const store = yield* DelegatedResultsStore.DelegatedResultsStore; + assert.deepStrictEqual(yield* store.parentsWithUnreadResults(SINCE), [ + ThreadId.make("p-no-end"), + ThreadId.make("p-none"), + ThreadId.make("p-owed"), + ]); + }), + ); + + it.effect("lists the linked children a parent's tasks follow, settled ones too", () => + Effect.gen(function* () { + const parent = "p-linked"; + yield* insert({ + id: "r-1", + thread: parent, + childThreadId: null, + remote: { environmentId: "env-b", threadId: "remote-1" }, + }); + yield* insert({ + id: "r-2", + thread: parent, + childThreadId: null, + delivery: "acknowledged", + remote: { environmentId: "env-c", threadId: "remote-2" }, + }); + // A local child, and another parent's linked child. + yield* insert({ id: "local", thread: parent }); + yield* insert({ + id: "r-other", + thread: "p-other", + childThreadId: null, + remote: { environmentId: "env-b", threadId: "remote-3" }, + }); + const store = yield* DelegatedResultsStore.DelegatedResultsStore; + const children = yield* store.remoteChildrenOf(ThreadId.make(parent)); + assert.deepStrictEqual( + children?.toSorted((a, b) => (a.threadId < b.threadId ? -1 : 1)), + [ + { environmentId: EnvironmentId.make("env-b"), threadId: ThreadId.make("remote-1") }, + { environmentId: EnvironmentId.make("env-c"), threadId: ThreadId.make("remote-2") }, + ], + ); + }), + ); + + it.effect("reads a thread's sent ids of one kind from its messages and accepted receipts", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const message = (id: string, thread: string) => sql` + INSERT INTO orchestration_v2_projection_messages ( + message_id, thread_id, run_id, node_id, role, streaming, created_at, updated_at, payload_json + ) VALUES (${id}, ${thread}, NULL, NULL, 'user', 0, ${RECENT}, ${RECENT}, '{}') + `; + const receipt = (id: string, thread: string, status: string) => sql` + INSERT INTO orchestration_v2_command_receipts ( + command_id, thread_id, command_type, accepted_at, result_sequence, status, error + ) VALUES (${id}, ${thread}, 'message.dispatch', ${RECENT}, 1, ${status}, NULL) + `; + yield* message("delegated-orphan:p-sent:aaaaaaaaaaaa", "p-sent"); + yield* receipt("delegated-orphan:p-sent:aaaaaaaaaaaa:0", "p-sent", "accepted"); + // Accepted, but its message row is gone: the receipt still records it. + yield* receipt("delegated-orphan:p-sent:bbbbbbbbbbbb:0", "p-sent", "accepted"); + // A refused command told nobody. + yield* receipt("delegated-orphan:p-sent:cccccccccccc:0", "p-sent", "rejected"); + // Another kind of message, and another thread's reminder. + yield* message("delegated-reminder:p-sent:dddd", "p-sent"); + yield* message("delegated-orphan:p-other:eeeeeeeeeeee", "p-other"); + const store = yield* DelegatedResultsStore.DelegatedResultsStore; + const ids = yield* store.sentIds(ThreadId.make("p-sent"), "delegated-orphan"); + assert.deepStrictEqual(ids?.toSorted(), [ + "delegated-orphan:p-sent:aaaaaaaaaaaa", + "delegated-orphan:p-sent:aaaaaaaaaaaa:0", + "delegated-orphan:p-sent:bbbbbbbbbbbb:0", + ]); + }), + ); +}); diff --git a/apps/server/src/fork/delegatedResults/DelegatedResultsStore.ts b/apps/server/src/fork/delegatedResults/DelegatedResultsStore.ts new file mode 100644 index 0000000000000..f1bce931b3cb5 --- /dev/null +++ b/apps/server/src/fork/delegatedResults/DelegatedResultsStore.ts @@ -0,0 +1,117 @@ +/** + * Fork (issues 386, 625): the reads of the delegated result reminder that upstream's stores do not + * offer. They read only indexed columns and the payload of the few rows they match, never a + * parent's whole task list or history. + */ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/sql/SqlClient"; + +export interface RemoteChildKey { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; +} + +export class DelegatedResultsStore extends Context.Service< + DelegatedResultsStore, + { + /** + * Parents with a `delegate_task` task that ended at or after `sinceIso` and whose result is + * still owed or has no delivery state. Found at start, also when no wake was reserved for it. + */ + readonly parentsWithUnreadResults: (sinceIso: string) => Effect.Effect>; + /** The linked threads this parent's tasks follow, settled ones too; null: unreadable now. */ + readonly remoteChildrenOf: ( + parentThreadId: ThreadId, + ) => Effect.Effect | null>; + /** + * The ids of this thread's messages, and of its accepted command receipts, that start with + * `prefix`: what was sent to it under ids of that kind. Null: unreadable now. + */ + readonly sentIds: ( + threadId: ThreadId, + prefix: string, + ) => Effect.Effect | null>; + } +>()("t3/fork/delegatedResults/DelegatedResultsStore") {} + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + const parentsWithUnreadResults: DelegatedResultsStore["Service"]["parentsWithUnreadResults"] = ( + sinceIso, + ) => + sql<{ readonly thread_id: string }>` + SELECT DISTINCT thread_id FROM orchestration_v2_projection_subagents + WHERE status IN ('completed', 'failed', 'cancelled', 'interrupted') + AND COALESCE(completion_delivery_state, '') IN ('pending', 'claimed', '') + AND origin = 'app_owned' + AND COALESCE(completed_at, updated_at) >= ${sinceIso} + ORDER BY thread_id + `.pipe( + Effect.map((rows) => rows.map((row) => ThreadId.make(row.thread_id))), + Effect.catchCause((cause) => + Effect.logWarning("delegated result reminder: unread results cannot be read", { + cause, + }).pipe(Effect.as([])), + ), + ); + + const remoteChildrenOf: DelegatedResultsStore["Service"]["remoteChildrenOf"] = (parentThreadId) => + sql<{ readonly environment_id: string | null; readonly remote_thread_id: string | null }>` + SELECT + json_extract(payload_json, '$.remoteChild.environmentId') AS environment_id, + json_extract(payload_json, '$.remoteChild.threadId') AS remote_thread_id + FROM orchestration_v2_projection_subagents + WHERE thread_id = ${parentThreadId} + AND origin = 'app_owned' + AND child_thread_id IS NULL + `.pipe( + Effect.map((rows) => + rows.flatMap((row) => + row.environment_id === null || row.remote_thread_id === null + ? [] + : [ + { + environmentId: EnvironmentId.make(row.environment_id), + threadId: ThreadId.make(row.remote_thread_id), + }, + ], + ), + ), + Effect.catchCause((cause) => + Effect.logWarning("delegated result reminder: a parent's linked tasks cannot be read", { + threadId: parentThreadId, + cause, + }).pipe(Effect.as(null)), + ), + ); + + const sentIds: DelegatedResultsStore["Service"]["sentIds"] = (threadId, prefix) => + sql<{ readonly id: string }>` + SELECT message_id AS id FROM orchestration_v2_projection_messages + WHERE thread_id = ${threadId} AND substr(message_id, 1, ${prefix.length}) = ${prefix} + UNION + SELECT command_id AS id FROM orchestration_v2_command_receipts + WHERE thread_id = ${threadId} + AND status = 'accepted' + AND substr(command_id, 1, ${prefix.length}) = ${prefix} + `.pipe( + Effect.map((rows) => rows.map((row) => row.id)), + Effect.catchCause((cause) => + Effect.logWarning( + "delegated result reminder: a thread's earlier reminders cannot be read", + { + threadId, + cause, + }, + ).pipe(Effect.as(null)), + ), + ); + + return DelegatedResultsStore.of({ parentsWithUnreadResults, remoteChildrenOf, sentIds }); +}); + +export const layer = Layer.effect(DelegatedResultsStore, make); diff --git a/apps/server/src/fork/delegatedResults/delegatedResults.test.ts b/apps/server/src/fork/delegatedResults/delegatedResults.test.ts index a1a5cc05042d4..e997afb9e7e48 100644 --- a/apps/server/src/fork/delegatedResults/delegatedResults.test.ts +++ b/apps/server/src/fork/delegatedResults/delegatedResults.test.ts @@ -4,13 +4,22 @@ import { EnvironmentId, NodeId, ThreadId } from "@t3tools/contracts"; import { countByPlace, isRemindable, + type LinkedThread, localActivity, + legacyOrphanSetId, + orphanBatches, + orphanBatchIds, + orphanKey, + orphanReminderText, + ORPHANS_PER_MESSAGE, + orphansOf, parentBusy, REMINDER_MAX_AGE_MS, reminderIds, reminderText, remoteActivity, resultDelivery, + toldOrphanKeys, } from "./delegatedResults.ts"; import { at, delegatedTask, NOW, PARENT } from "./delegatedResults.testkit.ts"; @@ -32,7 +41,7 @@ describe("resultDelivery", () => { ).toBe("settled"); } }); - it("says none for an old row with no delivery record", () => { + it("says none for an old row, or a wait task, with no delivery record", () => { expect(resultDelivery(delegatedTask({ completionDelivery: undefined }))).toBe("none"); }); }); @@ -50,8 +59,15 @@ describe("isRemindable", () => { }); expect(isRemindable(remote, NOW)).toBe(true); }); - it("never reminds an old row from before delivery records", () => { - expect(isRemindable(delegatedTask({ completionDelivery: undefined }), NOW)).toBe(false); + it("reminds a recent result with no delivery record: a wait whose acknowledgement failed", () => { + expect(isRemindable(delegatedTask({ completionDelivery: undefined }), NOW)).toBe(true); + }); + it("never reminds an old row from before delivery records (older than a day)", () => { + const old = delegatedTask({ + completionDelivery: undefined, + completedAt: at(NOW - REMINDER_MAX_AGE_MS - 1), + }); + expect(isRemindable(old, NOW)).toBe(false); }); it("never reminds a result older than a day (an abandoned thread)", () => { const old = delegatedTask({ completedAt: at(NOW - REMINDER_MAX_AGE_MS - 1) }); @@ -129,6 +145,18 @@ describe("activity and slots", () => { } as const; expect(parentBusy(idle)).toBe(false); expect(parentBusy({ ...idle, status: "queued" })).toBe(true); + expect(parentBusy({ ...idle, pendingRuntimeRequest: {} as never })).toBe(true); + }); + it("does not treat background work the parent left running as busy", () => { + const idle = { + activityRunStatus: null, + status: "idle", + pendingRuntimeRequest: null, + pendingBackgroundTasks: [{} as never], + } as const; + // Upstream counts it as a live child shell; no run end may come for hours. + expect(localActivity(idle as never)).toBe("live"); + expect(parentBusy(idle)).toBe(false); }); }); @@ -138,5 +166,125 @@ describe("reminderText", () => { expect(text).toContain("2 delegated tasks"); expect(text.indexOf("t-1")).toBeLessThan(text.indexOf("t-2")); expect(text).toContain("task_status"); + // Upstream's wake may land just before it. + expect(text).toContain("may not have reached"); + }); +}); + +const HERE = EnvironmentId.make("env-here"); +const THERE = EnvironmentId.make("env-b"); +const linked = ( + threadId: string, + from: { environmentId?: EnvironmentId; threadId?: ThreadId } = {}, + environmentId = THERE, +): LinkedThread => ({ + environmentId, + threadId: ThreadId.make(threadId), + title: `title ${threadId}`, + delegatedFrom: { + environmentId: from.environmentId ?? HERE, + threadId: from.threadId ?? PARENT, + }, +}); + +describe("orphansOf", () => { + it("keeps a thread there launched from this parent that no task follows", () => { + const listed = [linked("r-1"), linked("r-2")]; + const followed = [{ environmentId: THERE, threadId: ThreadId.make("r-1") }]; + expect(orphansOf(listed, HERE, PARENT, followed).map((t) => t.threadId)).toEqual(["r-2"]); + }); + it("never counts a thread launched from another environment's thread of the same id", () => { + const listed = [linked("r-1", { environmentId: EnvironmentId.make("env-c") })]; + expect(orphansOf(listed, HERE, PARENT, [])).toEqual([]); + }); + it("never counts another parent's thread, or one followed in another environment's name", () => { + expect( + orphansOf([linked("r-1", { threadId: ThreadId.make("other") })], HERE, PARENT, []), + ).toEqual([]); + // Same thread id, but the task follows it in another environment: still an orphan here. + const followedElsewhere = [ + { environmentId: EnvironmentId.make("env-c"), threadId: ThreadId.make("r-1") }, + ]; + expect(orphansOf([linked("r-1")], HERE, PARENT, followedElsewhere)).toHaveLength(1); + }); +}); + +describe("orphan reminder", () => { + it("has the same ids for the same orphans in any order, other ids than a task reminder", () => { + const a = orphanBatchIds(PARENT, [linked("r-1"), linked("r-2")]); + const b = orphanBatchIds(PARENT, [linked("r-2"), linked("r-1")]); + expect(a).toEqual(b); + expect(orphanBatchIds(PARENT, [linked("r-1")])).not.toEqual(a); + // The same thread id in another environment is another orphan. + expect(orphanKey(linked("r-1", {}, EnvironmentId.make("env-c")))).not.toBe( + orphanKey(linked("r-1")), + ); + expect(a.messageId).not.toEqual( + reminderIds(PARENT, [NodeId.make("r-1"), NodeId.make("r-2")]).messageId, + ); + }); + + it("reads back each orphan a message or command id named, for this parent only", () => { + const ids = orphanBatchIds(PARENT, [linked("r-1"), linked("r-2")]); + const other = orphanBatchIds(ThreadId.make("other"), [linked("r-3")]); + expect(toldOrphanKeys(PARENT, [ids.messageId])).toEqual( + new Set([orphanKey(linked("r-1")), orphanKey(linked("r-2"))]), + ); + // A receipt's command id says the same; another parent's ids and other kinds say nothing. + expect(toldOrphanKeys(PARENT, [ids.commandId])).toEqual( + toldOrphanKeys(PARENT, [ids.messageId]), + ); + expect(toldOrphanKeys(PARENT, [other.messageId, "delegated-reminder:x:0123456789ab"])).toEqual( + new Set(), + ); + }); + + it("names only the orphans not told yet: a grown set sends just the new ones", () => { + const first = orphanBatchIds(PARENT, [linked("r-1")]); + const grown = [linked("r-1"), linked("r-2"), linked("r-3")]; + const batches = orphanBatches(PARENT, grown, [first.messageId]); + expect(batches.map((batch) => batch.map((orphan) => orphan.threadId))).toEqual([ + ["r-2", "r-3"], + ]); + // Told in two messages, it names none; an empty record names all. + const second = orphanBatchIds(PARENT, batches[0]!); + expect(orphanBatches(PARENT, grown, [first.commandId, second.messageId])).toEqual([]); + expect(orphanBatches(PARENT, grown, []).flat()).toHaveLength(3); + }); + + it("splits many orphans into messages that each name every one of theirs", () => { + const many = Array.from({ length: ORPHANS_PER_MESSAGE + 2 }, (_, index) => + linked(`r-${String(index).padStart(2, "0")}`), + ); + const batches = orphanBatches(PARENT, many, []); + expect(batches.map((batch) => batch.length)).toEqual([ORPHANS_PER_MESSAGE, 2]); + expect(orphanReminderText(batches[0]!)).not.toContain("more"); + }); + + it("takes the first scan's one message for exactly this set as told", () => { + const set = [linked("r-1"), linked("r-2")]; + // That format's id, as the first scan made it (sha256 of the sorted keys, 16 hex digits). + expect(legacyOrphanSetId(PARENT, set)).toBe("delegated-orphans:thread-parent:9f652a107b5cc2f3"); + expect(orphanBatches(PARENT, set, [legacyOrphanSetId(PARENT, set)])).toEqual([]); + // A grown set has another digest: each orphan is named once more, then by its own key. + expect( + orphanBatches(PARENT, [...set, linked("r-3")], [legacyOrphanSetId(PARENT, set)]).flat(), + ).toHaveLength(3); + }); + + it("names each orphan with its environment and the tools that read and stop it", () => { + const text = orphanReminderText([linked("r-2"), linked("r-1")]); + expect(text).toContain("2 threads"); + expect(text.indexOf("r-1")).toBeLessThan(text.indexOf("r-2")); + expect(text).toContain(`in ${THERE}`); + expect(text).toContain("t3_thread_read"); + expect(text).toContain("t3_thread_interrupt"); + }); + + it("does not say why the record is missing: a stop and a failed record read the same", () => { + const one = orphanReminderText([linked("r-1")]); + expect(one).toContain("1 thread in linked environments was launched"); + expect(one).toContain("delegate_task did not record it here"); + expect(one).not.toContain("server stopped"); }); }); diff --git a/apps/server/src/fork/delegatedResults/delegatedResults.ts b/apps/server/src/fork/delegatedResults/delegatedResults.ts index 34cacc82c3b47..9d39d79e59b0d 100644 --- a/apps/server/src/fork/delegatedResults/delegatedResults.ts +++ b/apps/server/src/fork/delegatedResults/delegatedResults.ts @@ -1,7 +1,7 @@ /** - * Fork (issue 386): pure rules of `task_list` and the delegated result reminder. Upstream's + * Fork (issues 386, 625): pure rules of `task_list` and the delegated result reminder. Upstream's * `completionDelivery.state`: `pending`/`claimed` = owed, `delivered`/`acknowledged`/`disposed` = - * settled, no state = a row from before it was kept (nothing says whether it was seen). + * settled. No state (an old row, or a wait that never acknowledged it): nobody knows it was seen. */ // @effect-diagnostics-next-line nodeBuiltinImport:off -- The reminder id must be a deterministic digest. import * as NodeCrypto from "node:crypto"; @@ -17,7 +17,7 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; -/** Where a task's result stands for its parent. `none`: no delivery state was recorded. */ +/** Where a task's result stands for its parent. `none`: no delivery state (see above). */ export type ResultDelivery = "owed" | "settled" | "none"; /** Live: running there now. Ended: not running. Unknown: its environment could not be read. */ @@ -99,18 +99,25 @@ export const countByPlace = ( export const REMINDER_MAX_AGE_MS = 24 * 60 * 60 * 1000; /** - * Owed and recent enough to remind: ended with an owed delivery state in the last day. Tasks with - * no delivery state (rows from before upstream kept one) are never reminded. + * Unread and recent enough to remind: ended in the last day with its result owed or with no + * delivery state. The reminder checks only an idle parent, so a `mode: "wait"` task whose wait is + * still blocking is never reminded; one whose wait acknowledged it is settled. Rows from before + * upstream kept a delivery state are all older than a day. */ export const isRemindable = (task: OrchestrationV2Subagent, nowMs: number): boolean => { - if (!isDelegatedTask(task) || resultDelivery(task) !== "owed") return false; + const delivery = resultDelivery(task); + if (!isDelegatedTask(task) || (delivery !== "owed" && delivery !== "none")) return false; const ended = task.completedAt ?? task.updatedAt; return nowMs - DateTime.toEpochMillis(ended) <= REMINDER_MAX_AGE_MS; }; -/** Busy: the parent is working or holds a question, so its own turn end comes first. */ +/** + * Busy: the parent's run is going or holds a question, so its own run end comes first and checks + * again. Background work the parent left running (a monitor, a command) does not hold it: upstream + * counts that as a live child, but no run end may come for hours, and a queued message is safe. + */ export const parentBusy = (shell: Parameters[0]): boolean => - isLiveSubagentShell(shell); + isLiveSubagentShell({ ...shell, pendingBackgroundTasks: [] }); /** Gone: archived or deleted. It is never reminded. */ export const parentGone = ( @@ -119,6 +126,16 @@ export const parentGone = ( const REMINDER_PREFIX = "delegated-reminder:"; +const digestOf = (keys: ReadonlyArray) => + NodeCrypto.createHash("sha256") + .update([...new Set(keys)].toSorted().join("\n")) + .digest("hex"); + +const idsFor = (prefix: string, parentThreadId: ThreadId, keys: ReadonlyArray) => { + const messageId = `${prefix}${parentThreadId}:${digestOf(keys).slice(0, 16)}`; + return { messageId, commandId: `${messageId}:0` }; +}; + /** * The same owed tasks always give the same id, so a check that runs again, after a restart too, is * answered from the message or its receipt instead of sending a second reminder. @@ -126,14 +143,8 @@ const REMINDER_PREFIX = "delegated-reminder:"; export const reminderIds = ( parentThreadId: ThreadId, taskIds: ReadonlyArray, -): { readonly messageId: string; readonly commandId: string } => { - const digest = NodeCrypto.createHash("sha256") - .update([...new Set(taskIds)].toSorted().join("\n")) - .digest("hex") - .slice(0, 16); - const messageId = `${REMINDER_PREFIX}${parentThreadId}:${digest}`; - return { messageId, commandId: `${messageId}:0` }; -}; +): { readonly messageId: string; readonly commandId: string } => + idsFor(REMINDER_PREFIX, parentThreadId, taskIds); const MAX_LISTED = 10; @@ -146,9 +157,135 @@ export const reminderText = (tasks: ReadonlyArray): str const more = sorted.length > MAX_LISTED ? [`- and ${sorted.length - MAX_LISTED} more`] : []; const count = sorted.length === 1 ? "1 delegated task" : `${sorted.length} delegated tasks`; return [ - `[delegated results] ${count} ended, but the result never reached this thread:`, + // Upstream's wake can land between the check and this send, so "may not". + `[delegated results] ${count} ended, but the result may not have reached this thread:`, ...lines, ...more, "Read each with task_status (that marks it read). task_list lists every task with an unread result.", ].join("\n"); }; + +/** A thread in a linked environment whose `delegatedFrom` names a thread here. */ +export interface LinkedThread { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly title: string; + readonly delegatedFrom: { readonly environmentId: EnvironmentId; readonly threadId: ThreadId }; +} + +const linkedKey = (environmentId: EnvironmentId, threadId: ThreadId) => + `${environmentId}\n${threadId}`; + +/** + * Orphans: threads there launched from `parentThreadId` here that no task of it follows, so a + * launch whose record here never ran (the server stopped half-way through `delegate_task`). + */ +export const orphansOf = ( + listed: ReadonlyArray, + here: EnvironmentId, + parentThreadId: ThreadId, + followed: ReadonlyArray<{ readonly environmentId: EnvironmentId; readonly threadId: ThreadId }>, +): ReadonlyArray => { + const known = new Set(followed.map((child) => linkedKey(child.environmentId, child.threadId))); + return listed.filter( + (thread) => + thread.delegatedFrom.environmentId === here && + thread.delegatedFrom.threadId === parentThreadId && + !known.has(linkedKey(thread.environmentId, thread.threadId)), + ); +}; + +/** The start of every orphan reminder id, of both formats (see `legacyOrphanSetId`). */ +export const ORPHAN_ID_PREFIX = "delegated-orphan"; +const orphanPrefix = (parentThreadId: ThreadId) => `${ORPHAN_ID_PREFIX}:${parentThreadId}:`; +const ORPHAN_KEY = /^[0-9a-f]{12}$/; + +/** One orphan's own key: the same thread there always gives the same key. */ +export const orphanKey = (orphan: Pick): string => + digestOf([linkedKey(orphan.environmentId, orphan.threadId)]).slice(0, 12); + +/** At most this many orphans in one message, each listed by name. */ +export const ORPHANS_PER_MESSAGE = 10; + +/** + * A message's ids name the key of each orphan it tells about, so the parent's message and receipt + * ids are the record of who was told: an orphan is never told twice, also when the set grows + * (a machine that answers late) or the one-day window moves (a restart). + */ +export const orphanBatchIds = ( + parentThreadId: ThreadId, + batch: ReadonlyArray>, +) => { + const keys = [...new Set(batch.map(orphanKey))].toSorted(); + const messageId = `${orphanPrefix(parentThreadId)}${keys.join(".")}`; + return { messageId, commandId: `${messageId}:0` }; +}; + +/** The orphan keys that earlier message or command ids of this parent named. */ +export const toldOrphanKeys = ( + parentThreadId: ThreadId, + ids: ReadonlyArray, +): ReadonlySet => { + const prefix = orphanPrefix(parentThreadId); + const told = new Set(); + for (const id of ids) { + if (!id.startsWith(prefix)) continue; + for (const key of id.slice(prefix.length).replace(/:0$/, "").split(".")) { + if (ORPHAN_KEY.test(key)) told.add(key); + } + } + return told; +}; + +/** + * The id of the first orphan scan's one message for a whole orphan set (before each orphan had + * its own key). A message of that id for exactly this set means all of it was told. + */ +export const legacyOrphanSetId = ( + parentThreadId: ThreadId, + orphans: ReadonlyArray>, +): string => + idsFor( + `${ORPHAN_ID_PREFIX}s:`, + parentThreadId, + orphans.map((orphan) => linkedKey(orphan.environmentId, orphan.threadId)), + ).messageId; + +/** The orphans not told yet, sorted, in messages of at most `ORPHANS_PER_MESSAGE`. */ +export const orphanBatches = ( + parentThreadId: ThreadId, + orphans: ReadonlyArray, + sentIds: ReadonlyArray, +): ReadonlyArray> => { + if (orphans.length > 0 && sentIds.includes(legacyOrphanSetId(parentThreadId, orphans))) { + return []; + } + const told = toldOrphanKeys(parentThreadId, sentIds); + const fresh = orphans + .filter((orphan) => !told.has(orphanKey(orphan))) + .toSorted((a, b) => + linkedKey(a.environmentId, a.threadId) < linkedKey(b.environmentId, b.threadId) ? -1 : 1, + ); + const batches: Array> = []; + for (let start = 0; start < fresh.length; start += ORPHANS_PER_MESSAGE) { + batches.push(fresh.slice(start, start + ORPHANS_PER_MESSAGE)); + } + return batches; +}; + +/** One message's text; a batch is at most `ORPHANS_PER_MESSAGE` long, so every orphan is named. */ +export const orphanReminderText = (orphans: ReadonlyArray): string => { + const lines = orphans + .toSorted((a, b) => + linkedKey(a.environmentId, a.threadId) < linkedKey(b.environmentId, b.threadId) ? -1 : 1, + ) + .map((orphan) => `- ${orphan.threadId} "${orphan.title}" in ${orphan.environmentId}`); + const one = orphans.length === 1; + return [ + // Neutral on why: the server stopped during delegate_task, or the record failed (that stops + // the thread there, and the agent saw an error then). + `[delegated results] ${one ? "1 thread" : `${orphans.length} threads`} in linked environments ${one ? "was" : "were"} launched from this thread, but no task here follows ${one ? "it" : "them"} (delegate_task did not record ${one ? "it" : "them"} here), so ${one ? "its result" : "their results"} will not arrive:`, + ...lines, + "Read each with t3_thread_read and its environmentId; stop one you no longer need with t3_thread_interrupt.", + ].join("\n"); +}; diff --git a/apps/server/src/fork/delegatedResults/linkedThreads.ts b/apps/server/src/fork/delegatedResults/linkedThreads.ts new file mode 100644 index 0000000000000..b6e191635ecc5 --- /dev/null +++ b/apps/server/src/fork/delegatedResults/linkedThreads.ts @@ -0,0 +1,84 @@ +/** + * Fork (issues 386, 625): reads a linked environment's threads that a thread here delegated, page + * by page (`t3_thread_list` lists the most recently updated first, at most 100 a page), for + * `task_list` and the orphan scan at start. + */ +import type { + EnvironmentId, + OrchestratorMcpFailure, + OrchestratorMcpThreadListResult, +} from "@t3tools/contracts"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; + +import type { LinkedThread } from "./delegatedResults.ts"; + +export const LINKED_PAGE_SIZE = 100; + +export interface LinkedThreadsRead { + readonly threads: ReadonlyArray; + /** + * complete: every thread that may match was read. limit: the page limit was reached (a rerun + * reads the same pages). cut-short: a later page did not answer (a rerun may read the rest). + * unreachable: the first page did not answer. + */ + readonly state: "complete" | "limit" | "cut-short" | "unreachable"; +} + +export const listLinkedThreads = Effect.fn("delegatedResults.listLinkedThreads")(function* ( + page: ( + cursor: number, + ) => Effect.Effect, + environmentId: EnvironmentId, + options: { + readonly maxPages: number; + /** Each page's own limit; one that takes longer counts as no answer. */ + readonly timeout: Duration.Input; + /** Stop after a page whose oldest thread was last updated before this (epoch ms). */ + readonly sinceMs?: number; + /** Keep only threads made before this (epoch ms): a launch since then may still be recorded. */ + readonly createdBeforeMs?: number; + }, +) { + const threads: Array = []; + let cursor = 0; + for (let read = 0; read < options.maxPages; read += 1) { + const listed = yield* page(cursor).pipe( + Effect.timeoutOption(options.timeout), + Effect.orElseSucceed(() => ({ _tag: "None" }) as const), + ); + if (listed._tag === "None") { + return { + threads, + state: read === 0 ? "unreachable" : "cut-short", + } satisfies LinkedThreadsRead; + } + for (const thread of listed.value.threads) { + if (thread.delegatedFrom === undefined) continue; + if ( + options.createdBeforeMs !== undefined && + !(Date.parse(thread.createdAt) < options.createdBeforeMs) + ) { + continue; + } + threads.push({ + environmentId, + threadId: thread.threadId, + title: thread.title, + delegatedFrom: thread.delegatedFrom, + }); + } + const oldest = listed.value.threads.at(-1); + if ( + listed.value.nextCursor === null || + listed.value.nextCursor <= cursor || + (options.sinceMs !== undefined && + oldest !== undefined && + Date.parse(oldest.updatedAt) < options.sinceMs) + ) { + return { threads, state: "complete" } satisfies LinkedThreadsRead; + } + cursor = listed.value.nextCursor; + } + return { threads, state: "limit" } satisfies LinkedThreadsRead; +}); diff --git a/apps/server/src/fork/forkLayers.ts b/apps/server/src/fork/forkLayers.ts index 982f534448201..acaf8244dc823 100644 --- a/apps/server/src/fork/forkLayers.ts +++ b/apps/server/src/fork/forkLayers.ts @@ -17,6 +17,7 @@ import * as UsagePacer from "../usage/UsagePacer.ts"; import * as UsageReadingsHistory from "../usage/UsageReadingsHistory.ts"; import * as WorkAdmission from "../workAdmission.ts"; import * as ActivityProbe from "./activityProbe.ts"; +import * as DelegatedOrphanScan from "./delegatedResults/DelegatedOrphanScan.ts"; import * as DelegatedResultReminder from "./delegatedResults/DelegatedResultReminder.ts"; import * as ForkFeatures from "./forkFeatures.ts"; import * as HandoffOwner from "./handoff/handoffOwner.ts"; @@ -82,6 +83,8 @@ export const layerRoutes = Layer.mergeAll( ActivityProbe.layer, // GET /api/fork/features: what this version does for a linked machine; its token only. ForkFeatures.layer, + // Linked threads a parent here launched but whose task the last stop lost: one reminder each. + DelegatedOrphanScan.layer, ); /** diff --git a/apps/server/src/mcp/toolkits/waits/taskList.test.ts b/apps/server/src/mcp/toolkits/waits/taskList.test.ts index a27eb7a67a2d5..2148ebfc245b1 100644 --- a/apps/server/src/mcp/toolkits/waits/taskList.test.ts +++ b/apps/server/src/mcp/toolkits/waits/taskList.test.ts @@ -1,7 +1,8 @@ /** - * Fork (issue 386): `task_list` through the registered tool. The state question: a linked - * environment that went away (its tasks read as unknown, never as live slots), settled results - * (left out unless asked), old rows with no delivery record, and an empty thread. + * Fork (issues 386, 625): `task_list` through the registered tool. The state question: a linked + * environment that went away (its tasks read as unknown, never as live slots, and it is asked + * once), a stop still owed there, settled results (left out unless asked), old rows with no + * delivery record, an environment with more threads than one page, and an empty thread. */ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, expect, it } from "@effect/vitest"; @@ -27,11 +28,15 @@ import { Waits } from "../../../waits/WaitWorker.ts"; import * as McpHttpServer from "../../McpHttpServer.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as McpToolAccessTestkit from "../../McpToolAccess.testkit.ts"; -import { REMOTE_READ_TIMEOUT } from "./taskList.ts"; +import { ORPHAN_PAGES, REMOTE_READ_TIMEOUT } from "./taskList.ts"; const HERE = EnvironmentId.make("env-here"); const ENV_UP = EnvironmentId.make("env-up"); const ENV_GONE = EnvironmentId.make("env-gone"); +/** Answers every page with more to come: more threads than `task_list` reads. */ +const ENV_BIG = EnvironmentId.make("env-big"); +/** Answers its first page, never its second. */ +const ENV_CUT = EnvironmentId.make("env-cut"); const remoteTask = (id: string, environmentId: EnvironmentId, status = "running") => delegatedTask({ @@ -89,8 +94,12 @@ interface World { readonly children?: ReadonlyArray; } +/** Reads of a run there, per environment; tests read it. */ +const runStatusReads = new Map(); + const toolsLayer = (world: World) => { let omitSettled: boolean | undefined; + runStatusReads.clear(); const services = Layer.mergeAll( ServerSettingsService.layerTest({}), Layer.mock(ThreadManagement.ThreadManagementService)({ @@ -130,14 +139,17 @@ const toolsLayer = (world: World) => { Layer.mock(Waits)({}), Layer.mock(RemoteDelegation.RemoteDelegation)({ // The environment that went away never answers; the other one says the run is going. - runStatus: (_scope, task) => - task.remoteChild?.environmentId === ENV_GONE ? Effect.never : Effect.succeed("running"), + runStatus: (_scope, task) => { + const environmentId = task.remoteChild!.environmentId; + runStatusReads.set(environmentId, (runStatusReads.get(environmentId) ?? 0) + 1); + return environmentId === ENV_GONE ? Effect.never : Effect.succeed("running"); + }, }), Layer.mock(PeerForwarding.PeerForwarding)({ links: () => Effect.succeed({ environmentId: HERE, - links: [ENV_UP, ENV_GONE].map((environmentId) => ({ + links: [ENV_UP, ENV_GONE, ENV_BIG, ENV_CUT].map((environmentId) => ({ environmentId, label: environmentId, status: "reachable" as const, @@ -147,40 +159,86 @@ const toolsLayer = (world: World) => { })), notListed: "", }), - call: ((_scope: unknown, _tool: unknown, environmentId: EnvironmentId) => - environmentId === ENV_GONE + call: (( + _scope: unknown, + _tool: unknown, + environmentId: EnvironmentId, + params: { readonly cursor: number }, + ) => + environmentId === ENV_GONE || (environmentId === ENV_CUT && params.cursor > 0) ? Effect.never - : Effect.succeed({ - projectId: null, - currentThreadId: null, - nextCursor: null, - total: 3, - threads: [ - // Followed by a task here. - { - threadId: ThreadId.make("remote-r-up"), - title: "known", - delegatedFrom: { environmentId: HERE, threadId: PARENT, title: "p" }, - }, - // Launched by this thread, but its record never ran. - { - threadId: ThreadId.make("remote-lost"), - title: "lost one", - delegatedFrom: { environmentId: HERE, threadId: PARENT, title: "p" }, - }, - // Another thread's child. - { - threadId: ThreadId.make("remote-other"), - title: "other", - delegatedFrom: { - environmentId: HERE, - threadId: ThreadId.make("thread-other"), - title: "o", - }, - }, - { threadId: ThreadId.make("remote-own"), title: "its own thread" }, - ], - })) as never, + : environmentId === ENV_CUT + ? Effect.succeed({ + projectId: null, + currentThreadId: null, + nextCursor: 100, + total: 200, + threads: [], + }) + : environmentId === ENV_BIG + ? Effect.succeed({ + projectId: null, + currentThreadId: null, + nextCursor: params.cursor + 100, + total: 10_000, + threads: [{ threadId: ThreadId.make(`big-${params.cursor}`), title: "busy" }], + }) + : params.cursor > 0 + ? // The second page: an older thread this one launched, also lost. + Effect.succeed({ + projectId: null, + currentThreadId: null, + nextCursor: null, + total: 101, + threads: [ + { + threadId: ThreadId.make("remote-lost-older"), + title: "older lost one", + delegatedFrom: { environmentId: HERE, threadId: PARENT, title: "p" }, + }, + ], + }) + : Effect.succeed({ + projectId: null, + currentThreadId: null, + nextCursor: 100, + total: 101, + threads: [ + // Followed by a task here. + { + threadId: ThreadId.make("remote-r-up"), + title: "known", + delegatedFrom: { environmentId: HERE, threadId: PARENT, title: "p" }, + }, + // Launched by this thread, but its record never ran. + { + threadId: ThreadId.make("remote-lost"), + title: "lost one", + delegatedFrom: { environmentId: HERE, threadId: PARENT, title: "p" }, + }, + // Another thread's child. + { + threadId: ThreadId.make("remote-other"), + title: "other", + delegatedFrom: { + environmentId: HERE, + threadId: ThreadId.make("thread-other"), + title: "o", + }, + }, + // Launched by a thread of the same id in another environment. + { + threadId: ThreadId.make("remote-foreign"), + title: "foreign", + delegatedFrom: { + environmentId: EnvironmentId.make("env-other"), + threadId: PARENT, + title: "p", + }, + }, + { threadId: ThreadId.make("remote-own"), title: "its own thread" }, + ], + })) as never, }), ).pipe(Layer.provideMerge(NodeServices.layer)); return McpHttpServer.layerForkTools.pipe( @@ -320,14 +378,67 @@ describe("task_list", () => { it.effect("finds a thread launched in a linked environment that no task follows", () => Effect.gen(function* () { const listed = yield* withTimeouts(callTaskList({ checkLinked: true })); + // Its second page too; never one launched by another environment's thread of that id. expect(listed.orphans).toEqual([ { environmentId: ENV_UP, threadId: "remote-lost", title: "lost one" }, + { environmentId: ENV_UP, threadId: "remote-lost-older", title: "older lost one" }, ]); expect(listed.unreachable).toEqual([ENV_GONE]); expect(listed.next).toContain("t3_thread_read"); + // The environment with more threads than it reads is named. + expect(listed.next).toContain(`newest ${ORPHAN_PAGES * 100} threads of ${ENV_BIG}.`); + // One whose later page did not answer is named for that, not for the page limit. + expect(listed.next).toContain(`A later page of ${ENV_CUT} did not answer`); + // Settled tasks are read to know what is followed, but listed only when asked. + expect(listed.tasks.map((task) => task.taskId)).not.toContain("l-read"); }).pipe(Effect.provide(toolsLayer(mixedWorld))), ); + it.effect("reads a remote task there while the stop it owes there is not sent", () => + Effect.gen(function* () { + const listed = yield* callTaskList({}); + const byId = Object.fromEntries(listed.tasks.map((task) => [task.taskId, task])); + expect(byId["r-stop"]).toMatchObject({ activity: "live", where: ENV_UP }); + expect(byId["r-stopped"]).toMatchObject({ activity: "ended", where: ENV_UP }); + expect(listed.slots).toEqual([{ where: ENV_UP, live: 1, unknown: 0 }]); + // Only the one whose stop is owed was read there. + expect(runStatusReads.get(ENV_UP)).toBe(1); + }).pipe( + Effect.provide( + toolsLayer({ + tasks: [ + { + ...remoteTask("r-stop", ENV_UP, "completed"), + remoteChild: { ...remoteTask("r-stop", ENV_UP).remoteChild!, stoppedThere: false }, + }, + { + ...remoteTask("r-stopped", ENV_UP, "completed"), + remoteChild: { ...remoteTask("r-stopped", ENV_UP).remoteChild!, stoppedThere: true }, + }, + ], + shells: new Map(), + }), + ), + ), + ); + + it.effect("waits for an environment that does not answer once, not once per task", () => + Effect.gen(function* () { + // Eight open tasks there, four read at a time: after the first timeouts the rest are + // unknown without a read (it was two rounds of reads, eight in all). + const listed = yield* withTimeouts(callTaskList({})); + expect(listed.slots).toEqual([{ where: ENV_GONE, live: 0, unknown: 8 }]); + expect(runStatusReads.get(ENV_GONE)).toBe(4); + }).pipe( + Effect.provide( + toolsLayer({ + tasks: Array.from({ length: 8 }, (_, index) => remoteTask(`g-${index}`, ENV_GONE)), + shells: new Map(), + }), + ), + ), + ); + it.effect("an empty thread lists nothing and owes nothing", () => Effect.gen(function* () { const listed = yield* callTaskList({}); diff --git a/apps/server/src/mcp/toolkits/waits/taskList.ts b/apps/server/src/mcp/toolkits/waits/taskList.ts index d716cc980cb6d..a804335355781 100644 --- a/apps/server/src/mcp/toolkits/waits/taskList.ts +++ b/apps/server/src/mcp/toolkits/waits/taskList.ts @@ -16,14 +16,20 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { + type ChildActivity, countByPlace, isDelegatedTask, isEndedTask, localActivity, + orphansOf, remoteActivity, resultDelivery, taskWhere, } from "../../../fork/delegatedResults/delegatedResults.ts"; +import { + LINKED_PAGE_SIZE, + listLinkedThreads, +} from "../../../fork/delegatedResults/linkedThreads.ts"; import * as PeerForwarding from "../../../peer/PeerForwarding.ts"; import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts"; import type { McpThreadInvocationScope } from "../../McpInvocationContext.ts"; @@ -64,7 +70,7 @@ export const TaskListResult = Schema.Struct({ tasks: Schema.Array(TaskListEntry), /** Ended tasks whose result never reached this thread. */ owed: Schema.Number, - /** Ended tasks with no delivery record (from before it was kept): read them to be sure. */ + /** Ended tasks with no delivery record (an old row, or a wait that never acknowledged it): read them to be sure. */ unrecorded: Schema.Number, /** Children running now per place, from their live status; `unknown` could not be read. */ slots: Schema.Array(Schema.Struct({ where: Where, live: Schema.Number, unknown: Schema.Number })), @@ -80,9 +86,14 @@ export const TaskListResult = Schema.Struct({ }); export type TaskListResult = typeof TaskListResult.Type; -/** The longest one linked environment may take to answer before it counts as unknown. */ +/** + * The longest one linked environment may take to answer before it counts as unknown. After one + * read there times out, its other tasks read as unknown without asking again. + */ export const REMOTE_READ_TIMEOUT = Duration.seconds(5); const REMOTE_READ_CONCURRENCY = 4; +/** Pages of `t3_thread_list` read per linked environment for orphans: its newest 500 threads. */ +export const ORPHAN_PAGES = 5; const iso = (time: DateTime.Utc | null) => (time === null ? null : DateTime.formatIso(time)); @@ -111,19 +122,30 @@ export const listTasks = Effect.fn("waits.taskList")(function* (input: TaskListI const listed = input.includeSettled === true ? all : all.filter((task) => resultDelivery(task) !== "settled"); - const activityOf = (task: OrchestrationV2Subagent) => + // Environments where one read timed out: their other tasks are not asked again. + const silent = new Set(); + const remoteRead = (task: OrchestrationV2Subagent): Effect.Effect => { + const environmentId = task.remoteChild!.environmentId; + if (silent.has(environmentId)) return Effect.succeed("unknown"); + return remote.runStatus(threadScope, task).pipe( + Effect.timeoutOption(REMOTE_READ_TIMEOUT), + Effect.map((status) => { + if (status._tag === "None") silent.add(environmentId); + return remoteActivity(status._tag === "Some" ? status.value : null); + }), + ); + }; + const activityOf = (task: OrchestrationV2Subagent): Effect.Effect => task.childThreadId !== null ? threads.getThreadShell(task.childThreadId).pipe( Effect.map(localActivity), Effect.orElseSucceed(() => "unknown" as const), ) - : // A follower already recorded the end of a remote task; only an open one is read there. - isEndedTask(task) + : // A follower recorded the end of a remote task; it is read there only while it is open, + // or while the stop it owes there is not sent yet (its thread may still run there). + isEndedTask(task) && task.remoteChild?.stoppedThere !== false ? Effect.succeed("ended" as const) - : remote.runStatus(threadScope, task).pipe( - Effect.timeoutOption(REMOTE_READ_TIMEOUT), - Effect.map((status) => remoteActivity(status._tag === "Some" ? status.value : null)), - ); + : remoteRead(task); const tasks = yield* Effect.forEach( listed, @@ -162,9 +184,22 @@ export const listTasks = Effect.fn("waits.taskList")(function* (input: TaskListI const owed = tasks.filter((task) => task.result === "owed").length; const unrecorded = tasks.filter((task) => task.result === "none").length; - const linked: Partial> = - input.checkLinked === true ? yield* findOrphans(threadScope, caller.id, all) : {}; + const { + partial, + cutShort, + ...linked + }: Partial> & { + readonly partial?: ReadonlyArray; + readonly cutShort?: ReadonlyArray; + } = input.checkLinked === true ? yield* findOrphans(threadScope, caller.id, all) : {}; const orphanCount = linked.orphans?.length ?? 0; + const partialNote = + (partial === undefined || partial.length === 0 + ? "" + : ` Orphans were looked for only in the newest ${ORPHAN_PAGES * LINKED_PAGE_SIZE} threads of ${partial.join(", ")}.`) + + (cutShort === undefined || cutShort.length === 0 + ? "" + : ` A later page of ${cutShort.join(", ")} did not answer, so orphans there may be missing.`); return { tasks, owed, @@ -172,72 +207,67 @@ export const listTasks = Effect.fn("waits.taskList")(function* (input: TaskListI slots: countByPlace([...tasks, ...otherActivity]), ...linked, next: - owed + unrecorded + orphanCount === 0 + (owed + unrecorded + orphanCount === 0 ? "Nothing unread. Slots show the children running now." : `Read each owed or unrecorded task with task_status; that marks it read.${ orphanCount === 0 ? "" : " An orphan runs there with no task here: read it with t3_thread_read and its environmentId." - }`, + }`) + partialNote, } satisfies TaskListResult; }); /** * Threads in linked environments whose `delegatedFrom` names this thread but that no task here - * follows: a launch whose record never ran. Reads the newest 100 threads of each environment. + * follows: a launch whose record never ran. Reads the newest `ORPHAN_PAGES` pages of each + * environment's threads; one with more is named in `partial`, one whose later page did not answer + * in `cutShort`. */ const findOrphans = Effect.fn("waits.taskList.orphans")(function* ( scope: McpThreadInvocationScope, callerId: ThreadId, tasks: ReadonlyArray, ) { + const none = { orphans: [], unreachable: [], partial: [], cutShort: [] }; // An agent of a linked environment would be pointed at this machine's links. - if (scope.linkedEnvironments === false || scope.client?.linked === true) { - return { orphans: [], unreachable: [] }; - } + if (scope.linkedEnvironments === false || scope.client?.linked === true) return none; const forwarding = yield* PeerForwarding.PeerForwarding; const links = yield* forwarding.links(scope).pipe(Effect.orElseSucceed(() => null)); - if (links === null) return { orphans: [], unreachable: [] }; - const known = new Set( - tasks.flatMap((task) => - task.remoteChild === undefined - ? [] - : [`${task.remoteChild.environmentId}\n${task.remoteChild.threadId}`], - ), + if (links === null) return none; + const followed = tasks.flatMap((task) => + task.remoteChild === undefined ? [] : [task.remoteChild], ); const unreachable: Array = []; + const partial: Array = []; + const cutShort: Array = []; const orphans: Array<{ environmentId: EnvironmentId; threadId: ThreadId; title: string }> = []; yield* Effect.forEach( links.links.filter((link) => link.status !== "expired"), (link) => - forwarding - .call(scope, OrchestratorToolkit.tools.t3_thread_list, link.environmentId, { - allProjects: true, - limit: 100, - }) - .pipe( - Effect.timeoutOption(REMOTE_READ_TIMEOUT), - Effect.map((listed) => { - if (listed._tag === "None") return void unreachable.push(link.environmentId); - for (const thread of listed.value.threads) { - const from = thread.delegatedFrom; - if ( - from !== undefined && - from.environmentId === scope.environmentId && - from.threadId === callerId && - !known.has(`${link.environmentId}\n${thread.threadId}`) - ) { - orphans.push({ - environmentId: link.environmentId, - threadId: thread.threadId, - title: thread.title, - }); - } - } + listLinkedThreads( + (cursor) => + forwarding.call(scope, OrchestratorToolkit.tools.t3_thread_list, link.environmentId, { + allProjects: true, + limit: LINKED_PAGE_SIZE, + cursor, }), - Effect.orElseSucceed(() => void unreachable.push(link.environmentId)), - ), + link.environmentId, + { maxPages: ORPHAN_PAGES, timeout: REMOTE_READ_TIMEOUT }, + ).pipe( + Effect.map((read) => { + if (read.state === "unreachable") unreachable.push(link.environmentId); + if (read.state === "limit") partial.push(link.environmentId); + if (read.state === "cut-short") cutShort.push(link.environmentId); + for (const orphan of orphansOf(read.threads, scope.environmentId, callerId, followed)) { + orphans.push({ + environmentId: orphan.environmentId, + threadId: orphan.threadId, + title: orphan.title, + }); + } + }), + ), { concurrency: REMOTE_READ_CONCURRENCY, discard: true }, ); - return { orphans, unreachable }; + return { orphans, unreachable, partial, cutShort }; }); diff --git a/apps/server/src/mcp/toolkits/waits/tools.ts b/apps/server/src/mcp/toolkits/waits/tools.ts index 0b82cb1c4156d..5f70a008c3924 100644 --- a/apps/server/src/mcp/toolkits/waits/tools.ts +++ b/apps/server/src/mcp/toolkits/waits/tools.ts @@ -82,7 +82,7 @@ const WaitCancelTool = Tool.make("wait_cancel", { // Fork (issue 386): the caller's delegated tasks with unread results and live slot use. const TaskListTool = Tool.make("task_list", { description: - "List this thread's delegated tasks with an unread result (local and linked) and its running children per place, from live status. Marks nothing read; task_status does.", + "List this thread's delegated tasks with unread results and live child counts per place. Read only; task_status marks read.", parameters: TaskListInput, success: TaskListResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/provider/forkAgentContextBudget.json b/apps/server/src/provider/forkAgentContextBudget.json index cbc6e8fba205d..d41e64837f367 100644 --- a/apps/server/src/provider/forkAgentContextBudget.json +++ b/apps/server/src/provider/forkAgentContextBudget.json @@ -1,3 +1,3 @@ { - "mcpToolListJsonBytes": 2451 + "mcpToolListJsonBytes": 2406 } diff --git a/docs/fork/README.md b/docs/fork/README.md index d1b3d31a6c0f6..3e6b8cb245dbb 100644 --- a/docs/fork/README.md +++ b/docs/fork/README.md @@ -179,10 +179,38 @@ thread and how often each kind is checked. The same switch turns on the delegated result reminder: when a `delegate_task` child ends and its result has still not reached the idle parent a couple of minutes later (upstream's own wake did not -land), the parent gets one queued message naming those tasks. It is sent once per set of owed tasks, -also across restarts, never to a busy, stopped or archived thread, and never for results older than -a day. `task_list` (always there, read only) lists a thread's tasks with an unread result and how -many of its children run now per place, from their live status. +land), the parent gets one queued message naming those tasks. A result counts as unread while it is +owed, and also when it has no delivery state at all: an old row, or a `mode: "wait"` task whose +acknowledgement failed. The message is sent once per set of unread tasks, also across restarts, +never to a stopped or archived thread, and never for results older than a day. A parent is busy, and +waits for its own run end, while a run of it goes or a question of it is open (an open question +holds the reminder until that run ends); background work it left running does not hold it. At +start, every parent with a result unread from the last day is checked once, also when upstream had +reserved no wake for it. Upstream's wake can land just before the reminder, so the message says the +result _may_ not have arrived. + +The server makes a thread in a linked environment before it records the task here, so a server that +stops in between leaves a thread there that no task here follows. At start (a few minutes after, +and again every ten minutes while an environment does not answer fully or a parent cannot be read, +six scans at most), each linked environment's threads made before the start (with a minute's +tolerance for a clock that runs ahead) and changed in the last day are read. A parent that one +names as its `delegatedFrom` but no task of it follows gets a queued message naming them, with the +tools to read and stop them. Each such thread is named once: a machine that answers late, or a +later start, names only the ones not named yet. A busy parent (a question to a person included) +reads it after its current run. What is still unchecked after the last scan is logged by name and +scanned again at the next start. Off on a held start (`--hold-work`) and while Linked environments +is off. This scan stands in for an upstream gap: upstream's `delegate_task` also launches there +before it records here, and its fix (record before the launch) is not merged yet. When an upstream +sync brings that fix, slim the scan to threads lost before it, or drop it. + +`task_list` (always there, read only) lists a thread's tasks with an unread result and how many of +its children run now per place, from their live status; a remote task whose stop there is still +owed is read there too. An environment that does not answer once is not asked again in the same +call. With `checkLinked`, it also lists such orphans, from each environment's newest 500 threads +(`next` names an environment with more, and one whose later page did not answer). Which slot view to trust: `task_list` for this +thread's own children (it reads each one's live status), `wait_list` for whether `delegate_task` +will be let through (it shows what the slot gate counts against `server.agentSlots`, every agent on +the machine or account). Switch on: **Settings → Fork**, section Waits (shown while a server is connected), "Let agents wait for events", or in `settings.json`: `"waitFor": { "enabled": true }`.