diff --git a/apps/server/src/fork/agentSlotRoutes.test.ts b/apps/server/src/fork/agentSlotRoutes.test.ts index e022542d0419c..b7d46e6fe9b40 100644 --- a/apps/server/src/fork/agentSlotRoutes.test.ts +++ b/apps/server/src/fork/agentSlotRoutes.test.ts @@ -20,8 +20,10 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import * as Metric from "effect/Metric"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; +import * as SqlError from "effect/sql/SqlError"; import * as TestClock from "effect/testing/TestClock"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; @@ -33,6 +35,7 @@ import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts import * as Scheduler from "../scheduling/Scheduler.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ServerSettings from "../serverSettings.ts"; +import * as SlotGate from "../slots/SlotGate.ts"; import * as AgentSlotRoutes from "./agentSlotRoutes.ts"; import { type SlotWorld, slotGateOver, slotWorld } from "./slotWorld.testkit.ts"; @@ -85,6 +88,8 @@ interface Harness { readonly webhookWorker: Effect.Effect; /** One pass of the fork's waiting-note cleanup. */ readonly notesWorker: Effect.Effect; + /** The server's gate the runs ask (with \`slots\`). */ + readonly gate: Option.Option; } interface Prompt { @@ -182,7 +187,9 @@ const boot = (body: (harness: Harness) => Effect.Effect, options: Op return yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const sql = yield* SqlClient.SqlClient; + const gate = yield* Effect.serviceOption(SlotGate.SlotGate); return yield* body({ + gate, service, sql, started, @@ -192,7 +199,9 @@ const boot = (body: (harness: Harness) => Effect.Effect, options: Op notesWorker: Effect.suspend(() => workers.get("agent-slot-notes")!), }); }).pipe( - Effect.provide(Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(dependencies)))), + Effect.provide( + Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provideMerge(dependencies))), + ), ); }); @@ -842,7 +851,7 @@ describe("scheduled runs through the agent slot gate (#706)", () => { ({ name, stop }) => { const slots = slotWorld([BUSY]); return withService( - ({ service, started, webhookWorker }) => + ({ service, started, webhookWorker, gate }) => Effect.gen(function* () { yield* TestClock.setTime(START); const { task } = yield* service.upsert( @@ -857,6 +866,8 @@ describe("scheduled runs through the agent slot gate (#706)", () => { slots.running = []; yield* workerRounds(webhookWorker, 4); assert.deepEqual(started, []); + // Given up: its place in the gate's line is gone too. + assert.deepEqual(yield* Option.getOrThrow(gate).line, []); if (name === "paused") { const { delivery } = yield* service.getWebhookDelivery({ id: task.id, @@ -1199,7 +1210,7 @@ describe("scheduled runs through the agent slot gate (#706)", () => { () => { const slots = slotWorld([BUSY]); return withService( - ({ service, sql, started, prompts }) => + ({ service, sql, started, prompts, gate }) => Effect.gen(function* () { yield* TestClock.setTime(START); const { task } = yield* service.upsert( @@ -1212,6 +1223,8 @@ describe("scheduled runs through the agent slot gate (#706)", () => { yield* sql`UPDATE scheduled_task_webhook_pending SET task_json = json_set(task_json, '$.prompt', 'Older words.') WHERE delivery_id = ${first.deliveryId}`; + // Out of the line, or it would go first (oldest): the worker no longer asks for it. + yield* Option.getOrThrow(gate).leave([`webhook-delivery:${first.deliveryId}`]); // A direct delivery with a slot free runs at once; the stored row is not its own. slots.running = []; const direct = yield* service.triggerWebhook(webhookRequest(task, '{"n":2}')); @@ -1395,6 +1408,290 @@ describe("scheduled runs through the agent slot gate (#706)", () => { ); }); +describe("a scheduled run or webhook delivery that waits keeps its place in the gate's line (#738)", () => { + const LATER = ThreadId.make("thread:later-waiter"); + /** A wake-up of another idle thread that joins the line after the run (asks every 5 s). */ + const laterWaiter = (gate: Harness["gate"]) => + Option.getOrThrow(gate).acquire({ + callerThreadId: LATER, + targetInstanceId: INSTANCE, + intoThreadId: LATER, + keepsHeld: true, + place: { key: "later-waiter" }, + }); + const lineOf = (gate: Harness["gate"]) => Option.getOrThrow(gate).line; + + it.effect( + "full: a waiter that joins later does not pass a scheduled run; the run starts first", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, poll, started, gate }) => + Effect.gen(function* () { + const task = yield* dueTask(service); + yield* poll; + assert.deepEqual(started, []); + assert.deepEqual(yield* lineOf(gate), [`scheduled-run:${task.id}`]); + yield* TestClock.adjust("5 seconds"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + // Runs again: each poll keeps its place (no second place). + yield* poll; + assert.deepEqual(yield* lineOf(gate), [`scheduled-run:${task.id}`, "later-waiter"]); + // A slot frees and the later waiter asks first: the run is ahead of it. + slots.running = []; + yield* TestClock.adjust("1 second"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* TestClock.adjust("4 seconds"); + yield* poll; + assert.deepEqual(launches(started).length, 1); + // Let through: its place is gone; the waiter is next. + assert.deepEqual(yield* lineOf(gate), ["later-waiter"]); + }), + { slots }, + ); + }, + ); + + it.effect( + "a run due before a waiter joined is ahead of it, though the poll asks after it", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, poll, started, gate }) => + Effect.gen(function* () { + const task = yield* dueTask(service); + // The waiter asks a second after the due time, before the poll does. + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* poll; + assert.deepEqual(yield* lineOf(gate), [`scheduled-run:${task.id}`, "later-waiter"]); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* poll; + assert.equal(launches(started).length, 1); + }), + { slots }, + ); + }, + ); + + it.effect("full: a waiter that joins later does not pass a stored webhook delivery", () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, started, webhookWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + // Refused and stored; the worker holds it, asking every 5 s in its place. + yield* workerRounds(webhookWorker, 2); + assert.deepEqual(yield* lineOf(gate), [`webhook-delivery:${accepted.deliveryId}`]); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + assert.deepEqual(yield* lineOf(gate), ["later-waiter"]); + }), + { slots }, + ); + }); + + it.effect( + "a stored delivery's place goes when its task is deleted before the worker runs it", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, webhookWorker, notesWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + yield* settle; + // Refused and stored: its place waits for the worker. + assert.deepEqual(yield* lineOf(gate), [`webhook-delivery:${accepted.deliveryId}`]); + yield* service.delete({ id: task.id } as never); + // The worker drops it without asking the gate; the sweep sees it is no longer pending. + yield* webhookWorker; + yield* settle; + yield* notesWorker; + assert.deepEqual(yield* lineOf(gate), []); + }), + { slots }, + ); + }, + ); + + it.effect( + "a stored delivery is ahead of a waiter that joined after it was stored, though the worker asks after it", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, started, webhookWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + yield* settle; + // A restart: the line is memory only; the stored row keeps its due time. + yield* Option.getOrThrow(gate).leave([`webhook-delivery:${accepted.deliveryId}`]); + yield* TestClock.adjust("1 second"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* TestClock.adjust("1 second"); + yield* webhookWorker; + yield* settle; + assert.deepEqual(yield* lineOf(gate), [ + `webhook-delivery:${accepted.deliveryId}`, + "later-waiter", + ]); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + }), + { slots }, + ); + }, + ); + + it.effect("a direct delivery waiting in memory keeps its place through the 5 s sweep", () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, sql, started, notesWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + // A row with its id is there already, not pending: it is not stored, it waits in memory. + const deliveryId = "delivery:relay:held"; + yield* sql`INSERT INTO scheduled_task_webhook_pending + (delivery_id, task_id, coalesce_key, due_at, state, task_json, request_json, delivery_ids_json) + VALUES (${deliveryId}, ${task.id}, '', ${START}, 'running', '{}', '{}', '[]')`; + const accepted = yield* service.triggerWebhook({ + ...webhookRequest(task), + relayDeliveryId: "held", + }); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + assert.equal(accepted.deliveryId, deliveryId); + yield* settle; + yield* TestClock.adjust("1 second"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + // The sweep finds no pending row for it, but it still waits: its place stays. + yield* notesWorker; + assert.deepEqual(yield* lineOf(gate), [`webhook-delivery:${deliveryId}`, "later-waiter"]); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + }), + { slots }, + ); + }); + + it.effect("a stored delivery whose row cannot be read is logged and still keeps a place", () => { + const slots = slotWorld([BUSY]); + const readFails = { value: false }; + const warnings: Array = []; + const logger = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(JSON.stringify(message)); + }); + return boot( + ({ service, started, webhookWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + yield* settle; + // A restart, then the worker's read of the row's place fails: its task's place, now. + yield* Option.getOrThrow(gate).leave([`webhook-delivery:${accepted.deliveryId}`]); + readFails.value = true; + yield* TestClock.adjust("1 second"); + yield* webhookWorker; + yield* settle; + assert.lengthOf( + warnings.filter((warning) => warning.includes("place in line")), + 1, + ); + assert.deepEqual(yield* lineOf(gate), [`webhook-task:${task.id}`]); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + assert.deepEqual(yield* lineOf(gate), ["later-waiter"]); + }), + { slots }, + ).pipe( + Effect.updateService( + SqlClient.SqlClient, + (sql) => + new Proxy(sql, { + apply: (target, self, args: Array) => + readFails.value && + Array.isArray(args[0]) && + args[0].join("?").includes("SELECT delivery_id, due_at FROM") + ? Effect.fail( + new SqlError.SqlError({ + reason: new SqlError.LockTimeoutError({ cause: null }), + }), + ) + : Reflect.apply(target, self, args), + }), + ), + Effect.provide( + Layer.mergeAll( + SqlitePersistence.layerMemory, + Logger.layer([logger], { mergeWithExisting: true }), + ), + ), + ); + }); + + it.effect( + "a waiting run's place goes when its task is deleted or paused (no slot kept for it)", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, poll, notesWorker, gate }) => + Effect.gen(function* () { + const deleted = yield* dueTask(service); + const paused = yield* service.upsert( + yield* taskInput({ + title: "Second", + schedule: { type: "interval", everyMs: 60_000 }, + }), + ); + yield* TestClock.adjust("2 minutes"); + yield* poll; + assert.sameMembers( + [...(yield* lineOf(gate))], + [`scheduled-run:${deleted.id}`, `scheduled-run:${paused.task.id}`], + ); + // Still due: the sweep keeps both. + yield* notesWorker; + assert.lengthOf(yield* lineOf(gate), 2); + yield* service.delete({ id: deleted.id } as never); + yield* service.setEnabled({ id: paused.task.id, enabled: false } as never); + yield* notesWorker; + assert.deepEqual(yield* lineOf(gate), []); + // A later start is not held back by them. + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "allow"); + }), + { slots }, + ); + }, + ); +}); + describe("refusal texts fit who reads them (#706)", () => { const full = { type: "full" as const, diff --git a/apps/server/src/fork/agentSlotRoutes.ts b/apps/server/src/fork/agentSlotRoutes.ts index a53a7ed104994..5ed9f53fb4ce9 100644 --- a/apps/server/src/fork/agentSlotRoutes.ts +++ b/apps/server/src/fork/agentSlotRoutes.ts @@ -13,7 +13,7 @@ import { ScheduledTaskError, type ThreadId, } from "@t3tools/contracts"; -import { slotRefusalReason, type SlotDecision } from "@t3tools/shared/agentSlots"; +import { slotRefusalReason, slotRunningText, type SlotDecision } from "@t3tools/shared/agentSlots"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -59,6 +59,8 @@ export interface RouteStart { readonly intoThreadId?: ThreadId | undefined; /** The caller is the woken thread itself: its held slot stays held (`SlotGate.acquire`). */ readonly keepsHeld?: boolean | undefined; + /** Its place in the gate's one line: a start that asks again while refused. */ + readonly place?: SlotGate.SlotPlace | undefined; } const ENDED: RouteSlot = { type: "allow", end: () => Effect.void }; @@ -83,6 +85,7 @@ const admit = ( ...(start.alsoNeeds === undefined ? {} : { alsoNeeds: start.alsoNeeds }), ...(start.intoThreadId === undefined ? {} : { intoThreadId: start.intoThreadId }), ...(start.keepsHeld === undefined ? {} : { keepsHeld: start.keepsHeld }), + ...(start.place === undefined ? {} : { place: start.place }), }); if (outcome.type === "refuse") return outcome; const token = outcome.token; @@ -99,8 +102,6 @@ const admit = ( const slotsFull = (message: string) => new OrchestratorMcpFailure({ code: "slots_full", message }); -const agents = (count: number) => (count === 1 ? "1 agent is" : `${count} agents are`); - const unreadReason = (problem: SlotGate.CountReadProblem) => problem === "timeout" ? `the read took longer than ${Duration.toSeconds(SlotGate.COUNT_READ_TIMEOUT)} seconds` @@ -109,9 +110,9 @@ const unreadReason = (problem: SlotGate.CountReadProblem) => /** {@link slotRefusalReason}, for a reader on another machine ("there", not "this machine"). */ const reasonThere = (decision: FullDecision) => decision.rule === "machine" - ? `${agents(decision.used)} running there, and the limit is ${decision.max}` + ? `${slotRunningText(decision, " there")}, and the limit is ${decision.max}` : decision.rule === "account" - ? `${agents(decision.used)} running there on account ${decision.instanceId}, and the limit is ${decision.max}` + ? `${slotRunningText(decision, ` there on account ${decision.instanceId}`)}, and the limit is ${decision.max}` : `only ${decision.used} MB of memory is free there, and a new agent needs at least ${decision.max} MB`; /** Why, in one clause, for a reader on this machine (a scheduled task's history, run now, logs). */ @@ -181,8 +182,22 @@ export interface SendSlot { readonly instanceId: ProviderInstanceId; /** The receiver is the caller (a wait's message): its held slot stays held. */ readonly keepsHeld?: boolean | undefined; + /** A wake-up that asks again while refused: its place in the gate's one line. */ + readonly place?: SlotGate.SlotPlace | undefined; + /** + * The caller ends the slot (`end`) when its dispatch ended. Until then the start counts whatever + * the start grace, so a send that waits after the gate step still has its slot when it starts. + */ + readonly endsItself?: boolean | undefined; } +/** Waiters that no longer need a slot leave the gate's line (nothing without a gate). */ +export const leaveLine = ( + gate: Option.Option, + keys: ReadonlyArray, +): Effect.Effect => + Option.match(gate, { onNone: () => Effect.void, onSome: (slots) => slots.leave(keys) }); + /** * A message into an existing thread: into one that shows a run it only queues; into one that * shows none it takes a slot like `delegate_task`. One gate step decides both, so a run that @@ -196,8 +211,9 @@ export const admitSend = ( callerThreadId: send.callerThreadId, targetInstanceId: send.instanceId, intoThreadId: send.threadId, - childThreadId: send.threadId, + ...(send.endsItself === true ? {} : { childThreadId: send.threadId }), ...(send.keepsHeld === undefined ? {} : { keepsHeld: send.keepsHeld }), + ...(send.place === undefined ? {} : { place: send.place }), }); /** @@ -367,6 +383,17 @@ const CurrentDirectDelivery = Context.Reference( { defaultValue: () => undefined }, ); +/** + * Places in the gate's line (`SlotGate.SlotPlace`): a scheduled run that waits keeps one per task, + * at its due time; a webhook delivery one per delivery, at the time it was stored (direct: its + * first ask). Each leaves when its run starts (the gate), when it is given up or its task deleted + * (`waitForWebhookSlot`, and `leaveEndedPlaces` every 5 s). + */ +const SCHEDULED_PLACE = "scheduled-run:"; +const DELIVERY_PLACE = "webhook-delivery:"; +/** A stored delivery whose row cannot be read: its task's (one stored delivery runs at a time). */ +const DELIVERY_TASK_PLACE = "webhook-task:"; + /** * What a run takes from its task's row, decoded as `ScheduledTaskService` decodes it (so a prompt * or title with spaces at either end compares as the trimmed text a run uses). Not the run's @@ -466,16 +493,107 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( * The slot is named for the run's thread when its dispatch ends (`end`), bound or new, so a * dispatch that failed frees it at once; `ended` frees it on a path that never got there. */ - const take = (target: { - readonly instanceId: ProviderInstanceId; - readonly intoThreadId: ThreadId | undefined; - }) => + const take = ( + target: { + readonly instanceId: ProviderInstanceId; + readonly intoThreadId: ThreadId | undefined; + }, + place: SlotGate.SlotPlace | undefined, + ) => admit(gate, { callerThreadId: null, targetInstanceId: target.instanceId, intoThreadId: target.intoThreadId, + ...(place === undefined ? {} : { place }), }); + /** Webhook deliveries waiting in `waitForWebhookSlot` now: their wait leaves the line itself. */ + const waitingDeliveries = new Set(); + + /** + * The run's place in line: a run on its schedule at its due time, a webhook delivery at the + * time it was stored. "Run now" asks once and has none. A stored delivery is the one its task's + * worker runs now (one at a time per task, in the worker's order); unreadable, it is logged and + * keeps its task's place from this ask (`waitForWebhookSlot` asks in it, and leaves it). + */ + const placeOf = Effect.fnUntraced(function* ( + task: ScheduledTask, + trigger: "scheduled" | "manual" | "webhook", + ) { + if (trigger === "manual") return undefined; + if (trigger === "scheduled") { + const dueMs = task.nextRunAt === null ? Number.NaN : Date.parse(task.nextRunAt); + return { + key: `${SCHEDULED_PLACE}${task.id}`, + ...(Number.isFinite(dueMs) ? { sinceMs: dueMs } : {}), + } satisfies SlotGate.SlotPlace; + } + const direct = yield* CurrentDirectDelivery; + if (direct !== undefined) return { key: `${DELIVERY_PLACE}${direct.deliveryId}` }; + const [stored] = yield* sql<{ delivery_id: string; due_at: number }>` + SELECT delivery_id, due_at FROM scheduled_task_webhook_pending + WHERE task_id = ${task.id} AND state IN ('dispatching', 'running', 'completed', 'failed') + ORDER BY due_at, rowid LIMIT 1 + `.pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not read a waiting webhook delivery's place in line", { + taskId: task.id, + cause, + }).pipe(Effect.as([])), + ), + ); + return stored === undefined + ? ({ key: `${DELIVERY_TASK_PLACE}${task.id}` } satisfies SlotGate.SlotPlace) + : ({ + key: `${DELIVERY_PLACE}${stored.delivery_id}`, + sinceMs: Number(stored.due_at), + } satisfies SlotGate.SlotPlace); + }); + + const leave = (place: SlotGate.SlotPlace | undefined) => + place === undefined ? Effect.void : leaveLine(gate, [place.key]); + + /** + * Places whose wait ended without the gate seeing it: a run on its schedule no longer due + * (paused, deleted, edited, rescheduled), a stored delivery no longer pending (its task deleted). + */ + const leaveEndedPlaces = Effect.gen(function* () { + if (Option.isNone(gate)) return; + const keys = yield* gate.value.line; + const taskIds = keys.flatMap((key) => + key.startsWith(SCHEDULED_PLACE) ? [key.slice(SCHEDULED_PLACE.length)] : [], + ); + const deliveryIds = keys.flatMap((key) => + key.startsWith(DELIVERY_PLACE) && !waitingDeliveries.has(key) + ? [key.slice(DELIVERY_PLACE.length)] + : [], + ); + if (taskIds.length === 0 && deliveryIds.length === 0) return; + const nowIso = DateTime.formatIso(DateTime.toUtc(yield* DateTime.now)); + const due = new Set( + taskIds.length === 0 + ? [] + : (yield* sql<{ task_id: string }>` + SELECT task_id FROM scheduled_tasks + WHERE task_id IN ${sql.in(taskIds)} AND enabled = 1 AND next_run_at IS NOT NULL + AND next_run_at <= ${nowIso} AND last_run_status <> 'running' + `).map((row) => row.task_id), + ); + const pending = new Set( + deliveryIds.length === 0 + ? [] + : (yield* sql<{ delivery_id: string }>` + SELECT delivery_id FROM scheduled_task_webhook_pending + WHERE delivery_id IN ${sql.in(deliveryIds)} AND state IN ('pending', 'dispatching') + `).map((row) => row.delivery_id), + ); + const ended = [ + ...taskIds.filter((id) => !due.has(id)).map((id) => `${SCHEDULED_PLACE}${id}`), + ...deliveryIds.filter((id) => !pending.has(id)).map((id) => `${DELIVERY_PLACE}${id}`), + ]; + if (ended.length > 0) yield* gate.value.leave(ended); + }); + const allowed = ( task: ScheduledTask, end: (threadId: ThreadId | null) => Effect.Effect, @@ -669,32 +787,51 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( * row up again, once). Dropped only as a queued delivery is (task deleted, replaced, paused, not a * webhook). Saved while it waits, it is handed back to the worker to run on the task as saved. */ - const waitForWebhookSlot = Effect.fnUntraced(function* (task: ScheduledTask, first: Refused) { - yield* Effect.logInfo("Webhook delivery waits for an agent slot", { - taskId: task.id, - reason: reasonHere(first.refusal), - }); - while (true) { - yield* Effect.sleep(WEBHOOK_SLOT_RETRY); - const gone = yield* webhookGone(task).pipe(Effect.orElseSucceed(() => null)); - if (gone !== null) return yield* Effect.fail(new WebhookSkipped({ reason: gone })); - const since = yield* savedSince(task); - // Not known (the row cannot be read): asked again at the next retry, with no slot taken. - if (since === "unknown") continue; - if (since === "saved") { - if ((yield* CurrentDirectDelivery) === undefined) return yield* keepForWorker(task); - if (!(yield* storeDirect(task, yield* DateTime.now))) - return yield* Effect.fail(new WebhookSkipped({ reason: SAVED_WHILE_WAITING_NOT_STORED })); - return handedToWorker(task); + const waitForWebhookSlot = Effect.fnUntraced( + function* (task: ScheduledTask, first: Refused, place: SlotGate.SlotPlace | undefined) { + yield* Effect.logInfo("Webhook delivery waits for an agent slot", { + taskId: task.id, + reason: reasonHere(first.refusal), + }); + while (true) { + yield* Effect.sleep(WEBHOOK_SLOT_RETRY); + const gone = yield* webhookGone(task).pipe(Effect.orElseSucceed(() => null)); + if (gone !== null) return yield* Effect.fail(new WebhookSkipped({ reason: gone })); + const since = yield* savedSince(task); + // Not known (the row cannot be read): asked again at the next retry, with no slot taken. + if (since === "unknown") continue; + // Handed back, its place goes on release; the worker's ask rejoins at the row's due time. + if (since === "saved") { + if ((yield* CurrentDirectDelivery) === undefined) return yield* keepForWorker(task); + if (!(yield* storeDirect(task, yield* DateTime.now))) + return yield* Effect.fail( + new WebhookSkipped({ reason: SAVED_WHILE_WAITING_NOT_STORED }), + ); + return handedToWorker(task); + } + // Checked again as it would start: the owner may have widened the project meanwhile. + const refused = task.threadId === null ? yield* profiles.runRefusal(task) : null; + if (refused !== null) return yield* Effect.fail(new WebhookSkipped({ reason: refused })); + // A bound thread that shows a run only queues: the gate lets it pass (`intoThreadId`). + // Asked in its place, so a waiter that joined later never passes it. + const slot = yield* take(yield* targetOf(task), place); + if (slot.type === "allow") return allowed(task, slot.end); } - // Checked again as it would start: the owner may have widened the project meanwhile. - const refused = task.threadId === null ? yield* profiles.runRefusal(task) : null; - if (refused !== null) return yield* Effect.fail(new WebhookSkipped({ reason: refused })); - // A bound thread that shows a run only queues: the gate lets it pass (`intoThreadId`). - const slot = yield* take(yield* targetOf(task)); - if (slot.type === "allow") return allowed(task, slot.end); - } - }); + }, + // Skipped, failed, stopped or handed back: its place goes too (let through, the gate dropped + // it already). + (effect, _task, _first, place) => + place === undefined + ? effect + : Effect.acquireUseRelease( + Effect.sync(() => waitingDeliveries.add(place.key)), + () => effect, + () => + Effect.sync(() => waitingDeliveries.delete(place.key)).pipe( + Effect.andThen(leave(place)), + ), + ), + ); /** * Last before the run is marked running. Refused, a run on its schedule stays due (asked again @@ -723,6 +860,8 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( // A new thread wider than the profile the task was saved with starts nothing: on its schedule // a failed run with the reason (moved to its next time), "run now" fails, a webhook skips. const profileRefusal = task.threadId === null ? yield* profiles.runRefusal(task) : null; + // Refused for its profile, a place it had from an earlier ask goes at the next sweep (its task + // is no longer due, its delivery no longer pending: `leaveEndedPlaces`). if (profileRefusal !== null) { if (trigger === "manual") { return yield* new ScheduledTaskError({ message: profileRefusal, taskId: task.id }); @@ -743,7 +882,8 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( } as ScheduledSlot; } if (Option.isNone(gate)) return { skipped: null, end: () => Effect.void } as ScheduledSlot; - const slot = yield* take(yield* targetOf(task)); + const place = yield* placeOf(task, trigger); + const slot = yield* take(yield* targetOf(task), place); if (slot.type === "allow") return allowed(task, slot.end); if (trigger === "manual") { return yield* new ScheduledTaskError({ @@ -753,7 +893,7 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( } if (trigger === "webhook") { if (yield* storeDirect(task, now)) return handedToWorker(task); - return yield* waitForWebhookSlot(task, slot); + return yield* waitForWebhookSlot(task, slot, place); } yield* keepDue(task, now, slot.refusal).pipe( Effect.catchCause((cause) => @@ -801,7 +941,17 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( AND (enabled = 0 OR next_run_at IS NULL OR next_run_at > ${nowIso}) `; }); - yield* scheduler.register("agent-slot-notes", clearStaleNotes); + yield* scheduler.register( + "agent-slot-notes", + Effect.andThen( + leaveEndedPlaces.pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not check the line places of waiting scheduled runs", { cause }), + ), + ), + clearStaleNotes, + ), + ); return { admit: admitRun, diff --git a/apps/server/src/fork/continuationSlot.test.ts b/apps/server/src/fork/continuationSlot.test.ts index d253830334f10..2d891cc5f5a7a 100644 --- a/apps/server/src/fork/continuationSlot.test.ts +++ b/apps/server/src/fork/continuationSlot.test.ts @@ -15,6 +15,7 @@ import { } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -54,8 +55,14 @@ interface World { /** Threads the gate refuses whatever is free. */ full: Set; asks: Array; + /** The gate's calls after an ask: `started `, `release `, `leave `. */ + calls: Array; + /** Threads whose ask takes this long (a slow count read), then is refused. */ + slow: Set; /** Threads gone (`null` shell), and whether a shell read fails. */ gone: Set; + /** Threads the gate sees live but upstream's worker then reads archived. */ + archivedForUpstream: Set; unreadable: boolean; logs: Array; } @@ -64,7 +71,10 @@ const newWorld = (free: number): World => ({ free, full: new Set(), asks: [], + calls: [], + slow: new Set(), gone: new Set(), + archivedForUpstream: new Set(), unreadable: false, logs: [], }); @@ -72,19 +82,25 @@ const newWorld = (free: number): World => ({ const gateOf = (world: World) => Layer.mock(SlotGate.SlotGate)({ acquire: (input) => - Effect.sync((): SlotGate.SlotAcquire => { + Effect.gen(function* () { world.asks.push(input); - if (world.free === 0 || world.full.has(input.callerThreadId!)) - return { - type: "refuse", - message: "full", - refusal: { type: "unread", problem: "timeout" }, - }; + const refused: SlotGate.SlotAcquire = { + type: "refuse", + message: "full", + refusal: { type: "unread", problem: "timeout" }, + }; + if (world.slow.has(input.callerThreadId!)) { + yield* Effect.sleep("5 seconds"); + return refused; + } + if (world.free === 0 || world.full.has(input.callerThreadId!)) return refused; world.free -= 1; - return { type: "allow", token: world.asks.length }; + return { type: "allow", token: world.asks.length } satisfies SlotGate.SlotAcquire; }), - started: () => Effect.void, - release: () => Effect.void, + started: (token, childThreadId) => + Effect.sync(() => world.calls.push(`started ${childThreadId} ${token}`)), + release: (token) => Effect.sync(() => world.calls.push(`release ${token}`)), + leave: (keys) => Effect.sync(() => world.calls.push(...keys.map((key) => `leave ${key}`))), }); const projection = { @@ -115,7 +131,7 @@ const server = ( ), getThreadRecords: (threadId) => Effect.succeed( - world.gone.has(threadId) + world.gone.has(threadId) || world.archivedForUpstream.has(threadId) ? { ...projection, thread: { archivedAt: "2026-11-21T00:00:00.000Z", deletedAt: null } } : projection, ) as never, @@ -152,6 +168,12 @@ const offer = (value: ProviderContinuationRequests.ProviderContinuationRequest) Effect.flatMap((requests) => requests.offer(value)), ); +/** Lets forked fibers run (scheduler turns, no clock moved). */ +const settle = Effect.forEach(Array.from({ length: 50 }), () => Effect.yieldNow, { discard: true }); + +/** The gate's slot calls, without the line's. */ +const slotCalls = (world: World) => world.calls.filter((call) => !call.startsWith("leave ")); + const waitLogs = (world: World) => world.logs.filter((log) => log.includes("provider continuation waits for an agent slot")); @@ -182,7 +204,10 @@ describe("provider continuations take an agent slot", () => { targetInstanceId: INSTANCE, intoThreadId: A, keepsHeld: true, + place: { key: "continuation:0" }, }); + // Every ask in its one place in the gate's line. + assert.isTrue(world.asks.every((asked) => asked.place?.key === "continuation:0")); world.free = 1; yield* TestClock.adjust("5 seconds"); const sent = yield* Queue.take(dispatched); @@ -214,6 +239,69 @@ describe("provider continuations take an agent slot", () => { }), ); + it.effect("a slow count read for one thread holds up no other thread's continuation", () => + Effect.gen(function* () { + const world = newWorld(5); + world.slow.add(A); + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + yield* offer(request(A)); + yield* offer(request(B)); + // No time passes: B is not queued behind A's 5-second ask. + yield* settle; + assert.strictEqual(yield* Queue.size(dispatched), 1); + const sent = yield* Queue.take(dispatched); + assert.strictEqual(sent.type === "message.dispatch" && sent.threadId, B); + // A's ask ends (it cannot be cut off half-way), so the server can stop. + yield* TestClock.adjust("5 seconds"); + }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); + }), + ); + + it.effect( + "a granted slot is held until upstream's dispatch of it ends, however long it waits", + () => + Effect.gen(function* () { + const world = newWorld(1); + const dispatched = yield* Queue.unbounded(); + const lockFree = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* offer( + request(A, { + // Behind its own thread's earlier dispatch (its parent's lock) for 2 minutes. + dispatchIfCurrent: (effect) => + Deferred.await(lockFree).pipe(Effect.andThen(Effect.asSome(effect))), + }), + ); + yield* TestClock.adjust("2 minutes"); + assert.lengthOf(world.asks, 1); + // Asked and granted, not ended: the start counts whatever the start grace. + assert.deepStrictEqual(slotCalls(world), []); + yield* Deferred.succeed(lockFree, undefined); + yield* Queue.take(dispatched); + yield* settle; + assert.deepStrictEqual(slotCalls(world), [`started ${A} 1`, "release 1"]); + }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); + }), + ); + + it.effect("a granted request upstream drops (its thread archived meanwhile) frees its slot", () => + Effect.gen(function* () { + const world = newWorld(1); + const dispatched = yield* Queue.unbounded(); + const cleared: Array = []; + yield* Effect.gen(function* () { + // Granted on a live shell; upstream then reads the thread archived and clears the latch. + world.archivedForUpstream.add(A); + yield* offer(request(A, { clearIfCurrent: () => Effect.sync(() => cleared.push(A)) })); + yield* settle; + assert.deepStrictEqual(cleared, [A]); + assert.strictEqual(yield* Queue.size(dispatched), 0); + assert.deepStrictEqual(slotCalls(world), [`started ${A} 1`, "release 1"]); + }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); + }), + ); + it.effect("a thread that cannot be read waits, then is sent once", () => Effect.gen(function* () { const world = newWorld(5); @@ -241,7 +329,7 @@ describe("provider continuations take an agent slot", () => { const cleared: Array = []; yield* Effect.gen(function* () { yield* offer(request(A, { clearIfCurrent: () => Effect.sync(() => cleared.push(A)) })); - yield* TestClock.adjust("0 seconds"); + yield* settle; assert.deepStrictEqual(cleared, [A]); assert.deepStrictEqual(world.asks, []); assert.strictEqual(yield* Queue.size(dispatched), 0); @@ -254,7 +342,7 @@ describe("provider continuations take an agent slot", () => { const world = newWorld(0); const inner = yield* Queue.unbounded(); - const gated = ContinuationSlot.gateContinuations( + const gated = yield* ContinuationSlot.gateContinuations( { offer: (value) => Queue.offer(inner, value).pipe(Effect.asVoid), take: Queue.take(inner), @@ -291,6 +379,8 @@ describe("provider continuations take an agent slot", () => { yield* TestClock.adjust("12 seconds"); }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); const asked = world.asks.length; + // Its place in the line goes with it. + assert.include(world.calls, "leave continuation:0"); world.free = 5; yield* TestClock.adjust("1 minute"); assert.strictEqual(yield* Queue.size(dispatched), 0); @@ -308,7 +398,7 @@ describe("provider continuations take an agent slot", () => { }).pipe(Effect.provide(server(world, dispatched, null)), Effect.scoped); const inner = { offer: () => Effect.void, take: Effect.never }; assert.strictEqual( - ContinuationSlot.gateContinuations( + yield* ContinuationSlot.gateContinuations( inner, Option.none(), () => Effect.succeed(null), diff --git a/apps/server/src/fork/continuationSlot.ts b/apps/server/src/fork/continuationSlot.ts index ec0f5a85475f5..8bf6e77b14e34 100644 --- a/apps/server/src/fork/continuationSlot.ts +++ b/apps/server/src/fork/continuationSlot.ts @@ -1,7 +1,7 @@ /** - * Fork: a provider continuation into an idle thread takes an agent slot. Asked at the queue, not - * around the dispatch: an adapter's `dispatchIfCurrent` holds its own lock while it dispatches. - * A refused request goes back on the queue; the adapter's latch stays set, so it offers no second. + * Fork: a provider continuation into an idle thread takes an agent slot, asked off upstream's queue + * (never inside `dispatchIfCurrent`, which holds the adapter's lock), each request in its own fiber + * and place in the gate's line. The slot is held until upstream's dispatch of it ends. */ import type { ProviderInstanceId, ThreadId } from "@t3tools/contracts"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; @@ -9,11 +9,12 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import type * as Scope from "effect/Scope"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as SlotGate from "../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "./agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "./agentSlotRoutes.ts"; type Request = ProviderContinuationRequests.ProviderContinuationRequest; interface Requests { @@ -27,69 +28,104 @@ export const CONTINUATION_SLOT_RETRY = Duration.seconds(5); /** `null`: no such thread, or it is gone. */ export type ContinuationThread = { readonly instanceId: ProviderInstanceId } | null; +/** + * The request holding its slot (`end`) until upstream's dispatch of it ends, whichever way, so a + * run that waits behind its thread's earlier dispatch still has it. One never dispatched frees it + * at the gate's dispatch limit. + */ +const holdingSlot = ( + request: Request, + end: (childThreadId: ThreadId | null) => Effect.Effect, +): Request => { + const dispatchIfCurrent = request.dispatchIfCurrent; + const clearIfCurrent = request.clearIfCurrent; + const ended = end(request.threadId); + return { + ...request, + dispatchIfCurrent: (effect: Effect.Effect) => + (dispatchIfCurrent === undefined ? Effect.asSome(effect) : dispatchIfCurrent(effect)).pipe( + Effect.ensuring(ended), + ), + ...(clearIfCurrent === undefined + ? {} + : { clearIfCurrent: () => clearIfCurrent().pipe(Effect.ensuring(ended)) }), + }; +}; + /** * `take` hands out a request into an idle thread only with a slot. Upstream's at once: a delegated * result (docs/fork/README.md) and a gone thread (upstream drops it and clears the adapter's latch). - * An unreadable thread waits, as a refused one does. + * An unreadable thread waits, as a refused one does. No ask ever holds up upstream's queue: each + * request asks in its own fiber, so a slow count delays only that request. */ export const gateContinuations = ( requests: Requests, gate: Option.Option, threadOf: (threadId: ThreadId) => Effect.Effect, scope: Scope.Scope, -): Requests => { - if (Option.isNone(gate)) return requests; - /** Logged once until handed out. */ - const waiting = new Set(); +): Effect.Effect => { + if (Option.isNone(gate)) return Effect.succeed(requests); + let places = 0; - /** `null` when the request may go to upstream now, else why it waits. */ - const waitsFor = Effect.fn("ContinuationSlot.waitsFor")(function* (request: Request) { - if (request.delegatedCompletion !== undefined) return null; + /** The request for upstream when it may go now (holding its slot), else why it waits. */ + const ask = Effect.fn("ContinuationSlot.ask")(function* ( + request: Request, + place: SlotGate.SlotPlace, + ) { const thread = yield* threadOf(request.threadId).pipe( Effect.orElseSucceed(() => "unreadable" as const), ); if (thread === "unreadable") return "the thread could not be read to check for a slot"; - if (thread === null) return null; + if (thread === null) return request; const slot = yield* admitSend(gate, { callerThreadId: request.threadId, threadId: request.threadId, instanceId: thread.instanceId, keepsHeld: true, + place, + endsItself: true, }); - return slot.type === "refuse" ? reasonHere(slot.refusal) : null; + return slot.type === "refuse" ? reasonHere(slot.refusal) : holdingSlot(request, slot.end); }); - const putBack = Effect.fn("ContinuationSlot.putBack")(function* ( - request: Request, - detail: string, - ) { - if (!waiting.has(request)) { - waiting.add(request); - yield* Effect.logInfo("provider continuation waits for an agent slot", { - threadId: request.threadId, - providerThreadId: request.providerThreadId, - detail, - }); - } - yield* Effect.sleep(CONTINUATION_SLOT_RETRY).pipe( - Effect.andThen(requests.offer(request)), - Effect.forkIn(scope), - ); - }); - - const take: Effect.Effect = Effect.gen(function* () { - while (true) { - const request = yield* requests.take; - const detail = yield* waitsFor(request); - if (detail === null) { - waiting.delete(request); - return request; + /** One request, until it may go: asked again every `CONTINUATION_SLOT_RETRY`, logged once. */ + const untilSlot = Effect.fn("ContinuationSlot.untilSlot")( + function* (request: Request, place: SlotGate.SlotPlace, ready: Queue.Queue) { + let logged = false; + while (true) { + const answer = yield* ask(request, place); + if (typeof answer !== "string") { + yield* Queue.offer(ready, answer); + return; + } + if (!logged) { + logged = true; + yield* Effect.logInfo("provider continuation waits for an agent slot", { + threadId: request.threadId, + providerThreadId: request.providerThreadId, + detail: answer, + }); + } + yield* Effect.sleep(CONTINUATION_SLOT_RETRY); } - yield* putBack(request, detail); - } - }); + }, + // Stopped while it waits (the server stops): nothing is sent later, and its place goes. + (effect, _request: Request, place: SlotGate.SlotPlace) => + Effect.ensuring(effect, leaveLine(gate, [place.key])), + ); - return { offer: requests.offer, take }; + return Effect.gen(function* () { + const ready = yield* Queue.unbounded(); + // Upstream's queue is only read here, and never waits on an ask. + const handOut = (request: Request): Effect.Effect => + request.delegatedCompletion !== undefined + ? Effect.asVoid(Queue.offer(ready, request)) + : Effect.asVoid( + Effect.forkIn(untilSlot(request, { key: `continuation:${places++}` }, ready), scope), + ); + yield* requests.take.pipe(Effect.flatMap(handOut), Effect.forever, Effect.forkIn(scope)); + return { offer: requests.offer, take: Queue.take(ready) }; + }); }; /** The queue the adapters offer to (memoized by reference), gated for upstream's worker only. */ @@ -99,7 +135,7 @@ export const layerOver = Layer.effect( const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; const threads = yield* ThreadManagement.ThreadManagementService; const gate = yield* Effect.serviceOption(SlotGate.SlotGate); - return gateContinuations( + return yield* gateContinuations( requests, gate, (threadId) => diff --git a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts index 5b392893fd148..a571e682722c2 100644 --- a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts +++ b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts @@ -80,6 +80,8 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = /** Answers its first page only. */ const cutShort = new Set(); const linksOff = { value: false }; + /** Every send fails as a store that cannot be read now does (tried again, same id). */ + const sendFails = { value: false }; const pageReads: Array<{ environmentId: EnvironmentId; cursor: number }> = []; const sent: Array<{ threadId: ThreadId; messageId: string; text: string }> = []; const receipts = new Map(); @@ -113,6 +115,7 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = silent, cutShort, linksOff, + sendFails, pageReads, sent, receipts, @@ -137,24 +140,32 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = ), ), sendToThread: (input) => - refusing.has(input.commandId) - ? Effect.sync(() => refused.add(input.commandId)).pipe( - Effect.andThen( - Effect.fail({ - _tag: "ThreadManagementCommandInvariantError", - message: "refused", - } as never), - ), - ) - : Effect.sync(() => { - sent.push({ + sendFails.value + ? Effect.fail( + new ThreadManagement.ThreadManagementProjectionLoadError({ + projectId: input.projectId, threadId: input.threadId, - messageId: input.messageId!, - text: input.text, - }); - receipts.set(input.commandId, input.threadId); - return {} as never; - }), + cause: new Error("store busy"), + }), + ) + : refusing.has(input.commandId) + ? Effect.sync(() => refused.add(input.commandId)).pipe( + Effect.andThen( + Effect.fail({ + _tag: "ThreadManagementCommandInvariantError", + message: "refused", + } as never), + ), + ) + : Effect.sync(() => { + sent.push({ + threadId: input.threadId, + messageId: input.messageId!, + text: input.text, + }); + receipts.set(input.commandId, input.threadId); + return {} as never; + }), }), Layer.mock(CommandReceiptStore.CommandReceiptStoreV2)({ getByCommandId: (commandId: CommandId) => @@ -775,6 +786,8 @@ describe("DelegatedOrphanScan and agent slots", () => { callerThreadId: PARENT, intoThreadId: PARENT, keepsHeld: true, + // Asked again every 5 s through the wait service's delivery, in one place in line. + place: { key: expect.stringMatching(/^wait-delivery:/) }, }), ]); }), @@ -929,6 +942,79 @@ describe("DelegatedOrphanScan and agent slots", () => { }), ); + it.effect("a send that keeps failing is logged as a failed send, not an unreadable parent", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.slots.state.all = true; + const warnings: Array = []; + const logger = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(JSON.stringify(message)); + }); + const fiber = yield* (yield* service(world)) + .run() + .pipe( + Effect.provide(Logger.layer([logger], { mergeWithExisting: false })), + Effect.forkChild, + ); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + yield* settle; + // A slot frees, but every send fails. + world.slots.state.all = false; + world.sendFails.value = true; + yield* TestClock.adjust(SLOT_RETRY); + yield* settle; + for (let step = 0; step < OPTIONS.attempts; step++) { + yield* TestClock.adjust(OPTIONS.retryDelay); + yield* settle; + } + expect(fiber.pollUnsafe()).toBeDefined(); + expect(warnings.filter((warning) => warning.includes("its send failed"))).toHaveLength(1); + expect(warnings.filter((warning) => warning.includes("cannot be read"))).toHaveLength(0); + expect( + warnings.filter((warning) => warning.includes("left to the next start: its send kept")), + ).toHaveLength(1); + expect(world.sent).toHaveLength(0); + }), + ); + + it.effect( + "the give-up warning lists a parent whose send kept failing apart from unread ones", + () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + // A slot is free, but every send to the parent fails: each scan tries it again. + world.sendFails.value = true; + const warnings: Array = []; + const logger = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(JSON.stringify(message)); + }); + const fiber = yield* (yield* service(world)) + .run() + .pipe( + Effect.provide(Logger.layer([logger], { mergeWithExisting: false })), + Effect.forkChild, + ); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + for (let attempt = 1; attempt <= OPTIONS.attempts; attempt += 1) { + yield* settle; + if (attempt < OPTIONS.attempts) yield* TestClock.adjust(OPTIONS.retryDelay); + } + yield* Fiber.join(fiber); + const left = warnings.filter((warning) => warning.includes("work unchecked")); + expect(left).toHaveLength(1); + const fields = (JSON.parse(left[0]!) as Array).find( + (part): part is Record => typeof part === "object" && part !== null, + ); + expect(fields?.failedSendParentThreadIds).toEqual([PARENT]); + expect(fields?.unreadableParentThreadIds).toEqual([]); + expect(world.sent).toHaveLength(0); + }), + ); + it.effect( "a refused message before one that waits for a slot is logged once, also past a day", () => diff --git a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts index c6c9414f19cca..e39cc61e87bba 100644 --- a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts +++ b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts @@ -76,7 +76,8 @@ export type ParentOutcome = /** No orphan, or every one was named before. */ | { readonly type: "none" } | { readonly type: "gone" } - | { readonly type: "unreadable" } + /** The parent could not be read, or (`sendFailed`, why) a reminder's send failed: tried again. */ + | { readonly type: "unreadable"; readonly sendFailed?: string } /** * The agent slot gate refused (nothing of that message was sent): `run` asks again for this * parent alone, with the threads this scan found. @@ -164,7 +165,7 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( // A busy parent is not held: its run's end changes nothing about an orphan, and the message // waits in its queue until that run (a question to a person included) is over. const sent: Array = []; - let retry = false; + let sendFailed: string | undefined; let refused: string | undefined; for (const batch of batches) { const ids = orphanBatchIds(parentThreadId, batch); @@ -195,14 +196,14 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( refused = result.detail; break; case "retry": - retry = true; + sendFailed = result.detail; break; case "waitsForSlot": // The next batches go to the same idle thread: they wait too. Sent ones are recorded. return { type: "waitsForSlot", detail: result.detail, linked } satisfies ParentOutcome; } } - if (retry) return { type: "unreadable" } satisfies ParentOutcome; + if (sendFailed !== undefined) return { type: "unreadable", sendFailed } satisfies ParentOutcome; if (refused !== undefined) return { type: "held", detail: refused } satisfies ParentOutcome; return { type: "sent", threadIds: sent } satisfies ParentOutcome; }); @@ -275,9 +276,14 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( return { type: "scanned", unreachable, cutShort, partial, parents } satisfies ScanOutcome; }); - const unreadableParents = (outcome: ScanOutcome) => + /** Parents tried again: unreadable ones, or (`sendFailed`) ones whose reminder's send failed. */ + const unreadableParents = (outcome: ScanOutcome, sendFailed?: boolean) => [...outcome.parents.entries()] - .filter(([, parent]) => parent.type === "unreadable") + .filter( + ([, parent]) => + parent.type === "unreadable" && + (sendFailed === undefined || (parent.sendFailed !== undefined) === sendFailed), + ) .map(([threadId]) => threadId); /** Another pass while an environment did not answer (fully) or a parent could not be read. */ @@ -313,13 +319,22 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( } if (outcome.type !== "waitsForSlot" && outcome.type !== "unreadable") return; unreadable = outcome.type === "unreadable" ? unreadable + 1 : 0; + // A failed send is told as one, not as a parent that cannot be read. + const sendFailed = outcome.type === "unreadable" ? outcome.sendFailed : undefined; if (unreadable === 1) - yield* Effect.logWarning("delegated orphan reminder: its parent cannot be read", { - threadId: parentThreadId, - }); + yield* sendFailed === undefined + ? Effect.logWarning("delegated orphan reminder: its parent cannot be read", { + threadId: parentThreadId, + }) + : Effect.logWarning("delegated orphan reminder: its send failed", { + threadId: parentThreadId, + detail: sendFailed, + }); if (unreadable >= options.attempts) return yield* Effect.logWarning( - "delegated orphan reminder left to the next start: its parent stayed unreadable", + sendFailed === undefined + ? "delegated orphan reminder left to the next start: its parent stayed unreadable" + : "delegated orphan reminder left to the next start: its send kept failing", { threadId: parentThreadId, tries: unreadable }, ); yield* Effect.sleep(outcome.type === "waitsForSlot" ? slotRetryDelay : options.retryDelay); @@ -365,7 +380,8 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( scans: options.attempts, unansweredEnvironmentIds: outcome.unreachable, cutShortEnvironmentIds: outcome.cutShort, - unreadableParentThreadIds: unreadableParents(outcome), + unreadableParentThreadIds: unreadableParents(outcome, false), + failedSendParentThreadIds: unreadableParents(outcome, true), }, ); } diff --git a/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts b/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts index 43f125345efed..4dc99488b4d7f 100644 --- a/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts +++ b/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts @@ -47,6 +47,8 @@ export const makeSlots = () => { const full = new Set(); const state = { all: false }; const asks: Array = []; + /** Keys that left the gate's line. */ + const left: Array = []; const layer = Layer.mock(SlotGate.SlotGate)({ acquire: (input) => Effect.sync((): SlotGate.SlotAcquire => { @@ -59,8 +61,9 @@ export const makeSlots = () => { }), started: () => Effect.void, release: () => Effect.void, + leave: (keys) => Effect.sync(() => left.push(...keys)), }); - return { full, state, asks, layer }; + return { full, state, asks, left, layer }; }; /** diff --git a/apps/server/src/fork/handoff/HandoffService.test.ts b/apps/server/src/fork/handoff/HandoffService.test.ts index 1b8aa914c2f0c..dc9fccf66d3d0 100644 --- a/apps/server/src/fork/handoff/HandoffService.test.ts +++ b/apps/server/src/fork/handoff/HandoffService.test.ts @@ -102,7 +102,8 @@ const harness = (state: { yield* TestClock.setTime(NOW_MS); const events = yield* Queue.unbounded(); const sent: Array = []; - const reads = { count: 0 }; + /** `text`: the reads that build a handoff's text (its transcript part). */ + const reads = { count: 0, text: 0 }; const ended = new Map(); const milestones: Array<{ readonly name: string; @@ -147,9 +148,10 @@ const harness = (state: { }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(state.data.thread), - getThreadRecords: () => + getThreadRecords: (_threadId, fields) => Effect.sync(() => { reads.count++; + if ((fields as ReadonlyArray).includes("attempts")) reads.text++; return state.data; }), }), @@ -529,7 +531,11 @@ describe("HandoffService", () => { /** A gate with `free` slots: each allowed start takes one; the test frees them. */ const countingGate = (free: number) => { - const slots = { free, asks: [] as Array }; + const slots = { + free, + asks: [] as Array, + left: [] as Array, + }; const layer = Layer.mock(SlotGate.SlotGate)({ acquire: (input) => Effect.sync((): SlotGate.SlotAcquire => { @@ -545,6 +551,7 @@ describe("HandoffService", () => { }), started: () => Effect.void, release: () => Effect.void, + leave: (keys) => Effect.sync(() => slots.left.push(...keys)), }); return { slots, layer }; }; @@ -553,25 +560,77 @@ describe("HandoffService", () => { Effect.gen(function* () { const gate = countingGate(0); const state = { data: projection({ usedTokens: 160_000 }), settings: on, gate: gate.layer }; - const { sent, emit } = yield* harness(state); + const { sent, emit, reads } = yield* harness(state); yield* emit(runUpdated(run()), "readThread", 1); yield* settle; yield* TestClock.adjust(MINUTE_MS); expect(sent).toEqual([]); - // Asked again every 5 s, each time after the busy check, as the thread's own start. + // Asked again every 5 s, each time after the busy check, as the thread's own start, in + // its one place in the gate's line. expect(gate.slots.asks.length).toBeGreaterThanOrEqual(12); + const place = { key: `handoff:server:handoff:${threadId}:${runId}` }; expect(gate.slots.asks[0]).toEqual({ callerThreadId: threadId, targetInstanceId: modelSelection.instanceId, intoThreadId: threadId, keepsHeld: true, + place, }); + expect(gate.slots.asks.every((asked) => asked.place?.key === place.key)).toBe(true); + // Refused: its text (the transcript read) is not built at all. + expect(reads.text).toBe(0); gate.slots.free = 1; yield* TestClock.adjust(HandoffService.HANDOFF_SLOT_RETRY); yield* emit(null, "handOffWhenDue", 1); expect(messages(sent).map((message) => message.commandId)).toEqual([ `server:handoff:${threadId}:${runId}`, ]); + // Built once, with the slot. + expect(reads.text).toBe(1); + }), + ); + + it.effect("a waiting handoff whose thread gets busy leaves its place, and waits", () => + Effect.gen(function* () { + const gate = countingGate(0); + const data = projection({ usedTokens: 160_000 }); + const state = { data, settings: on, gate: gate.layer }; + const { sent, emit } = yield* harness(state); + yield* emit(runUpdated(run()), "readThread", 1); + yield* settle; + yield* TestClock.adjust(MINUTE_MS); + expect(gate.slots.left).toEqual([]); + // A question to the person opens: not due now, so it needs no slot and holds no place. + state.data = { + ...data, + runtimeRequests: [ + { id: RuntimeRequestId.make("request-1"), status: "pending" }, + ] as unknown as OrchestrationV2ThreadProjection["runtimeRequests"], + }; + yield* emit(threadChanged(state.data), "readThread", 14); + yield* settle; + expect(gate.slots.left).toEqual([`handoff:server:handoff:${threadId}:${runId}`]); + expect(sent).toEqual([]); + }), + ); + + it.effect("a waiting handoff replaced by a newer run's timer leaves its place", () => + Effect.gen(function* () { + const gate = countingGate(0); + const data = projection({ usedTokens: 160_000 }); + const state = { data, settings: on, gate: gate.layer }; + const { sent, emit, events } = yield* harness(state); + yield* emit(runUpdated(run()), "readThread", 1); + yield* settle; + yield* TestClock.adjust(MINUTE_MS); + expect(gate.slots.left).toEqual([]); + // A newer run completes: its own timer replaces the waiting one, which leaves the line. + const newer = run({ id: RunId.make("run-2"), ordinal: 2 }); + state.data = { ...data, runs: [...data.runs, newer] }; + yield* Queue.offer(events, runUpdated(newer)); + yield* settle; + expect(gate.slots.left).toContain(`handoff:server:handoff:${threadId}:${runId}`); + expect(sent).toEqual([]); }), ); diff --git a/apps/server/src/fork/handoff/HandoffService.ts b/apps/server/src/fork/handoff/HandoffService.ts index 6e18013461b75..ac9a66ab5e2a3 100644 --- a/apps/server/src/fork/handoff/HandoffService.ts +++ b/apps/server/src/fork/handoff/HandoffService.ts @@ -30,8 +30,10 @@ import { forkParked } from "../../serverActivation.ts"; import * as ServerSettings from "../../serverSettings.ts"; import * as SlotGate from "../../slots/SlotGate.ts"; import { type HandoffTrigger, handoffDecision } from "./handoffDecision.ts"; +import { leaveLine } from "../agentSlotRoutes.ts"; import { dispatchHandoff, + handoffPlace, type HandoffWaitsForSlot, readHandoffRecords, } from "./handoffDispatch.ts"; @@ -104,15 +106,6 @@ export const make = Effect.gen(function* () { ids: { readonly commandId: string; readonly messageId: string }, read: NonNullable>>, ) { - const recent = read.records.runs - .toSorted((left, right) => right.ordinal - left.ordinal) - .slice(0, TRANSCRIPT_RUNS) - .map((recentRun) => recentRun.id); - const more = yield* projections.getThreadRecords( - threadId, - ["attempts", "subagents", "turnItems"], - { turnItemRunIds: recent, omitSettledSubagents: true }, - ); const firstSends = new Set( read.records.runs.flatMap((other) => { const parsed = parseHandoffMessageId(threadId, other.userMessageId); @@ -127,12 +120,24 @@ export const make = Effect.gen(function* () { commandId: CommandId.make(ids.commandId), messageId: MessageId.make(ids.messageId), threadId, - text: handoffMessageText({ - records: { ...read.records, ...more }, - sourceRunId: run.id, - number: firstSends.size + 1, - nowIso: DateTime.formatIso(yield* DateTime.now), - modelSelection, + // Built only with a slot: a refused handoff asks again every 5 s and reads nothing more. + text: Effect.gen(function* () { + const recent = read.records.runs + .toSorted((left, right) => right.ordinal - left.ordinal) + .slice(0, TRANSCRIPT_RUNS) + .map((recentRun) => recentRun.id); + const more = yield* projections.getThreadRecords( + threadId, + ["attempts", "subagents", "turnItems"], + { turnItemRunIds: recent, omitSettledSubagents: true }, + ); + return handoffMessageText({ + records: { ...read.records, ...more }, + sourceRunId: run.id, + number: firstSends.size + 1, + nowIso: DateTime.formatIso(yield* DateTime.now), + modelSelection, + }); }), attachments: [], createdBy: "agent", @@ -144,8 +149,19 @@ export const make = Effect.gen(function* () { ); }); - const handOffWhenDue = Effect.fn("HandoffService.handOffWhenDue")( - function* (threadId: ThreadId, sourceRunId: RunId) { + /** A handoff's place in the gate's line while it waits for a slot; left when it stops waiting. */ + interface Waiting { + key: string | undefined; + } + const leavePlace = (waiting: Waiting) => + Effect.suspend(() => { + const key = waiting.key; + waiting.key = undefined; + return key === undefined ? Effect.void : leaveLine(gate, [key]); + }); + + const handOffLoop = Effect.fn("HandoffService.handOffWhenDue")( + function* (threadId: ThreadId, sourceRunId: RunId, waiting: Waiting) { let runId = sourceRunId; let trigger: HandoffTrigger = "turn-end"; let slotWaitLogged = false; @@ -202,6 +218,8 @@ export const make = Effect.gen(function* () { nowMs, }); if (decision.kind !== "handoff") yield* endSettle; + // Not due now: it needs no slot, so it holds no place from the waiters behind it. + if (decision.kind !== "handoff") yield* leavePlace(waiting); if (decision.kind === "skip") return; if (decision.kind === "sleep") { yield* Effect.sleep(Duration.millis(decision.untilMs - nowMs)); @@ -256,6 +274,8 @@ export const make = Effect.gen(function* () { read, ); if (!waitsForSlot(outcome)) return; + if (waiting.key !== handoffPlace(decision.commandId).key) yield* leavePlace(waiting); + waiting.key = handoffPlace(decision.commandId).key; // Settled already: it decides again and checks the relay's view, without a new settle. if (!slotWaitLogged) yield* logSlotWait(threadId, outcome); slotWaitLogged = true; @@ -270,6 +290,12 @@ export const make = Effect.gen(function* () { ), ); + /** Replaced by a newer run's timer, or stopped: its place in the line goes with it. */ + const handOffWhenDue = (threadId: ThreadId, sourceRunId: RunId) => { + const waiting: Waiting = { key: undefined }; + return handOffLoop(threadId, sourceRunId, waiting).pipe(Effect.ensuring(leavePlace(waiting))); + }; + /** A handoff run that ended: keep the thread read, or send the same text once more if it failed. */ const handoffRunEnded = Effect.fn("HandoffService.handoffRunEnded")( function* ( @@ -345,11 +371,18 @@ export const make = Effect.gen(function* () { // The busy check runs again at each try: a thread that moved on drops it. let outcome = yield* resend; if (waitsForSlot(outcome)) yield* logSlotWait(threadId, outcome); - while (waitsForSlot(outcome)) { - yield* Effect.sleep(HANDOFF_SLOT_RETRY); - if ((yield* readThread(threadId)) === null) return; - outcome = yield* resend; - } + // Its place in the line goes when it stops waiting, whichever way. + yield* Effect.gen(function* () { + while (waitsForSlot(outcome)) { + yield* Effect.sleep(HANDOFF_SLOT_RETRY); + if ((yield* readThread(threadId)) === null) return; + outcome = yield* resend; + } + }).pipe( + Effect.ensuring( + leaveLine(gate, [handoffPlace(`server:${run.userMessageId}${HANDOFF_RETRY_SUFFIX}`).key]), + ), + ); }, (effect, run) => effect.pipe( diff --git a/apps/server/src/fork/handoff/handoffDispatch.ts b/apps/server/src/fork/handoff/handoffDispatch.ts index a6e9e8f643f96..4e9f1826724ec 100644 --- a/apps/server/src/fork/handoff/handoffDispatch.ts +++ b/apps/server/src/fork/handoff/handoffDispatch.ts @@ -9,7 +9,7 @@ import * as Effect from "effect/Effect"; import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; import * as SlotGate from "../../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "../agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "../agentSlotRoutes.ts"; import { type HandoffRecords, handoffDispatchDropped } from "./handoffFacts.ts"; export type HandoffDispatchCommand = Extract< @@ -59,30 +59,46 @@ export interface HandoffWaitsForSlot { readonly detail: string; } +/** A handoff's place in the gate's line, while it waits for a slot. */ +export const handoffPlace = (commandId: string): SlotGate.SlotPlace => ({ + key: `handoff:${commandId}`, +}); + /** * Sends a handoff message only to a quiet thread at its source run (see the file comment), with an * agent slot when the server has a gate. One uninterruptible step: a replaced timer or a shutdown - * cannot stop it between the send and the cancel. + * cannot stop it between the send and the cancel. A text given as an effect is built only once + * the gate gave a slot, so a refused handoff reads nothing more. */ -export const dispatchHandoff = Effect.fn("HandoffDispatch.dispatch")(function* ( - command: HandoffDispatchCommand, +export const dispatchHandoff = Effect.fn("HandoffDispatch.dispatch")(function* < + E = never, + R = never, +>( + command: Omit & { + readonly text: string | Effect.Effect; + }, ) { const orchestrator = yield* Orchestrator.OrchestratorV2; const projections = yield* ProjectionStore.ProjectionStoreV2; + const gate = yield* Effect.serviceOption(SlotGate.SlotGate); + const place = handoffPlace(command.commandId); const records = yield* readHandoffRecords(command.threadId); if (handoffDispatchDropped(command, records)) { + yield* leaveLine(gate, [place.key]); yield* Effect.logInfo("handoff dropped: the thread moved on", { threadId: command.threadId }); return "dropped" as const; } - const slot = yield* admitSend(yield* Effect.serviceOption(SlotGate.SlotGate), { + const slot = yield* admitSend(gate, { callerThreadId: command.threadId, threadId: command.threadId, instanceId: (command.modelSelection ?? records.thread.modelSelection).instanceId, keepsHeld: true, + place, }); if (slot.type === "refuse") return { type: "waitsForSlot", detail: reasonHere(slot.refusal) } satisfies HandoffWaitsForSlot; - yield* orchestrator.dispatch(command); + const text = typeof command.text === "string" ? command.text : yield* command.text; + yield* orchestrator.dispatch({ ...command, text }); const { runs } = yield* projections.getThreadRecords(command.threadId, ["runs"], { runStatuses: ["queued"], }); diff --git a/apps/server/src/fork/limitResumeSlot.test.ts b/apps/server/src/fork/limitResumeSlot.test.ts index 76ab6107774d3..6c4a95dfcc0ae 100644 --- a/apps/server/src/fork/limitResumeSlot.test.ts +++ b/apps/server/src/fork/limitResumeSlot.test.ts @@ -170,12 +170,15 @@ describe("usage-limit resumes take an agent slot", () => { (thread) => limitRecoveryCommand(thread, true, RESET_MS)?.commandId, ), ); - // Each asked as the thread's own start. + // Each asked as the thread's own start, in its place in the gate's line. expect(gate.state.asks[0]).toEqual({ callerThreadId: ThreadId.make("thread:a"), targetInstanceId: INSTANCE, intoThreadId: ThreadId.make("thread:a"), keepsHeld: true, + place: { + key: `limit-resume:${limitRecoveryCommand(armed("a"), true, RESET_MS)?.commandId}`, + }, }); }).pipe( Effect.provide( diff --git a/apps/server/src/fork/limitResumeSlot.ts b/apps/server/src/fork/limitResumeSlot.ts index 11cf0948510a1..5a5e781b6bea5 100644 --- a/apps/server/src/fork/limitResumeSlot.ts +++ b/apps/server/src/fork/limitResumeSlot.ts @@ -67,6 +67,8 @@ export const gateLimitResumes = ( threadId: command.threadId, instanceId, keepsHeld: true, + // Asked again every tick (5 s) while refused: it keeps its place in the line. + place: { key: `limit-resume:${command.commandId}` }, }); if (slot.type === "refuse") return yield* skipped(command.commandId, command.threadId, reasonHere(slot.refusal)); diff --git a/apps/server/src/fork/pullRequestWatchSlot.test.ts b/apps/server/src/fork/pullRequestWatchSlot.test.ts index 4e2ecb75e15a2..2c70a715cb4dc 100644 --- a/apps/server/src/fork/pullRequestWatchSlot.test.ts +++ b/apps/server/src/fork/pullRequestWatchSlot.test.ts @@ -26,6 +26,7 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -164,6 +165,8 @@ describe("pull request watch wake-ups take an agent slot", () => { targetInstanceId: INSTANCE, intoThreadId: THREAD, keepsHeld: true, + // Its place in the gate's line, kept over 2 missed sweeps (2 min each). + place: { key: `pull-request-watch:${THREAD}`, staleAfter: Duration.minutes(5) }, }, ]); }), @@ -292,6 +295,8 @@ describe("pull request watch wake-ups take an agent slot", () => { yield* gone.wrapped.dispatch(command); expect(gone.sent).toEqual([command]); expect(gone.slots.asks).toEqual([]); + // Its wake is not owed any more: its place in the gate's line goes with it. + expect(gone.slots.left).toEqual([`pull-request-watch:${THREAD}`]); }), ); diff --git a/apps/server/src/fork/pullRequestWatchSlot.ts b/apps/server/src/fork/pullRequestWatchSlot.ts index 0227bb9e66d57..76a63217451c0 100644 --- a/apps/server/src/fork/pullRequestWatchSlot.ts +++ b/apps/server/src/fork/pullRequestWatchSlot.ts @@ -10,6 +10,7 @@ import type { ThreadPullRequestKey, } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -18,7 +19,7 @@ import * as Schema from "effect/Schema"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as SlotGate from "../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "./agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "./agentSlotRoutes.ts"; type WatchSync = Extract< OrchestrationV2ServerCommand, @@ -38,6 +39,9 @@ export class PullRequestWatchWakeWaitsForSlot extends Schema.TaggedError( const instanceId = yield* instanceOf(command.threadId).pipe( Effect.orElseSucceed(() => "unreadable" as const), ); - if (instanceId === null) return yield* orchestrator.dispatch(command); + const place = { key: `pull-request-watch:${command.threadId}`, staleAfter: WATCH_PLACE_STALE }; + // No such thread: its wake is not owed, so its place goes (upstream refuses the command). + if (instanceId === null) + return yield* leaveLine(gate, [place.key]).pipe( + Effect.andThen(orchestrator.dispatch(command)), + ); if (instanceId === "unreadable") return yield* waitsForSlot(command, "the thread could not be read to check for a slot"); const slot = yield* admitSend(gate, { @@ -97,6 +106,9 @@ export const gatePullRequestWatchWakes = ( threadId: command.threadId, instanceId, keepsHeld: true, + // Asked again at upstream's next sweep (2 min) while refused: it keeps its place in line, + // but holds a free slot only `SlotGate.LINE_HOLD` (15 s) without asking. + place, }); const keys = watchesOf(command).map((watch) => watchKey(command.threadId, watch)); if (slot.type === "refuse") { diff --git a/apps/server/src/fork/workerWakeSlots.test.ts b/apps/server/src/fork/workerWakeSlots.test.ts index e6d025cbd8264..38a9bb31a10f2 100644 --- a/apps/server/src/fork/workerWakeSlots.test.ts +++ b/apps/server/src/fork/workerWakeSlots.test.ts @@ -15,6 +15,7 @@ import { RuntimeRequestId, ThreadId, } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -23,6 +24,7 @@ import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as Tracer from "effect/Tracer"; @@ -56,6 +58,8 @@ const WAKE_COMMAND = `server:delegated-task-question:${PARENT}:${REQUEST}`; interface World { free: number; asks: Array; + /** Keys that left the gate's line. */ + left: Array; /** The parent: idle, gone (archived), or not readable now. */ parent: "idle" | "gone" | "unreadable"; requestStatus: "pending" | "resolved"; @@ -68,6 +72,7 @@ interface World { const newWorld = (free: number): World => ({ free, asks: [], + left: [], parent: "idle", requestStatus: "pending", taskStatus: "running", @@ -91,8 +96,11 @@ const gateOf = (world: World) => }), started: () => Effect.void, release: () => Effect.void, + leave: (keys) => Effect.sync(() => world.left.push(...keys)), }); +const QUESTION_PLACE = { key: `question-wake:${WAKE_COMMAND}` }; + const logsOf = (world: World) => Logger.layer( [ @@ -166,16 +174,84 @@ const upstreamOf = (world: World) => }), ); -/** One server: the wrapper over upstream, with the state folder `dir` (a restart reuses it). */ -const server = (world: World, dir: string) => +/** One server: the wrapper over upstream, on `disk` (a restart reuses it). */ +const server = (world: World, disk: Disk) => WorkerWakeSlots.layerOver.pipe( + // The held-wakes file is on the test's disk; the server's folders are real. + Layer.provide(disk.layer), Layer.provideMerge( - Layer.mergeAll(upstreamOf(world), gateOf(world), ServerConfig.layerTest(process.cwd(), dir)), + Layer.mergeAll( + upstreamOf(world), + gateOf(world), + ServerConfig.layerTest(process.cwd(), disk.dir), + ServerSettings.layerTest({}), + ), ), Layer.provideMerge(logsOf(world)), Layer.provideMerge(NodeServices.layer), ); +/** + * The wrapper's file system: in memory, so the test clock alone orders its reads and writes. The + * server's folders are real (`dir`). `exists` of the held-wakes file fails while `failing.exists` + * is set; every rename and sync is recorded in `ops` (by base name). + */ +interface Disk { + readonly dir: string; + readonly files: Map; + readonly ops: Array; + readonly failing: { exists: boolean; rename: boolean }; + readonly layer: Layer.Layer; +} + +const baseName = (file: string) => file.slice(file.lastIndexOf("/") + 1); +const diskError = (tag: "NotFound" | "PermissionDenied", method: string, file: string) => + PlatformError.systemError({ _tag: tag, module: "FileSystem", method, pathOrDescriptor: file }); + +const diskOver = (dir: string): Disk => { + const files = new Map(); + const ops: Array = []; + const failing = { exists: false, rename: false }; + const layer = FileSystem.layerNoop({ + exists: (file) => + failing.exists && baseName(file) === WorkerWakeSlots.HELD_WAKES_FILE + ? Effect.fail(diskError("PermissionDenied", "exists", file)) + : Effect.succeed(files.has(file)), + readFileString: (file) => + Effect.suspend(() => { + const text = files.get(file); + return text === undefined + ? Effect.fail(diskError("NotFound", "readFileString", file)) + : Effect.succeed(text); + }), + rename: (from, to) => + Effect.suspend(() => { + if (failing.rename && baseName(from) === WorkerWakeSlots.HELD_WAKES_FILE) + return Effect.fail(diskError("PermissionDenied", "rename", from)); + const text = files.get(from); + if (text === undefined) return Effect.fail(diskError("NotFound", "rename", from)); + ops.push(`rename ${baseName(from)} ${baseName(to)}`); + files.delete(from); + files.set(to, text); + return Effect.void; + }), + open: (file, options) => + Effect.sync(() => { + if (options?.flag === "w") files.set(file, ""); + return { + writeAll: (bytes: Uint8Array) => + Effect.sync(() => { + files.set(file, (files.get(file) ?? "") + new TextDecoder().decode(bytes)); + }), + sync: Effect.sync(() => { + ops.push(`sync ${baseName(file)}`); + }), + } as unknown as FileSystem.File; + }), + }); + return { dir, files, ops, failing, layer }; +}; + const questionAsked: OrchestrationV2DomainEvent = { id: EventId.make("event:question"), type: "runtime-request.updated", @@ -193,28 +269,27 @@ const ask = DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor.pipe( const wakes = (world: World) => world.sent.filter((command) => command.commandId === WAKE_COMMAND); -const heldFile = (dir: string) => +const heldFile = (disk: Disk) => Effect.gen(function* () { const path = yield* Path.Path; const config = yield* ServerConfig.pipe( - Effect.provide(ServerConfig.layerTest(process.cwd(), dir)), + Effect.provide(ServerConfig.layerTest(process.cwd(), disk.dir)), ); return path.join(config.stateDir, WorkerWakeSlots.HELD_WAKES_FILE); }).pipe(Effect.provide(NodeServices.layer)); -const heldIds = (dir: string) => +const heldIds = (disk: Disk) => Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const file = yield* heldFile(dir); - if (!(yield* fs.exists(file))) return []; - const parsed = JSON.parse(yield* fs.readFileString(file)) as { - wakes: ReadonlyArray<{ commandId: string }>; - }; + const text = disk.files.get(yield* heldFile(disk)); + if (text === undefined) return []; + const parsed = JSON.parse(text) as { wakes: ReadonlyArray<{ commandId: string }> }; return parsed.wakes.map((wake) => wake.commandId); - }).pipe(Effect.provide(NodeServices.layer)); + }); +/** A real folder for the server's own directories, and the held-wakes file in memory. */ const tempDir = FileSystem.FileSystem.pipe( Effect.flatMap((fs) => fs.makeTempDirectoryScoped({ prefix: "t3-fork-held-wakes-" })), + Effect.map(diskOver), Effect.provide(NodeServices.layer), ); @@ -237,12 +312,15 @@ describe("a delegated task's question to an idle parent takes an agent slot", () targetInstanceId: ProviderInstanceId.make("claudeAgent"), intoThreadId: PARENT, keepsHeld: true, + place: QUESTION_PLACE, }); // Upstream's reactor asks again (a later event): still one held, never two. yield* ask; yield* TestClock.adjust("1 minute"); assert.deepStrictEqual(wakes(world), []); assert.isAtLeast(world.asks.length, 12); + // Every ask in its one place in the gate's line. + assert.isTrue(world.asks.every((asked) => asked.place?.key === QUESTION_PLACE.key)); assert.lengthOf(waitLogs(world), 1); world.free = 1; yield* TestClock.adjust("5 seconds"); @@ -281,7 +359,8 @@ describe("a delegated task's question to an idle parent takes an agent slot", () ); assert.deepStrictEqual(wakes(world), []); world.free = 1; - yield* TestClock.adjust("5 seconds").pipe( + // The next server asks at once, so the file's wakes take their places in line first. + yield* TestClock.adjust("0 seconds").pipe( Effect.provide(server(world, dir)), Effect.scoped, ); @@ -304,15 +383,14 @@ describe("a delegated task's question to an idle parent takes an agent slot", () const world = newWorld(0); yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); const file = yield* heldFile(dir); - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const held = yield* fs.readFileString(file).pipe(Effect.provide(NodeServices.layer)); + const held = dir.files.get(file)!; world.free = 1; yield* TestClock.adjust("5 seconds").pipe( Effect.provide(server(world, dir)), Effect.scoped, ); // The write after the send is lost: the file still names it. - yield* fs.writeFileString(file, held).pipe(Effect.provide(NodeServices.layer)); + dir.files.set(file, held); world.free = 1; yield* TestClock.adjust("5 seconds").pipe( Effect.provide(server(world, dir)), @@ -342,6 +420,8 @@ describe("a delegated task's question to an idle parent takes an agent slot", () world.free = 5; yield* TestClock.adjust("5 seconds"); assert.deepStrictEqual(yield* heldIds(dir), []); + // It holds no place in the line either. + assert.include(world.left, QUESTION_PLACE.key); yield* TestClock.adjust("1 minute"); assert.deepStrictEqual(wakes(world), []); }).pipe(Effect.provide(server(world, dir)), Effect.scoped); @@ -374,8 +454,6 @@ describe("a delegated task's question to an idle parent takes an agent slot", () "old, empty, broken or missing file: nothing sent from it; a bad one is kept; a new hold writes it", () => Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); for (const contents of [ null, "", @@ -386,8 +464,7 @@ describe("a delegated task's question to an idle parent takes an agent slot", () const dir = yield* tempDir; const world = newWorld(0); const file = yield* heldFile(dir); - if (contents !== null) - yield* fs.writeFileString(file, contents).pipe(Effect.provide(NodeServices.layer)); + if (contents !== null) dir.files.set(file, contents); yield* Effect.gen(function* () { yield* TestClock.adjust("10 seconds"); assert.deepStrictEqual(world.sent, []); @@ -395,19 +472,16 @@ describe("a delegated task's question to an idle parent takes an agent slot", () assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); }).pipe(Effect.provide(server(world, dir)), Effect.scoped); // A file that could not be read is kept beside the new one, and named in the log. - const kept = (yield* fs - .readDirectory(path.dirname(file)) - .pipe(Effect.provide(NodeServices.layer))).filter((name) => - name.startsWith(`${WorkerWakeSlots.HELD_WAKES_FILE}.corrupt-`), - ); + const kept = [...dir.files.keys()] + .map(baseName) + .filter((name) => name.startsWith(`${WorkerWakeSlots.HELD_WAKES_FILE}.corrupt-`)); if (contents === null || contents.startsWith('{"version":1')) { assert.deepStrictEqual(kept, []); continue; } assert.lengthOf(kept, 1); - const keptAs = path.join(path.dirname(file), kept[0]!); assert.strictEqual( - yield* fs.readFileString(keptAs).pipe(Effect.provide(NodeServices.layer)), + dir.files.get(`${file}.corrupt-${kept[0]!.split("-").at(-1)}`), contents, ); assert.isTrue( @@ -461,6 +535,7 @@ describe("a delegated task's question to an idle parent takes an agent slot", () }), threadOf: () => Effect.succeed({ instanceId: ProviderInstanceId.make("claudeAgent") }), questionOpen: () => Effect.succeed(true), + compactionDue: () => Effect.succeed(false), store: { load: Effect.succeed([w1]), save: (held) => @@ -502,16 +577,8 @@ describe("a delegated task's question to an idle parent takes an agent slot", () const world = newWorld(0); yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); const file = yield* heldFile(dir); - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const stored = JSON.parse( - yield* fs.readFileString(file).pipe(Effect.provide(NodeServices.layer)), - ) as { version: 1; wakes: Array }; - yield* fs - .writeFileString( - file, - JSON.stringify({ ...stored, wakes: [{ type: "?" }, ...stored.wakes] }), - ) - .pipe(Effect.provide(NodeServices.layer)); + const stored = JSON.parse(dir.files.get(file)!) as { version: 1; wakes: Array }; + dir.files.set(file, JSON.stringify({ ...stored, wakes: [{ type: "?" }, ...stored.wakes] })); world.free = 1; yield* TestClock.adjust("5 seconds").pipe(Effect.provide(server(world, dir)), Effect.scoped); assert.lengthOf(wakes(world), 1); @@ -519,6 +586,154 @@ describe("a delegated task's question to an idle parent takes an agent slot", () }).pipe(Effect.scoped), ); + it.effect( + "a file whose check fails is not missing: nothing written over it, sent once it reads", + () => + Effect.gen(function* () { + const dir = yield* tempDir; + const world = newWorld(0); + yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + dir.failing.exists = true; + world.free = 1; + yield* Effect.gen(function* () { + yield* TestClock.adjust("20 seconds"); + assert.deepStrictEqual(wakes(world), []); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + assert.lengthOf( + world.logs.filter((log) => log.includes("held wakes file could not be read")), + 1, + ); + dir.failing.exists = false; + yield* TestClock.adjust("5 seconds"); + assert.lengthOf(wakes(world), 1); + assert.deepStrictEqual(yield* heldIds(dir), []); + }).pipe(Effect.provide(server(world, dir)), Effect.scoped); + }).pipe(Effect.scoped), + ); + + it.effect( + "a broken file that cannot be set aside is kept as it is: nothing written over it, read again", + () => + Effect.gen(function* () { + const dir = yield* tempDir; + const world = newWorld(0); + const file = yield* heldFile(dir); + dir.files.set(file, "{not json"); + dir.failing.rename = true; + yield* Effect.gen(function* () { + yield* ask; + yield* TestClock.adjust("20 seconds"); + // Held in memory; the broken file is untouched, and the failure logged once. + assert.strictEqual(dir.files.get(file), "{not json"); + assert.lengthOf( + world.logs.filter((log) => log.includes("held wakes file could not be read")), + 1, + ); + // It can be set aside now: kept as .corrupt-, and the held wake is written. + dir.failing.rename = false; + yield* TestClock.adjust("5 seconds"); + const kept = [...dir.files.keys()] + .map(baseName) + .filter((name) => name.startsWith(`${WorkerWakeSlots.HELD_WAKES_FILE}.corrupt-`)); + assert.lengthOf(kept, 1); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + }).pipe(Effect.provide(server(world, dir)), Effect.scoped); + }).pipe(Effect.scoped), + ); + + it.effect( + "a file that cannot be read yet: a new hold is kept in memory, the file's own wakes go first", + () => + Effect.gen(function* () { + const wake = (name: string) => + ({ + type: "message.dispatch", + commandId: `server:delegated-task-question:${PARENT}:${name}`, + messageId: `message:${name}`, + threadId: PARENT, + }) as never as Extract; + const [fromFile, held] = [wake("from-file"), wake("held")]; + const world = newWorld(0); + let readable = false; + const saves: Array> = []; + const slots = yield* WorkerWakeSlots.makeWorkerWakeSlots({ + orchestrator: { dispatch: () => Effect.die("unused") } as never, + gate: Option.some(yield* SlotGate.SlotGate.pipe(Effect.provide(gateOf(world)))), + threadOf: () => Effect.succeed({ instanceId: ProviderInstanceId.make("claudeAgent") }), + questionOpen: () => Effect.succeed(true), + compactionDue: () => Effect.succeed(false), + store: { + load: Effect.suspend(() => + readable + ? Effect.succeed([fromFile]) + : Effect.fail(diskError("PermissionDenied", "exists", "held")), + ), + save: (wakes) => Effect.sync(() => saves.push(wakes.map((one) => one.commandId))), + }, + }); + yield* slots.orchestrator.dispatch(held); + assert.deepStrictEqual( + slots.held().map((one) => one.commandId), + [held.commandId], + ); + assert.deepStrictEqual(saves, []); + readable = true; + yield* slots.retry; + assert.deepStrictEqual( + slots.held().map((one) => one.commandId), + [fromFile.commandId, held.commandId], + ); + assert.deepStrictEqual(saves.at(-1), [fromFile.commandId, held.commandId]); + }).pipe(Effect.provide(logsOf(newWorld(0)))), + ); + + it.effect("a save is synced before its rename, and the folder after it", () => + Effect.gen(function* () { + const dir = yield* tempDir; + const world = newWorld(0); + const ops = dir.ops; + yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); + const file = WorkerWakeSlots.HELD_WAKES_FILE; + const rename = ops.indexOf(`rename ${file}.tmp ${file}`); + assert.isAbove(rename, -1); + assert.include(ops.slice(0, rename), `sync ${file}.tmp`); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const folder = baseName(path.dirname(yield* heldFile(dir))); + assert.include(ops.slice(rename + 1), `sync ${folder}`); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + }).pipe(Effect.scoped), + ); + + it.effect("on the real disk: a save is read back whole, and a missing file is none", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const folder = yield* fs.makeTempDirectoryScoped({ prefix: "t3-fork-held-wakes-real-" }); + const store = yield* WorkerWakeSlots.fileStore( + path.join(folder, WorkerWakeSlots.HELD_WAKES_FILE), + ); + assert.deepStrictEqual(yield* store.load, []); + const wake = { + type: "message.dispatch", + commandId: WAKE_COMMAND, + messageId: `message:delegated-task-question:${PARENT}:${REQUEST}`, + threadId: PARENT, + text: "A question waits.", + attachments: [], + createdBy: "agent", + creationSource: "server", + dispatchMode: { type: "start_immediately" }, + } as never as Extract; + yield* store.save([wake]); + assert.deepStrictEqual( + (yield* store.load).map((one) => one.commandId), + [WAKE_COMMAND], + ); + assert.deepStrictEqual(yield* fs.readDirectory(folder), [WorkerWakeSlots.HELD_WAKES_FILE]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("no gate: upstream's own orchestrator", () => Effect.gen(function* () { const inner = { dispatch: () => Effect.die("unused") } as never; @@ -527,6 +742,7 @@ describe("a delegated task's question to an idle parent takes an agent slot", () gate: Option.none(), threadOf: () => Effect.succeed(null), questionOpen: () => Effect.succeed(true), + compactionDue: () => Effect.succeed(false), store: { load: Effect.succeed([]), save: () => Effect.void }, }); assert.strictEqual(slots.orchestrator, inner); @@ -534,8 +750,18 @@ describe("a delegated task's question to an idle parent takes an agent slot", () ); }); -/** Upstream's idle compaction worker for the handoff test thread, behind the wrapper. */ -const compaction = (world: World) => +/** + * Upstream's idle compaction worker for the handoff test thread, behind the wrapper. Its window + * runs from now to 2 minutes on; `thread.data` is what both upstream and the fork's check read. + * Returns when upstream's worker is done with the run. + */ +const compaction = ( + world: World, + thread: { data: ReturnType }, + due?: ( + command: Extract, + ) => Effect.Effect, +) => Effect.gen(function* () { yield* TestClock.setTime(NOW_MS); const decided = yield* Deferred.make(); @@ -560,7 +786,6 @@ const compaction = (world: World) => occurredAt: at(), payload: run(), } as OrchestrationV2DomainEvent); - const data = projection({ usedTokens: 250_000, cacheExpiresInMs: 2 * MINUTE_MS }); const slots = yield* WorkerWakeSlots.makeWorkerWakeSlots({ orchestrator: { streamDomainEvents: Stream.fromQueue(events), @@ -571,8 +796,16 @@ const compaction = (world: World) => }), } as never, gate: Option.some(yield* SlotGate.SlotGate.pipe(Effect.provide(gateOf(world)))), - threadOf: () => Effect.succeed({ instanceId: data.thread.modelSelection.instanceId }), + threadOf: () => Effect.succeed({ instanceId: thread.data.thread.modelSelection.instanceId }), questionOpen: () => Effect.succeed(true), + compactionDue: + due ?? + ((command) => + Clock.currentTimeMillis.pipe( + Effect.map((nowMs) => + WorkerWakeSlots.compactionWindowOpen(thread.data, command, nowMs), + ), + )), store: { load: Effect.succeed([]), save: () => Effect.void }, }); yield* IdleCompactionService.make.pipe( @@ -581,8 +814,8 @@ const compaction = (world: World) => Effect.provide( Layer.mergeAll( Layer.mock(ProjectionStore.ProjectionStoreV2)({ - getThread: () => Effect.succeed(data.thread), - getThreadRecords: () => Effect.succeed(data), + getThread: () => Effect.sync(() => thread.data.thread), + getThreadRecords: () => Effect.sync(() => thread.data), }), Layer.mock(ServerSettings.ServerSettingsService)({ getSettings: Effect.succeed({ @@ -595,33 +828,217 @@ const compaction = (world: World) => ), Effect.withTracer(tracer), ); - yield* Deferred.await(decided); + yield* TestClock.adjust("0 seconds"); + return decided; }); +const COMPACTION_PLACE = { key: `idle-compaction:server:idle-compact:${threadId}:${runId}` }; +const warm = () => ({ data: projection({ usedTokens: 250_000, cacheExpiresInMs: 2 * MINUTE_MS }) }); + describe("upstream's idle compaction takes an agent slot", () => { it.effect("a free slot: compacted as upstream does, asked as the thread's own start", () => Effect.gen(function* () { const world = newWorld(1); - yield* compaction(world); + yield* Deferred.await(yield* compaction(world, warm())); assert.lengthOf(world.sent, 1); assert.deepStrictEqual(world.asks[0], { callerThreadId: threadId, targetInstanceId: ProviderInstanceId.make("claudeAgent"), intoThreadId: threadId, keepsHeld: true, + place: COMPACTION_PLACE, }); }), ); - it.effect("full: skipped and logged, never sent late (the prompt cache it saves expires)", () => + it.effect( + "full, then a slot frees while the window is open: asked every 5 s in its place, sent once", + () => + Effect.gen(function* () { + const world = newWorld(0); + const decided = yield* compaction(world, warm()); + yield* TestClock.adjust("30 seconds"); + assert.deepStrictEqual(world.sent, []); + assert.isAtLeast(world.asks.length, 6); + assert.isTrue(world.asks.every((asked) => asked.place?.key === COMPACTION_PLACE.key)); + assert.lengthOf( + world.logs.filter((log) => log.includes("idle compaction waits for an agent slot")), + 1, + ); + world.free = 1; + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.lengthOf(world.sent, 1); + yield* TestClock.adjust("1 hour"); + assert.lengthOf(world.sent, 1); + }), + ); + + it.effect("full until the cache window closes: skipped, logged, never sent late", () => Effect.gen(function* () { const world = newWorld(0); - yield* compaction(world); + const decided = yield* compaction(world, warm()); + yield* TestClock.adjust("2 minutes"); + yield* Deferred.await(decided); assert.deepStrictEqual(world.sent, []); - assert.isTrue(world.logs.some((log) => log.includes("idle compaction skipped"))); + assert.isTrue( + world.logs.some((log) => + log.includes("idle compaction skipped: no agent slot while its cache window was open"), + ), + ); + assert.include(world.left, COMPACTION_PLACE.key); world.free = 5; yield* TestClock.adjust("1 hour"); assert.deepStrictEqual(world.sent, []); }), ); + + it.effect("full until under a minute of its window is left: skipped then, not sent cold", () => + Effect.gen(function* () { + const world = newWorld(0); + const decided = yield* compaction(world, warm()); + // The window ends 2 minutes on; a slot that frees with less than a minute left is not used. + yield* TestClock.adjust("55 seconds"); + assert.isFalse(world.left.includes(COMPACTION_PLACE.key)); + world.free = 5; + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.deepStrictEqual(world.sent, []); + assert.include(world.left, COMPACTION_PLACE.key); + }), + ); + + it.effect("a check that fails: skipped, logged with its cause, not as a full gate", () => + Effect.gen(function* () { + const world = newWorld(0); + const decided = yield* compaction(world, warm(), () => Effect.fail("records unreadable")); + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.deepStrictEqual(world.sent, []); + assert.isTrue( + world.logs.some((log) => + log.includes("idle compaction skipped: whether it is still due could not be read"), + ), + ); + assert.isFalse(world.logs.some((log) => log.includes("no agent slot while its cache"))); + assert.include(world.left, COMPACTION_PLACE.key); + }), + ); + + it.effect("full, and the thread is no longer idle: skipped at the next check, never sent", () => + Effect.gen(function* () { + const world = newWorld(0); + const thread = warm(); + const decided = yield* compaction(world, thread); + // A question to the person is open now: no window, so no compaction. + thread.data = { + ...thread.data, + runtimeRequests: [{ ...question(world), status: "pending" }] as never, + }; + world.free = 5; + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.deepStrictEqual(world.sent, []); + assert.include(world.left, COMPACTION_PLACE.key); + }), + ); +}); + +describe("the server's check that a waiting idle compaction is still due", () => { + const commandFor = (run: string) => + ({ + type: "message.dispatch", + commandId: `server:idle-compact:${threadId}:${run}`, + threadId, + }) as never as Extract; + + it("open only for the run upstream's window is for, with a minute of it left", () => { + const records = warm().data; + // The window runs to 2 minutes on (NOW_MS + 2 min). + assert.isTrue(WorkerWakeSlots.compactionWindowOpen(records, commandFor(runId), NOW_MS)); + // Another run's compaction (a newer run completed since): not this window's. + assert.isFalse(WorkerWakeSlots.compactionWindowOpen(records, commandFor("run-older"), NOW_MS)); + const endMs = NOW_MS + 2 * MINUTE_MS; + const minLeft = WorkerWakeSlots.COMPACTION_MIN_LEFT_MS; + assert.isTrue( + WorkerWakeSlots.compactionWindowOpen(records, commandFor(runId), endMs - minLeft - 1), + ); + assert.isFalse( + WorkerWakeSlots.compactionWindowOpen(records, commandFor(runId), endMs - minLeft), + ); + }); + + it.effect("reads the settings, the project, and upstream's records as its worker does", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const thread = warm(); + const queries: Array = []; + let idleCompaction: "beforeCacheExpires" | "off" = "beforeCacheExpires"; + let projectOff = false; + const projections = { + getThread: () => Effect.sync(() => thread.data.thread), + getThreadRecords: (id: ThreadId, kinds: unknown, options: unknown) => + Effect.sync(() => { + queries.push({ id, kinds, options }); + return thread.data; + }), + } as never as ProjectionStore.ProjectionStoreV2["Service"]; + const settings = { + getSettings: Effect.sync(() => ({ + ...DEFAULT_SERVER_SETTINGS, + idleCompaction, + ...(projectOff + ? { + projectSettingsOverrides: { + [thread.data.thread.projectId]: { + providerInstances: { + [thread.data.thread.providerInstanceId]: { enabled: false }, + }, + }, + }, + } + : {}), + })), + } as never as ServerSettings.ServerSettingsService["Service"]; + const due = WorkerWakeSlots.compactionDueOver(settings, projections); + assert.isTrue(yield* due(commandFor(runId))); + assert.isFalse(yield* due(commandFor("run-older"))); + // Upstream's query for its window: the same records, kinds and filters. + const active = [thread.data.thread.activeProviderThreadId]; + assert.deepStrictEqual(queries[0], { + id: threadId, + kinds: [ + "runs", + "messages", + "providerThreads", + "providerTurns", + "runtimeRequests", + "turnItems", + "contextTransfers", + ], + options: { + messageRunIds: [runId], + turnItemStatuses: ["pending", "running", "waiting"], + runStatuses: ["preparing", "starting", "running", "rolled_back"], + includeLatestRun: true, + providerThreadIds: active, + providerTurnThreadIds: active, + latestProviderTurnUsageOnly: true, + runtimeRequestStatuses: ["pending"], + contextTransferStatuses: ["pending"], + }, + }); + // Idle compaction turned off since: no longer due, and nothing more read. + idleCompaction = "off"; + const before = queries.length; + assert.isFalse(yield* due(commandFor(runId))); + assert.strictEqual(queries.length, before); + idleCompaction = "beforeCacheExpires"; + projectOff = true; + const enabled = yield* due(commandFor(runId)); + projectOff = false; + assert.isFalse(enabled); + assert.isTrue(yield* due(commandFor(runId))); + }), + ); }); diff --git a/apps/server/src/fork/workerWakeSlots.ts b/apps/server/src/fork/workerWakeSlots.ts index 9ac8820677adf..44858ca5cb09b 100644 --- a/apps/server/src/fork/workerWakeSlots.ts +++ b/apps/server/src/fork/workerWakeSlots.ts @@ -1,13 +1,17 @@ /** * Fork: upstream's idle compaction and a delegated task's question to its idle parent start a run, - * so each takes an agent slot. A refused compaction is skipped (it only pays before the prompt - * cache expires). A refused question wake is kept in `HELD_WAKES_FILE` and sent once with a slot. + * so each takes an agent slot, with a place in the gate's line. A refused compaction asks again + * while the thread stays idle and its prompt cache window is open, then is skipped. A refused + * question wake is kept in `HELD_WAKES_FILE` and sent once with a slot. */ import { type CommandId, OrchestrationV2Command, type OrchestrationV2ServerCommand, type ProviderInstanceId, + resolveIdleCompaction, + resolveProjectProviderInstanceEnabled, + type RunId, type RuntimeRequestId, type ThreadId, } from "@t3tools/contracts"; @@ -18,15 +22,21 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import { ServerConfig } from "../config.ts"; +import { + idleCompactionWindow, + type IdleCompactionRecords, +} from "../orchestration-v2/IdleCompaction.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import { forkParked } from "../serverActivation.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as SlotGate from "../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "./agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "./agentSlotRoutes.ts"; type MessageDispatch = Extract; @@ -34,6 +44,9 @@ type MessageDispatch = Extract command.type === "message.dispatch" && command.commandId.startsWith(QUESTION_COMMAND); +/** The completed run an idle compaction is for, from upstream's command id. */ +const compactionRunOf = (command: MessageDispatch): RunId | null => { + const prefix = `${COMPACTION_COMMAND}${command.threadId}:`; + return command.commandId.startsWith(prefix) + ? (command.commandId.slice(prefix.length) as RunId) + : null; +}; + +/** + * A retried compaction needs this much of its cache window left: half the lead upstream's worker + * sends it with (`LEAD_MS`, 2 min, in `orchestration-v2/IdleCompaction.ts`), the time a session + * takes to resume and send it while the cache is warm. Later it would most likely read cold. + */ +export const COMPACTION_MIN_LEFT_MS = 60_000; + +/** + * A refused idle compaction is still due: its thread is idle at its run, and the prompt cache + * window is open, by the rule upstream's worker sent it by (`idleCompactionWindow`), with at + * least `COMPACTION_MIN_LEFT_MS` of it left. + */ +export const compactionWindowOpen = ( + records: IdleCompactionRecords, + command: MessageDispatch, + nowMs: number, +): boolean => { + const runId = compactionRunOf(command); + const window = idleCompactionWindow(records, nowMs); + return runId !== null && window?.runId === runId && nowMs < window.endMs - COMPACTION_MIN_LEFT_MS; +}; + +/** Places in the gate's line: one per held question wake and per waiting compaction. */ +const questionPlace = (command: MessageDispatch): SlotGate.SlotPlace => ({ + key: `question-wake:${command.commandId}`, +}); +const compactionPlace = (command: MessageDispatch): SlotGate.SlotPlace => ({ + key: `idle-compaction:${command.commandId}`, +}); + /** The question a wake is about, from upstream's ids (`DelegatedTaskQuestionReactor.ts`). */ const questionOf = (command: MessageDispatch) => { const source = command.notification?.source; @@ -70,13 +121,17 @@ const encodeCommand = Schema.encodeSync(OrchestrationV2Command); /** The parent as the gate needs it: `null` when gone (deleted or archived) or unknown. */ export type WakeThread = { readonly instanceId: ProviderInstanceId } | null; -/** Where the held wakes are kept; `null` keeps them in memory only (tests). */ +/** Where the held wakes are kept. */ export interface HeldWakesStore { - readonly load: Effect.Effect>; + /** + * What is held; none when there is no file. Fails when the file cannot be checked or read (an + * I/O error, never "missing"): its wakes may be there, so nothing is written over it. + */ + readonly load: Effect.Effect, PlatformError.PlatformError>; readonly save: (wakes: ReadonlyArray) => Effect.Effect; } -export interface WorkerWakeInput { +export interface WorkerWakeInput { readonly orchestrator: Orchestrator.OrchestratorV2["Service"]; readonly gate: Option.Option; readonly threadOf: (threadId: ThreadId) => Effect.Effect; @@ -85,6 +140,11 @@ export interface WorkerWakeInput { parentThreadId: ThreadId, question: NonNullable>, ) => Effect.Effect; + /** + * Whether a refused idle compaction is still due: its thread idle at its run, and the prompt + * cache window open (`idleCompactionWindow`, as upstream's worker reads it). + */ + readonly compactionDue: (command: MessageDispatch) => Effect.Effect; readonly store: HeldWakesStore; } @@ -92,21 +152,54 @@ export interface WorkerWakeInput { * Upstream's orchestrator with the two gated, and `retry`, which sends the held wakes that have a * slot now (the layer runs it every `HELD_WAKE_RETRY`). With no gate it is upstream's own. */ -export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* ( - input: WorkerWakeInput, +export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* ( + input: WorkerWakeInput, ) { const { orchestrator, gate } = input; if (Option.isNone(gate)) return { orchestrator, retry: Effect.void, held: (): ReadonlyArray => [] }; const held = new Map(); - for (const wake of yield* input.store.load) held.set(wake.commandId, wake); - if (held.size > 0) - yield* Effect.logInfo("held delegated task questions loaded: sent when a slot frees", { - count: held.size, - }); + /** + * The file was read (or there is none). Until then nothing is written over it: a file that + * cannot be checked or read may hold wakes. Each retry reads it again. + */ + let loaded = false; + let loadLogged = false; const saving = yield* Semaphore.make(1); // One save at a time, its snapshot taken under the lock: the last write is what is held now. - const save = saving.withPermits(1)(Effect.suspend(() => input.store.save([...held.values()]))); + const save = saving.withPermits(1)( + Effect.suspend(() => (loaded ? input.store.save([...held.values()]) : Effect.void)), + ); + const load = Effect.gen(function* () { + const fromFile = yield* input.store.load.pipe( + Effect.tapCause((cause) => + loadLogged + ? Effect.void + : Effect.logWarning( + "held wakes file could not be read: nothing is written over it, read again in 5 s", + { cause }, + ), + ), + Effect.option, + ); + if (Option.isNone(fromFile)) { + loadLogged = true; + return; + } + // The file's wakes first, in its order (their place in the gate's line), then any held since. + const since = [...held.values()]; + held.clear(); + for (const wake of [...fromFile.value, ...since]) { + if (!held.has(wake.commandId)) held.set(wake.commandId, wake); + } + loaded = true; + if (fromFile.value.length > 0) + yield* Effect.logInfo("held delegated task questions loaded: sent when a slot frees", { + count: fromFile.value.length, + }); + if (since.length > 0) yield* save; + }); + yield* load; /** Held wakes whose parent or question could not be read: logged once. */ const unreadLogged = new Set(); const unread = Effect.fn("WorkerWakeSlots.unread")(function* (command: MessageDispatch) { @@ -119,36 +212,76 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < return false; }); - /** `null`: it may start now; else why it waits. */ - const slotFor = Effect.fn("WorkerWakeSlots.slotFor")(function* (command: MessageDispatch) { + /** `null`: it may start now; else why it waits (in its place in the gate's line). */ + const slotFor = Effect.fn("WorkerWakeSlots.slotFor")(function* ( + command: MessageDispatch, + place: SlotGate.SlotPlace, + ) { const thread = yield* input .threadOf(command.threadId) .pipe(Effect.orElseSucceed(() => "unreadable" as const)); if (thread === "unreadable") return "the thread could not be read to check for a slot"; - if (thread === null) return null; + if (thread === null) { + yield* leaveLine(gate, [place.key]); + return null; + } const slot = yield* admitSend(gate, { callerThreadId: command.threadId, threadId: command.threadId, instanceId: thread.instanceId, keepsHeld: true, + place, }); return slot.type === "refuse" ? reasonHere(slot.refusal) : null; }); - const compaction = Effect.fn("WorkerWakeSlots.compaction")(function* (command: MessageDispatch) { - const detail = yield* slotFor(command); - if (detail === null) return yield* orchestrator.dispatch(command); - yield* Effect.logInfo("idle compaction skipped: it waits for no agent slot", { - threadId: command.threadId, - detail, - }); - return NOTHING_STORED; - }); + /** + * Held in upstream's own timer (a newer completed run replaces it, and a stop ends it): asked + * again every `COMPACTION_SLOT_RETRY` while it is still due, else skipped. Late is worse than + * never: after the window a compaction pays a full uncached read. + */ + const compaction = Effect.fn("WorkerWakeSlots.compaction")( + function* (command: MessageDispatch) { + const place = compactionPlace(command); + let detail = yield* slotFor(command, place); + if (detail === null) return yield* orchestrator.dispatch(command); + yield* Effect.logInfo("idle compaction waits for an agent slot", { + threadId: command.threadId, + detail, + }); + while (detail !== null) { + yield* Effect.sleep(COMPACTION_SLOT_RETRY); + // Checked again in the fork before each ask: still idle at its run, window still open. + // A check that fails skips it too (fail-safe), logged as what it is, with its cause. + const due = yield* input + .compactionDue(command) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "idle compaction skipped: whether it is still due could not be read", + { threadId: command.threadId, cause }, + ).pipe(Effect.as("unread" as const)), + ), + ); + if (due === "unread") return NOTHING_STORED; + if (!due) { + yield* Effect.logInfo( + "idle compaction skipped: no agent slot while its cache window was open", + { threadId: command.threadId }, + ); + return NOTHING_STORED; + } + detail = yield* slotFor(command, place); + } + return yield* orchestrator.dispatch(command); + }, + (effect, command) => Effect.ensuring(effect, leaveLine(gate, [compactionPlace(command).key])), + ); const question = Effect.fn("WorkerWakeSlots.question")(function* (command: MessageDispatch) { // Held already: the retry sends it, once. if (held.has(command.commandId)) return NOTHING_STORED; - const detail = yield* slotFor(command); + const detail = yield* slotFor(command, questionPlace(command)); if (detail === null) return yield* orchestrator.dispatch(command); held.set(command.commandId, command); yield* save; @@ -161,16 +294,19 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < /** One held wake: `true` when it leaves the file (sent, or no longer owed). */ const retryOne = Effect.fn("WorkerWakeSlots.retryOne")(function* (command: MessageDispatch) { + const place = questionPlace(command); + // No longer owed: it leaves the line too, so it holds no slot from the waiters behind it. + const owedNoMore = leaveLine(gate, [place.key]).pipe(Effect.as(true)); const thread = yield* input.threadOf(command.threadId).pipe(Effect.option); if (Option.isNone(thread)) return yield* unread(command); - if (thread.value === null) return true; + if (thread.value === null) return yield* owedNoMore; const asked = questionOf(command); if (asked !== null) { const open = yield* input.questionOpen(command.threadId, asked).pipe(Effect.option); if (Option.isNone(open)) return yield* unread(command); - if (!open.value) return true; + if (!open.value) return yield* owedNoMore; } - if ((yield* slotFor(command)) !== null) return false; + if ((yield* slotFor(command, place)) !== null) return false; yield* orchestrator.dispatch(command).pipe( Effect.catchCause((cause) => Effect.logWarning("waking the parent of a delegated task question failed", { @@ -183,6 +319,7 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < }); const retry = Effect.gen(function* () { + if (!loaded) yield* load; let changed = false; for (const command of held.values()) { if (!(yield* retryOne(command))) continue; @@ -209,25 +346,23 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < }); /** - * The held wakes in `filePath`. Missing is none. A file or entry that cannot be read is logged and - * left out: the next save writes what is held then. A file that cannot be read is first moved to - * `.corrupt-`, so nothing is written over it. + * The held wakes in `filePath`: missing is none, a failed check or read is an error. A file that + * does not decode is moved to `.corrupt-`; a bad entry is logged and left out. A save is + * synced before its rename and the folder after, so a power loss leaves one whole file. */ -const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (filePath: string) { +export const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (filePath: string) { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const load = Effect.gen(function* () { - if (!(yield* fs.exists(filePath).pipe(Effect.orElseSucceed(() => false)))) return []; - const text = yield* fs.readFileString(filePath).pipe(Effect.option); - const file = Option.flatMap(text, decodeFile); + if (!(yield* fs.exists(filePath))) return []; + const file = decodeFile(yield* fs.readFileString(filePath)); if (Option.isNone(file)) { const keptAs = `${filePath}.corrupt-${yield* Clock.currentTimeMillis}`; - const kept = yield* fs.rename(filePath, keptAs).pipe( - Effect.as(keptAs), - Effect.orElseSucceed(() => null), - ); + // Not set aside: a read error (the caller keeps it, saves nothing over it, reads it again). + yield* fs.rename(filePath, keptAs); yield* Effect.logWarning("held wakes file unreadable; its wakes are not sent", { filePath, - keptAs: kept, + keptAs, }); return []; } @@ -252,8 +387,20 @@ const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (filePath: st version: 1, wakes: wakes.map((wake) => encodeCommand(wake)), }); - yield* fs.writeFileString(temporary, text); + yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(temporary, { flag: "w" }); + yield* file.writeAll(new TextEncoder().encode(text)); + yield* file.sync; + }), + ); yield* fs.rename(temporary, filePath); + // The rename itself, on disk. A folder cannot be opened on every system (Windows): skipped. + yield* Effect.scoped( + fs + .open(path.dirname(filePath), { flag: "r" }) + .pipe(Effect.flatMap((folder) => folder.sync)), + ).pipe(Effect.ignore); }, // Not written: they are still held in memory and written with the next change. Effect.catchCause((cause) => Effect.logWarning("held wakes not written", { filePath, cause })), @@ -261,6 +408,54 @@ const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (filePath: st return { load, save } satisfies HeldWakesStore; }); +/** + * The server's check that a refused idle compaction is still due: idle compaction on for the + * thread's provider (`beforeCacheExpires`), the provider on for its project, and the window open + * over the records upstream's worker reads (a copy of its query in + * `orchestration-v2/IdleCompactionService.ts`: keep the two the same). + */ +export const compactionDueOver = ( + settings: ServerSettings.ServerSettingsService["Service"], + projections: ProjectionStore.ProjectionStoreV2["Service"], +) => + Effect.fn("WorkerWakeSlots.compactionDue")(function* (command: MessageDispatch) { + const runId = compactionRunOf(command); + const current = yield* settings.getSettings; + const thread = yield* projections.getThread(command.threadId); + if ( + runId === null || + resolveIdleCompaction(current, thread.providerInstanceId) !== "beforeCacheExpires" || + !resolveProjectProviderInstanceEnabled(current, thread.projectId, thread.providerInstanceId) + ) + return false; + const active = thread.activeProviderThreadId === null ? [] : [thread.activeProviderThreadId]; + // The records upstream's idle compaction worker reads for its window. + const records = yield* projections.getThreadRecords( + command.threadId, + [ + "runs", + "messages", + "providerThreads", + "providerTurns", + "runtimeRequests", + "turnItems", + "contextTransfers", + ], + { + messageRunIds: [runId], + turnItemStatuses: ["pending", "running", "waiting"], + runStatuses: ["preparing", "starting", "running", "rolled_back"], + includeLatestRun: true, + providerThreadIds: active, + providerTurnThreadIds: active, + latestProviderTurnUsageOnly: true, + runtimeRequestStatuses: ["pending"], + contextTransferStatuses: ["pending"], + }, + ); + return compactionWindowOpen(records, command, yield* Clock.currentTimeMillis); + }); + /** Upstream's orchestrator for the runtime's workers, gated as the file's doc says. */ export const layerOver = Layer.effect( Orchestrator.OrchestratorV2, @@ -268,6 +463,7 @@ export const layerOver = Layer.effect( const orchestrator = yield* Orchestrator.OrchestratorV2; const projections = yield* ProjectionStore.ProjectionStoreV2; const config = yield* ServerConfig; + const settings = yield* ServerSettings.ServerSettingsService; const path = yield* Path.Path; const slots = yield* makeWorkerWakeSlots({ orchestrator, @@ -290,12 +486,14 @@ export const layerOver = Layer.effect( (task) => asked.taskIds.includes(task.id) && SETTLED_TASK.has(task.status), ); }), + compactionDue: compactionDueOver(settings, projections), store: yield* fileStore(path.join(config.stateDir, HELD_WAKES_FILE)), }); // A stop waits for a send's file write: a send it cut off would be sent again (receipts dedupe). + // The first pass runs at once, so wakes from the file take their places in line first. yield* forkParked( - Effect.sleep(HELD_WAKE_RETRY).pipe( - Effect.andThen(Effect.uninterruptible(slots.retry)), + Effect.uninterruptible(slots.retry).pipe( + Effect.andThen(Effect.sleep(HELD_WAKE_RETRY)), Effect.forever, ), ); diff --git a/apps/server/src/slots/SlotGate.test.ts b/apps/server/src/slots/SlotGate.test.ts index aaade95d2eafe..e21ecb37206bf 100644 --- a/apps/server/src/slots/SlotGate.test.ts +++ b/apps/server/src/slots/SlotGate.test.ts @@ -15,6 +15,7 @@ import { import { type AgentSlotsSettings, WaitToolFailure } from "@t3tools/contracts/waitFor"; import type { DeepPartial } from "@t3tools/shared/Struct"; import { DISPATCH_SAFETY_MS, type SlotReservation } from "@t3tools/shared/agentSlots"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -2160,3 +2161,490 @@ describe("slot waits granted in one sweep: every wake-up granted is let through" }), ); }); + +describe("SlotGate: one line for every waiting start (issue 738)", () => { + const waitsOn = (patch: Partial) => ({ + ...on(patch), + waitFor: { enabled: true }, + }); + /** A wake-up of `thread` that asks again while refused, in its place `key` (or none). */ + const wake = (thread: string, key: string | null, targetInstanceId = MAIN) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ + callerThreadId: ThreadId.make(thread), + targetInstanceId, + intoThreadId: ThreadId.make(thread), + keepsHeld: true, + ...(key === null ? {} : { place: { key } }), + }), + ), + ); + /** A start without a place (`delegate_task` from `thread`). */ + const start = (thread: string, targetInstanceId = MAIN) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ callerThreadId: ThreadId.make(thread), targetInstanceId }), + ), + ); + const line = SlotGate.SlotGate.pipe(Effect.flatMap((gate) => gate.line)); + + it.effect("steady load: the oldest waiter starts first, never one that asks first", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + for (const name of ["w0", "w1", "w2"]) + assert.strictEqual((yield* wake(name, name)).type, "refuse"); + const started: Array = []; + for (let round = 0; round < 6; round += 1) { + // The last run ends; a new waiter joins every round (steady load). + world.threads = []; + const waiting = yield* line; + // Everyone else asks first: a plain start, then the younger waiters, newest first. + assert.strictEqual((yield* start("newcomer")).type, "refuse"); + for (const name of waiting.slice(1).toReversed()) + assert.strictEqual((yield* wake(name, name)).type, "refuse"); + const oldest = waiting[0]!; + assert.strictEqual((yield* wake(oldest, oldest)).type, "allow"); + started.push(oldest); + world.threads = [{ id: oldest, instanceId: MAIN }]; + assert.strictEqual((yield* wake(`w${round + 3}`, `w${round + 3}`)).type, "refuse"); + } + assert.deepStrictEqual(started, ["w0", "w1", "w2", "w3", "w4", "w5"]); + }), + ); + }), + ); + + it.effect("a waiter whose own account is full holds no slot from the others", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: SECOND }]; + yield* run( + world, + on({ maxPerMachine: 3, maxPerAccount: { [SECOND]: 1 } }), + Effect.gen(function* () { + assert.strictEqual((yield* wake("w", "w", SECOND)).type, "refuse"); + // Two machine slots are free; the waiter can take neither, so both stay free. + assert.strictEqual((yield* start("a")).type, "allow"); + assert.strictEqual((yield* start("b")).type, "allow"); + assert.strictEqual((yield* start("c")).type, "refuse"); + }), + ); + }), + ); + + it.effect( + "a waiter whose thread moved to a full account holds no slot from the others (each ask's account)", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [ + { id: "busy", instanceId: SECOND }, + { id: "other", instanceId: MAIN }, + ]; + yield* run( + world, + on({ maxPerMachine: 2, maxPerAccount: { [SECOND]: 1 } }), + Effect.gen(function* () { + assert.strictEqual((yield* wake("w", "w", MAIN)).type, "refuse"); + // A machine slot frees; the waiter's thread now runs on SECOND, which is full. + world.threads = [{ id: "busy", instanceId: SECOND }]; + const asks: Array = []; + for (let round = 0; round < 12; round += 1) { + asks.push((yield* wake("w", "w", SECOND)).type); + asks.push((yield* start("a")).type); + if (asks.at(-1) === "allow") break; + yield* TestClock.adjust("5 seconds"); + } + assert.deepStrictEqual(asks, ["refuse", "allow"]); + }), + ); + }), + ); + + it.effect("a slot held for a slot wait is its thread's: a waiter behind does not take it", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + const holder = ThreadId.make("thread:holder"); + yield* run( + world, + waitsOn({ maxPerMachine: 2, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const [held] = yield* gate.grant([{ key: "hold", threadId: holder }]); + assert.strictEqual(held?.type, "granted"); + yield* gate.keep(["hold"]); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // The holder's agent starts on its own slot, though a waiter is in line. + assert.strictEqual((yield* start(holder)).type, "allow"); + }), + ); + }), + ); + + it.effect("slot waits share the line, each at its own age", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + const younger = { key: "young", threadId: ThreadId.make("young"), sinceMs: nowMs + 1 }; + const older = { key: "old", threadId: ThreadId.make("old"), sinceMs: nowMs - 1 }; + assert.deepStrictEqual( + (yield* gate.grant([older, younger])).map((grant) => grant.type), + ["full", "full"], + ); + assert.deepStrictEqual(yield* line, ["old", "w", "young"]); + world.threads = []; + // The waiter asks first, but an older slot wait is in line: it waits. + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + assert.deepStrictEqual( + (yield* gate.grant([older])).map((grant) => grant.type), + ["granted"], + ); + assert.deepStrictEqual(yield* line, ["w", "young"]); + }), + ); + }), + ); + + it.effect("a slot wait younger than a waiting wake-up waits behind it", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + const younger = { key: "young", threadId: ThreadId.make("young"), sinceMs: nowMs + 1 }; + world.threads = []; + // The sweep asks first, but the wake-up is older: the slot is its. + const [refused] = yield* gate.grant([younger]); + assert.strictEqual(refused?.type, "full"); + // Its refusal counts the wake-up as waiting in line, not as a running agent. + if (refused?.type === "full") assert.strictEqual(refused.decision.inLine, 1); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + }), + ); + }), + ); + + it.effect("a waiter that left, or stopped asking, or shows a run, holds nothing", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + // Left: its wake is gone (its thread deleted, its question answered). + assert.strictEqual((yield* wake("gone", "gone")).type, "refuse"); + yield* gate.leave(["gone"]); + assert.deepStrictEqual(yield* line, []); + // Stopped asking (a route that died): it leaves after LINE_STALE. + assert.strictEqual((yield* wake("quiet", "quiet")).type, "refuse"); + world.threads = []; + yield* TestClock.adjust("29 seconds"); + assert.strictEqual((yield* start("a")).type, "refuse"); + yield* TestClock.adjust("2 seconds"); + assert.deepStrictEqual(yield* line, []); + // Shows a run: its message only queues, so it needs no slot. + world.threads = [{ id: "busy", instanceId: MAIN }]; + assert.strictEqual((yield* wake("running", "running")).type, "refuse"); + world.threads = [{ id: "running", instanceId: MAIN }]; + assert.deepStrictEqual(yield* line, ["running"]); + world.threads = []; + assert.strictEqual((yield* start("b")).type, "refuse"); + world.threads = [{ id: "running", instanceId: MAIN }]; + yield* TestClock.adjust("31 seconds"); + world.threads = []; + assert.strictEqual((yield* start("b")).type, "allow"); + }), + ); + }), + ); + + it.effect( + "restart: the line is memory only; a new gate has none, and a waiter joins at its next ask", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run(world, on({ maxPerMachine: 1 }), wake("w", "w")); + world.threads = []; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.deepStrictEqual(yield* line, []); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + assert.deepStrictEqual(yield* line, []); + }), + ); + }), + ); + + it.effect( + "a free slot kept for a waiter that does not come back within LINE_HOLD goes to the next; the first keeps its place", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + // A waiter that asks seldom (a pull request watch: every 2 min), with a long stale time. + const seldom = SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ + callerThreadId: ThreadId.make("seldom"), + targetInstanceId: MAIN, + intoThreadId: ThreadId.make("seldom"), + keepsHeld: true, + place: { key: "seldom", staleAfter: "5 minutes" }, + }), + ), + ); + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.strictEqual((yield* seldom).type, "refuse"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // The slot frees: it is kept for the oldest waiter, which does not ask now. + world.threads = []; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + yield* TestClock.adjust(SlotGate.LINE_HOLD); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // Not back within its hold: the next waiter takes the slot, not left unused. + yield* TestClock.adjust("1 second"); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + // It keeps its place: first again once it asks, before a waiter that joined later. + assert.deepStrictEqual(yield* line, ["seldom"]); + world.threads = [{ id: "w", instanceId: MAIN }]; + assert.strictEqual((yield* wake("later", "later")).type, "refuse"); + assert.strictEqual((yield* seldom).type, "refuse"); + world.threads = []; + assert.strictEqual((yield* wake("later", "later")).type, "refuse"); + assert.strictEqual((yield* seldom).type, "allow"); + }), + ); + }), + ); + + it.effect( + "two places of one thread count from its oldest: a waiter between them never blocks it", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + // Thread t waits twice (a question wake and a continuation), x between the two. + assert.strictEqual((yield* wake("t", "t:first")).type, "refuse"); + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + assert.strictEqual((yield* wake("t", "t:second")).type, "refuse"); + world.threads = []; + // x waits behind t; t's younger place is let through at t's oldest place in line. + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + assert.strictEqual((yield* wake("t", "t:second")).type, "allow"); + }), + ); + }), + ); + + it.effect("a waiter that could not use a slot for a while is kept a fresh one when it can", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + const seldom = SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ + callerThreadId: ThreadId.make("seldom"), + targetInstanceId: MAIN, + intoThreadId: ThreadId.make("seldom"), + keepsHeld: true, + place: { key: "seldom", staleAfter: "5 minutes" }, + }), + ), + ); + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.strictEqual((yield* seldom).type, "refuse"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // A slot frees for a moment (kept for the oldest), then is taken again. + world.threads = []; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + world.threads = [{ id: "busy", instanceId: MAIN }]; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // Much later a slot frees again: its 15 s start now, not at the first offer. + yield* TestClock.adjust("20 seconds"); + world.threads = []; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + }), + ); + }), + ); + + it.effect("a slot wait counts from its thread's oldest place in line", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + // Thread t waits as a wake-up, x after it, then t's slot wait (younger than x). + assert.strictEqual((yield* wake("t", "t:wake")).type, "refuse"); + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + const slotWait = { key: "t:slot", threadId: ThreadId.make("t"), sinceMs: nowMs + 1 }; + assert.strictEqual((yield* gate.grant([slotWait]))[0]?.type, "full"); + world.threads = []; + // x asks first; the free slot is t's (its oldest place), and its slot wait takes it. + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + assert.strictEqual((yield* gate.grant([slotWait]))[0]?.type, "granted"); + }), + ); + }), + ); + + it.effect( + "a free slot kept for a slot wait is held over the next sweep (2 sweeps), then not", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + const older = { key: "old", threadId: ThreadId.make("old"), sinceMs: nowMs - 1 }; + assert.strictEqual((yield* gate.grant([older]))[0]?.type, "full"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + world.threads = []; + // Kept for the slot wait, asked at sweeps (every 15 s): 20 s on it is still its. + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + yield* TestClock.adjust("20 seconds"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // Two sweeps (30 s) without it: the waiter behind goes. + yield* TestClock.adjust("11 seconds"); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + }), + ); + }), + ); + + it.effect("starts that make a new thread (scheduled runs) each have a place of their own", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + const newRun = (key: string) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ callerThreadId: null, targetInstanceId: MAIN, place: { key } }), + ), + ); + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.strictEqual((yield* newRun("run-a")).type, "refuse"); + assert.strictEqual((yield* newRun("run-b")).type, "refuse"); + world.threads = []; + // The younger run asks first: the older is ahead of it, not the same start. + assert.strictEqual((yield* newRun("run-b")).type, "refuse"); + assert.strictEqual((yield* newRun("run-a")).type, "allow"); + }), + ); + }), + ); + + it.effect( + "a refusal tells the waiters in line apart from the running agents (as wait_list)", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + world.threads = []; + const refused = yield* start("a"); + assert.strictEqual(refused.type, "refuse"); + if (refused.type !== "refuse") return; + assert.strictEqual( + refused.message, + "No free agent slot: 0 agents are running on this machine and 1 start is waiting in line ahead of this one, and the limit is 1. Try again later. Or do the work yourself.", + ); + assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 0, max: 1 }); + }), + ); + }), + ); + + it.effect( + "memory floor only: a waiter that stops asking is dropped from memory at the next ask", + () => + Effect.gen(function* () { + const world = newWorld(); + world.freeMemoryBytes = 500 * MB; + yield* run( + world, + on({ minFreeMemoryMb: 1000 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + assert.strictEqual(yield* gate.lineKept, 1); + // It never asks again (its route ended without leaving); another start asks later. + yield* TestClock.adjust("31 seconds"); + world.freeMemoryBytes = 16_000 * MB; + assert.strictEqual((yield* start("a")).type, "allow"); + assert.strictEqual(yield* gate.lineKept, 0); + // A sweep of slot waits drops one the same way. + world.freeMemoryBytes = 500 * MB; + assert.strictEqual((yield* wake("v", "v")).type, "refuse"); + yield* TestClock.adjust("31 seconds"); + yield* gate.grant([{ key: "s", threadId: ThreadId.make("s") }]); + assert.strictEqual(yield* gate.lineKept, 1); + }), + ); + }), + ); + + it.effect("off or no limit: a waiter is let through and leaves the line", () => + Effect.gen(function* () { + const world = newWorld(); + yield* run( + world, + on({}), + Effect.gen(function* () { + assert.deepStrictEqual(yield* wake("w", "w"), { type: "allow", token: null }); + assert.deepStrictEqual(yield* line, []); + }), + ); + }), + ); +}); diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index f1e2c43ce1b2b..db185164cb0b5 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -106,6 +106,8 @@ export interface SlotRequest { * it needs no wake-up, so its hold counts no wake-up still to start (`pendingWakes`). */ readonly awake?: boolean | undefined; + /** When the wait was made: its place in the gate's one line (`SlotPlace`); else now. */ + readonly sinceMs?: number | undefined; } export type SlotGrant = @@ -133,6 +135,27 @@ export interface SlotUsage { readonly freeMemoryMb: number | null; } +/** + * A start that asks again while refused (a wake-up, a scheduled run, a webhook delivery): its place + * in the gate's one line, kept from its first ask, so a freed slot goes to the oldest waiter. It + * leaves when allowed, by `leave`, or after `staleAfter` without asking (default `LINE_STALE`). + */ +export interface SlotPlace { + readonly key: string; + readonly staleAfter?: Duration.Input | undefined; + /** When it became due (a scheduled run's due time, a stored delivery's): its age; else now. */ + readonly sinceMs?: number | undefined; +} + +/** A waiter that has not asked again for this long has left the line (6 asks of 5 s). */ +const LINE_STALE = Duration.seconds(30); +/** + * A waiter a free slot is kept for holds it this long without asking again (3 asks of 5 s); then + * the waiters behind it go first until it asks again. It keeps its place. A waiter that asks + * seldom (a pull request watch: every 2 min) or that died so leaves no slot idle for long. + */ +export const LINE_HOLD = Duration.seconds(15); + export interface SlotAcquireInput { readonly callerThreadId: ThreadId | null; readonly targetInstanceId: ProviderInstanceId; @@ -152,6 +175,11 @@ export interface SlotAcquireInput { * against it and are not used up, so the agent it wakes up for still has its slot. */ readonly keepsHeld?: boolean | undefined; + /** + * Its place in the line, for a start that asks again while refused. Without one the start asks + * from the back of the line: it never takes a slot a waiter in line is owed. + */ + readonly place?: SlotPlace | undefined; } export class SlotGate extends Context.Service< @@ -192,6 +220,17 @@ export class SlotGate extends Context.Service< * the thread held before counts again. */ readonly drop: (keys: ReadonlyArray) => Effect.Effect; + /** Waiters that no longer need a slot (gone, answered, stopped) leave the line now. */ + readonly leave: (keys: ReadonlyArray) => Effect.Effect; + /** + * Every slot wait still waiting, by key (the wait service's whole list, at each sweep): any + * other slot wait in the line ended (met, cancelled, handed over) and leaves it. + */ + readonly slotWaitsWaiting: (keys: ReadonlyArray) => Effect.Effect; + /** The line's keys, oldest first (waiters that went stale are left out). */ + readonly line: Effect.Effect>; + /** How many places memory keeps, stale ones not dropped yet too (dropped at every ask). */ + readonly lineKept: Effect.Effect; /** * For `wait_list`: null while slots are off, or when the running agents or the held slots * cannot be read. @@ -231,6 +270,45 @@ type CountRead = } | { readonly type: "unread"; readonly problem: CountReadProblem }; +/** One waiter in the line. `instanceId` null: a slot wait with no account (the machine only). */ +interface LineEntry { + readonly key: string; + /** A route's wake-up, or a slot wait (granted only on a whole read of the held slots). */ + readonly kind: "wake" | "slotWait"; + readonly threadId: string; + /** The account of its last ask: a thread moved to another account is modelled on that one. */ + instanceId: string | null; + readonly sinceMs: number; + /** Joins at the same time keep their order. */ + readonly order: number; + lastAskMs: number; + staleMs: number; + /** How long a free slot is kept for it while it does not ask (`LINE_HOLD`). */ + holdMs: number; + /** Since when a free slot is kept for it, untouched by its own ask since; `null`: none is. */ + offeredAtMs: number | null; +} + +const staleMsOf = (place: SlotPlace) => + Duration.toMillis(Duration.fromInputUnsafe(place.staleAfter ?? LINE_STALE)); + +/** + * A count refusal says how many of its `used` are waiters in line ahead of it (`inLine`), as + * `wait_list` shows only what runs, starts or is held. + */ +const withLine = ( + refused: Extract, + ahead: ReadonlyArray, +): Extract => { + const inLine = + refused.rule === "machine" + ? ahead.length + : refused.rule === "account" + ? ahead.filter((entry) => entry.instanceId === refused.instanceId).length + : 0; + return inLine === 0 ? refused : { ...refused, inLine }; +}; + const make = Effect.gen(function* () { const serverSettings = yield* ServerSettings.ServerSettingsService; const activeRunShells = yield* ActiveRunShells.ActiveRunShells; @@ -268,6 +346,132 @@ const make = Effect.gen(function* () { const usedLater = new Set(); /** Why the store last could not be read; warned once until the reason changes or a read works. */ let storeProblem: string | null = null; + /** + * The one line of starts that wait for a slot, by key: every route's waiter (`SlotPlace`) and + * every slot wait (`grant`), oldest first. Memory only: after a restart each waiter joins again + * at its next ask (a slot wait at its own age). + */ + const line = new Map(); + let lineOrder = 0; + + /** Joins the line, or keeps the place it has (asked again). */ + const joinLine = (entry: { + readonly key: string; + readonly kind: LineEntry["kind"]; + readonly threadId: string; + readonly instanceId: string | null; + readonly sinceMs: number; + readonly nowMs: number; + readonly staleMs: number; + readonly holdMs: number; + }): LineEntry => { + const seen = line.get(entry.key); + if (seen !== undefined) { + seen.lastAskMs = entry.nowMs; + seen.instanceId = entry.instanceId; + seen.staleMs = entry.staleMs; + seen.holdMs = entry.holdMs; + // It came back: a slot kept for it is its own ask's now. + seen.offeredAtMs = null; + return seen; + } + const joined: LineEntry = { + key: entry.key, + kind: entry.kind, + threadId: entry.threadId, + instanceId: entry.instanceId, + sinceMs: entry.sinceMs, + order: lineOrder++, + lastAskMs: entry.nowMs, + staleMs: entry.staleMs, + holdMs: entry.holdMs, + offeredAtMs: null, + }; + line.set(entry.key, joined); + return joined; + }; + + const isStale = (entry: LineEntry, nowMs: number) => nowMs - entry.lastAskMs > entry.staleMs; + /** A waiter that stopped asking has left the line: dropped at every ask, whatever the limits. */ + const pruneLine = (nowMs: number) => { + for (const [key, entry] of line) if (isStale(entry, nowMs)) line.delete(key); + }; + /** The live line, oldest first. */ + const lineInOrder = (nowMs: number) => { + pruneLine(nowMs); + return [...line.values()].sort( + (left, right) => left.sinceMs - right.sinceMs || left.order - right.order, + ); + }; + /** The waiters ahead of `threadId`'s oldest place (one thread, one place in line). */ + const beforeThread = (ordered: ReadonlyArray, threadId: string) => { + const first = ordered.findIndex((entry) => entry.threadId === threadId); + return first < 0 ? ordered : ordered.slice(0, first); + }; + + /** + * The waiters ahead of a start, oldest first, as starts: each that would get a slot now keeps it + * from those behind; one that would not (its account full) holds none. Left out: the asker's own + * thread, a thread that shows a run (it only queues), one counted already (held, or named), and + * one a slot was kept for longer than its hold without it asking (`LINE_HOLD`). + */ + const lineAhead = (input: { + readonly ahead: ReadonlyArray; + readonly threadId: string | null; + readonly limits: SlotLimits; + readonly running: ReadonlyArray; + readonly visible: ReadonlySet; + readonly kept: ReadonlyArray; + readonly held: HeldRead; + readonly accountOnly: (threadId: string) => ReadonlyArray; + readonly freeMemoryBytes: number | null; + readonly nowMs: number; + }): ReadonlyArray => { + const counted = new Set([ + ...input.held.held.map((reservation) => reservation.threadId), + ...input.kept.flatMap((entry) => + [entry.childThreadId, entry.queuedInto].filter((id) => id !== undefined), + ), + ]); + if (input.threadId !== null) counted.add(input.threadId); + const taken: Array = []; + for (const entry of input.ahead) { + if (counted.has(entry.threadId) || input.visible.has(entry.threadId)) continue; + // A slot wait is granted only when the held slots read whole: until then it takes none. + if (entry.kind === "slotWait" && !input.held.whole) continue; + // A slot kept for it that it did not come back for: the next waiter goes (till it asks). + if (entry.offeredAtMs !== null && input.nowMs - entry.offeredAtMs > entry.holdMs) continue; + counted.add(entry.threadId); + const others = onePerThread( + input.held.held.filter((reservation) => reservation.threadId !== entry.threadId), + ); + const decision = slotDecision({ + limits: entry.instanceId === null ? { ...input.limits, account: () => null } : input.limits, + running: input.running.filter((agent) => agent.threadId !== entry.threadId), + starting: [ + ...input.kept, + ...pendingWakes(others, input.visible, input.kept, input.held.woken), + ...taken, + ], + reservations: others, + accountOnly: input.accountOnly(entry.threadId), + targetInstanceId: entry.instanceId ?? "", + freeMemoryBytes: input.freeMemoryBytes, + }); + if (decision.type === "allow") { + taken.push({ + token: -10_000 - taken.length, + instanceId: entry.instanceId ?? "", + atMs: input.nowMs, + childThreadId: entry.threadId, + }); + entry.offeredAtMs ??= input.nowMs; + } else { + entry.offeredAtMs = null; + } + } + return taken; + }; /** * The one count read in flight, joined by every caller that needs the count while it runs. @@ -622,13 +826,36 @@ const make = Effect.gen(function* () { const limits = Option.isNone(settings) ? null : effectiveSlotLimits(settings.value.agentSlots); + const place = input.place; if (Option.isNone(settings) || limits === null) { + if (place !== undefined) line.delete(place.key); return asDecided({ type: "allow", token: null }, []); } const waitsOn = settings.value.waitFor.enabled; const nowMs = yield* Clock.currentTimeMillis; const counted = limitsCount(limits, input.targetInstanceId); const into = input.intoThreadId; + const askingThread = into ?? input.callerThreadId; + pruneLine(nowMs); + // A waiter keeps its place from its first ask, refused or not. One for a thread not made yet + // (a scheduled run's new thread) is that start's own, by its key. + const own = + place === undefined + ? undefined + : joinLine({ + key: place.key, + kind: "wake", + threadId: askingThread ?? place.key, + instanceId: input.targetInstanceId, + sinceMs: place.sinceMs ?? nowMs, + nowMs, + staleMs: staleMsOf(place), + holdMs: Duration.toMillis(LINE_HOLD), + }); + const allowed = (outcome: SlotAcquire, used: ReadonlyArray) => { + if (place !== undefined) line.delete(place.key); + return asDecided(outcome, used); + }; const count = counted || into !== undefined ? yield* readCount : undefined; // Fails closed: a count it needs but cannot read refuses before anything is // recorded, so the refused call starts nothing, uses no held slot and writes nothing. @@ -651,7 +878,7 @@ const make = Effect.gen(function* () { ...current, { token, instanceId: input.targetInstanceId, atMs: nowMs, queuedInto: into }, ]); - return asDecided({ type: "allow", token }, []); + return allowed({ type: "allow", token }, []); } // The running agents only where a count limit applies (no count means none to check). const read = counted ? shown : undefined; @@ -668,6 +895,24 @@ const make = Effect.gen(function* () { // Other held threads not woken yet: each wake-up is a run of its own. const wakes = pendingWakes(others, visible, kept, heldRead.woken); const memoryBytes = yield* freeMemoryBytes(limits); + // The waiters ahead of it in the line (all of them for a start without a place) come first. + const ordered = read === undefined ? [] : lineInOrder(nowMs); + const ahead = + read === undefined + ? [] + : lineAhead({ + // From its thread's oldest place: two places of one thread never block each other. + ahead: own === undefined ? ordered : beforeThread(ordered, own.threadId), + threadId: own?.threadId ?? askingThread, + limits, + running: read.running, + visible, + kept, + held: heldRead, + accountOnly: (threadId) => hiddenHeld(waitsOn, nowMs, threadId), + freeMemoryBytes: memoryBytes, + nowMs, + }); const decisionWith = ( targetInstanceId: ProviderInstanceId, starting: ReadonlyArray, @@ -682,26 +927,30 @@ const make = Effect.gen(function* () { targetInstanceId, freeMemoryBytes: memoryBytes, }); - const decision = decisionWith(input.targetInstanceId, [...kept, ...wakes]); + const decision = decisionWith(input.targetInstanceId, [...kept, ...wakes, ...ahead]); const alsoNeeds = input.alsoNeeds ?? []; const full = ( refused: Extract, free: number, - ): SlotAcquire => ({ - type: "refuse", - message: slotRefusalText(refused, waitsOn), - refusal: { - type: "full", - decision: refused, - waitsOn, - ...(alsoNeeds.length === 0 ? {} : { batch: { needed: alsoNeeds.length + 1, free } }), - }, - }); + ): SlotAcquire => { + const decision = withLine(refused, ahead); + return { + type: "refuse", + message: slotRefusalText(decision, waitsOn), + refusal: { + type: "full", + decision, + waitsOn, + ...(alsoNeeds.length === 0 ? {} : { batch: { needed: alsoNeeds.length + 1, free } }), + }, + }; + }; if (decision.type === "refuse") return asDecided(full(decision, 0), []); // The later starts of the batch, counted as starting one after another (nothing recorded). const starting: Array = [ ...kept, ...wakes, + ...ahead, { token: -1, instanceId: input.targetInstanceId, atMs: nowMs }, ]; for (const [index, instanceId] of alsoNeeds.entries()) { @@ -726,7 +975,7 @@ const make = Effect.gen(function* () { const caller = input.keepsHeld === true ? null : input.callerThreadId; const used = caller === null ? [] : yield* useUp(reservations, caller); if (!heldRead.whole && caller !== null) unreadStarts.add(caller); - return asDecided({ type: "allow", token }, used); + return allowed({ type: "allow", token }, used); }, (decided, input) => observeBeside(decided, (result) => [result.outcome.type], { @@ -759,6 +1008,7 @@ const make = Effect.gen(function* () { } if (requests.length === 0) return []; const nowMs = yield* Clock.currentTimeMillis; + pruneLine(nowMs); /** A wait without an account waits for the machine limit and the memory floor only. */ const limitsFor = (request: SlotRequest): SlotLimits => request.account === undefined ? { ...limits, account: () => null } : limits; @@ -777,7 +1027,24 @@ const make = Effect.gen(function* () { let reservations = heldRead.held; const grants: Array = []; const grantedThreads = new Set(); - for (const request of requests) { + // Every slot wait is in the line at its own age; one the sweep stopped asking for leaves it. + const sweepMs = settings.value.waitFor.unitSweepSeconds * 1000; + const staleMs = Math.max(Duration.toMillis(LINE_STALE), 3 * sweepMs); + // Asked at each sweep: a slot kept for it is held over one sweep it may just miss. + const holdMs = Math.max(Duration.toMillis(LINE_HOLD), 2 * sweepMs); + const places = requests.map((request) => + joinLine({ + key: request.key, + kind: "slotWait", + threadId: request.threadId, + instanceId: request.account ?? null, + sinceMs: request.sinceMs ?? nowMs, + nowMs, + staleMs, + holdMs, + }), + ); + for (const [index, request] of requests.entries()) { // One wake per thread in a call: its other slot waits ask again next time. if (grantedThreads.has(request.threadId)) { grants.push({ type: "sameThread" }); @@ -798,13 +1065,33 @@ const make = Effect.gen(function* () { reservations.filter((reservation) => reservation.threadId !== request.threadId), ); const visible = read?.visible ?? new Set(); + // The waiters ahead of it in the line (route wake-ups and older slot waits) come first. + const ordered = read === undefined ? [] : lineInOrder(nowMs); + const place = places[index]; + const ahead = + read === undefined || place === undefined || !ordered.includes(place) + ? [] + : lineAhead({ + ahead: beforeThread(ordered, request.threadId), + threadId: request.threadId, + // Each waiter ahead by its own limits (its account's too), not this wait's. + limits, + running: read.running, + visible, + kept, + held: { ...heldRead, held: reservations }, + accountOnly: (threadId) => + hiddenHeld(settings.value.waitFor.enabled, nowMs, threadId), + freeMemoryBytes: memoryBytes, + nowMs, + }); const decision = slotDecision({ limits: own, running: (read?.running ?? []).filter((agent) => agent.threadId !== request.threadId), // Every other held thread not woken yet needs a slot for its wake-up as well as its // agent, the ones granted earlier in this call too. Its own wake-up is // not counted, as a thread's own run never counts against its own start. - starting: [...kept, ...pendingWakes(others, visible, kept, heldRead.woken)], + starting: [...kept, ...pendingWakes(others, visible, kept, heldRead.woken), ...ahead], // Its own held slot is not counted: a grant moves it here, so the thread keeps one. reservations: others, // Read again for each request: a grant earlier in this call may have hidden holds. @@ -814,7 +1101,7 @@ const make = Effect.gen(function* () { }); if (decision.type === "refuse") { // Refused: a slot it already holds stays as it is. - grants.push({ type: "full", decision }); + grants.push({ type: "full", decision: withLine(decision, ahead) }); continue; } // Asking again moves its hold but never extends it: a thread that keeps asking without @@ -849,6 +1136,7 @@ const make = Effect.gen(function* () { // A slot held for 0 s is not held after this call; the wait is still woken. if (reserveMs > 0) yield* Ref.update(held, (current) => [...current, reservation]); if (request.awake === true) awakeKeys.set(request.key, untilMs); + line.delete(request.key); grantedThreads.add(request.threadId); grants.push({ type: "granted", untilMs }); } @@ -959,6 +1247,28 @@ const make = Effect.gen(function* () { keep: Effect.fn("SlotGate.keep")(function* (keys: ReadonlyArray) { yield* clearStored(yield* keepLocked(keys)); }), + leave: Effect.fn("SlotGate.leave")((keys: ReadonlyArray) => + Effect.sync(() => { + for (const key of keys) line.delete(key); + }), + ), + slotWaitsWaiting: Effect.fn("SlotGate.slotWaitsWaiting")((keys: ReadonlyArray) => + Effect.sync(() => { + const waiting = new Set(keys); + for (const [key, entry] of line) { + if (entry.kind === "slotWait" && !waiting.has(key)) line.delete(key); + } + }), + ), + line: Clock.currentTimeMillis.pipe( + Effect.map((nowMs) => + [...line.values()] + .filter((entry) => !isStale(entry, nowMs)) + .sort((left, right) => left.sinceMs - right.sinceMs || left.order - right.order) + .map((entry) => entry.key), + ), + ), + lineKept: Effect.sync(() => line.size), drop: Effect.fn("SlotGate.drop")(function* (keys: ReadonlyArray) { if (keys.length === 0) return; for (const key of keys) { diff --git a/apps/server/src/waits/WaitDelivery.ts b/apps/server/src/waits/WaitDelivery.ts index 2f41beb46e3b6..b0ac9d75b7acb 100644 --- a/apps/server/src/waits/WaitDelivery.ts +++ b/apps/server/src/waits/WaitDelivery.ts @@ -180,6 +180,8 @@ export const make = Effect.fn("WaitDelivery.make")(function* ( threadId: message.threadId, instanceId: shell!.modelSelection.instanceId, keepsHeld: true, + // Asked again every WAKE_SLOT_RETRY_MS while refused: it keeps its place in the line. + place: { key: `wait-delivery:${message.commandId}` }, }); if (slot.type === "refuse") return { type: "waitsForSlot", detail: reasonHere(slot.refusal) }; return yield* threads diff --git a/apps/server/src/waits/WaitService.slots.test.ts b/apps/server/src/waits/WaitService.slots.test.ts index e0ebf8e07d313..a55c0a6463686 100644 --- a/apps/server/src/waits/WaitService.slots.test.ts +++ b/apps/server/src/waits/WaitService.slots.test.ts @@ -543,6 +543,9 @@ describe("WaitService slot waits", () => { }), ); yield* gate.release(allowed.type === "allow" ? allowed.token : null); + // The probe for T2 above is no stored wait: the sweep's list says so, and it leaves + // the gate's line (a real waiting slot wait would go first, issue 738). + yield* gate.slotWaitsWaiting([]); // T1's start used its slot up: the slot is free for another thread, not held twice. assert.strictEqual(yield* delegate(OUTSIDER), "allow"); // Cleared in the file too, not only hidden in this gate's memory. diff --git a/apps/server/src/waits/WaitService.ts b/apps/server/src/waits/WaitService.ts index 195bc875f7c91..f91fc8a1b09bb 100644 --- a/apps/server/src/waits/WaitService.ts +++ b/apps/server/src/waits/WaitService.ts @@ -1731,10 +1731,10 @@ export const make = Effect.fn("WaitService.make")(function* (input: { const grantSweep = Effect.fnUntraced(function* (settings: ServerSettings) { const gate = input.slots; if (gate === undefined) return 0; - const waiting = yield* lock.withPermits(1)( + const read = yield* lock.withPermits(1)( Effect.gen(function* () { const loaded = yield* ensureLoaded(true); - if (!loaded.ok) return []; + if (!loaded.ok) return null; return loaded.contents.waits .flatMap((wait) => (wait.kind === "slot" && wait.state === "waiting" ? [wait] : [])) .sort((a, b) => @@ -1748,6 +1748,11 @@ export const make = Effect.fn("WaitService.make")(function* (input: { ); }), ); + // Unread: the gate's line keeps the slot waits it has (they end by time if they ended). + if (read === null) return 0; + const waiting = read; + // The slot waits that ended since (met, cancelled, handed over) leave the gate's line. + yield* gate.slotWaitsWaiting(waiting.map((wait) => wait.id)); if (waiting.length === 0) return 0; // Outside the lock: the gate reads the store's held slots under it. const grants = yield* gate.grant( @@ -1755,6 +1760,8 @@ export const make = Effect.fn("WaitService.make")(function* (input: { key: wait.id, threadId: wait.threadId, account: wait.spec.account, + // Its place in the gate's one line, beside the routes' waiting wake-ups. + sinceMs: Date.parse(wait.createdAt), })), ); const answers = waiting.map((wait, index) => { diff --git a/docs/fork/README.md b/docs/fork/README.md index 464be4741f86c..e124d0bec010e 100644 --- a/docs/fork/README.md +++ b/docs/fork/README.md @@ -403,8 +403,13 @@ These routes start a new agent run and go through the same gate, with the same c waiting delivery is in the store, so a restart keeps it and it runs once: a direct one is moved there when it is first refused. If that store fails, the delivery waits in memory; a save then stores it, and if the store fails again it ends without a run, saying why. A scheduled run or a webhook - takes a slot only when no agent that waits for one holds it; there is no order between routes, - so on a host kept full it can wait long. + takes a slot only when no agent that waits for one holds it. A run on its schedule and a webhook + delivery that wait keep a place in the gate's one line (below), a run at its due time and a + delivery at the time it was stored, so a waiter that joins later never passes them. The place + goes when the run starts, when the delivery is given up (task paused, deleted, replaced or + retriggered) or handed back to the stored-delivery worker (which asks again at the row's due + time), and within 5 s of its task being paused, deleted or no longer due (moved to a later + time). "Run now" asks once, from the back of the line. - A task bound to a thread whose run is active only queues its prompt there: no new run, so no slot and no wait. A bound thread that is idle starts a run and is gated, counted on the thread's current provider. @@ -465,9 +470,31 @@ These routes start a new agent run and go through the same gate, with the same c failed and armed, and upstream's recovery worker asks again on its next tick (every 5 s; logged once per resume). An armed resume has no time limit, a restart finds it from the stored thread, and its id is fixed per failed run and reset, so it is sent once. -- There is no order between wake-ups, and none between them and other routes: a wake-up holds no - place in line and asks again on its own timer, so a busy host can pass one over again and again - while other starts take the slots that free. +- One line, oldest first ([#738](https://github.com/scuttletoby/t3code/issues/738)): every + wake-up that asks again while refused keeps a place in the gate's line from its first ask (the + PR-watch wake-up, the usage-limit resume, a wait's message, a waiting scheduled run or webhook + delivery, and the four below), and so does every slot wait, at its own age. A slot that frees + goes to the oldest waiter that can use it: a start without a place (`delegate_task`, a send, + "run now") and every younger waiter count the waiters ahead of them as starts. A thread with two + places (a question wake and a continuation) counts from its oldest. A waiter whose own account is + full (the account of its last ask: its thread may have moved) holds nothing from the others, a + thread that shows a run only queues, and a thread with a held slot starts on it. A free slot is kept for the oldest waiter for 15 s (a slot wait: 2 + sweeps) without it asking; if it does not come back by then, the waiters behind it go first until + it asks again, still in its place. So a waiter that asks seldom (the PR-watch wake-up, at + upstream's 2-minute sweep) or that died leaves a slot idle 15 s at most; the PR-watch wake-up + gets a slot only when one is free as it asks, as before the line. The fork could hold a refused + PR-watch wake-up and ask every 5 s itself (as it does for idle compaction); it does not, because + upstream's reactor waits for that dispatch, so holding it would delay the other watches' news, + and a retry of its own would race upstream's next sync for the thread. A waiter leaves the line + when it is let through, when its wake is no longer owed (its thread gone, its question answered, a + handoff that is no longer due), or when it has not asked again for 30 s (a PR-watch wake-up: 5 + minutes; a slot wait: 3 sweeps); places that stopped asking are dropped at the next ask, under + any limit. A refusal names the waiters in line ahead apart from the running agents ("2 agents + are running on this machine and 1 start is waiting in line ahead of this one"), since + `wait_list` shows only what runs, starts or is held. The line is memory only: after a restart + each waiter joins again at its next ask. Held question wakes ask at once, in their file's order, + but at the time of that ask (the file keeps no age): a slot wait made before the restart, or a + scheduled run due before it, comes first. - The bound-thread and message routes decide "running or idle" from the gate's own count in the same step that takes the slot. When the thread's run ends between that check and the send, the queued message starts the next run: the call then holds a slot until that run shows, for the @@ -499,17 +526,20 @@ only queues: no slot. - The fork's context handoff, and its one resend of a failed handoff run (`apps/server/src/fork/handoff/handoffDispatch.ts`, after its busy check). Refused, nothing is sent: it asks again every 5 s or at the thread's next event (logged once), deciding again each - time without a new settle, and a thread that moved on drops it. Like every handoff timer it is + time without a new settle, and a thread that moved on drops it. Its text (a read of the last + runs' turns) is built only once it has a slot. Like every handoff timer it is memory only: a restart drops a waiting handoff, and the thread's next turn end decides again. A waiting resend of a failed handoff run is dropped too, with no log: the failed run's text stays in the thread, and the next turn end decides again. - Upstream's provider continuation ("Background task completed": an agent that ended its turn while its own background work ran starts again when that work ends; - `apps/server/src/fork/continuationSlot.ts`). The slot is asked when upstream's worker takes the - request from its queue, never inside an adapter's `dispatchIfCurrent`, which holds the - adapter's own lock. Refused, the request goes back on the queue and is asked again every 5 s - (logged once); the adapter offers no second one meanwhile, and other threads' continuations - pass. A delegated result is left as above. It waits in memory, as upstream's queue does: a + `apps/server/src/fork/continuationSlot.ts`). The slot is asked off upstream's queue, each + request in a fiber of its own, never inside an adapter's `dispatchIfCurrent`, which holds the + adapter's own lock: a slow count read delays only that request, never upstream's queue. Refused, + it is asked again every 5 s (logged once); the adapter offers no second one meanwhile. A granted + slot is held until upstream's dispatch of that request ends (it may wait behind its own thread's + earlier dispatch), then counts until its run shows, for the start grace at most; a request + upstream never dispatches holds it 10 minutes at most. A delegated result is left as above. It waits in memory, as upstream's queue does: a restart loses it, as the restart ends the provider session whose background work it reports (Claude's buffered output lives in that process). - A delegated task's question to its idle parent (upstream's question reactor; @@ -518,19 +548,20 @@ only queues: no slot. leaves the file once sent, or when the question is answered, its task ends or the parent is archived or deleted. A restart reads the file and sends it once; upstream's fixed command id keeps a send cut off before the file write from making a second message. Saves run one at a - time, each writing what is held then. A file that cannot be read is moved to - `fork-held-wakes.json.corrupt-` beside it and logged, so the next save does not write over - it; an entry that cannot be read is logged and left out. A held wake whose parent or question - cannot be read waits, logged once. -- Upstream's idle compaction (same file), after the handoff owner's check. Refused, it is skipped - (logged), not held: it only pays off while the prompt cache is warm, minutes before it expires, - and a late one costs a full uncached read. The next completed run arms a new one, as after a - restart. It is asked once: a thread whose window always falls while the slots are full does not - compact. - -Like the wake-ups above, the handoff, the continuation and the question wake hold no place in -line: each asks again on its own 5 s timer, so under steady load a slot that frees between two -asks can go to another start, and one can wait without limit. Nothing is lost or sent twice. + time, each writing what is held then, synced to disk before the rename and the folder after it. + A file that does not decode is moved to `fork-held-wakes.json.corrupt-` beside it and + logged, so the next save does not write over it (when that move fails, the file is handled as + one that cannot be read, below: kept as it is, nothing written over it); an entry that does not decode is logged and + left out. A file that cannot be checked or read (an I/O error, not "missing") is logged once and + never written over: new holds stay in memory, and every 5 s it is read again, its wakes then + going first. A held wake whose parent or question cannot be read waits, logged once. +- Upstream's idle compaction (same file), after the handoff owner's check. Refused, it is held in + upstream's own timer and asked again every 5 s in its place in line (logged once), while the + fork reads again that the thread is still idle at that run and at least a minute of its prompt + cache window is left (upstream's `idleCompactionWindow`; half the 2-minute lead upstream sends + it with). Once not, it is skipped (logged): a late compaction costs a full uncached read. A + check that cannot be read skips it too, logged with its cause. A newer completed run replaces it, as upstream's timer does; a + restart drops it (upstream). The calling thread is left out of the count (it waits on its child). A start no thread on this machine asks for (a scheduled run, the owner's client, a call from another machine) has no caller, so diff --git a/packages/shared/src/agentSlots.ts b/packages/shared/src/agentSlots.ts index aa0558972148a..cf5753d4f495a 100644 --- a/packages/shared/src/agentSlots.ts +++ b/packages/shared/src/agentSlots.ts @@ -247,6 +247,11 @@ export type SlotDecision = /** The limit (count rules) or the floor in MB (memory). */ readonly max: number; readonly instanceId: string; + /** + * Of `used`, the starts waiting in the gate's line ahead of this one (count rules): they are + * not running yet, so `wait_list` does not show them. + */ + readonly inLine?: number | undefined; }; export interface SlotDecisionInput { @@ -325,15 +330,29 @@ export function slotDecision(input: SlotDecisionInput): SlotDecision { return { type: "allow" }; } +/** + * "N agents are running{where}", and the starts waiting in line ahead of this one apart + * (`inLine`), so the count matches `wait_list`. + */ +export function slotRunningText( + decision: Extract, + where: string, +): string { + const agents = (count: number) => (count === 1 ? "1 agent is" : `${count} agents are`); + const inLine = decision.inLine ?? 0; + if (inLine === 0) return `${agents(decision.used)} running${where}`; + const starts = inLine === 1 ? "1 start is" : `${inLine} starts are`; + return `${agents(decision.used - inLine)} running${where} and ${starts} waiting in line ahead of this one`; +} + /** Why a slot is not free, without a full stop (held slots count as running agents). */ export function slotRefusalReason( decision: Extract, ): string { - const agents = (count: number) => (count === 1 ? "1 agent is" : `${count} agents are`); return decision.rule === "machine" - ? `${agents(decision.used)} running on this machine, and the limit is ${decision.max}` + ? `${slotRunningText(decision, " on this machine")}, and the limit is ${decision.max}` : decision.rule === "account" - ? `${agents(decision.used)} running on account ${decision.instanceId}, and the limit is ${decision.max}` + ? `${slotRunningText(decision, ` on account ${decision.instanceId}`)}, and the limit is ${decision.max}` : `only ${decision.used} MB of memory is free, and a new agent needs at least ${decision.max} MB`; }