diff --git a/.github/fork-seams.json b/.github/fork-seams.json index 4a577ac4b153e..62c7913a1e012 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -69,6 +69,27 @@ "continueThreadsAfterServerUpdate: false,", "automaticUpdates: false," ] + }, + "packages/shared/src/AgentScope.ts": { + "why": "The slot observer (apps/server/src/slots/SlotObserver.ts) reads an agent's memory from its own systemd scope; one optional method on upstream's interface asks for it.", + "added": [ + "/** Fork: the thread's own systemd scope and slice, for reading its memory; none when shared. */", + "readonly ownScope?: (", + "threadId: string,", + ") => Effect.Effect<{ unit: string; slice: string } | undefined>;" + ] + }, + "apps/server/src/process/agentScope.ts": { + "why": "The scope map is private to upstream's make(); the fork's ownScope (see packages/shared/src/AgentScope.ts) is a 7-line lookup in it.", + "added": [ + "ownScope: (threadId) =>", + "Effect.sync(() => {", + "const tracked = scopeByThread.get(threadId);", + "return tracked === undefined || tracked.shared", + "? undefined", + ": { unit: tracked.unit, slice: ownedSlice };", + "})," + ] } }, "//pending": "Upstream files whose fork lines predate counting every upstream file, each with its ceiling: the most fork lines it may hold against upstream (counted from upstream, as fork-follow runs the check). To move into fork files or pin as seams; lower a ceiling when lines move out and take a file off when none are left. A pull request may change them but not grow them.", diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index 244ee6149aed8..0dd55c83a4b66 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -41,7 +41,10 @@ import { makeSlotObserver } from "./SlotObserver.ts"; export type SlotToken = number; export type SlotAcquire = - /** `token` is `null` when the gate is off or has no limit (nothing to release). */ + /** + * `token` is `null` when the gate is off or has no limit (nothing to release), unless the + * capacity rule beside it keeps the start itself (`observe` mode). + */ | { readonly type: "allow"; readonly token: SlotToken | null } | { readonly type: "refuse"; readonly message: string }; @@ -200,23 +203,30 @@ const make = Effect.gen(function* () { /** * Every live held slot: granted in memory and kept in the wait store, one per wait (counted one * per thread, `onePerThread`). None while waits are off (nothing of the store is read then). A - * store that cannot be read holds none. + * store that cannot be read holds none. Taken now; the store's part is read when the returned + * effect runs, without the used and hidden keys of now (the capacity rule reads it later). */ - const liveHeld = Effect.fn("SlotGate.liveHeld")(function* (waitsOn: boolean, nowMs: number) { + const heldAt = Effect.fn("SlotGate.heldAt")(function* (waitsOn: boolean, nowMs: number) { for (const [key, untilMs] of usedKeys) if (untilMs <= nowMs) usedKeys.delete(key); // A grant never kept nor dropped (a cut-off call) hides the old holds only until they end. for (const [key, olds] of replacing) { if (olds.every((old) => old.untilMs <= nowMs)) replacing.delete(key); } const memory = yield* Ref.updateAndGet(held, (current) => liveReservations(current, nowMs)); - if (!waitsOn) return []; - const fromStore = stored === null ? [] : yield* stored.read; - const hidden = hiddenKeys(); - return liveReservations([...memory, ...fromStore], nowMs).filter( - (reservation) => !usedKeys.has(reservation.key) && !hidden.has(reservation.key), + if (!waitsOn) return Effect.succeed>([]); + const skipped = new Set([...usedKeys.keys(), ...hiddenKeys()]); + const source = stored; + return (source === null ? Effect.succeed([]) : source.read).pipe( + Effect.map((fromStore) => + liveReservations([...memory, ...fromStore], nowMs).filter( + (reservation) => !skipped.has(reservation.key), + ), + ), ); }); + const liveHeld = (waitsOn: boolean, nowMs: number) => Effect.flatten(heldAt(waitsOn, nowMs)); + /** * The live holds that re-asks not kept yet hide, of threads other than `threadId`. They still * count against their own account (never the machine, where the new hold counts), so a slot @@ -245,6 +255,7 @@ const make = Effect.gen(function* () { startGraceMs: settings.value.agentSlots.startGraceSeconds * 1000, // Off or without a limit, the gate keeps no records of new starts. gateRecords: effectiveSlotLimits(settings.value.agentSlots) !== null, + waitsOn: settings.value.waitFor.enabled, }) : Option.none(), ), @@ -253,26 +264,50 @@ const make = Effect.gen(function* () { /** * Runs a decision and, in `observe` mode, hands today's answers to the rule beside it with the * records as they were before the decision changed them. Forked there: never changed, never - * delayed. Inside the semaphore, so observations come in decision order. + * delayed (the wait store's held slots are read by the rule, off this path). Inside the + * semaphore, so observations come in decision order. */ const observeBeside = Effect.fn("SlotGate.observeBeside")(function* ( decision: Effect.Effect, answers: (result: A) => ReadonlyArray<"allow" | "refuse">, + /** + * `acquire` only. An allow the gate keeps no record of (`untracked`) gets a token of the + * rule's own, so `started` and `release` reach the rule's record of that start. + */ + caller?: { + readonly threadId: string; + readonly instanceId: string; + readonly untracked: (result: A) => boolean; + readonly withToken: (result: A, token: SlotToken) => A; + }, ) { const observe = yield* observing; if (Option.isNone(observe)) return yield* decision; + const { startGraceMs, gateRecords, waitsOn } = observe.value; const atMs = yield* Clock.currentTimeMillis; const starting = yield* Ref.get(entries); - const heldBefore = yield* Ref.get(held); + const heldBefore = yield* heldAt(waitsOn, atMs); const result = yield* decision; + // Only a start the gate did not record: never a token of the gate's own (settings changed + // between the two reads). + const ownToken = + !gateRecords && caller !== undefined && caller.untracked(result) + ? yield* Ref.getAndUpdate(nextToken, (value) => value + 1) + : null; yield* observer.observe({ today: answers(result), atMs, starting, held: heldBefore, - ...observe.value, + callerThreadId: caller?.threadId, + startGraceMs, + gateRecords, + ownStart: + ownToken === null || caller === undefined + ? undefined + : { token: ownToken, instanceId: caller.instanceId, atMs }, }); - return result; + return ownToken === null || caller === undefined ? result : caller.withToken(result, ownToken); }); /** Used up: they stop counting (memory ones go; stored ones are skipped until their end). */ @@ -319,6 +354,9 @@ const make = Effect.gen(function* () { ), ); + /** A decision and the caller's held slots it used up (typed whole, for `observeBeside`). */ + const asDecided = (outcome: SlotAcquire, used: ReadonlyArray) => ({ outcome, used }); + /** * The decision, under the semaphore, and the caller's held slot it used up (if any). The * caller's own held slots never count against it; they are not a pass past a limit either. @@ -333,10 +371,7 @@ const make = Effect.gen(function* () { ? null : effectiveSlotLimits(settings.value.agentSlots); if (Option.isNone(settings) || limits === null) { - return { - outcome: { type: "allow", token: null } satisfies SlotAcquire, - used: [] as ReadonlyArray, - }; + return asDecided({ type: "allow", token: null }, []); } const waitsOn = settings.value.waitFor.enabled; const nowMs = yield* Clock.currentTimeMillis; @@ -357,13 +392,7 @@ const make = Effect.gen(function* () { freeMemoryBytes: yield* freeMemoryBytes(limits), }); if (decision.type === "refuse") { - return { - outcome: { - type: "refuse", - message: slotRefusalText(decision, waitsOn), - } satisfies SlotAcquire, - used: [] as ReadonlyArray, - }; + return asDecided({ type: "refuse", message: slotRefusalText(decision, waitsOn) }, []); } const token = yield* Ref.getAndUpdate(nextToken, (value) => value + 1); yield* Ref.update(entries, (current) => [ @@ -372,9 +401,15 @@ const make = Effect.gen(function* () { ]); // The agent it held a slot for starts now: its held slot is used up, with every older copy. const used = yield* useUp(reservations, input.callerThreadId); - return { outcome: { type: "allow", token } satisfies SlotAcquire, used }; + return asDecided({ type: "allow", token }, used); }, - (decided) => observeBeside(decided, (result) => [result.outcome.type]), + (decided, input) => + observeBeside(decided, (result) => [result.outcome.type], { + threadId: input.callerThreadId, + instanceId: input.targetInstanceId, + untracked: (result) => result.outcome.type === "allow" && result.outcome.token === null, + withToken: (result, token) => asDecided({ type: "allow", token }, result.used), + }), // One at a time: the next caller sees the entry the last one added. semaphore.withPermits(1), ); @@ -575,12 +610,13 @@ const make = Effect.gen(function* () { for (const key of keys) replacing.delete(key); yield* Ref.update(held, (current) => current.filter((entry) => !keys.includes(entry.key))); }), + // A token is the gate's or the capacity rule's (`observeBeside`); each ignores the other's. started: Effect.fn("SlotGate.started")((token: SlotToken | null, childThreadId: ThreadId) => token === null ? Effect.void : Ref.update(entries, (current) => current.map((entry) => (entry.token === token ? { ...entry, childThreadId } : entry)), - ), + ).pipe(Effect.andThen(observer.started(token, childThreadId))), ), release: Effect.fn("SlotGate.release")(function* (token: SlotToken | null) { if (token === null) return; @@ -594,6 +630,7 @@ const make = Effect.gen(function* () { : [entry.endedAtMs === undefined ? { ...entry, endedAtMs: nowMs } : entry], ), ); + yield* observer.released(token, nowMs); }), }); }); diff --git a/apps/server/src/slots/SlotObserver.test.ts b/apps/server/src/slots/SlotObserver.test.ts index c778fa2ba0ec5..055f505aae452 100644 --- a/apps/server/src/slots/SlotObserver.test.ts +++ b/apps/server/src/slots/SlotObserver.test.ts @@ -161,6 +161,12 @@ const acquire = SlotGate.SlotGate.pipe( Effect.flatMap((gate) => gate.acquire({ callerThreadId: caller, targetInstanceId: MAIN })), ); +/** The dispatch ended without naming a child (it failed before one existed). */ +const release = (answer: SlotGate.SlotAcquire) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => gate.release(answer.type === "allow" ? answer.token : null)), + ); + /** Lets forked observations and the fake host's promises finish (scheduler turns, no real wait). */ const settle = Effect.forEach(Array.from({ length: 20 }), () => Effect.yieldNow, { discard: true }); @@ -222,7 +228,10 @@ describe("SlotGate observe mode", () => { world, {}, Effect.gen(function* () { - assert.deepStrictEqual(yield* acquire, { type: "allow", token: null }); + const answer = yield* acquire; + // A token for the rule's own record of the start (the gate keeps none while off). + assert.strictEqual(answer.type, "allow"); + assert.isNumber(answer.type === "allow" ? answer.token : null); return yield* statusAfter(1); }), ); @@ -232,6 +241,26 @@ describe("SlotGate observe mode", () => { }), ); + it.effect("the observation starts after the decision returns, never on the caller's stack", () => + Effect.gen(function* () { + const world = newWorld(); + // The fake projection read is synchronous: forked to start at once, it would run before + // the caller got its answer (inside the gate's lock). + const readsAtAnswer = yield* run( + world, + observe(), + Effect.gen(function* () { + assert.strictEqual((yield* acquire).type, "allow"); + const reads = world.snapshotReads; + yield* statusAfter(1); + return reads; + }), + ); + assert.strictEqual(readsAtAnswer, 0); + assert.strictEqual(world.snapshotReads, 1); + }), + ); + it.effect("the same decisions in fixed and observe mode, even with a hanging machine", () => Effect.gen(function* () { // Two slots: allow, allow, refuse; a release frees one; a slot wait is granted, then full. @@ -415,7 +444,7 @@ describe("SlotGate observe mode", () => { world, observe(), Effect.gen(function* () { - yield* acquire; + yield* release(yield* acquire); yield* TestClock.adjust(CALM_WAIT); yield* acquire; return yield* statusAfter(2); @@ -444,25 +473,232 @@ describe("SlotGate observe mode", () => { }), ); - it.effect("slots off: an allow counts as starting for the start grace, then goes", () => + it.effect("slots off: a start counts until its dispatch ends, then for the start grace", () => Effect.gen(function* () { const world = newWorld(); - const [soon, later] = yield* run( + const [during, grace, gone] = yield* run( world, observe({ startGraceSeconds: 30 }), Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + // A dispatch that has not named its child yet counts, however long it takes. + const first = yield* acquire; + yield* TestClock.adjust(Duration.seconds(45)); yield* acquire; + const during = yield* statusAfter(2); + // The child is named and the dispatch ends; it shows no run yet (a queued run). + const token = first.type === "allow" ? first.token : null; + yield* gate.started(token, ThreadId.make("child-queued")); + yield* gate.release(token); yield* TestClock.adjust(Duration.seconds(29)); + yield* release(yield* acquire); + const grace = yield* statusAfter(3); + yield* TestClock.adjust(Duration.seconds(2)); yield* acquire; - const soon = yield* statusAfter(2); - yield* TestClock.adjust(Duration.seconds(20)); + return [during, grace, yield* statusAfter(4)] as const; + }), + ); + assert.strictEqual(during.last?.pieces, 1); + // At 74 s: the queued child is 29 s past its dispatch's end, inside its 30 s grace; the + // second start (45 s) never ended. + assert.strictEqual(grace.last?.pieces, 2); + // At 76 s: the queued child's grace is over; the third start ended with no child at once. + assert.strictEqual(gone.last?.pieces, 1); + }), + ); + + it.effect("slots off: a start whose child shows its run counts once, as that run (#353)", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const first = yield* acquire; + const token = first.type === "allow" ? first.token : null; + yield* gate.started(token, ThreadId.make("child")); + // The child shows its run (300 MB in its own scope) inside the 30 s window. + world.threads = [{ id: "child", unit: "t3code-claude-cccc0000.scope" }]; + scopeFiles(world, "t3code-claude-cccc0000.scope", 300); + yield* TestClock.adjust(Duration.seconds(5)); + // A start whose dispatch ends with no child, so only the first agent is left. + yield* release(yield* acquire); + const running = yield* statusAfter(2); + // Its dispatch ends after the run showed: still counted once, as the run. + yield* gate.release(token); + yield* TestClock.adjust(Duration.seconds(5)); yield* acquire; - return [soon, yield* statusAfter(3)] as const; + return [running, yield* statusAfter(3)] as const; }), ); - assert.strictEqual(soon.last?.pieces, 1); - // At 49 s: the allow at 29 s still starts; the one at 0 s is past its 30 s grace. - assert.strictEqual(later.last?.pieces, 1); + // Not also as starting: that would be 2 pieces, owing 768 MB more. + for (const observed of status) assert.deepInclude(observed.last, { pieces: 1, owedMb: 468 }); + }), + ); + + it.effect("slots off: a start whose run showed and ended stops counting, grace or not", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const first = yield* acquire; + const token = first.type === "allow" ? first.token : null; + yield* gate.started(token, ThreadId.make("child")); + yield* gate.release(token); + // Its run shows, then ends, all inside the 30 s grace of its dispatch's end. + world.threads = [{ id: "child", unit: "t3code-claude-cccc0000.scope" }]; + scopeFiles(world, "t3code-claude-cccc0000.scope", 300); + yield* TestClock.adjust(Duration.seconds(5)); + yield* release(yield* acquire); + yield* statusAfter(2); + world.threads = []; + yield* TestClock.adjust(Duration.seconds(5)); + yield* release(yield* acquire); + return yield* statusAfter(3); + }), + ); + // A finished agent: not counted again as starting for the rest of its grace. + assert.strictEqual(status.last?.pieces, 0); + }), + ); + + it.effect("slots off: a child named while the observation waits is not counted twice", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const first = yield* acquire; + yield* statusAfter(1); + yield* TestClock.adjust(Duration.seconds(5)); + // Decided while the first start has no child yet; observed after it shows its run. + yield* acquire; + yield* gate.started(first.type === "allow" ? first.token : null, ThreadId.make("child")); + world.threads = [{ id: "child", unit: "t3code-claude-cccc0000.scope" }]; + scopeFiles(world, "t3code-claude-cccc0000.scope", 300); + return yield* statusAfter(2); + }), + ); + assert.deepInclude(status.last, { pieces: 1, owedMb: 468 }); + }), + ); + + it.effect("slots off: a dispatch that ends with no child stops counting at once", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + yield* release(yield* acquire); + yield* TestClock.adjust(Duration.seconds(1)); + yield* acquire; + return yield* statusAfter(2); + }), + ); + assert.deepInclude(status.last, { pieces: 0 }); + }), + ); + + it.effect("held slots in the wait store count as the gate counts them (a restart, #353)", () => + Effect.gen(function* () { + const HOLD = { key: "w_held", threadId: "waiter", untilMs: 60_000 }; + const cases = [ + // After a restart the store has the only copy of a delivered slot wait's hold. + { name: "slots off", agentSlots: {}, holds: [HOLD], caller, pieces: 1, reads: 1 }, + { + name: "slots on", + agentSlots: { enabled: true, maxPerMachine: 5 }, + holds: [HOLD], + caller, + pieces: 1, + // The gate's own decision reads it too. + reads: 2, + }, + // The holder's own delegation uses its hold up: counted once, as the new start. + { + name: "the holder", + agentSlots: {}, + holds: [HOLD], + caller: "waiter", + pieces: 0, + reads: 1, + }, + { + name: "an ended hold", + agentSlots: {}, + holds: [{ ...HOLD, untilMs: 0 }], + caller, + pieces: 0, + reads: 1, + }, + { name: "an empty store", agentSlots: {}, holds: [], caller, pieces: 0, reads: 1 }, + // Waits off: the gate counts no hold, and nothing of the store is read. + { + name: "waits off", + agentSlots: {}, + holds: [HOLD], + caller, + pieces: 0, + reads: 0, + waitsOff: true, + }, + ]; + for (const entry of cases) { + const world = newWorld(); + let reads = 0; + const status = yield* run( + world, + { + agentSlots: { mode: "observe", ...entry.agentSlots }, + waitFor: { enabled: entry.waitsOff !== true }, + }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ + read: Effect.sync(() => { + reads += 1; + return entry.holds; + }), + used: () => Effect.void, + }); + const answer = yield* gate.acquire({ + callerThreadId: ThreadId.make(entry.caller), + targetInstanceId: MAIN, + }); + assert.strictEqual(answer.type, "allow", entry.name); + return yield* statusAfter(1); + }), + ); + assert.strictEqual(status.last?.pieces, entry.pieces, entry.name); + assert.strictEqual(reads, entry.reads, entry.name); + } + }), + ); + + it.effect("a wait store read that hangs never delays the decision; the rule reads slow", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + { ...observe(), waitFor: { enabled: true } }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ read: Effect.never, used: () => Effect.void }); + // No clock is moved before the answer: a decision that waited would never end here. + assert.strictEqual((yield* acquire).type, "allow"); + yield* settle; + yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + return yield* statusAfter(1); + }), + ); + assert.deepInclude(status.last, { rule: "refuse", reason: "unreadable", readings: "slow" }); }), ); @@ -474,7 +710,7 @@ describe("SlotGate observe mode", () => { observe(), Effect.gen(function* () { yield* TestClock.adjust(CALM_WAIT); - yield* acquire; + yield* release(yield* acquire); yield* TestClock.adjust(CALM_WAIT); const both = yield* Effect.all([acquire, acquire], { concurrency: "unbounded" }); assert.deepStrictEqual( diff --git a/apps/server/src/slots/SlotObserver.ts b/apps/server/src/slots/SlotObserver.ts index d1c6e55d4716d..b3642623b6351 100644 --- a/apps/server/src/slots/SlotObserver.ts +++ b/apps/server/src/slots/SlotObserver.ts @@ -38,6 +38,7 @@ import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FiberSet from "effect/FiberSet"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Semaphore from "effect/Semaphore"; @@ -61,10 +62,21 @@ export interface SlotObserveInput { readonly today: ReadonlyArray; readonly atMs: number; readonly starting: ReadonlyArray; - readonly held: ReadonlyArray; + /** + * The live held slots as the gate counts them (memory and wait store), as of the decision. Read + * in the observer's fiber, so a slow wait store never delays the decision. + */ + readonly held: Effect.Effect>; + /** `acquire`'s caller: its own held slot is used up by the start it asks for (counted once). */ + readonly callerThreadId?: string | undefined; readonly startGraceMs: number; /** The gate keeps `starting` (slots on with a limit); else the rule keeps its own allows. */ readonly gateRecords: boolean; + /** + * While the gate keeps no records: the allowed start, under a token of its own that the gate's + * `started` and `release` hand on (`started`, `released` here). + */ + readonly ownStart?: StartingEntry | undefined; } /** systemd nests `a-b-c.slice` as `a.slice/a-b.slice/a-b-c.slice`. */ @@ -104,10 +116,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( let waiting = 0; let hanging = 0; /** - * While the gate keeps no records (slots off), the times of the agents allowed lately: each - * counts as starting for `startGraceMs`, the time a new agent takes to show a run. + * While the gate keeps no records (slots off), the agents allowed lately, kept as the gate keeps + * its own (`liveStarting`): each counts until its dispatch ends, then for `startGraceMs` unless + * its child shows a run (then it counts as that run, once). A dispatch that ends with no child + * stops counting at once. */ - let ownStarts: ReadonlyArray = []; + let ownStarts: ReadonlyArray = []; const readFacts = async (pieces: ReadonlyArray): Promise => { const machine = await readMachine(host); @@ -159,30 +173,50 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( const run = Effect.fn("SlotObserver.run")(function* ( input: SlotObserveInput, - ownStarting: number, + ownBefore: ReadonlyArray, ) { const running = yield* readRunning; + // Off the decision path; a store read past the time limit leaves the pieces unknown. + const heldRead = + running === undefined + ? Option.none() + : yield* Effect.timeoutOption(input.held, OBSERVE_READ_TIMEOUT); let pieces: ReadonlyArray | null = null; - if (running !== undefined) { + if (running !== undefined && Option.isSome(heldRead)) { + // The starts of before the decision as they are now, read with the runs of now: a child + // named or a dispatch ended while this waited is seen (else a child that shows its run + // meanwhile would count twice). + const before = new Set(ownBefore.map((entry) => entry.token)); + const own = ownStarts.filter((entry) => before.has(entry.token)); + // A child seen with a run counts as running from now on, as in the gate's own records. + ownStarts = ownStarts.filter( + (entry) => entry.childThreadId === undefined || !running.visible.has(entry.childThreadId), + ); const starting = liveStarting( - input.starting, + [...input.starting, ...own], running.visible, input.atMs, input.startGraceMs, ); - const held = onePerThread(liveReservations(input.held, input.atMs)); + const held = onePerThread( + liveReservations(heldRead.value, input.atMs).filter( + (reservation) => reservation.threadId !== input.callerThreadId, + ), + ); const scopes = yield* Effect.forEach([...running.visible], (threadId) => agentScope.ownScope === undefined ? Effect.undefined : agentScope.ownScope(threadId), ); pieces = [ ...scopes.map((scope) => ({ scope })), ...[...starting, ...held].map(() => ({ scope: undefined })), - ...Array.from({ length: ownStarting }, () => ({ scope: undefined })), ]; } const read = pieces === null - ? { facts: null, readings: "failed" as SlotReadings } + ? { + facts: null, + readings: (running === undefined ? "failed" : "slow") as SlotReadings, + } : yield* readings(pieces); const { decisions, pressure: next } = observeSlotAnswers(CAPACITY_RULE, { facts: read.facts, @@ -221,20 +255,17 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( // Kept at the call, before any skip, so a skipped allow still counts for the next ones. ownStarts = input.gateRecords ? [] - : ownStarts.filter((atMs) => atMs <= input.atMs && input.atMs - atMs < input.startGraceMs); - const ownStarting = ownStarts.length; - if (!input.gateRecords) { - ownStarts = [ - ...ownStarts, - ...input.today.filter((answer) => answer === "allow").map(() => input.atMs), - ]; + : liveStarting(ownStarts, new Set(), input.atMs, input.startGraceMs); + const ownBefore = ownStarts; + if (!input.gateRecords && input.ownStart !== undefined) { + ownStarts = [...ownStarts, input.ownStart]; } if (waiting >= OBSERVE_MAX_WAITING) { return Ref.update(status, (current) => ({ ...current, skipped: current.skipped + 1 })); } waiting += 1; return lock - .withPermits(1)(run(input, ownStarting)) + .withPermits(1)(run(input, ownBefore)) .pipe( Effect.catchCause((cause) => Effect.logWarning("agent slots: the capacity rule could not be observed", { cause }), @@ -244,13 +275,32 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( waiting -= 1; }), ), - FiberSet.run(fibers), + // Not on the caller's stack: it runs once the decision is answered, whatever it reads. + FiberSet.run(fibers, { startImmediately: false }), Effect.asVoid, ); }).pipe(Effect.ignoreCause); return { observe, + /** The gate's `started` with one of the rule's own tokens: the child that start names. */ + started: (token: number, childThreadId: string) => + Effect.sync(() => { + ownStarts = ownStarts.map((entry) => + entry.token === token ? { ...entry, childThreadId } : entry, + ); + }), + /** The gate's `release`: the dispatch ended (as the gate ends its own entries). */ + released: (token: number, atMs: number) => + Effect.sync(() => { + ownStarts = ownStarts.flatMap((entry) => + entry.token !== token + ? [entry] + : entry.childThreadId === undefined + ? [] + : [entry.endedAtMs === undefined ? { ...entry, endedAtMs: atMs } : entry], + ); + }), status: Ref.get(status), }; }); diff --git a/apps/server/src/waits/WaitService.slots.test.ts b/apps/server/src/waits/WaitService.slots.test.ts index 6db8d8abfd694..d9a3762a2eef7 100644 --- a/apps/server/src/waits/WaitService.slots.test.ts +++ b/apps/server/src/waits/WaitService.slots.test.ts @@ -578,6 +578,24 @@ describe("WaitService slot waits", () => { }), ); + it.effect("wait_list shows the capacity rule in observe mode only (#353)", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW); + for (const mode of ["observe", "fixed"] as const) { + const world = newWorld(); + const store = disk(); + const send = fakeDelivery(); + const listed = yield* Effect.gen(function* () { + const { delegate, service } = yield* startService(store.fs, send.delivery); + assert.strictEqual(yield* delegate(T1), "allow"); + return yield* service.list({ threadId: T1, projectId: PROJECT }); + }).pipe(Effect.provide(serverLayer(world, slotsOn(2, { agentSlots: { mode } })))); + if (mode === "observe") assert.deepInclude(listed.slotRule, { mode: "observe" }); + else assert.isUndefined(listed.slotRule); + } + }), + ); + it.effect( "one thread holds at most one slot: repeated met slot waits do not hoard every free slot", () =>