From 93e821c4ab17f18e78d5a2b50d3e0428faa19482 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:22:42 +0100 Subject: [PATCH] slots observer: a start uses its thread's held slot up with slots off (#381) With slots off the gate never uses a held slot up, so it stayed in the wait store and counted beside the holder's own start: one agent counted twice until the hold ended. The rule now marks the holds of the threads its own starts are for as used, by key, until they end, as the gate's useUp does with slots on. Observe data only; no decision changes. Also: the held slots doc says the wait store is read when the observation runs, not at the decision; tests for the rule's memory cleanup (own starts, used holds; mutation M10 of the #371 lens). Co-Authored-By: Claude Opus 5.5 --- apps/server/src/slots/SlotObserver.test.ts | 163 +++++++++++++++++++++ apps/server/src/slots/SlotObserver.ts | 43 +++++- packages/shared/src/agentSlots.ts | 6 +- 3 files changed, 203 insertions(+), 9 deletions(-) diff --git a/apps/server/src/slots/SlotObserver.test.ts b/apps/server/src/slots/SlotObserver.test.ts index 055f505aae452..1fec049554691 100644 --- a/apps/server/src/slots/SlotObserver.test.ts +++ b/apps/server/src/slots/SlotObserver.test.ts @@ -16,6 +16,7 @@ import { AgentScope, type AgentScopeShape } from "@t3tools/shared/AgentScope"; import type { CapacityFileRead, CapacityHost } from "@t3tools/shared/capacityReadings"; import * as HostProcess from "@t3tools/shared/HostProcess"; import type { DeepPartial } from "@t3tools/shared/Struct"; +import { DISPATCH_SAFETY_MS } from "@t3tools/shared/agentSlots"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -26,10 +27,12 @@ import * as HostResources from "../resourceTelemetry/HostResources.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as SlotGate from "./SlotGate.ts"; import { + makeSlotObserver, OBSERVE_MAX_HANGING, OBSERVE_MAX_WAITING, OBSERVE_READ_TIMEOUT, SlotCapacityHost, + type SlotObserveInput, slicePath, } from "./SlotObserver.ts"; @@ -167,6 +170,9 @@ const release = (answer: SlotGate.SlotAcquire) => Effect.flatMap((gate) => gate.release(answer.type === "allow" ? answer.token : null)), ); +/** A dispatch that ends with no child; its answer. */ +const acquireAndRelease = Effect.tap(acquire, release).pipe(Effect.map((answer) => answer.type)); + /** Lets forked observations and the fake host's promises finish (scheduler turns, no real wait). */ const settle = Effect.forEach(Array.from({ length: 20 }), () => Effect.yieldNow, { discard: true }); @@ -682,6 +688,89 @@ describe("SlotGate observe mode", () => { }), ); + it.effect("a held slot its holder's start used up counts once, slots off as on (#381)", () => + Effect.gen(function* () { + const HOLD = { key: "w_held", threadId: "waiter", untilMs: 100_000 }; + const waiter = ThreadId.make("waiter"); + const cases = [ + { name: "slots on", agentSlots: { enabled: true, maxPerMachine: 5 }, slowFirst: false }, + // Slots off: the gate never uses the hold up, so it stays in the store. + { name: "slots off", agentSlots: {}, slowFirst: false }, + // The holder's own observation reads the store too late: the next one still sees the start. + { name: "slots off, first read slow", agentSlots: {}, slowFirst: true }, + ]; + for (const entry of cases) { + const world = newWorld(); + let reads = 0; + const counts = yield* run( + world, + { + agentSlots: { mode: "observe", startGraceSeconds: 30, ...entry.agentSlots }, + waitFor: { enabled: true }, + }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ + read: Effect.suspend(() => { + reads += 1; + return entry.slowFirst && reads === 1 ? Effect.never : Effect.succeed([HOLD]); + }), + used: () => Effect.void, + }); + const answers: Array = []; + // The holder delegates; another thread delegates 5 s later. + const own = yield* gate.acquire({ callerThreadId: waiter, targetInstanceId: MAIN }); + answers.push(own.type); + if (entry.slowFirst) { + yield* settle; + yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + yield* statusAfter(1); + } + yield* TestClock.adjust(Duration.seconds(5)); + answers.push(yield* acquireAndRelease); + const started = (yield* statusAfter(2)).last?.pieces; + // Its child shows its run: counted as that run, the hold still used up. + const token = own.type === "allow" ? own.token : null; + yield* gate.started(token, ThreadId.make("child")); + world.threads = [{ id: "child" }]; + yield* TestClock.adjust(Duration.seconds(5)); + answers.push(yield* acquireAndRelease); + const running = (yield* statusAfter(3)).last?.pieces; + // The child's run ends: nothing of it counts, nor the hold it used. + yield* gate.release(token); + world.threads = []; + yield* TestClock.adjust(Duration.seconds(5)); + answers.push(yield* acquireAndRelease); + const ended = (yield* statusAfter(4)).last?.pieces; + // Observe data only: every answer stays today's allow. + assert.deepStrictEqual(answers, ["allow", "allow", "allow", "allow"], entry.name); + return [started, running, ended]; + }), + ); + assert.deepStrictEqual(counts, [1, 1, 0], entry.name); + } + // The holder's dispatch ends with no child before anyone else asks: nothing counts. + for (const agentSlots of [{ enabled: true, maxPerMachine: 5 }, {}]) { + const status = yield* run( + newWorld(), + { agentSlots: { mode: "observe", ...agentSlots }, waitFor: { enabled: true } }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ + read: Effect.succeed([HOLD]), + used: () => Effect.void, + }); + yield* release(yield* gate.acquire({ callerThreadId: waiter, targetInstanceId: MAIN })); + yield* TestClock.adjust(Duration.seconds(5)); + yield* acquireAndRelease; + return yield* statusAfter(2); + }), + ); + assert.strictEqual(status.last?.pieces, 0, JSON.stringify(agentSlots)); + } + }), + ); + it.effect("a wait store read that hangs never delays the decision; the rule reads slow", () => Effect.gen(function* () { const world = newWorld(); @@ -755,6 +844,80 @@ describe("SlotGate observe mode", () => { }), ); + it.effect("the rule's memory keeps only what still counts (#381)", () => + Effect.gen(function* () { + const world = newWorld(); + const HOLD = { key: "w_held", threadId: "waiter", untilMs: 60_000 }; + const observer = yield* makeSlotObserver(Effect.succeed({ visible: new Set() })).pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(SlotCapacityHost, hostOf(world)), + Layer.succeed(AgentScope, scopesOf(world)), + Layer.succeed(HostProcess.UserId, UID), + Layer.succeed(HostProcess.Platform, "linux"), + ), + ), + ); + let decided = 0; + /** One decision, observed to the end. */ + const decide = (input: Partial & { atMs: number }) => + Effect.gen(function* () { + yield* observer.observe({ + today: ["allow"], + starting: [], + held: Effect.succeed([HOLD]), + startGraceMs: 30_000, + gateRecords: false, + ...input, + }); + decided += 1; + for (let turn = 0; (yield* observer.status).observed < decided; turn += 1) { + if (turn === 200) return yield* Effect.die(new Error("not observed")); + yield* settle; + } + }); + // Slots off: the holder starts (a dispatch that never ends: a lost release). + yield* decide({ + atMs: 0, + callerThreadId: "waiter", + ownStart: { token: 1, instanceId: MAIN, atMs: 0 }, + }); + assert.deepStrictEqual(yield* observer.kept, { starts: 1, usedHolds: 1 }, "kept"); + // Past its hold's end and the dispatch's safety limit: the next decision lets both go. + yield* decide({ atMs: DISPATCH_SAFETY_MS, held: Effect.succeed([]) }); + assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "ended"); + // Slots switched on (the gate keeps the records): the rule's own go at once. + yield* decide({ + atMs: DISPATCH_SAFETY_MS + 1, + callerThreadId: "waiter", + ownStart: { token: 2, instanceId: MAIN, atMs: DISPATCH_SAFETY_MS + 1 }, + held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), + }); + yield* decide({ atMs: DISPATCH_SAFETY_MS + 2, gateRecords: true }); + assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "slots on"); + // Switched on while a slots-off observation still waits: the slots-on one counts the hold + // as the gate does (the gate never used it up). + const ON = DISPATCH_SAFETY_MS + 3; + yield* observer.observe({ + today: ["allow"], + atMs: ON, + starting: [], + held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), + callerThreadId: "waiter", + startGraceMs: 30_000, + gateRecords: false, + ownStart: { token: 3, instanceId: MAIN, atMs: ON }, + }); + decided += 1; + yield* decide({ + atMs: ON, + gateRecords: true, + held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), + }); + assert.strictEqual((yield* observer.status).last?.pieces, 1, "switched on"); + }).pipe(Effect.scoped), + ); + it("nests a systemd slice the way systemd does", () => { assert.strictEqual( slicePath("app-t3code-agents-0123.slice"), diff --git a/apps/server/src/slots/SlotObserver.ts b/apps/server/src/slots/SlotObserver.ts index b3642623b6351..d63f194eede27 100644 --- a/apps/server/src/slots/SlotObserver.ts +++ b/apps/server/src/slots/SlotObserver.ts @@ -63,8 +63,10 @@ export interface SlotObserveInput { readonly atMs: number; readonly starting: ReadonlyArray; /** - * The live held slots as the gate counts them (memory and wait store), as of the decision. Read - * in the observer's fiber, so a slow wait store never delays the decision. + * The live held slots as the gate counts them: memory as of the decision, the wait store as of + * when the observation runs (read in the observer's fiber, so a slow store never delays the + * decision). An observation that waits behind others can miss a hold a later decision used up + * meanwhile, or count one granted after its decision: one agent off, for a few seconds. */ readonly held: Effect.Effect>; /** `acquire`'s caller: its own held slot is used up by the start it asks for (counted once). */ @@ -85,6 +87,9 @@ export function slicePath(slice: string): string { return parts.map((_, index) => `/${parts.slice(0, index + 1).join("-")}.slice`).join(""); } +/** One of the rule's own starts, and the thread that asked for it (its holds are used up). */ +type OwnStart = StartingEntry & { readonly threadId?: string | undefined }; + interface OwnPiece { readonly scope: { readonly unit: string; readonly slice: string } | undefined; } @@ -121,7 +126,14 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( * its child shows a run (then it counts as that run, once). A dispatch that ends with no child * stops counting at once. */ - let ownStarts: ReadonlyArray = []; + let ownStarts: ReadonlyArray = []; + /** + * While the gate keeps no records: held slots of the threads those starts were for, by key, until + * they end. A start uses its thread's hold up, as the gate's `useUp` does with slots on (with slots + * off the gate leaves it in the store), so the start and the hold never both count, not even once + * the start stops counting (its child shows its run, or its dispatch ends). + */ + const usedHolds = new Map(); const readFacts = async (pieces: ReadonlyArray): Promise => { const machine = await readMachine(host); @@ -198,9 +210,23 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( input.atMs, input.startGraceMs, ); + const live = liveReservations(heldRead.value, input.atMs); + if (!input.gateRecords) { + // Also the starts this decision counts (a skipped or slow observation of a start's own + // decision leaves its hold to the next one). + const startedThreads = new Set(own.map((entry) => entry.threadId)); + if (input.ownStart !== undefined) startedThreads.add(input.callerThreadId); + for (const reservation of live) { + if (startedThreads.has(reservation.threadId)) { + usedHolds.set(reservation.key, reservation.untilMs); + } + } + } const held = onePerThread( - liveReservations(heldRead.value, input.atMs).filter( - (reservation) => reservation.threadId !== input.callerThreadId, + live.filter( + (reservation) => + reservation.threadId !== input.callerThreadId && + (input.gateRecords || !usedHolds.has(reservation.key)), ), ); const scopes = yield* Effect.forEach([...running.visible], (threadId) => @@ -256,9 +282,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( ownStarts = input.gateRecords ? [] : liveStarting(ownStarts, new Set(), input.atMs, input.startGraceMs); + for (const [key, untilMs] of usedHolds) { + if (input.gateRecords || untilMs <= input.atMs) usedHolds.delete(key); + } const ownBefore = ownStarts; if (!input.gateRecords && input.ownStart !== undefined) { - ownStarts = [...ownStarts, input.ownStart]; + ownStarts = [...ownStarts, { ...input.ownStart, threadId: input.callerThreadId }]; } if (waiting >= OBSERVE_MAX_WAITING) { return Ref.update(status, (current) => ({ ...current, skipped: current.skipped + 1 })); @@ -302,5 +331,7 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( ); }), status: Ref.get(status), + /** What the rule keeps in memory (for tests: nothing past the point it stops counting). */ + kept: Effect.sync(() => ({ starts: ownStarts.length, usedHolds: usedHolds.size })), }; }); diff --git a/packages/shared/src/agentSlots.ts b/packages/shared/src/agentSlots.ts index 33d921931f9b1..8faa1b04da11b 100644 --- a/packages/shared/src/agentSlots.ts +++ b/packages/shared/src/agentSlots.ts @@ -119,12 +119,12 @@ export interface StartingEntry { * never get more slots than are free (a lost end stops after `DISPATCH_SAFETY_MS`). After the * dispatch, a child that shows no run yet (a queued run) counts for the start grace. */ -export function liveStarting( - entries: ReadonlyArray, +export function liveStarting( + entries: ReadonlyArray, visibleRunIds: ReadonlySet, nowMs: number, startGraceMs: number, -): ReadonlyArray { +): ReadonlyArray { return entries.filter((entry) => { if (entry.childThreadId !== undefined && visibleRunIds.has(entry.childThreadId)) return false; return entry.endedAtMs === undefined