From 96eb99a88d9935bf3377c1383f0b77ede85f2a6b Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:46:02 +0100 Subject: [PATCH] slots observer: count each hold once with slots off in every order (#381) With slots off the gate leaves a held slot in the store, so the observer marks a hold used once its thread starts. It marked only from starts a later good store read still saw in memory, so two orders counted one agent twice: the holder's own read slow or failed and its dispatch ended with no child (read 1, slots on 0), or its observation was skipped and its child showed a run first (read 2, should be 1). Now the caller's thread is noted at the decision, before the skip check, and the next observation with a good store read uses its holds up and drops the note. Slots on drops every note; an unread one goes after the dispatch safety limit. Observe data only: no decision changes. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/slots/SlotObserver.test.ts | 166 +++++++++++++++++++-- apps/server/src/slots/SlotObserver.ts | 32 ++-- 2 files changed, 175 insertions(+), 23 deletions(-) diff --git a/apps/server/src/slots/SlotObserver.test.ts b/apps/server/src/slots/SlotObserver.test.ts index 1fec049554691..0043a2e17fb63 100644 --- a/apps/server/src/slots/SlotObserver.test.ts +++ b/apps/server/src/slots/SlotObserver.test.ts @@ -198,6 +198,24 @@ const TWO_RUNNING = [{ id: "a" }, { id: "b" }]; /** The calm wait after a (re)start: the level is "elevated" until this much calm has passed. */ const CALM_WAIT = Duration.seconds(61); +/** The observer alone, its projection read as `readRunning` (the order rows drive it by hand). */ +const observerFor = ( + world: World, + readRunning: Effect.Effect< + { readonly visible: ReadonlySet } | undefined + > = Effect.succeed({ visible: new Set() }), +) => + makeSlotObserver(readRunning).pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(SlotCapacityHost, hostOf(world)), + Layer.succeed(AgentScope, scopesOf(world)), + Layer.succeed(HostProcess.UserId, UID), + Layer.succeed(HostProcess.Platform, "linux"), + ), + ), + ); + describe("SlotGate observe mode", () => { it.effect("fixed mode, old files and unknown modes read nothing for the rule", () => Effect.gen(function* () { @@ -771,6 +789,99 @@ describe("SlotGate observe mode", () => { }), ); + it.effect("slots off: a start uses its thread's hold up in every order", () => + Effect.gen(function* () { + const HOLD = { key: "w_held", threadId: "waiter", untilMs: 100_000 }; + // What another thread's decision 5 s after the holder's start counts. Slots on reads the + // same, as the gate uses the hold up at the holder's decision (the gate test above: 0 once + // the dispatch ended with no child, 1 while the child runs). + const cases = [ + // The holder's own store read is slow, or its projection read fails; then its dispatch + // ends with no child, so the start stops counting before any read sees it. + { own: "slow", after: "no child", pieces: 0 }, + { own: "fails", after: "no child", pieces: 0 }, + // The holder's observation is skipped (the queue is full); its child shows its run + // before the next decision, while the queued observations still read. + { own: "skipped", after: "child runs", pieces: 1 }, + // The child's run shows before the next store read: the start no longer counts, but it + // still used its hold up (counted as the child's run, once). + { own: "slow", after: "child runs", pieces: 1 }, + ] as const; + const got: Array = []; + for (const entry of cases) { + const name = `own ${entry.own}, ${entry.after}`; + const world = newWorld(); + let visible = new Set(); + let runningFails = false; + const observer = yield* observerFor( + world, + Effect.sync(() => (runningFails ? undefined : { visible })), + ); + const decide = (atMs: number, patch: Partial) => + observer.observe({ + today: ["allow"], + atMs, + starting: [], + held: Effect.succeed([HOLD]), + startGraceMs: 30_000, + gateRecords: false, + ...patch, + }); + const done = (count: number) => + Effect.gen(function* () { + for (let turn = 0; turn < 200; turn += 1) { + const status = yield* observer.status; + if (status.observed + status.skipped >= count) return status; + yield* settle; + } + return yield* Effect.die(new Error(`${name}: fewer than ${count} observations`)); + }); + let decisions = 0; + if (entry.own === "skipped") { + // Other threads' observations fill the queue; the first one's store read takes 1 s. + for (let index = 0; index < OBSERVE_MAX_WAITING; index += 1) { + yield* decide(0, { + callerThreadId: `other-${index}`, + held: Effect.as(Effect.sleep(Duration.seconds(1)), [HOLD]), + }); + decisions += 1; + } + } + runningFails = entry.own === "fails"; + yield* decide(0, { + callerThreadId: "waiter", + ownStart: { token: 1, instanceId: MAIN, atMs: 0 }, + held: entry.own === "slow" ? Effect.never : Effect.succeed([HOLD]), + }); + decisions += 1; + yield* settle; + if (entry.own === "slow") yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + if (entry.own !== "skipped") yield* done(decisions); + runningFails = false; + if (entry.after === "no child") { + yield* observer.released(1, 1_000); + } else { + yield* observer.started(1, "child"); + visible = new Set(["child"]); + } + for (let index = 0; entry.own === "skipped" && index < OBSERVE_MAX_WAITING; index += 1) { + yield* TestClock.adjust(Duration.seconds(1)); + yield* settle; + } + yield* done(decisions); + yield* decide(5_000, { callerThreadId: "other" }); + const status = yield* done(decisions + 1); + assert.strictEqual(status.skipped, entry.own === "skipped" ? 1 : 0, name); + assert.strictEqual(status.last?.readings, "ok", name); + got.push(`${name}: ${status.last?.pieces}`); + } + assert.deepStrictEqual( + got, + cases.map((entry) => `own ${entry.own}, ${entry.after}: ${entry.pieces}`), + ); + }).pipe(Effect.scoped), + ); + it.effect("a wait store read that hangs never delays the decision; the rule reads slow", () => Effect.gen(function* () { const world = newWorld(); @@ -848,16 +959,7 @@ describe("SlotGate observe mode", () => { Effect.gen(function* () { const world = newWorld(); const HOLD = { key: "w_held", threadId: "waiter", untilMs: 60_000 }; - const observer = yield* makeSlotObserver(Effect.succeed({ visible: new Set() })).pipe( - Effect.provide( - Layer.mergeAll( - Layer.succeed(SlotCapacityHost, hostOf(world)), - Layer.succeed(AgentScope, scopesOf(world)), - Layer.succeed(HostProcess.UserId, UID), - Layer.succeed(HostProcess.Platform, "linux"), - ), - ), - ); + const observer = yield* observerFor(world); let decided = 0; /** One decision, observed to the end. */ const decide = (input: Partial & { atMs: number }) => @@ -882,10 +984,10 @@ describe("SlotGate observe mode", () => { callerThreadId: "waiter", ownStart: { token: 1, instanceId: MAIN, atMs: 0 }, }); - assert.deepStrictEqual(yield* observer.kept, { starts: 1, usedHolds: 1 }, "kept"); + assert.deepStrictEqual(yield* observer.kept, { starts: 1, usedHolds: 1, notes: 0 }, "kept"); // Past its hold's end and the dispatch's safety limit: the next decision lets both go. yield* decide({ atMs: DISPATCH_SAFETY_MS, held: Effect.succeed([]) }); - assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "ended"); + assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0, notes: 0 }, "ended"); // Slots switched on (the gate keeps the records): the rule's own go at once. yield* decide({ atMs: DISPATCH_SAFETY_MS + 1, @@ -894,7 +996,11 @@ describe("SlotGate observe mode", () => { held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), }); yield* decide({ atMs: DISPATCH_SAFETY_MS + 2, gateRecords: true }); - assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "slots on"); + assert.deepStrictEqual( + yield* observer.kept, + { starts: 0, usedHolds: 0, notes: 0 }, + "slots on", + ); // Switched on while a slots-off observation still waits: the slots-on one counts the hold // as the gate does (the gate never used it up). const ON = DISPATCH_SAFETY_MS + 3; @@ -915,6 +1021,40 @@ describe("SlotGate observe mode", () => { held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), }); assert.strictEqual((yield* observer.status).last?.pieces, 1, "switched on"); + // A start's note that no store read could take (every read hangs): it goes after the + // dispatch's safety limit, or at once when slots switch on. + const slow = yield* observerFor(world); + let slowDecided = 0; + const decideSlow = (input: Partial & { atMs: number }) => + Effect.gen(function* () { + yield* slow.observe({ + today: ["allow"], + starting: [], + held: Effect.never, + startGraceMs: 30_000, + gateRecords: false, + ...input, + }); + slowDecided += 1; + yield* settle; + yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + for (let turn = 0; (yield* slow.status).observed < slowDecided; turn += 1) { + if (turn === 200) return yield* Effect.die(new Error("not observed")); + yield* settle; + } + }); + const startAt = (atMs: number, token: number) => ({ + atMs, + callerThreadId: "waiter", + ownStart: { token, instanceId: MAIN, atMs }, + }); + yield* decideSlow(startAt(0, 1)); + assert.deepStrictEqual(yield* slow.kept, { starts: 1, usedHolds: 0, notes: 1 }, "unread"); + yield* decideSlow({ atMs: DISPATCH_SAFETY_MS }); + assert.deepStrictEqual(yield* slow.kept, { starts: 0, usedHolds: 0, notes: 0 }, "limit"); + yield* decideSlow(startAt(DISPATCH_SAFETY_MS + 1, 2)); + yield* decideSlow({ atMs: DISPATCH_SAFETY_MS + 2, gateRecords: true }); + assert.deepStrictEqual(yield* slow.kept, { starts: 0, usedHolds: 0, notes: 0 }, "slots on"); }).pipe(Effect.scoped), ); diff --git a/apps/server/src/slots/SlotObserver.ts b/apps/server/src/slots/SlotObserver.ts index d63f194eede27..cce3db0e040e9 100644 --- a/apps/server/src/slots/SlotObserver.ts +++ b/apps/server/src/slots/SlotObserver.ts @@ -6,6 +6,7 @@ import type { SlotRuleStatus } from "@t3tools/contracts/waitFor"; import { AgentScope } from "@t3tools/shared/AgentScope"; import { + DISPATCH_SAFETY_MS, liveReservations, liveStarting, onePerThread, @@ -87,9 +88,6 @@ export function slicePath(slice: string): string { return parts.map((_, index) => `/${parts.slice(0, index + 1).join("-")}.slice`).join(""); } -/** One of the rule's own starts, and the thread that asked for it (its holds are used up). */ -type OwnStart = StartingEntry & { readonly threadId?: string | undefined }; - interface OwnPiece { readonly scope: { readonly unit: string; readonly slice: string } | undefined; } @@ -126,7 +124,7 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( * its child shows a run (then it counts as that run, once). A dispatch that ends with no child * stops counting at once. */ - let ownStarts: ReadonlyArray = []; + let ownStarts: ReadonlyArray = []; /** * While the gate keeps no records: held slots of the threads those starts were for, by key, until * they end. A start uses its thread's hold up, as the gate's `useUp` does with slots on (with slots @@ -134,6 +132,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( * the start stops counting (its child shows its run, or its dispatch ends). */ const usedHolds = new Map(); + /** + * Slots off: threads allowed a start, noted at the decision (before any skip). The next good store + * read uses their holds up, whatever became of the start meanwhile, and drops the notes. Slots on + * drops them all; a note no read took goes after `DISPATCH_SAFETY_MS`. + */ + const startedThreads = new Map(); const readFacts = async (pieces: ReadonlyArray): Promise => { const machine = await readMachine(host); @@ -212,15 +216,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( ); const live = liveReservations(heldRead.value, input.atMs); if (!input.gateRecords) { - // Also the starts this decision counts (a skipped or slow observation of a start's own - // decision leaves its hold to the next one). - const startedThreads = new Set(own.map((entry) => entry.threadId)); - if (input.ownStart !== undefined) startedThreads.add(input.callerThreadId); for (const reservation of live) { if (startedThreads.has(reservation.threadId)) { usedHolds.set(reservation.key, reservation.untilMs); } } + startedThreads.clear(); } const held = onePerThread( live.filter( @@ -285,9 +286,16 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( for (const [key, untilMs] of usedHolds) { if (input.gateRecords || untilMs <= input.atMs) usedHolds.delete(key); } + for (const [threadId, atMs] of startedThreads) { + if (input.gateRecords || input.atMs - atMs >= DISPATCH_SAFETY_MS) { + startedThreads.delete(threadId); + } + } const ownBefore = ownStarts; if (!input.gateRecords && input.ownStart !== undefined) { - ownStarts = [...ownStarts, { ...input.ownStart, threadId: input.callerThreadId }]; + ownStarts = [...ownStarts, input.ownStart]; + if (input.callerThreadId !== undefined) + startedThreads.set(input.callerThreadId, input.atMs); } if (waiting >= OBSERVE_MAX_WAITING) { return Ref.update(status, (current) => ({ ...current, skipped: current.skipped + 1 })); @@ -332,6 +340,10 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( }), status: Ref.get(status), /** What the rule keeps in memory (for tests: nothing past the point it stops counting). */ - kept: Effect.sync(() => ({ starts: ownStarts.length, usedHolds: usedHolds.size })), + kept: Effect.sync(() => ({ + starts: ownStarts.length, + usedHolds: usedHolds.size, + notes: startedThreads.size, + })), }; });