From a4d91d48ef4019c9e52a66a48cd0bfb94b23edab Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:44:45 +0000 Subject: [PATCH 1/2] slots: clear holds used up after a failed store read; warn once per reason A start allowed while the wait store could not be read has its stored hold used up at the next good read. That hold was only hidden in memory, so after a restart it counted again until it ended. The keys now go to a set that the next acquire, grant or usage clears in the store, outside the semaphore, as acquire already does for its own holds. A key leaves the set only after its write was tried, so a call cut off half-way leaves it for the next one. While the store cannot be read, the gate now warns once per reason instead of on every decision and observation; a good read resets it. Fixes #574 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/slots/SlotGate.test.ts | 170 +++++++++++++++++- apps/server/src/slots/SlotGate.ts | 50 +++++- .../src/waits/WaitService.slots.test.ts | 11 +- 3 files changed, 224 insertions(+), 7 deletions(-) diff --git a/apps/server/src/slots/SlotGate.test.ts b/apps/server/src/slots/SlotGate.test.ts index 8ddda38bff0fe..1937cc3de6cef 100644 --- a/apps/server/src/slots/SlotGate.test.ts +++ b/apps/server/src/slots/SlotGate.test.ts @@ -12,12 +12,14 @@ import { ServerSettingsError, ThreadId, } from "@t3tools/contracts"; -import type { AgentSlotsSettings } from "@t3tools/contracts/waitFor"; +import { type AgentSlotsSettings, WaitToolFailure } from "@t3tools/contracts/waitFor"; import type { DeepPartial } from "@t3tools/shared/Struct"; import { DISPATCH_SAFETY_MS, type SlotReservation } from "@t3tools/shared/agentSlots"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import { TestClock } from "effect/testing"; import * as OrchestratorV2 from "../orchestration-v2/Orchestrator.ts"; @@ -773,6 +775,172 @@ describe("SlotGate", () => { }), ); + /** A store whose read fails with `problem` while it is set; `used` clears a hold in it. */ + const flakyStore = (holds: ReadonlyArray) => { + const store = { + problem: null as string | null, + holds, + used: [] as Array, + /** Set: the next `used` call completes it, then hangs (a call cut off half-way). */ + hangNext: null as Deferred.Deferred | null, + reads: 0, + }; + const attach = (gate: SlotGate.SlotGate["Service"]) => + gate.attachReservations({ + read: Effect.suspend(() => { + store.reads += 1; + return store.problem === null + ? Effect.succeed(store.holds) + : Effect.fail( + new WaitToolFailure({ code: "store_unavailable", message: store.problem }), + ); + }), + used: (key) => + Effect.suspend(() => { + const hang = store.hangNext; + if (hang !== null) { + store.hangNext = null; + return Deferred.succeed(hang, undefined).pipe(Effect.andThen(Effect.never)); + } + store.used.push(key); + store.holds = store.holds.filter((entry) => entry.key !== key); + return Effect.void; + }), + }); + return { store, attach }; + }; + + it.effect( + "a start while the store cannot be read: the next good read clears its hold in the store (#574)", + () => + Effect.gen(function* () { + // Fixed mode: no observation reads the store beside the decisions. + const settings = waitsOn({ mode: "fixed", maxPerMachine: 2 }); + for (const path of ["acquire", "grant", "usage"] as const) { + const world = newWorld(); + const { store, attach } = flakyStore([ + { key: "stored", threadId: holder, untilMs: 90_000 }, + ]); + yield* run( + world, + settings, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* attach(gate); + store.problem = "a newer file"; + const start = yield* acquireAs(holder); + assert.strictEqual(start.type, "allow"); + // Its dispatch ends with no child: nothing of it is left starting. + yield* gate.release(start.type === "allow" ? start.token : null); + assert.deepStrictEqual(store.used, []); + store.problem = null; + // The next good read, on any path, uses the hold up and clears it in the store. + if (path === "acquire") assert.strictEqual((yield* acquireAs(other)).type, "allow"); + if (path === "grant") { + assert.deepStrictEqual(types(yield* grant({ key: "w-other", threadId: other })), [ + "granted", + ]); + } + if (path === "usage") { + assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 0, max: 2 }); + } + assert.deepStrictEqual(store.used, ["stored"], path); + // Runs again: it is not cleared twice (another start clears only its own hold). + yield* gate.usage; + yield* acquireAs(other); + assert.deepStrictEqual( + store.used.filter((key) => key === "stored"), + ["stored"], + path, + ); + }), + ); + // A restart (a new gate on the same store): the used hold does not count again. + yield* run( + world, + settings, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* attach(gate); + assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 0, max: 2 }, path); + }), + ); + } + }), + ); + + it.effect("a clear cut off half-way is tried again by the next call (#574)", () => + Effect.gen(function* () { + const world = newWorld(); + const { store, attach } = flakyStore([ + { key: "stored", threadId: holder, untilMs: 90_000 }, + ]); + yield* run( + world, + waitsOn({ mode: "fixed", maxPerMachine: 2 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* attach(gate); + store.problem = "a newer file"; + assert.strictEqual((yield* acquireAs(holder)).type, "allow"); + store.problem = null; + const clearing = yield* Deferred.make(); + store.hangNext = clearing; + // The good read uses the hold up; its clear hangs and the call is cut off. + const fiber = yield* Effect.forkChild(gate.usage); + yield* Deferred.await(clearing); + yield* Fiber.interrupt(fiber); + assert.deepStrictEqual(store.used, []); + // Hidden in memory meanwhile, and cleared by the next call. + assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 1, max: 2 }); + assert.deepStrictEqual(store.used, ["stored"]); + assert.deepStrictEqual(store.holds, []); + }), + ); + }), + ); + + it.effect("a store that cannot be read is warned about once per reason (#574)", () => + Effect.gen(function* () { + const world = newWorld(); + const warnings: Array = []; + const logger = Logger.make(({ logLevel, message }) => { + const text = String(Array.isArray(message) ? message[0] : message); + if (logLevel === "Warn") warnings.push(text); + }); + const storeWarnings = () => + warnings.filter((text) => text.includes("held in the wait store cannot be read")).length; + const { store, attach } = flakyStore([]); + yield* run( + world, + waitsOn({ mode: "fixed", maxPerMachine: 2 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* attach(gate); + store.problem = "a newer file"; + yield* acquireAs(other); + yield* grant({ key: "w1", threadId: holder }); + assert.isNull(yield* gate.usage); + yield* acquireAs(other); + assert.strictEqual(storeWarnings(), 1); + // Another reason: warned again. + store.problem = "a damaged file"; + yield* acquireAs(other); + yield* acquireAs(other); + assert.strictEqual(storeWarnings(), 2); + // Read again, then the same reason as before: a new spell of failures, warned again. + store.problem = null; + yield* gate.usage; + store.problem = "a damaged file"; + yield* gate.usage; + yield* gate.usage; + assert.strictEqual(storeWarnings(), 3); + assert.isAbove(store.reads, 7); + }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))), + ); + }), + ); + it.effect("usage: null while slots are off; counts running, starting and held", () => Effect.gen(function* () { const world = newWorld(); diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index 1e273f5f04465..1165642b213da 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -177,6 +177,15 @@ const make = Effect.gen(function* () { * so none comes in between; an uncounted one may, and counts against nothing. */ const unreadStarts = new Set(); + /** + * Stored holds that a good read used up for `unreadStarts`, not yet cleared in the store. Cleared + * outside the semaphore by the next acquire, grant or usage (`clearUsedLater`), as `acquire` + * clears its own; a key leaves only after its write was tried, so a call cut off half-way leaves + * it for the next one (clearing twice changes nothing). Lost on a restart, like `unreadStarts`. + */ + const usedLater = new Set(); + /** Why the store last could not be read; warned once until the reason changes or a read works. */ + let storeProblem: string | null = null; /** * Threads with an active run, settled or not (settling a thread does not stop its run), nobody @@ -236,6 +245,7 @@ const make = Effect.gen(function* () { if (!waitsOn) { // Stored holds do not count while waits are off; a start then never uses them up either. unreadStarts.clear(); + storeProblem = null; return Effect.succeed({ held: [], whole: true }); } const skipped = new Set([...usedKeys.keys(), ...hiddenKeys()]); @@ -248,16 +258,22 @@ const make = Effect.gen(function* () { for (const reservation of fromStore) { if (!unreadStarts.has(reservation.threadId)) continue; usedKeys.set(reservation.key, reservation.untilMs); + usedLater.add(reservation.key); skipped.add(reservation.key); } unreadStarts.clear(); + storeProblem = null; return { held: live([...memory, ...fromStore]), whole: true }; }), - Effect.catch((cause) => - Effect.logWarning("agent slots: the slots held in the wait store cannot be read", { + Effect.catch((cause) => { + const unread = Effect.succeed({ held: live(memory), whole: false }); + // Every decision and observation reads the store: one warning per reason, not per read. + if (cause.message === storeProblem) return unread; + storeProblem = cause.message; + return Effect.logWarning("agent slots: the slots held in the wait store cannot be read", { cause, - }).pipe(Effect.as({ held: live(memory), whole: false })), - ), + }).pipe(Effect.andThen(unread)); + }), ); }); @@ -394,6 +410,21 @@ const make = Effect.gen(function* () { ), ); + /** + * Clears the stored holds a later good read used up (`usedLater`), outside the semaphore. Each + * key leaves the set once its write was tried (a failed write is logged; its copy ends by itself). + */ + const clearUsedLater = Effect.suspend(() => { + const keys = [...usedLater]; + return clearStored(keys).pipe( + Effect.andThen( + Effect.sync(() => { + for (const key of keys) usedLater.delete(key); + }), + ), + ); + }); + /** A decision and the caller's held slots it used up (typed whole, for `observeBeside`). */ const asDecided = (outcome: SlotAcquire, used: ReadonlyArray) => ({ outcome, used }); @@ -466,10 +497,11 @@ const make = Effect.gen(function* () { // Cleared in the store after the decision, outside the semaphore; a failed write only means // the stored copy ends by itself (it no longer counts here either way). yield* clearStored(used); + yield* clearUsedLater; return outcome; }); - const grant = Effect.fn("SlotGate.grant")( + const grantLocked = Effect.fn("SlotGate.grantLocked")( function* (requests: ReadonlyArray) { const settings = yield* readSettings; const limits = Option.isNone(settings) @@ -581,6 +613,12 @@ const make = Effect.gen(function* () { semaphore.withPermits(1), ); + const grant = Effect.fn("SlotGate.grant")(function* (requests: ReadonlyArray) { + const grants = yield* grantLocked(requests); + yield* clearUsedLater; + return grants; + }); + const keepLocked = Effect.fn("SlotGate.keepLocked")(function* (keys: ReadonlyArray) { const replaced: Array = []; for (const key of keys) { @@ -605,6 +643,8 @@ const make = Effect.gen(function* () { ? [] : liveStarting(yield* Ref.get(entries), read.visible, nowMs, limits.startGraceMs); const heldRead = yield* liveHeld(settings.value.waitFor.enabled, nowMs); + // Not under the semaphore: a hold this read used up is cleared in the store here. + yield* clearUsedLater; if (!heldRead.whole) return null; const reservations = onePerThread(heldRead.held); // Holds hidden by a re-ask not kept yet: on their own account only, as `decide` counts them. diff --git a/apps/server/src/waits/WaitService.slots.test.ts b/apps/server/src/waits/WaitService.slots.test.ts index ca762e3fe2790..b489bb3341187 100644 --- a/apps/server/src/waits/WaitService.slots.test.ts +++ b/apps/server/src/waits/WaitService.slots.test.ts @@ -499,7 +499,7 @@ describe("WaitService slot waits", () => { ); it.effect( - "a store that cannot be loaded is not an empty one: no grant, used up later (#415)", + "a store that cannot be loaded is not an empty one: no grant, used up later and cleared (#415, #574)", () => Effect.gen(function* () { yield* TestClock.setTime(NOW); @@ -547,6 +547,15 @@ describe("WaitService slot waits", () => { yield* gate.release(allowed.type === "allow" ? allowed.token : null); // T1's start used its slot up: the slot is free for another thread, not held twice. assert.strictEqual(yield* delegate(OUTSIDER), "allow"); + // Cleared in the file too, not only hidden in this gate's memory. + const used = stored(store.files).find((wait) => wait.id === "w_aaaaaaaaaa"); + assert.strictEqual(used?.state, "delivered"); + assert.isUndefined(used?.reservedUntil); + }).pipe(Effect.provide(serverLayer(world, slotsOn(1)))); + // A restart: a new gate on the same file does not count T1's used slot again. + yield* Effect.gen(function* () { + const { delegate } = yield* startService(store.fs, send.delivery); + assert.strictEqual(yield* delegate(T2), "allow"); }).pipe(Effect.provide(serverLayer(world, slotsOn(1)))); }), ); From c4f36bb5ce18f3408e0bb32c86e7ddc6ad3f115c Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:23:02 +0000 Subject: [PATCH 2/2] slots: test that waits off resets the store warning; clearUsedLater doc (#592 review) Co-Authored-By: Claude Opus 5.5 --- apps/server/src/slots/SlotGate.test.ts | 8 ++++++++ apps/server/src/slots/SlotGate.ts | 2 ++ 2 files changed, 10 insertions(+) diff --git a/apps/server/src/slots/SlotGate.test.ts b/apps/server/src/slots/SlotGate.test.ts index 1937cc3de6cef..ce9bd7a5818a2 100644 --- a/apps/server/src/slots/SlotGate.test.ts +++ b/apps/server/src/slots/SlotGate.test.ts @@ -935,6 +935,14 @@ describe("SlotGate", () => { yield* gate.usage; yield* gate.usage; assert.strictEqual(storeWarnings(), 3); + // Waits off, then on again with the same reason: a new spell, warned again. + const settings = yield* ServerSettings.ServerSettingsService; + yield* settings.updateSettings({ waitFor: { enabled: false } }); + yield* acquireAs(other); + yield* settings.updateSettings({ waitFor: { enabled: true } }); + yield* gate.usage; + yield* gate.usage; + assert.strictEqual(storeWarnings(), 4); assert.isAbove(store.reads, 7); }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))), ); diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index 1165642b213da..fdf4decb08214 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -413,6 +413,8 @@ const make = Effect.gen(function* () { /** * Clears the stored holds a later good read used up (`usedLater`), outside the semaphore. Each * key leaves the set once its write was tried (a failed write is logged; its copy ends by itself). + * With no store attached nothing is written and the keys still leave: such a hold would count + * again after a restart until it ends, the safe direction (fewer starts, never more). */ const clearUsedLater = Effect.suspend(() => { const keys = [...usedLater];