Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
303 changes: 300 additions & 3 deletions apps/server/src/fork/agentSlotRoutes.test.ts

Large diffs are not rendered by default.

226 changes: 188 additions & 38 deletions apps/server/src/fork/agentSlotRoutes.ts

Large diffs are not rendered by default.

118 changes: 104 additions & 14 deletions apps/server/src/fork/continuationSlot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
} from "@t3tools/contracts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
Expand Down Expand Up @@ -54,8 +55,14 @@ interface World {
/** Threads the gate refuses whatever is free. */
full: Set<ThreadId>;
asks: Array<SlotGate.SlotAcquireInput>;
/** The gate's calls after an ask: `started <thread>`, `release <token>`, `leave <key>`. */
calls: Array<string>;
/** Threads whose ask takes this long (a slow count read), then is refused. */
slow: Set<ThreadId>;
/** Threads gone (`null` shell), and whether a shell read fails. */
gone: Set<ThreadId>;
/** Threads the gate sees live but upstream's worker then reads archived. */
archivedForUpstream: Set<ThreadId>;
unreadable: boolean;
logs: Array<string>;
}
Expand All @@ -64,27 +71,36 @@ const newWorld = (free: number): World => ({
free,
full: new Set(),
asks: [],
calls: [],
slow: new Set(),
gone: new Set(),
archivedForUpstream: new Set(),
unreadable: false,
logs: [],
});

const gateOf = (world: World) =>
Layer.mock(SlotGate.SlotGate)({
acquire: (input) =>
Effect.sync((): SlotGate.SlotAcquire => {
Effect.gen(function* () {
world.asks.push(input);
if (world.free === 0 || world.full.has(input.callerThreadId!))
return {
type: "refuse",
message: "full",
refusal: { type: "unread", problem: "timeout" },
};
const refused: SlotGate.SlotAcquire = {
type: "refuse",
message: "full",
refusal: { type: "unread", problem: "timeout" },
};
if (world.slow.has(input.callerThreadId!)) {
yield* Effect.sleep("5 seconds");
return refused;
}
if (world.free === 0 || world.full.has(input.callerThreadId!)) return refused;
world.free -= 1;
return { type: "allow", token: world.asks.length };
return { type: "allow", token: world.asks.length } satisfies SlotGate.SlotAcquire;
}),
started: () => Effect.void,
release: () => Effect.void,
started: (token, childThreadId) =>
Effect.sync(() => world.calls.push(`started ${childThreadId} ${token}`)),
release: (token) => Effect.sync(() => world.calls.push(`release ${token}`)),
leave: (keys) => Effect.sync(() => world.calls.push(...keys.map((key) => `leave ${key}`))),
});

const projection = {
Expand Down Expand Up @@ -115,7 +131,7 @@ const server = (
),
getThreadRecords: (threadId) =>
Effect.succeed(
world.gone.has(threadId)
world.gone.has(threadId) || world.archivedForUpstream.has(threadId)
? { ...projection, thread: { archivedAt: "2026-11-21T00:00:00.000Z", deletedAt: null } }
: projection,
) as never,
Expand Down Expand Up @@ -152,6 +168,12 @@ const offer = (value: ProviderContinuationRequests.ProviderContinuationRequest)
Effect.flatMap((requests) => requests.offer(value)),
);

/** Lets forked fibers run (scheduler turns, no clock moved). */
const settle = Effect.forEach(Array.from({ length: 50 }), () => Effect.yieldNow, { discard: true });

/** The gate's slot calls, without the line's. */
const slotCalls = (world: World) => world.calls.filter((call) => !call.startsWith("leave "));

const waitLogs = (world: World) =>
world.logs.filter((log) => log.includes("provider continuation waits for an agent slot"));

Expand Down Expand Up @@ -182,7 +204,10 @@ describe("provider continuations take an agent slot", () => {
targetInstanceId: INSTANCE,
intoThreadId: A,
keepsHeld: true,
place: { key: "continuation:0" },
});
// Every ask in its one place in the gate's line.
assert.isTrue(world.asks.every((asked) => asked.place?.key === "continuation:0"));
world.free = 1;
yield* TestClock.adjust("5 seconds");
const sent = yield* Queue.take(dispatched);
Expand Down Expand Up @@ -214,6 +239,69 @@ describe("provider continuations take an agent slot", () => {
}),
);

it.effect("a slow count read for one thread holds up no other thread's continuation", () =>
Effect.gen(function* () {
const world = newWorld(5);
world.slow.add(A);
const dispatched = yield* Queue.unbounded<OrchestrationV2ServerCommand>();
yield* Effect.gen(function* () {
yield* offer(request(A));
yield* offer(request(B));
// No time passes: B is not queued behind A's 5-second ask.
yield* settle;
assert.strictEqual(yield* Queue.size(dispatched), 1);
const sent = yield* Queue.take(dispatched);
assert.strictEqual(sent.type === "message.dispatch" && sent.threadId, B);
// A's ask ends (it cannot be cut off half-way), so the server can stop.
yield* TestClock.adjust("5 seconds");
}).pipe(Effect.provide(server(world, dispatched)), Effect.scoped);
}),
);

it.effect(
"a granted slot is held until upstream's dispatch of it ends, however long it waits",
() =>
Effect.gen(function* () {
const world = newWorld(1);
const dispatched = yield* Queue.unbounded<OrchestrationV2ServerCommand>();
const lockFree = yield* Deferred.make<void>();
yield* Effect.gen(function* () {
yield* offer(
request(A, {
// Behind its own thread's earlier dispatch (its parent's lock) for 2 minutes.
dispatchIfCurrent: (effect) =>
Deferred.await(lockFree).pipe(Effect.andThen(Effect.asSome(effect))),
}),
);
yield* TestClock.adjust("2 minutes");
assert.lengthOf(world.asks, 1);
// Asked and granted, not ended: the start counts whatever the start grace.
assert.deepStrictEqual(slotCalls(world), []);
yield* Deferred.succeed(lockFree, undefined);
yield* Queue.take(dispatched);
yield* settle;
assert.deepStrictEqual(slotCalls(world), [`started ${A} 1`, "release 1"]);
}).pipe(Effect.provide(server(world, dispatched)), Effect.scoped);
}),
);

it.effect("a granted request upstream drops (its thread archived meanwhile) frees its slot", () =>
Effect.gen(function* () {
const world = newWorld(1);
const dispatched = yield* Queue.unbounded<OrchestrationV2ServerCommand>();
const cleared: Array<ThreadId> = [];
yield* Effect.gen(function* () {
// Granted on a live shell; upstream then reads the thread archived and clears the latch.
world.archivedForUpstream.add(A);
yield* offer(request(A, { clearIfCurrent: () => Effect.sync(() => cleared.push(A)) }));
yield* settle;
assert.deepStrictEqual(cleared, [A]);
assert.strictEqual(yield* Queue.size(dispatched), 0);
assert.deepStrictEqual(slotCalls(world), [`started ${A} 1`, "release 1"]);
}).pipe(Effect.provide(server(world, dispatched)), Effect.scoped);
}),
);

it.effect("a thread that cannot be read waits, then is sent once", () =>
Effect.gen(function* () {
const world = newWorld(5);
Expand Down Expand Up @@ -241,7 +329,7 @@ describe("provider continuations take an agent slot", () => {
const cleared: Array<ThreadId> = [];
yield* Effect.gen(function* () {
yield* offer(request(A, { clearIfCurrent: () => Effect.sync(() => cleared.push(A)) }));
yield* TestClock.adjust("0 seconds");
yield* settle;
assert.deepStrictEqual(cleared, [A]);
assert.deepStrictEqual(world.asks, []);
assert.strictEqual(yield* Queue.size(dispatched), 0);
Expand All @@ -254,7 +342,7 @@ describe("provider continuations take an agent slot", () => {
const world = newWorld(0);
const inner =
yield* Queue.unbounded<ProviderContinuationRequests.ProviderContinuationRequest>();
const gated = ContinuationSlot.gateContinuations(
const gated = yield* ContinuationSlot.gateContinuations(
{
offer: (value) => Queue.offer(inner, value).pipe(Effect.asVoid),
take: Queue.take(inner),
Expand Down Expand Up @@ -291,6 +379,8 @@ describe("provider continuations take an agent slot", () => {
yield* TestClock.adjust("12 seconds");
}).pipe(Effect.provide(server(world, dispatched)), Effect.scoped);
const asked = world.asks.length;
// Its place in the line goes with it.
assert.include(world.calls, "leave continuation:0");
world.free = 5;
yield* TestClock.adjust("1 minute");
assert.strictEqual(yield* Queue.size(dispatched), 0);
Expand All @@ -308,7 +398,7 @@ describe("provider continuations take an agent slot", () => {
}).pipe(Effect.provide(server(world, dispatched, null)), Effect.scoped);
const inner = { offer: () => Effect.void, take: Effect.never };
assert.strictEqual(
ContinuationSlot.gateContinuations(
yield* ContinuationSlot.gateContinuations(
inner,
Option.none(),
() => Effect.succeed(null),
Expand Down
124 changes: 80 additions & 44 deletions apps/server/src/fork/continuationSlot.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,20 @@
/**
* Fork: a provider continuation into an idle thread takes an agent slot. Asked at the queue, not
* around the dispatch: an adapter's `dispatchIfCurrent` holds its own lock while it dispatches.
* A refused request goes back on the queue; the adapter's latch stays set, so it offers no second.
* Fork: a provider continuation into an idle thread takes an agent slot, asked off upstream's queue
* (never inside `dispatchIfCurrent`, which holds the adapter's lock), each request in its own fiber
* and place in the gate's line. The slot is held until upstream's dispatch of it ends.
*/
import type { ProviderInstanceId, ThreadId } from "@t3tools/contracts";
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Queue from "effect/Queue";
import type * as Scope from "effect/Scope";

import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts";
import * as SlotGate from "../slots/SlotGate.ts";
import { admitSend, reasonHere } from "./agentSlotRoutes.ts";
import { admitSend, leaveLine, reasonHere } from "./agentSlotRoutes.ts";

type Request = ProviderContinuationRequests.ProviderContinuationRequest;
interface Requests {
Expand All @@ -27,69 +28,104 @@ export const CONTINUATION_SLOT_RETRY = Duration.seconds(5);
/** `null`: no such thread, or it is gone. */
export type ContinuationThread = { readonly instanceId: ProviderInstanceId } | null;

/**
* The request holding its slot (`end`) until upstream's dispatch of it ends, whichever way, so a
* run that waits behind its thread's earlier dispatch still has it. One never dispatched frees it
* at the gate's dispatch limit.
*/
const holdingSlot = (
request: Request,
end: (childThreadId: ThreadId | null) => Effect.Effect<void>,
): Request => {
const dispatchIfCurrent = request.dispatchIfCurrent;
const clearIfCurrent = request.clearIfCurrent;
const ended = end(request.threadId);
return {
...request,
dispatchIfCurrent: <A, E, R>(effect: Effect.Effect<A, E, R>) =>
(dispatchIfCurrent === undefined ? Effect.asSome(effect) : dispatchIfCurrent(effect)).pipe(
Effect.ensuring(ended),
),
...(clearIfCurrent === undefined
? {}
: { clearIfCurrent: () => clearIfCurrent().pipe(Effect.ensuring(ended)) }),
};
};

/**
* `take` hands out a request into an idle thread only with a slot. Upstream's at once: a delegated
* result (docs/fork/README.md) and a gone thread (upstream drops it and clears the adapter's latch).
* An unreadable thread waits, as a refused one does.
* An unreadable thread waits, as a refused one does. No ask ever holds up upstream's queue: each
* request asks in its own fiber, so a slow count delays only that request.
*/
export const gateContinuations = <E>(
requests: Requests,
gate: Option.Option<SlotGate.SlotGate["Service"]>,
threadOf: (threadId: ThreadId) => Effect.Effect<ContinuationThread, E>,
scope: Scope.Scope,
): Requests => {
if (Option.isNone(gate)) return requests;
/** Logged once until handed out. */
const waiting = new Set<Request>();
): Effect.Effect<Requests> => {
if (Option.isNone(gate)) return Effect.succeed(requests);
let places = 0;

/** `null` when the request may go to upstream now, else why it waits. */
const waitsFor = Effect.fn("ContinuationSlot.waitsFor")(function* (request: Request) {
if (request.delegatedCompletion !== undefined) return null;
/** The request for upstream when it may go now (holding its slot), else why it waits. */
const ask = Effect.fn("ContinuationSlot.ask")(function* (
request: Request,
place: SlotGate.SlotPlace,
) {
const thread = yield* threadOf(request.threadId).pipe(
Effect.orElseSucceed(() => "unreadable" as const),
);
if (thread === "unreadable") return "the thread could not be read to check for a slot";
if (thread === null) return null;
if (thread === null) return request;
const slot = yield* admitSend(gate, {
callerThreadId: request.threadId,
threadId: request.threadId,
instanceId: thread.instanceId,
keepsHeld: true,
place,
endsItself: true,
});
return slot.type === "refuse" ? reasonHere(slot.refusal) : null;
return slot.type === "refuse" ? reasonHere(slot.refusal) : holdingSlot(request, slot.end);
});

const putBack = Effect.fn("ContinuationSlot.putBack")(function* (
request: Request,
detail: string,
) {
if (!waiting.has(request)) {
waiting.add(request);
yield* Effect.logInfo("provider continuation waits for an agent slot", {
threadId: request.threadId,
providerThreadId: request.providerThreadId,
detail,
});
}
yield* Effect.sleep(CONTINUATION_SLOT_RETRY).pipe(
Effect.andThen(requests.offer(request)),
Effect.forkIn(scope),
);
});

const take: Effect.Effect<Request> = Effect.gen(function* () {
while (true) {
const request = yield* requests.take;
const detail = yield* waitsFor(request);
if (detail === null) {
waiting.delete(request);
return request;
/** One request, until it may go: asked again every `CONTINUATION_SLOT_RETRY`, logged once. */
const untilSlot = Effect.fn("ContinuationSlot.untilSlot")(
function* (request: Request, place: SlotGate.SlotPlace, ready: Queue.Queue<Request>) {
let logged = false;
while (true) {
const answer = yield* ask(request, place);
if (typeof answer !== "string") {
yield* Queue.offer(ready, answer);
return;
}
if (!logged) {
logged = true;
yield* Effect.logInfo("provider continuation waits for an agent slot", {
threadId: request.threadId,
providerThreadId: request.providerThreadId,
detail: answer,
});
}
yield* Effect.sleep(CONTINUATION_SLOT_RETRY);
}
yield* putBack(request, detail);
}
});
},
// Stopped while it waits (the server stops): nothing is sent later, and its place goes.
(effect, _request: Request, place: SlotGate.SlotPlace) =>
Effect.ensuring(effect, leaveLine(gate, [place.key])),
);

return { offer: requests.offer, take };
return Effect.gen(function* () {
const ready = yield* Queue.unbounded<Request>();
// Upstream's queue is only read here, and never waits on an ask.
const handOut = (request: Request): Effect.Effect<void> =>
request.delegatedCompletion !== undefined
? Effect.asVoid(Queue.offer(ready, request))
: Effect.asVoid(
Effect.forkIn(untilSlot(request, { key: `continuation:${places++}` }, ready), scope),
);
yield* requests.take.pipe(Effect.flatMap(handOut), Effect.forever, Effect.forkIn(scope));
return { offer: requests.offer, take: Queue.take(ready) };
});
};

/** The queue the adapters offer to (memoized by reference), gated for upstream's worker only. */
Expand All @@ -99,7 +135,7 @@ export const layerOver = Layer.effect(
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
const threads = yield* ThreadManagement.ThreadManagementService;
const gate = yield* Effect.serviceOption(SlotGate.SlotGate);
return gateContinuations(
return yield* gateContinuations(
requests,
gate,
(threadId) =>
Expand Down
Loading
Loading