Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/server/scripts/kin-shadow-dry-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -682,7 +682,7 @@ export const runDryRun = Effect.fn("KinDryRun.run")(
let wakeups = 0;
const drain = Effect.fn("KinDryRun.drain")(function* (limit: number) {
while (true) {
const next = yield* queue.nextReady(orgId);
const next = (yield* queue.readyLeads(orgId))[0];
if (next === undefined || !Number.isFinite(next.readyAt) || next.readyAt > limit)
return;
if (++wakeups > 200)
Expand Down
89 changes: 52 additions & 37 deletions apps/server/src/kin/lead/KinEventQueue.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,10 @@ describe("KinEventQueue", () => {
yield* queue.enqueue(yield* event("ready", "routine", "a", "lead-a2"));
yield* sql`UPDATE kin_events SET not_before = 300000 WHERE event_id = 'deferred'`;
yield* TestClock.adjust("20 seconds");
assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a2");
assert.deepStrictEqual(
(yield* queue.readyLeads(orgId)).map((lead) => lead.agentId),
["lead-a2", "lead-a"],
);
const changed = yield* queue.changes(orgId);
const signal = yield* changed.pipe(Effect.forkChild);
yield* sql`UPDATE kin_agents SET retired_at = '2026-11-07T00:00:00.000Z' WHERE agent_id = 'lead-a2'`;
Expand All @@ -296,7 +299,10 @@ describe("KinEventQueue", () => {
assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'ready'`, [
{ state: "expired" },
]);
assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a");
assert.deepStrictEqual(
(yield* queue.readyLeads(orgId)).map((lead) => lead.agentId),
["lead-a"],
);
}).pipe(Effect.provide(layer)),
);

Expand Down Expand Up @@ -767,16 +773,18 @@ describe("KinEventQueue", () => {
);

it.effect(
"holds one writer per organization across the caller scope while other organizations proceed",
"holds one writer per lead across the caller scope while other leads and organizations proceed",
() =>
Effect.gen(function* () {
yield* seed();
yield* seed("a", "lead-a2");
yield* seed("b");
const queue = yield* KinEventQueue.KinEventQueue;
const sql = yield* SqlClient.SqlClient;
const acquired = yield* Deferred.make<void>();
const release = yield* Deferred.make<void>();
yield* queue.enqueue(yield* event("a-first"));
yield* queue.enqueue(yield* event("a2-first", "github", "a", "lead-a2"));
yield* queue.enqueue(yield* event("b-first", "github", "b"));
yield* TestClock.adjust("20 seconds");
const first = yield* Effect.scoped(
Expand All @@ -798,10 +806,19 @@ describe("KinEventQueue", () => {
(yield* Fiber.join(other)).map((entry) => entry.eventId),
["b-first"],
);
const otherLead = yield* Effect.scoped(queue.claim(input("a2-first", "a", "lead-a2"))).pipe(
Effect.forkChild,
);
assert.deepStrictEqual(
(yield* Fiber.join(otherLead)).map((entry) => entry.eventId),
["a2-first"],
);
yield* finish("a2-first");
yield* TestClock.adjust("20 seconds");
assert.deepStrictEqual(yield* sql`SELECT wakeup_id FROM kin_wakeups WHERE org_id = 'a'`, [
{ wakeup_id: "a-first" },
]);
assert.deepStrictEqual(
yield* sql`SELECT wakeup_id FROM kin_wakeups WHERE org_id = 'a' AND status = 'running' ORDER BY wakeup_id`,
[{ wakeup_id: "a-first" }],
);
yield* Deferred.succeed(release, undefined);
yield* Fiber.join(first);
assert.deepStrictEqual(
Expand All @@ -822,37 +839,35 @@ describe("KinEventQueue", () => {
}).pipe(Effect.provide(layer)),
);

it.effect(
"wakes empty queues on enqueue and releases the organization permit on interruption",
() =>
Effect.gen(function* () {
yield* seed();
const queue = yield* KinEventQueue.KinEventQueue;
const cancelled = yield* Effect.scoped(queue.claim(input("cancelled"))).pipe(
Effect.forkChild,
);
yield* seed("a", "lead-a2");
yield* queue.enqueue(yield* event("other-lead", "github", "a", "lead-a2"));
const otherLead = yield* Effect.scoped(
queue.claim(input("other-lead", "a", "lead-a2")),
).pipe(Effect.forkChild);
yield* TestClock.adjust("20 seconds");
assert.isDefined(otherLead.pollUnsafe());
assert.deepStrictEqual(
(yield* Fiber.join(otherLead)).map((entry) => entry.eventId),
["other-lead"],
);
yield* finish("other-lead");
yield* Fiber.interrupt(cancelled);
const waiting = yield* Effect.scoped(queue.claim(input("notified"))).pipe(Effect.forkChild);
yield* TestClock.adjust("1 second");
yield* queue.enqueue(yield* event("notification"));
yield* TestClock.adjust("20 seconds");
assert.deepStrictEqual(
(yield* Fiber.join(waiting)).map((entry) => entry.eventId),
["notification"],
);
}).pipe(Effect.provide(layer)),
it.effect("wakes empty queues on enqueue and releases the lead permit on interruption", () =>
Effect.gen(function* () {
yield* seed();
const queue = yield* KinEventQueue.KinEventQueue;
const cancelled = yield* Effect.scoped(queue.claim(input("cancelled"))).pipe(
Effect.forkChild,
);
yield* seed("a", "lead-a2");
yield* queue.enqueue(yield* event("other-lead", "github", "a", "lead-a2"));
const otherLead = yield* Effect.scoped(queue.claim(input("other-lead", "a", "lead-a2"))).pipe(
Effect.forkChild,
);
yield* TestClock.adjust("20 seconds");
assert.isDefined(otherLead.pollUnsafe());
assert.deepStrictEqual(
(yield* Fiber.join(otherLead)).map((entry) => entry.eventId),
["other-lead"],
);
yield* finish("other-lead");
yield* Fiber.interrupt(cancelled);
const waiting = yield* Effect.scoped(queue.claim(input("notified"))).pipe(Effect.forkChild);
yield* TestClock.adjust("1 second");
yield* queue.enqueue(yield* event("notification"));
yield* TestClock.adjust("20 seconds");
assert.deepStrictEqual(
(yield* Fiber.join(waiting)).map((entry) => entry.eventId),
["notification"],
);
}).pipe(Effect.provide(layer)),
);

it.effect(
Expand Down
48 changes: 30 additions & 18 deletions apps/server/src/kin/lead/KinEventQueue.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ export class KinEventQueueWakeupConflict extends Schema.TaggedError<KinEventQueu
) {}
export class KinEventQueueBusy extends Schema.TaggedError<KinEventQueueBusy>()(
"KinEventQueueBusy",
{ orgId: KinOrgId },
{ orgId: KinOrgId, agentId: KinAgentId },
) {}
const isUnknownAgent = Schema.is(KinEventQueueUnknownAgent);
const isWakeupConflict = Schema.is(KinEventQueueWakeupConflict);
Expand Down Expand Up @@ -141,10 +141,11 @@ export class KinEventQueue extends Context.Service<
>;
readonly reconsiderBudget: (orgId: KinOrgId) => Effect.Effect<void, PersistenceSqlError>;
readonly changes: (orgId: KinOrgId) => Effect.Effect<Effect.Effect<void>>;
readonly nextReady: (
/** Every lead with queued events, earliest first; a lead is claimable once `readyAt` passes. */
readonly readyLeads: (
orgId: KinOrgId,
) => Effect.Effect<
{ readonly agentId: KinAgentId; readonly readyAt: number } | undefined,
ReadonlyArray<{ readonly agentId: KinAgentId; readonly readyAt: number }>,
PersistenceSqlError | PersistenceDecodeError
>;
readonly expireAgent: (
Expand Down Expand Up @@ -210,18 +211,25 @@ const LATER_RETRY_MS = 60_000;
const make = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const laneOf = sql.literal(LANE_OF);
const organizations = new Map<
KinOrgId,
{ gate: Semaphore.Semaphore; changed: Deferred.Deferred<void> }
>();
const organizations = new Map<KinOrgId, { changed: Deferred.Deferred<void> }>();
// One running wake-up per lead: the in-process gate is keyed like the claim index.
const gates = new Map<string, Semaphore.Semaphore>();

const organization = (orgId: KinOrgId) => {
const existing = organizations.get(orgId);
if (existing !== undefined) return existing;
const entry = { gate: Semaphore.makeUnsafe(1), changed: Deferred.makeUnsafe<void>() };
const entry = { changed: Deferred.makeUnsafe<void>() };
organizations.set(orgId, entry);
return entry;
};
const gateFor = (orgId: KinOrgId, agentId: KinAgentId) => {
const key = `${orgId}\u0000${agentId}`;
const existing = gates.get(key);
if (existing !== undefined) return existing;
const gate = Semaphore.makeUnsafe(1);
gates.set(key, gate);
return gate;
};

const notify = Effect.fn("KinEventQueue.notify")(function* (orgId: KinOrgId) {
const entry = organizations.get(orgId);
Expand Down Expand Up @@ -391,6 +399,7 @@ const make = Effect.gen(function* () {
function* (input: ClaimInput) {
yield* decodeClaimInput(input);
const entry = organization(input.orgId);
const gate = gateFor(input.orgId, input.agentId);
return yield* Effect.acquireRelease(
Effect.interruptible(
Effect.uninterruptibleMask((restore) =>
Expand All @@ -409,7 +418,7 @@ const make = Effect.gen(function* () {
);
continue;
}
yield* restore(entry.gate.take(1));
yield* restore(gate.take(1));
const events = yield* sql
.withTransaction(
Effect.gen(function* () {
Expand All @@ -423,13 +432,16 @@ const make = Effect.gen(function* () {
orgId: input.orgId,
wakeupId: input.wakeupId,
});
// Cross-process exclusion rests on idx_kin_wakeups_running_org.
// Cross-process exclusion rests on idx_kin_wakeups_running_lead.
const started = yield* sql`
INSERT INTO kin_wakeups (org_id, wakeup_id, agent_id, model, started_at, status, event_ids)
VALUES (${input.orgId}, ${input.wakeupId}, ${input.agentId}, ${input.model}, ${now}, 'running', '[]')
ON CONFLICT (org_id) WHERE status = 'running' DO NOTHING RETURNING wakeup_id`;
ON CONFLICT (org_id, agent_id) WHERE status = 'running' DO NOTHING RETURNING wakeup_id`;
if (started.length === 0)
return yield* new KinEventQueueBusy({ orgId: input.orgId });
return yield* new KinEventQueueBusy({
orgId: input.orgId,
agentId: input.agentId,
});
const rows = yield* sql<{
json: string;
priority: "P0" | "P1" | "P2" | "P3";
Expand Down Expand Up @@ -470,15 +482,15 @@ const make = Effect.gen(function* () {
}),
)
.pipe(
Effect.onError(() => entry.gate.release(1)),
Effect.onError(() => gate.release(1)),
Effect.catchTags({ NoClaimableEvents: () => Effect.undefined }),
);
if (events !== undefined) return events;
}
}),
),
),
() => entry.gate.release(1),
() => gate.release(1),
);
},
Effect.mapError((cause) =>
Expand Down Expand Up @@ -692,7 +704,7 @@ const make = Effect.gen(function* () {
isUnknownAgent(cause) ? cause : toQueuePersistenceError("KinEventQueue.expireAgent")(cause),
),
);
const nextReady = Effect.fn("KinEventQueue.nextReady")(
const readyLeads = Effect.fn("KinEventQueue.readyLeads")(
function* (orgId: KinOrgId) {
const now = yield* Clock.currentTimeMillis;
const rows = yield* sql<{ agent_id: string }>`SELECT e.agent_id
Expand Down Expand Up @@ -723,9 +735,9 @@ const make = Effect.gen(function* () {
);
return candidates.toSorted(
(a, b) => a.readyAt - b.readyAt || a.agentId.localeCompare(b.agentId),
)[0];
);
},
Effect.mapError(toQueuePersistenceError("KinEventQueue.nextReady")),
Effect.mapError(toQueuePersistenceError("KinEventQueue.readyLeads")),
);
return KinEventQueue.of({
reconsiderBudget: Effect.fn("KinEventQueue.reconsiderBudget")(function* (orgId) {
Expand All @@ -743,7 +755,7 @@ const make = Effect.gen(function* () {
requeueUnfinished,
changes,
expireAgent,
nextReady,
readyLeads,
});
});

Expand Down
74 changes: 73 additions & 1 deletion apps/server/src/kin/lead/KinStop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { assert, describe, it } from "@effect/vitest";
import {
CommandId,
EnvironmentId,
KinAgentId,
KinPersonId,
KinWakeupId,
MessageId,
Expand Down Expand Up @@ -199,6 +200,73 @@ describe("KinStop", () => {
}).pipe(provideDeterministicTestRuntime),
);

it.effect("stops two leads' running wake-ups at once and starts none until resume", () =>
Effect.gen(function* () {
const enteredThreads = yield* Queue.unbounded<ThreadId>();
yield* Effect.gen(function* () {
yield* seed;
const sql = yield* SqlClient.SqlClient;
const queue = yield* KinEventQueue.KinEventQueue;
const runner = yield* WakeupRunner.WakeupRunner;
const stop = yield* KinStop.KinStop;
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
yield* stop.attach(yield* KinStop.makeFanOut(Option.none()));
yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at)
SELECT 'lead-z', org_id, display_name, role, owner_person_id, created_at FROM kin_agents WHERE agent_id = 'lead'`;
yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id)
SELECT 'lead-z', org_id, name, level_since, models_json, worker_projects_json, owner_chat_id FROM kin_leads WHERE agent_id = 'lead'`;
const otherLead = KinAgentId.make("lead-z");
yield* queue.enqueue(event("work"));
yield* queue.enqueue({ ...event("other-work"), agentId: otherLead });
const first = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild);
const second = yield* runner.runOnce(orgId, otherLead).pipe(Effect.forkChild);
yield* TestClock.adjust("20 seconds");
const threads = [yield* Queue.take(enteredThreads), yield* Queue.take(enteredThreads)];
assert.deepStrictEqual(
yield* sql`SELECT agent_id FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY agent_id`,
[{ agent_id: "lead" }, { agent_id: "lead-z" }],
);

const paused = yield* stop.pause(owner, { orgId });
assert.isTrue(paused.paused);
yield* worker.drain();
yield* Fiber.join(first);
yield* Fiber.join(second);
for (const threadId of threads)
assert.strictEqual(yield* lastRunStatus(threadId), "interrupted");
assert.deepStrictEqual(
yield* sql`SELECT status FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`,
[{ status: "released" }, { status: "released" }],
);
assert.deepStrictEqual(
yield* sql`SELECT state, attempts FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`,
[
{ state: "queued", attempts: 0 },
{ state: "queued", attempts: 0 },
],
);

yield* queue.enqueue(event("held"));
yield* TestClock.adjust("2 minutes");
yield* runner.runOnce(orgId, agentId);
yield* runner.runOnce(orgId, otherLead);
assert.lengthOf(
yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`,
2,
);
const resumed = yield* stop.resume(owner, { orgId });
assert.isFalse(resumed.paused);
}).pipe(
Effect.provide(
harness({
untilInterrupted: true,
onTurn: ({ threadId }) => Queue.offer(enteredThreads, threadId).pipe(Effect.asVoid),
}),
),
);
}).pipe(provideDeterministicTestRuntime),
);

it.effect("resumes once with one resume event and expires events queued over a day ago", () =>
Effect.gen(function* () {
yield* seed;
Expand Down Expand Up @@ -275,7 +343,11 @@ describe("KinStop", () => {
Deferred.await(ready).pipe(
Effect.flatMap((stop) => stop.pause(owner, { orgId })),
Effect.orDie,
Effect.as({ head: "abc", indexAsOf: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z") }),
Effect.as({
head: "abc",
indexAsOf: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z"),
read: "/private/tmp",
}),
);
yield* Effect.gen(function* () {
yield* seed;
Expand Down
Loading