From 2569f89e0bfc6ba38fd82e14b4f4e26e517a4d32 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:52:49 +0530 Subject: [PATCH 1/6] feat(kin): claim wake-ups per lead, not per organization The claim gate, the running-wake-up index and the cross-process conflict target move from org_id to (org_id, agent_id), so one lead's wake-up no longer blocks another lead's claim. K034 swaps the index on live databases; ids 32 and 33 are taken by open PRs that land first, so the Migrator never skips it. nextReady becomes readyLeads, listing every lead with queued events. Co-Authored-By: Muse Wonder Full 1.0 --- .../server/src/kin/lead/KinEventQueue.test.ts | 29 +++++++-- apps/server/src/kin/lead/KinEventQueue.ts | 48 ++++++++------ .../src/kin/persistence/KinMigrations.test.ts | 63 ++++++++++++++++++- .../src/kin/persistence/KinMigrations.ts | 2 + .../migrations/K034_PerLeadWakeups.ts | 10 +++ 5 files changed, 126 insertions(+), 26 deletions(-) create mode 100644 apps/server/src/kin/persistence/migrations/K034_PerLeadWakeups.ts diff --git a/apps/server/src/kin/lead/KinEventQueue.test.ts b/apps/server/src/kin/lead/KinEventQueue.test.ts index fd6fddda70edc..decc62ac2d24a 100644 --- a/apps/server/src/kin/lead/KinEventQueue.test.ts +++ b/apps/server/src/kin/lead/KinEventQueue.test.ts @@ -286,7 +286,10 @@ describe("KinEventQueue", () => { yield* queue.enqueue(yield* event("ready", "routine", "a", "lead-a2")); yield* sql`UPDATE kin_events SET not_before = 300000 WHERE event_id = 'deferred'`; yield* TestClock.adjust("20 seconds"); - assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a2"); + assert.deepStrictEqual( + (yield* queue.readyLeads(orgId)).map((lead) => lead.agentId), + ["lead-a2", "lead-a"], + ); const changed = yield* queue.changes(orgId); const signal = yield* changed.pipe(Effect.forkChild); yield* sql`UPDATE kin_agents SET retired_at = '2026-11-07T00:00:00.000Z' WHERE agent_id = 'lead-a2'`; @@ -296,7 +299,10 @@ describe("KinEventQueue", () => { assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'ready'`, [ { state: "expired" }, ]); - assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a"); + assert.deepStrictEqual( + (yield* queue.readyLeads(orgId)).map((lead) => lead.agentId), + ["lead-a"], + ); }).pipe(Effect.provide(layer)), ); @@ -767,16 +773,18 @@ describe("KinEventQueue", () => { ); it.effect( - "holds one writer per organization across the caller scope while other organizations proceed", + "holds one writer per lead across the caller scope while other leads and organizations proceed", () => Effect.gen(function* () { yield* seed(); + yield* seed("a", "lead-a2"); yield* seed("b"); const queue = yield* KinEventQueue.KinEventQueue; const sql = yield* SqlClient.SqlClient; const acquired = yield* Deferred.make(); const release = yield* Deferred.make(); yield* queue.enqueue(yield* event("a-first")); + yield* queue.enqueue(yield* event("a2-first", "github", "a", "lead-a2")); yield* queue.enqueue(yield* event("b-first", "github", "b")); yield* TestClock.adjust("20 seconds"); const first = yield* Effect.scoped( @@ -798,10 +806,19 @@ describe("KinEventQueue", () => { (yield* Fiber.join(other)).map((entry) => entry.eventId), ["b-first"], ); + const otherLead = yield* Effect.scoped(queue.claim(input("a2-first", "a", "lead-a2"))).pipe( + Effect.forkChild, + ); + assert.deepStrictEqual( + (yield* Fiber.join(otherLead)).map((entry) => entry.eventId), + ["a2-first"], + ); + yield* finish("a2-first"); yield* TestClock.adjust("20 seconds"); - assert.deepStrictEqual(yield* sql`SELECT wakeup_id FROM kin_wakeups WHERE org_id = 'a'`, [ - { wakeup_id: "a-first" }, - ]); + assert.deepStrictEqual( + yield* sql`SELECT wakeup_id FROM kin_wakeups WHERE org_id = 'a' AND status = 'running' ORDER BY wakeup_id`, + [{ wakeup_id: "a-first" }], + ); yield* Deferred.succeed(release, undefined); yield* Fiber.join(first); assert.deepStrictEqual( diff --git a/apps/server/src/kin/lead/KinEventQueue.ts b/apps/server/src/kin/lead/KinEventQueue.ts index 55f2de71fdfd7..abfff55ede4df 100644 --- a/apps/server/src/kin/lead/KinEventQueue.ts +++ b/apps/server/src/kin/lead/KinEventQueue.ts @@ -109,7 +109,7 @@ export class KinEventQueueWakeupConflict extends Schema.TaggedError()( "KinEventQueueBusy", - { orgId: KinOrgId }, + { orgId: KinOrgId, agentId: KinAgentId }, ) {} const isUnknownAgent = Schema.is(KinEventQueueUnknownAgent); const isWakeupConflict = Schema.is(KinEventQueueWakeupConflict); @@ -141,10 +141,11 @@ export class KinEventQueue extends Context.Service< >; readonly reconsiderBudget: (orgId: KinOrgId) => Effect.Effect; readonly changes: (orgId: KinOrgId) => Effect.Effect>; - readonly nextReady: ( + /** Every lead with queued events, earliest first; a lead is claimable once `readyAt` passes. */ + readonly readyLeads: ( orgId: KinOrgId, ) => Effect.Effect< - { readonly agentId: KinAgentId; readonly readyAt: number } | undefined, + ReadonlyArray<{ readonly agentId: KinAgentId; readonly readyAt: number }>, PersistenceSqlError | PersistenceDecodeError >; readonly expireAgent: ( @@ -210,18 +211,25 @@ const LATER_RETRY_MS = 60_000; const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const laneOf = sql.literal(LANE_OF); - const organizations = new Map< - KinOrgId, - { gate: Semaphore.Semaphore; changed: Deferred.Deferred } - >(); + const organizations = new Map }>(); + // One running wake-up per lead: the in-process gate is keyed like the claim index. + const gates = new Map(); const organization = (orgId: KinOrgId) => { const existing = organizations.get(orgId); if (existing !== undefined) return existing; - const entry = { gate: Semaphore.makeUnsafe(1), changed: Deferred.makeUnsafe() }; + const entry = { changed: Deferred.makeUnsafe() }; organizations.set(orgId, entry); return entry; }; + const gateFor = (orgId: KinOrgId, agentId: KinAgentId) => { + const key = `${orgId}\u0000${agentId}`; + const existing = gates.get(key); + if (existing !== undefined) return existing; + const gate = Semaphore.makeUnsafe(1); + gates.set(key, gate); + return gate; + }; const notify = Effect.fn("KinEventQueue.notify")(function* (orgId: KinOrgId) { const entry = organizations.get(orgId); @@ -391,6 +399,7 @@ const make = Effect.gen(function* () { function* (input: ClaimInput) { yield* decodeClaimInput(input); const entry = organization(input.orgId); + const gate = gateFor(input.orgId, input.agentId); return yield* Effect.acquireRelease( Effect.interruptible( Effect.uninterruptibleMask((restore) => @@ -409,7 +418,7 @@ const make = Effect.gen(function* () { ); continue; } - yield* restore(entry.gate.take(1)); + yield* restore(gate.take(1)); const events = yield* sql .withTransaction( Effect.gen(function* () { @@ -423,13 +432,16 @@ const make = Effect.gen(function* () { orgId: input.orgId, wakeupId: input.wakeupId, }); - // Cross-process exclusion rests on idx_kin_wakeups_running_org. + // Cross-process exclusion rests on idx_kin_wakeups_running_lead. const started = yield* sql` INSERT INTO kin_wakeups (org_id, wakeup_id, agent_id, model, started_at, status, event_ids) VALUES (${input.orgId}, ${input.wakeupId}, ${input.agentId}, ${input.model}, ${now}, 'running', '[]') - ON CONFLICT (org_id) WHERE status = 'running' DO NOTHING RETURNING wakeup_id`; + ON CONFLICT (org_id, agent_id) WHERE status = 'running' DO NOTHING RETURNING wakeup_id`; if (started.length === 0) - return yield* new KinEventQueueBusy({ orgId: input.orgId }); + return yield* new KinEventQueueBusy({ + orgId: input.orgId, + agentId: input.agentId, + }); const rows = yield* sql<{ json: string; priority: "P0" | "P1" | "P2" | "P3"; @@ -470,7 +482,7 @@ const make = Effect.gen(function* () { }), ) .pipe( - Effect.onError(() => entry.gate.release(1)), + Effect.onError(() => gate.release(1)), Effect.catchTags({ NoClaimableEvents: () => Effect.undefined }), ); if (events !== undefined) return events; @@ -478,7 +490,7 @@ const make = Effect.gen(function* () { }), ), ), - () => entry.gate.release(1), + () => gate.release(1), ); }, Effect.mapError((cause) => @@ -692,7 +704,7 @@ const make = Effect.gen(function* () { isUnknownAgent(cause) ? cause : toQueuePersistenceError("KinEventQueue.expireAgent")(cause), ), ); - const nextReady = Effect.fn("KinEventQueue.nextReady")( + const readyLeads = Effect.fn("KinEventQueue.readyLeads")( function* (orgId: KinOrgId) { const now = yield* Clock.currentTimeMillis; const rows = yield* sql<{ agent_id: string }>`SELECT e.agent_id @@ -723,9 +735,9 @@ const make = Effect.gen(function* () { ); return candidates.toSorted( (a, b) => a.readyAt - b.readyAt || a.agentId.localeCompare(b.agentId), - )[0]; + ); }, - Effect.mapError(toQueuePersistenceError("KinEventQueue.nextReady")), + Effect.mapError(toQueuePersistenceError("KinEventQueue.readyLeads")), ); return KinEventQueue.of({ reconsiderBudget: Effect.fn("KinEventQueue.reconsiderBudget")(function* (orgId) { @@ -743,7 +755,7 @@ const make = Effect.gen(function* () { requeueUnfinished, changes, expireAgent, - nextReady, + readyLeads, }); }); diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 7daa1c08aab06..ea35b9638ee40 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -138,6 +138,7 @@ describe("Kin migrations", () => { { migration_id: 29, name: "DropJudgeLog" }, { migration_id: 30, name: "ManyLeads" }, { migration_id: 31, name: "LeadDeliveries" }, + { migration_id: 34, name: "PerLeadWakeups" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -226,6 +227,7 @@ describe("Kin migrations", () => { { migration_id: 29, name: "DropJudgeLog" }, { migration_id: 30, name: "ManyLeads" }, { migration_id: 31, name: "LeadDeliveries" }, + { migration_id: 34, name: "PerLeadWakeups" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -275,7 +277,7 @@ describe("Kin migrations", () => { { name: "idx_kin_triggers_due" }, { name: "idx_kin_waiting_open_widget_person" }, { name: "idx_kin_waiting_person_resolved_at" }, - { name: "idx_kin_wakeups_running_org" }, + { name: "idx_kin_wakeups_running_lead" }, { name: "idx_kin_widgets_chat_message" }, { name: "idx_kin_widgets_type_closed_at" }, { name: "idx_kin_workers_status" }, @@ -337,6 +339,7 @@ describe("Kin migrations", () => { [29, "DropJudgeLog"], [30, "ManyLeads"], [31, "LeadDeliveries"], + [34, "PerLeadWakeups"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -401,6 +404,8 @@ describe("Kin migrations", () => { "resumed", ]) yield* sql`ALTER TABLE kin_wakeups DROP COLUMN ${sql(column)}`; + yield* sql`DROP INDEX idx_kin_wakeups_running_lead`; + yield* sql`CREATE UNIQUE INDEX idx_kin_wakeups_running_org ON kin_wakeups(org_id) WHERE status = 'running'`; yield* sql`DROP INDEX idx_kin_leads_org_domain`; for (const column of ["domain", "charter", "created_by_person_id"]) yield* sql`ALTER TABLE kin_leads DROP COLUMN ${sql(column)}`; @@ -449,6 +454,7 @@ describe("Kin migrations", () => { [29, "DropJudgeLog"], [30, "ManyLeads"], [31, "LeadDeliveries"], + [34, "PerLeadWakeups"], ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, @@ -505,6 +511,7 @@ describe("Kin migrations", () => { { migration_id: 29, name: "DropJudgeLog" }, { migration_id: 30, name: "ManyLeads" }, { migration_id: 31, name: "LeadDeliveries" }, + { migration_id: 34, name: "PerLeadWakeups" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); @@ -526,6 +533,8 @@ describe("Kin migrations", () => { "resumed", ]) yield* sql`ALTER TABLE kin_wakeups DROP COLUMN ${sql(column)}`; + yield* sql`DROP INDEX idx_kin_wakeups_running_lead`; + yield* sql`CREATE UNIQUE INDEX idx_kin_wakeups_running_org ON kin_wakeups(org_id) WHERE status = 'running'`; yield* sql`DROP INDEX idx_kin_leads_org_domain`; for (const column of ["domain", "charter", "created_by_person_id"]) yield* sql`ALTER TABLE kin_leads DROP COLUMN ${sql(column)}`; @@ -553,6 +562,7 @@ describe("Kin migrations", () => { [29, "DropJudgeLog"], [30, "ManyLeads"], [31, "LeadDeliveries"], + [34, "PerLeadWakeups"], ]); assert.deepStrictEqual( yield* sql`SELECT session_id, via FROM kin_auth_paired_sessions ORDER BY session_id`, @@ -566,6 +576,55 @@ describe("Kin migrations", () => { }).pipe(Effect.provide(layer)), ); + it.effect("K034 swaps the running wake-up index to per lead on a database with rows", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`DROP INDEX idx_kin_wakeups_running_lead`; + yield* sql`CREATE UNIQUE INDEX idx_kin_wakeups_running_org ON kin_wakeups(org_id) WHERE status = 'running'`; + yield* sql`DELETE FROM kin_sql_migrations WHERE migration_id = 34`; + yield* sql`INSERT INTO kin_orgs (org_id, name, created_at) VALUES ('org', 'Kin', ${timestamp})`; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) + VALUES ('owner', 'org', 'owner', 'Owner', ${timestamp})`; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, direct_key, last_seq, created_at) + VALUES ('direct', 'org', 'direct', 'owner:lead', 0, ${timestamp})`; + for (const agent of ["lead", "lead-2"]) { + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + VALUES (${agent}, 'org', 'Lead', 'lead', 'owner', ${timestamp})`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agent}, 'org', 'Lead', 0, '{}', '[]', 'direct')`; + } + const wakeup = (id: string, agent: string, status: string) => + sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, model, started_at, ended_at, status, event_ids) + VALUES (${id}, 'org', ${agent}, 'test', 1000, ${status === "running" ? null : 2000}, ${status}, '[]')`; + yield* wakeup("running", "lead", "running"); + yield* wakeup("done", "lead", "completed"); + yield* wakeup("done-2", "lead-2", "completed"); + + assert.deepStrictEqual(yield* runKinMigrations(), [[34, "PerLeadWakeups"]]); + assert.deepStrictEqual( + yield* sql`SELECT name FROM sqlite_master WHERE type = 'index' AND name GLOB 'idx_kin_wakeups_running*' ORDER BY name`, + [{ name: "idx_kin_wakeups_running_lead" }], + ); + assert.deepStrictEqual( + yield* sql`SELECT name FROM pragma_index_info('idx_kin_wakeups_running_lead') ORDER BY seqno`, + [{ name: "org_id" }, { name: "agent_id" }], + ); + yield* wakeup("also-running", "lead-2", "running"); + const overlap = yield* Effect.exit(wakeup("overlap", "lead", "running")); + assert.isTrue(Exit.isFailure(overlap)); + assert.include(String(overlap), "UNIQUE constraint failed"); + assert.deepStrictEqual( + yield* sql`SELECT wakeup_id, agent_id, status FROM kin_wakeups ORDER BY wakeup_id`, + [ + { wakeup_id: "also-running", agent_id: "lead-2", status: "running" }, + { wakeup_id: "done", agent_id: "lead", status: "completed" }, + { wakeup_id: "done-2", agent_id: "lead-2", status: "completed" }, + { wakeup_id: "running", agent_id: "lead", status: "running" }, + ], + ); + }).pipe(Effect.provide(layer)), + ); + it.effect("refuses to start when a registered migration sits below the latest applied one", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -576,7 +635,7 @@ describe("Kin migrations", () => { const exit = yield* Effect.exit(runKinMigrations()); assert.isTrue(Exit.isFailure(exit)); assert.include(String(exit), "5_OneChatOwner"); - assert.include(String(exit), "below the latest applied migration 31"); + assert.include(String(exit), "below the latest applied migration 34"); assert.deepStrictEqual(yield* kinHistory, history); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index 4f16af1406c0d..b22f3f234b8f5 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -33,6 +33,7 @@ import WakeupTimings from "./migrations/K028_WakeupTimings.ts"; import DropJudgeLog from "./migrations/K029_DropJudgeLog.ts"; import ManyLeads from "./migrations/K030_ManyLeads.ts"; import LeadDeliveries from "./migrations/K031_LeadDeliveries.ts"; +import PerLeadWakeups from "./migrations/K034_PerLeadWakeups.ts"; const table = "kin_sql_migrations"; const run = Migrator.make({}); @@ -68,6 +69,7 @@ const loader = Migrator.fromRecord({ "29_DropJudgeLog": DropJudgeLog, "30_ManyLeads": ManyLeads, "31_LeadDeliveries": LeadDeliveries, + "34_PerLeadWakeups": PerLeadWakeups, }); // The Migrator skips every id at or below the latest applied one, so a migration that diff --git a/apps/server/src/kin/persistence/migrations/K034_PerLeadWakeups.ts b/apps/server/src/kin/persistence/migrations/K034_PerLeadWakeups.ts new file mode 100644 index 0000000000000..93b2b5e93a076 --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K034_PerLeadWakeups.ts @@ -0,0 +1,10 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // One running wake-up per lead, not per organization; existing rows satisfy the narrower index. + yield* sql`CREATE UNIQUE INDEX idx_kin_wakeups_running_lead + ON kin_wakeups(org_id, agent_id) WHERE status = 'running'`; + yield* sql`DROP INDEX idx_kin_wakeups_running_org`; +}); From 05108da5813674702633d5dee7013e6f1e9b0617 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:59:11 +0530 Subject: [PATCH 2/6] feat(kin): run wake-ups for every ready lead at once The runner's organization loop forks one run per ready lead instead of awaiting the earliest, recovery skips leads already in flight, and memory staging is keyed per wake-up so two leads can write in the same second while the git commit and push stay serialized per repository. LeadMemory's receipt lock and ended set follow the same per-wake-up keying. Pause still stops every running wake-up in the organization. Co-Authored-By: Muse Wonder Full 1.0 --- apps/server/src/kin/lead/KinStop.test.ts | 69 ++++++ apps/server/src/kin/lead/LeadMemory.ts | 20 +- apps/server/src/kin/lead/WakeupRunner.test.ts | 48 ++++- apps/server/src/kin/lead/WakeupRunner.ts | 199 ++++++++++-------- .../src/kin/memory/MemoryWriter.test.ts | 45 +++- apps/server/src/kin/memory/MemoryWriter.ts | 25 +-- 6 files changed, 298 insertions(+), 108 deletions(-) diff --git a/apps/server/src/kin/lead/KinStop.test.ts b/apps/server/src/kin/lead/KinStop.test.ts index 197eabc3401dd..b14a5d607bdc5 100644 --- a/apps/server/src/kin/lead/KinStop.test.ts +++ b/apps/server/src/kin/lead/KinStop.test.ts @@ -1,6 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; import { CommandId, + KinAgentId, KinPersonId, KinWakeupId, MessageId, @@ -14,6 +15,7 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as DateTime from "effect/DateTime"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; @@ -165,6 +167,73 @@ describe("KinStop", () => { }).pipe(provideDeterministicTestRuntime), ); + it.effect("stops two leads' running wake-ups at once and starts none until resume", () => + Effect.gen(function* () { + const enteredThreads = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const stop = yield* KinStop.KinStop; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + yield* stop.attach(yield* KinStop.makeFanOut(Option.none())); + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + SELECT 'lead-z', org_id, display_name, role, owner_person_id, created_at FROM kin_agents WHERE agent_id = 'lead'`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + SELECT 'lead-z', org_id, name, level_since, models_json, worker_projects_json, owner_chat_id FROM kin_leads WHERE agent_id = 'lead'`; + const otherLead = KinAgentId.make("lead-z"); + yield* queue.enqueue(event("work")); + yield* queue.enqueue({ ...event("other-work"), agentId: otherLead }); + const first = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + const second = yield* runner.runOnce(orgId, otherLead).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); + const threads = [yield* Queue.take(enteredThreads), yield* Queue.take(enteredThreads)]; + assert.deepStrictEqual( + yield* sql`SELECT agent_id FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY agent_id`, + [{ agent_id: "lead" }, { agent_id: "lead-z" }], + ); + + const paused = yield* stop.pause(owner, { orgId }); + assert.isTrue(paused.paused); + yield* worker.drain(); + yield* Fiber.join(first); + yield* Fiber.join(second); + for (const threadId of threads) + assert.strictEqual(yield* lastRunStatus(threadId), "interrupted"); + assert.deepStrictEqual( + yield* sql`SELECT status FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, + [{ status: "released" }, { status: "released" }], + ); + assert.deepStrictEqual( + yield* sql`SELECT state, attempts FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [ + { state: "queued", attempts: 0 }, + { state: "queued", attempts: 0 }, + ], + ); + + yield* queue.enqueue(event("held")); + yield* TestClock.adjust("2 minutes"); + yield* runner.runOnce(orgId, agentId); + yield* runner.runOnce(orgId, otherLead); + assert.lengthOf( + yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, + 2, + ); + const resumed = yield* stop.resume(owner, { orgId }); + assert.isFalse(resumed.paused); + }).pipe( + Effect.provide( + harness({ + untilInterrupted: true, + onTurn: ({ threadId }) => Queue.offer(enteredThreads, threadId).pipe(Effect.asVoid), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + it.effect("resumes once with one resume event and expires events queued over a day ago", () => Effect.gen(function* () { yield* seed; diff --git a/apps/server/src/kin/lead/LeadMemory.ts b/apps/server/src/kin/lead/LeadMemory.ts index 03a10dea0d9ce..106501b10c34e 100644 --- a/apps/server/src/kin/lead/LeadMemory.ts +++ b/apps/server/src/kin/lead/LeadMemory.ts @@ -291,19 +291,23 @@ const make = Effect.gen(function* () { const writer = yield* Writer.MemoryWriter; const writerAuthority = yield* Writer.MemoryWriterAuthority; const sql = yield* SqlClient.SqlClient; - const locks = new Map(); + // Receipts are keyed per wake-up, so the lock that orders them is too. + const locks = new Map(); + const wakeKey = (orgId: KinOrgId, wakeupId: string) => `${orgId}\u0000${wakeupId}`; const locked = Effect.fn("LeadMemory.locked")(function* ( orgId: KinOrgId, + wakeupId: string, operation: Effect.Effect, ) { - let lock = locks.get(orgId); + const key = wakeKey(orgId, wakeupId); + let lock = locks.get(key); if (!lock) { lock = Semaphore.makeUnsafe(1); - locks.set(orgId, lock); + locks.set(key, lock); } return yield* lock.withPermits(1)(operation); }); - const ended = new Map(); + const ended = new Set(); const resolve = (scope: McpInvocationScope, allowEnded = false) => resolveLead(scope, allowEnded).pipe( Effect.provideService(KinIdentity, identity), @@ -447,6 +451,7 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope); return yield* locked( ctx.orgId, + ctx.wakeupId, Effect.gen(function* () { return yield* stage(yield* resolve(scope), input); }), @@ -456,6 +461,7 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope); return yield* locked( ctx.orgId, + ctx.wakeupId, Effect.gen(function* () { const ctx = yield* resolve(scope); yield* checkSource(ctx, input.sourceEventId); @@ -509,6 +515,7 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope); return yield* locked( ctx.orgId, + ctx.wakeupId, Effect.gen(function* () { const ctx = yield* resolve(scope); yield* checkSource(ctx, input.sourceEventId); @@ -584,17 +591,18 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope, true); yield* locked( ctx.orgId, + ctx.wakeupId, Effect.gen(function* () { const ctx = yield* resolve(scope, true); yield* receipt( ctx, "end", input, - ended.get(ctx.orgId) === ctx.wakeupId + ended.has(wakeKey(ctx.orgId, ctx.wakeupId)) ? Effect.void : writer.appendJournal(ctx.orgId, ctx.wakeupId, [`Summary: ${input.summary}`]), ); - ended.set(ctx.orgId, ctx.wakeupId); + ended.add(wakeKey(ctx.orgId, ctx.wakeupId)); }), ); }, Effect.mapError(failure)), diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 8dcd7a866cec7..9ca21aaa50a58 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -973,7 +973,7 @@ describe("WakeupRunner", () => { yield* queue.enqueue(event("busy")); yield* queue.enqueue({ ...event("other"), agentId: KinAgentId.make("lead-z") }); yield* TestClock.adjust("20 seconds"); - const next = yield* queue.nextReady(orgId); + const next = (yield* queue.readyLeads(orgId))[0]; assert.strictEqual(next?.agentId, "lead-z"); yield* Fiber.join( yield* (yield* WakeupRunner.WakeupRunner) @@ -989,6 +989,52 @@ describe("WakeupRunner", () => { }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), ); + it.effect("runs two leads' wake-ups at the same time", () => + Effect.gen(function* () { + const holdRunning = yield* Deferred.make(); + const enteredThreads = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + SELECT 'lead-z', org_id, display_name, role, owner_person_id, created_at FROM kin_agents WHERE agent_id = 'lead'`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + SELECT 'lead-z', org_id, name, level_since, models_json, worker_projects_json, owner_chat_id FROM kin_leads WHERE agent_id = 'lead'`; + yield* queue.enqueue(event("first")); + yield* queue.enqueue({ ...event("other"), agentId: KinAgentId.make("lead-z") }); + yield* runner.start; + yield* TestClock.adjust("20 seconds"); + const entered = [yield* Queue.take(enteredThreads), yield* Queue.take(enteredThreads)]; + assert.notStrictEqual(entered[0], entered[1]); + assert.deepStrictEqual( + yield* sql`SELECT agent_id FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY agent_id`, + [{ agent_id: "lead" }, { agent_id: "lead-z" }], + ); + yield* Deferred.succeed(holdRunning, undefined); + while (true) { + const changed = yield* queue.changes(orgId); + const running = + yield* sql`SELECT wakeup_id FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY wakeup_id`; + if (running.length === 0) break; + yield* changed; + } + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events ORDER BY event_id`, [ + { state: "done" }, + { state: "done" }, + ]); + }).pipe( + Effect.provide( + harness({ + holdRunning, + onTurn: ({ threadId }) => Queue.offer(enteredThreads, threadId).pipe(Effect.asVoid), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + it.effect("steers a second message in the lane into the running turn, in the same wake-up", () => Effect.gen(function* () { const holdRunning = yield* Deferred.make(); diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index a43f668a4a136..0d5488ae03d70 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -1005,87 +1005,93 @@ const make = Effect.gen(function* () { ), ); const recoverOrg = Effect.fn("WakeupRunner.recoverOrg")( - function* (orgId: KinOrgId) { + function* (orgId: KinOrgId, active: ReadonlySet = new Set()) { const rows = yield* sql<{ wakeupId: string; + agentId: string; }>`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY started_at, wakeup_id`; - for (const raw of rows) { - yield* Effect.scoped( - Effect.gen(function* () { - yield* Effect.acquireRelease(Effect.void, () => - queue - .finish({ orgId, wakeupId: KinWakeupId.make(raw.wakeupId), status: "failed" }) - .pipe(Effect.ignore), - ); - let row = yield* decodeRunning(raw); - yield* Effect.acquireRelease(Effect.void, (_, exit) => - cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), - ); - const existing = - row.threadId === null - ? [] - : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; - const projection = - row.threadId === null || existing.length === 0 - ? undefined - : yield* threads.getThreadProjection(row.threadId); - const run = projection?.runs.find((run) => - row.runId === null - ? run.userMessageId === KinIdentity.wakeupMessageId(row.wakeupId) - : run.id === row.runId, - ); - if (projection === undefined || run === undefined) { - const receipts = - yield* sql`SELECT 1 FROM kin_lead_tool_calls WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId} LIMIT 1`; - if (receipts.length > 0 && Option.isSome(writer)) { - yield* writer.value - .commitWakeup(row.orgId, row.wakeupId, "failed") - .pipe( - Effect.catchCause((cause) => - Effect.logError( - "Kin recovery commit failed; requeueing unfinished wake-up", - cause, + // Rows of leads this process is already driving are not abandoned; the rest recover together. + const pending = rows.filter((row) => !active.has(row.agentId)); + yield* Effect.forEach( + pending, + (raw) => + Effect.scoped( + Effect.gen(function* () { + yield* Effect.acquireRelease(Effect.void, () => + queue + .finish({ orgId, wakeupId: KinWakeupId.make(raw.wakeupId), status: "failed" }) + .pipe(Effect.ignore), + ); + let row = yield* decodeRunning(raw); + yield* Effect.acquireRelease(Effect.void, (_, exit) => + cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), + ); + const existing = + row.threadId === null + ? [] + : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; + const projection = + row.threadId === null || existing.length === 0 + ? undefined + : yield* threads.getThreadProjection(row.threadId); + const run = projection?.runs.find((run) => + row.runId === null + ? run.userMessageId === KinIdentity.wakeupMessageId(row.wakeupId) + : run.id === row.runId, + ); + if (projection === undefined || run === undefined) { + const receipts = + yield* sql`SELECT 1 FROM kin_lead_tool_calls WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId} LIMIT 1`; + if (receipts.length > 0 && Option.isSome(writer)) { + yield* writer.value + .commitWakeup(row.orgId, row.wakeupId, "failed") + .pipe( + Effect.catchCause((cause) => + Effect.logError( + "Kin recovery commit failed; requeueing unfinished wake-up", + cause, + ), ), - ), - ); + ); + } + if (projection === undefined) yield* clearMissingThread(row); + yield* queue.requeueUnfinished(row.orgId, row.wakeupId); + return; } - if (projection === undefined) yield* clearMissingThread(row); - yield* queue.requeueUnfinished(row.orgId, row.wakeupId); - return; - } - row = { ...row, runId: run.id }; - yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; - const record = yield* lead(row.orgId, row.agentId); - if (record.retiredAt !== null) { - yield* cleanup(row); - return yield* retire(record); - } - yield* identity.registerAgentThread({ - orgId: row.orgId, - agentId: row.agentId, - threadId: projection.thread.id, - }); - yield* registry.register(projection.thread.id, { ...row, level: record.level }); - if (Option.isSome(bridge)) - yield* bridge.value.begin({ ...row, threadId: projection.thread.id }); - yield* wait(row, projection.thread.projectId).pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) - ? cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore) - : Effect.void, - ), - ); - }), - ).pipe( - Effect.catchCause((cause) => - Effect.logError("Kin wake-up recovery failed", { - orgId, - wakeupId: raw.wakeupId, - cause, + row = { ...row, runId: run.id }; + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + const record = yield* lead(row.orgId, row.agentId); + if (record.retiredAt !== null) { + yield* cleanup(row); + return yield* retire(record); + } + yield* identity.registerAgentThread({ + orgId: row.orgId, + agentId: row.agentId, + threadId: projection.thread.id, + }); + yield* registry.register(projection.thread.id, { ...row, level: record.level }); + if (Option.isSome(bridge)) + yield* bridge.value.begin({ ...row, threadId: projection.thread.id }); + yield* wait(row, projection.thread.projectId).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore) + : Effect.void, + ), + ); }), + ).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin wake-up recovery failed", { + orgId, + wakeupId: raw.wakeupId, + cause, + }), + ), ), - ); - } + { concurrency: "unbounded", discard: true }, + ); }, Effect.mapError((cause) => isWakeupError(cause) @@ -1132,36 +1138,53 @@ const make = Effect.gen(function* () { ); const start = Effect.gen(function* () { const organizations = new Set(); + // Leads with a wake-up in flight, per organization, so each is driven once. + const activeByOrg = new Map>(); + const activeOf = (orgId: KinOrgId) => { + const existing = activeByOrg.get(orgId); + if (existing !== undefined) return existing; + const created = new Set(); + activeByOrg.set(orgId, created); + return created; + }; yield* Effect.gen(function* () { for (const orgId of yield* orgIds()) { if (organizations.has(orgId)) continue; organizations.add(orgId); yield* Effect.gen(function* () { const changed = yield* queue.changes(orgId); - yield* recoverOrg(orgId); - const unfinished = - yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running'`; - if (unfinished.length > 0) { - yield* Effect.raceFirst(changed, threadActivity(orgId)).pipe( - Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), - ); - return; - } + const active = activeOf(orgId); + yield* recoverOrg(orgId, active); const rows = yield* sql<{ agentId: string; }>`SELECT l.agent_id AS "agentId" FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NOT NULL ORDER BY l.agent_id`; for (const row of rows) yield* retire(yield* lead(orgId, KinAgentId.make(row.agentId))); - const next = (yield* paused(orgId)) ? undefined : yield* queue.nextReady(orgId); + const ready = (yield* paused(orgId)) ? [] : yield* queue.readyLeads(orgId); const now = yield* Clock.currentTimeMillis; - if (next !== undefined && next.readyAt <= now) { - yield* runOnce(orgId, next.agentId); - return; + for (const candidate of ready) { + if (candidate.readyAt > now || active.has(candidate.agentId)) continue; + active.add(candidate.agentId); + yield* runOnce(orgId, candidate.agentId).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin runner iteration failed", { + orgId, + agentId: candidate.agentId, + cause, + }).pipe(Effect.andThen(Effect.sleep("1 second"))), + ), + Effect.ensuring( + Effect.sync(() => { + active.delete(candidate.agentId); + }), + ), + Effect.forkScoped, + ); } yield* Effect.raceFirst(changed, threadActivity(orgId)).pipe( Effect.timeoutOrElse({ - duration: Math.min(60_000, Math.max(1, (next?.readyAt ?? Infinity) - now)), + duration: Math.min(60_000, Math.max(1, (ready[0]?.readyAt ?? Infinity) - now)), orElse: () => Effect.void, }), ); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 6db48050f8a4c..466d7d4ecc17a 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -33,6 +33,7 @@ it.live.each([ "undo", "recovered-no-end", "lost-session-retry", + "two-leads", "completed", "failed", "blocked", @@ -265,8 +266,12 @@ it.live.each([ : Effect.succeed({ orgId, wakeupId, - agentId, + agentId: + scenario === "two-leads" && wakeupId === "wake2" + ? KinAgentId.make("fixture-lead-2") + : agentId, level: + scenario === "two-leads" || scenario === "conflict" || scenario === "B5-journal" || scenario === "read-during-record" || @@ -356,6 +361,44 @@ it.live.each([ yield* git(paths.writer, ["config", "user.name", "Fixture Bot"]); yield* git(paths.writer, ["config", "user.email", "fixture@example.invalid"]); const original = yield* git(paths.writer, ["rev-parse", "HEAD"]); + if (scenario === "two-leads") { + const one = { + path: "claims/one.md", + sourceEventId: "owner", + content: fixtureItem({ id: "one", kind: "claim", title: "First claim", links: [] }), + }; + const two = { + path: "claims/two.md", + sourceEventId: "owner", + content: fixtureItem({ id: "two", kind: "claim", title: "Second claim", links: [] }), + }; + yield* writer.stage(orgId, "wake1", one); + // The second lead stages and journals while the first wake-up is still open. + yield* writer.stage(orgId, "wake2", two); + yield* writer.appendJournal(orgId, "wake1", ["First lead note."]); + yield* writer.appendJournal(orgId, "wake2", ["Second lead note."]); + // The writer checkout is still validated on the second lead's writes too. + yield* fs.rename(`${paths.writer}/.git/refs`, `${paths.writer}/.git/refs-moved`); + expect((yield* writer.stage(orgId, "wake2", two).pipe(Effect.result))._tag).toBe("Failure"); + yield* fs.rename(`${paths.writer}/.git/refs-moved`, `${paths.writer}/.git/refs`); + const first = yield* writer.commitWakeup(orgId, "wake1", "completed"); + const second = yield* writer.commitWakeup(orgId, "wake2", "completed"); + expect(yield* git(paths.writer, ["rev-parse", `${second.commit}^`])).toBe(first.commit); + expect(yield* git(paths.writer, ["show", `${second.commit}:claims/one.md`])).toContain( + "First claim", + ); + expect(yield* git(paths.writer, ["show", `${second.commit}:claims/two.md`])).toContain( + "Second claim", + ); + const journal = yield* git(paths.writer, [ + "show", + `${second.commit}:journal/2026/11-08.md`, + ]); + expect(journal).toContain("First lead note."); + expect(journal).toContain("Second lead note."); + expect(cards.map((card) => card.wakeupId)).toEqual(["wake1", "wake2"]); + return; + } if (scenario === "lost-session-retry") { failBranch = true; expect((yield* writer.appendJournal(orgId, "wake1", []).pipe(Effect.result))._tag).toBe( diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 9ece371802d57..a030e52dce9ee 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -292,8 +292,10 @@ const make = Effect.gen(function* () { Effect.provideService(VcsProcess, vcs), ); const locked = memory.withWriterLock; + // Staged changes are per wake-up, so concurrent leads never share a session; the + // git commit and push themselves stay serialized per repository under `locked`. const sessions = new Map< - KinOrgId, + string, { wakeupId: string; generation: number; @@ -302,6 +304,7 @@ const make = Effect.gen(function* () { supersessions: Map; } >(); + const sessionKey = (orgId: KinOrgId, wakeupId: string) => `${orgId}\u0000${wakeupId}`; const io = (orgId: KinOrgId, effect: Effect.Effect) => effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation: "writer", cause }))); const git = Effect.fn("MemoryWriter.git")(function* ( @@ -405,13 +408,12 @@ const make = Effect.gen(function* () { ) return yield* new MemoryWriteRejected({ reason: "Invalid server context" }); yield* journalEntry(ctx, "completed", []); - const previous = sessions.get(orgId); - if (previous && previous.wakeupId !== wakeupId) - return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); + const key = sessionKey(orgId, wakeupId); + const previous = sessions.get(key); const generation = yield* memory.historyGeneration(orgId); // A purge rewrote history after this wake-up staged; committing it could restore purged content. if (previous && previous.generation !== generation) { - sessions.delete(orgId); + sessions.delete(key); return yield* new MemorySyncConflict({ orgId, operation: "push", @@ -438,7 +440,7 @@ const make = Effect.gen(function* () { yield* recovery.notify({ orgId, branch: `wakeup:${wakeupId}`, reason }); } } - sessions.set(orgId, state); + sessions.set(key, state); return { ctx, state }; }); const safePath = Effect.fn("MemoryWriter.safePath")(function* ( @@ -1044,11 +1046,8 @@ const make = Effect.gen(function* () { discard: Effect.fn("MemoryWriter.discard")((orgId, wakeupId) => locked( orgId, - Effect.gen(function* () { - const state = sessions.get(orgId); - if (state && state.wakeupId !== wakeupId) - return yield* new MemoryWriteRejected({ reason: "Another wake-up owns the writer" }); - sessions.delete(orgId); + Effect.sync(() => { + sessions.delete(sessionKey(orgId, wakeupId)); }), ), ), @@ -1248,7 +1247,9 @@ const make = Effect.gen(function* () { items: yield* itemChanges(orgId, recordedFiles, changes), }); return { commit }; - }).pipe(Effect.ensuring(Effect.sync(() => sessions.delete(orgId)))); + }).pipe( + Effect.ensuring(Effect.sync(() => sessions.delete(sessionKey(orgId, wakeupId)))), + ); }), ), ), From b3c7960960f0cbc164eb90bc65e1fd2241f7fddb Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:13:29 +0530 Subject: [PATCH 3/6] docs(kin): wake-ups run per lead in the lead spec Co-Authored-By: Muse Wonder Full 1.0 --- .../server/src/kin/lead/KinEventQueue.test.ts | 60 +++++++++---------- docs/kin/specs/lead.md | 12 ++-- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/apps/server/src/kin/lead/KinEventQueue.test.ts b/apps/server/src/kin/lead/KinEventQueue.test.ts index decc62ac2d24a..bff2a89abc51f 100644 --- a/apps/server/src/kin/lead/KinEventQueue.test.ts +++ b/apps/server/src/kin/lead/KinEventQueue.test.ts @@ -839,37 +839,35 @@ describe("KinEventQueue", () => { }).pipe(Effect.provide(layer)), ); - it.effect( - "wakes empty queues on enqueue and releases the organization permit on interruption", - () => - Effect.gen(function* () { - yield* seed(); - const queue = yield* KinEventQueue.KinEventQueue; - const cancelled = yield* Effect.scoped(queue.claim(input("cancelled"))).pipe( - Effect.forkChild, - ); - yield* seed("a", "lead-a2"); - yield* queue.enqueue(yield* event("other-lead", "github", "a", "lead-a2")); - const otherLead = yield* Effect.scoped( - queue.claim(input("other-lead", "a", "lead-a2")), - ).pipe(Effect.forkChild); - yield* TestClock.adjust("20 seconds"); - assert.isDefined(otherLead.pollUnsafe()); - assert.deepStrictEqual( - (yield* Fiber.join(otherLead)).map((entry) => entry.eventId), - ["other-lead"], - ); - yield* finish("other-lead"); - yield* Fiber.interrupt(cancelled); - const waiting = yield* Effect.scoped(queue.claim(input("notified"))).pipe(Effect.forkChild); - yield* TestClock.adjust("1 second"); - yield* queue.enqueue(yield* event("notification")); - yield* TestClock.adjust("20 seconds"); - assert.deepStrictEqual( - (yield* Fiber.join(waiting)).map((entry) => entry.eventId), - ["notification"], - ); - }).pipe(Effect.provide(layer)), + it.effect("wakes empty queues on enqueue and releases the lead permit on interruption", () => + Effect.gen(function* () { + yield* seed(); + const queue = yield* KinEventQueue.KinEventQueue; + const cancelled = yield* Effect.scoped(queue.claim(input("cancelled"))).pipe( + Effect.forkChild, + ); + yield* seed("a", "lead-a2"); + yield* queue.enqueue(yield* event("other-lead", "github", "a", "lead-a2")); + const otherLead = yield* Effect.scoped(queue.claim(input("other-lead", "a", "lead-a2"))).pipe( + Effect.forkChild, + ); + yield* TestClock.adjust("20 seconds"); + assert.isDefined(otherLead.pollUnsafe()); + assert.deepStrictEqual( + (yield* Fiber.join(otherLead)).map((entry) => entry.eventId), + ["other-lead"], + ); + yield* finish("other-lead"); + yield* Fiber.interrupt(cancelled); + const waiting = yield* Effect.scoped(queue.claim(input("notified"))).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* queue.enqueue(yield* event("notification")); + yield* TestClock.adjust("20 seconds"); + assert.deepStrictEqual( + (yield* Fiber.join(waiting)).map((entry) => entry.eventId), + ["notification"], + ); + }).pipe(Effect.provide(layer)), ); it.effect( diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index c66ba5d15166c..216febb30842a 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -11,7 +11,7 @@ A lead is a T3 thread that Kin wakes up. Each wake-up: 3. lets the lead act through Kin tools only (post, delegate, request an action, read and write memory), 4. commits memory and the journal in one git commit, and exits. -An organization can have several leads, one per domain (engineering, growth, finance, and so on). Only one wake-up runs per organization at a time. Every wake-up counts against one daily budget for the organization. One switch stops everything. A lead starts at L0 (shadow) and moves up a level only after the owner reviews what it would have done. The exception is its own direct chat with the owner, where it works at L2 from creation. +An organization can have several leads, one per domain (engineering, growth, finance, and so on). One wake-up runs per lead at a time; different leads' wake-ups run concurrently. Every wake-up counts against one daily budget for the organization. One switch stops everything. A lead starts at L0 (shadow) and moves up a level only after the owner reviews what it would have done. The exception is its own direct chat with the owner, where it works at L2 from creation. ## What exists in T3 @@ -38,8 +38,8 @@ Kin services live in `apps/server/src/kin/`. ### Components (all **new**, in `apps/server/src/kin/lead/` unless noted) -- `KinEventQueue`: persists events, coalesces them, hands one batch at a time to the runner per organization. -- `WakeupRunner`: one fiber per organization. Checks the stop switch and budget, starts the wake-up, waits, cleans up. +- `KinEventQueue`: persists events, coalesces them, hands one batch at a time to the runner per lead. +- `WakeupRunner`: one loop per organization, forking one run per ready lead. Checks the stop switch and budget, starts each wake-up, waits, cleans up. - `LeadPrompt`: builds the first message of each wake-up. - `KinLeadToolkit`: the MCP tools the lead calls. - `Memory` (`apps/server/src/kin/memory/`): reads, the front matter cache, and the git writer. @@ -204,8 +204,8 @@ Coalescing: - A batch holds at most 50 events. Overflow stays queued for the next batch; no delivery references are dropped or marked handled. Same-repo summaries belong to the prompt builder, not the durable queue. - One wake-up answers one chat, its lane: a human message's or card tap's own chat, else the lead's `ownerChatId`. A claim takes the lane of the highest-priority eligible event and only events in that lane, so a batch never spans chats; other chats' events wait for their own wake-up. - A human message or card tap that arrives in the lane of a running wake-up is steered into it: the runner claims it into that wake-up (`KinEventQueue.steer`, unchanged event and provenance) and sends it with `sendToThread({ mode: "steer", messageId: "kin-steer-msg::", createdBy: "system", creationSource: "server" })`, quoted exactly as in the wake-up prompt. If no provider turn is running, it is sent with `mode: "queue"`; T3 also turns a steer that lands after the turn into a new run. Either way the wake-up adopts that run and waits for it. Each steer extends the deadline by 5 minutes, to at most 30 minutes; after 5 steers, or once the wake-up stops waiting for runs, events queue for the next wake-up. A steer that may have landed is never re-sent; one that did not is released to the queue. Budget admission applies to steered events as to claimed ones. -- The runner never starts a second session for the same organization. This is the one-writer rule: memory is written only inside a wake-up, and wake-ups are serial. -- Implementation: an organization permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events of one lane, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 calls `finish(timed_out | failed)` for real failures and timeouts; graceful scope interruption uses `requeueUnfinished` without consuming an attempt. Organization signal entries stay available for idle runner subscriptions. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. +- The runner never starts a second session for the same lead. This is the one-writer rule: memory is written only inside a wake-up, and one lead's wake-ups are serial; different leads' wake-ups run at the same time, with their memory commits serialized per repository. +- Implementation: a per-lead permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events of one lane, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 calls `finish(timed_out | failed)` for real failures and timeouts; graceful scope interruption uses `requeueUnfinished` without consuming an attempt. Organization signal entries stay available for idle runner subscriptions. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval` (`PAUSE_POLL_INTERVAL_MS`, 5 s), so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts, but only when the wake-up did nothing visible: no agent message from its thread since it started, shadow entry, staged memory or deferral receipt, worker or action request. Otherwise its events become `failed` at once and the owner gets a "stopped partway" notice in the lane, because a retry could repeat what already happened. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. `release` returns `{ released, failedEventIds }` and `finish` returns `{ failedEventIds, stranded }`, where `stranded` counts events not retried because something visible happened; the ids are the events that transitioned to terminal `failed` in that call, ordered by id. Unknown or foreign lead ids fail with `KinEventQueueUnknownAgent`, and reused wake-up ids fail with `KinEventQueueWakeupConflict`, not SQL constraint errors. Enqueue validates the lead and computes owner priority inside the insert transaction. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. @@ -215,7 +215,7 @@ Restart: recovery runs independently in each organization loop and logs failures ### A wake-up -Kin foreign keys include `org_id` so config, wake-up, worker, shadow-entry and pause relationships cannot cross organizations. K002 adds composite unique indexes on the referenced K001 identities without rewriting those records. The database enforces at most one `running` wake-up per organization with a partial unique index. Thread and run ids are nullable until launch returns; they never reference upstream tables. +Kin foreign keys include `org_id` so config, wake-up, worker, shadow-entry and pause relationships cannot cross organizations. K002 adds composite unique indexes on the referenced K001 identities without rewriting those records. The database enforces at most one `running` wake-up per lead with a partial unique index. Thread and run ids are nullable until launch returns; they never reference upstream tables. Table `kin_wakeups` (**new**): `org_id, wakeup_id, agent_id, thread_id, run_id, model, started_at, ended_at, status ('running' | 'completed' | 'failed' | 'interrupted' | 'timed_out' | 'released'), event_ids, tokens_json, tools_json, memory_commit`. K028 adds the reply-latency stages as epoch ms, NULL when a stage never happened: `enqueued_at` (earliest claimed event's `queued_at`), `prepared_at`, `launched_at`, `first_token_at` (first assistant text), `posted_at` (first reply the bridge posted), plus `resumed` (1 for a warm wake-up). `started_at` is the claim. The runner logs one `Kin wake-up latency` line per wake-up; `ops/kin/host/README.md` has the p50/p90 query. From f650012f5f997a523f8d24dc11c14df201390c72 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:17:20 +0530 Subject: [PATCH 4/6] fix(kin): dry-run drain reads readyLeads Co-Authored-By: Muse Wonder Full 1.0 --- apps/server/scripts/kin-shadow-dry-run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/scripts/kin-shadow-dry-run.ts b/apps/server/scripts/kin-shadow-dry-run.ts index da6d417e877e0..15fa7b6107f2b 100644 --- a/apps/server/scripts/kin-shadow-dry-run.ts +++ b/apps/server/scripts/kin-shadow-dry-run.ts @@ -663,7 +663,7 @@ export const runDryRun = Effect.fn("KinDryRun.run")( let wakeups = 0; const drain = Effect.fn("KinDryRun.drain")(function* (limit: number) { while (true) { - const next = yield* queue.nextReady(orgId); + const next = (yield* queue.readyLeads(orgId))[0]; if (next === undefined || !Number.isFinite(next.readyAt) || next.readyAt > limit) return; if (++wakeups > 200) From ff11798a63f1250a8c64b8edf11fbee96e58d10b Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:33:31 +0530 Subject: [PATCH 5/6] fix(kin): per-lead wake-ups hold up when leads run at once Review fixes for running one wake-up per lead: - The org loop waits for the first lead it can launch, not one already running, so it no longer spins every millisecond while a lead finishes. A finished run leaves `active` before it wakes the loop, and recovered runs are forked beside the loop and marked active. - A staged change keeps the branch head it was validated at. When another lead committed since, commitWakeup validates every change again at the new head (supersession and duplicate ids included) and refuses with MemorySyncConflict instead of overwriting the other lead's file. - Retries and recovery find the wake-up's commit by its subject in history since the wake-up started, not at the branch head, so a recovery after another lead's commit neither commits twice nor reports lost changes. - readPromptSnapshot reads the wake-up's pinned commit when another lead's commit moved the branch since prepare. - Each lead reads from its own memory-read- checkout, built beside it and swapped in with two renames, under its own project, so one lead's prepare never wipes another's cwd. A lead thread in the old shared project is ended and relaunched in the lead's own. - LeadMemory's receipt lock and kin_end set are keyed per lead, so they no longer grow with every wake-up. - The spec notes that budget admission doesn't reserve, so N concurrent leads can overshoot the daily limit by up to N wake-ups. Co-Authored-By: Muse Wonder Full 1.0 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/lead/KinStop.test.ts | 6 +- apps/server/src/kin/lead/LeadMemory.ts | 24 +- apps/server/src/kin/lead/WakeupRunner.test.ts | 169 +++++- .../src/kin/lead/WakeupRunner.testkit.ts | 35 +- apps/server/src/kin/lead/WakeupRunner.ts | 263 +++++---- apps/server/src/kin/lead/layer.test.ts | 6 +- apps/server/src/kin/memory/Memory.test.ts | 76 ++- apps/server/src/kin/memory/Memory.ts | 137 +++-- .../src/kin/memory/MemoryWriter.test.ts | 91 ++- apps/server/src/kin/memory/MemoryWriter.ts | 554 ++++++++++-------- docs/kin/specs/lead.md | 20 +- docs/operations/kin-memory-purge.md | 2 +- 12 files changed, 942 insertions(+), 441 deletions(-) diff --git a/apps/server/src/kin/lead/KinStop.test.ts b/apps/server/src/kin/lead/KinStop.test.ts index 4fd1b4da2ac68..ca6077e8472d1 100644 --- a/apps/server/src/kin/lead/KinStop.test.ts +++ b/apps/server/src/kin/lead/KinStop.test.ts @@ -343,7 +343,11 @@ describe("KinStop", () => { Deferred.await(ready).pipe( Effect.flatMap((stop) => stop.pause(owner, { orgId })), Effect.orDie, - Effect.as({ head: "abc", indexAsOf: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z") }), + Effect.as({ + head: "abc", + indexAsOf: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z"), + read: "/private/tmp", + }), ); yield* Effect.gen(function* () { yield* seed; diff --git a/apps/server/src/kin/lead/LeadMemory.ts b/apps/server/src/kin/lead/LeadMemory.ts index 106501b10c34e..675fbc1332914 100644 --- a/apps/server/src/kin/lead/LeadMemory.ts +++ b/apps/server/src/kin/lead/LeadMemory.ts @@ -291,15 +291,16 @@ const make = Effect.gen(function* () { const writer = yield* Writer.MemoryWriter; const writerAuthority = yield* Writer.MemoryWriterAuthority; const sql = yield* SqlClient.SqlClient; - // Receipts are keyed per wake-up, so the lock that orders them is too. + // A lead runs one wake-up at a time, so the lock that orders its receipts is per lead: + // keyed per wake-up, these maps would grow with every wake-up ever run. const locks = new Map(); - const wakeKey = (orgId: KinOrgId, wakeupId: string) => `${orgId}\u0000${wakeupId}`; + const agentKey = (orgId: KinOrgId, agentId: KinAgentId) => `${orgId}\u0000${agentId}`; const locked = Effect.fn("LeadMemory.locked")(function* ( orgId: KinOrgId, - wakeupId: string, + agentId: KinAgentId, operation: Effect.Effect, ) { - const key = wakeKey(orgId, wakeupId); + const key = agentKey(orgId, agentId); let lock = locks.get(key); if (!lock) { lock = Semaphore.makeUnsafe(1); @@ -307,7 +308,8 @@ const make = Effect.gen(function* () { } return yield* lock.withPermits(1)(operation); }); - const ended = new Set(); + // The last wake-up each lead ended, so an identical kin_end retry appends no second summary. + const ended = new Map(); const resolve = (scope: McpInvocationScope, allowEnded = false) => resolveLead(scope, allowEnded).pipe( Effect.provideService(KinIdentity, identity), @@ -451,7 +453,7 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope); return yield* locked( ctx.orgId, - ctx.wakeupId, + ctx.agentId, Effect.gen(function* () { return yield* stage(yield* resolve(scope), input); }), @@ -461,7 +463,7 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope); return yield* locked( ctx.orgId, - ctx.wakeupId, + ctx.agentId, Effect.gen(function* () { const ctx = yield* resolve(scope); yield* checkSource(ctx, input.sourceEventId); @@ -515,7 +517,7 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope); return yield* locked( ctx.orgId, - ctx.wakeupId, + ctx.agentId, Effect.gen(function* () { const ctx = yield* resolve(scope); yield* checkSource(ctx, input.sourceEventId); @@ -591,18 +593,18 @@ const make = Effect.gen(function* () { const ctx = yield* resolve(scope, true); yield* locked( ctx.orgId, - ctx.wakeupId, + ctx.agentId, Effect.gen(function* () { const ctx = yield* resolve(scope, true); yield* receipt( ctx, "end", input, - ended.has(wakeKey(ctx.orgId, ctx.wakeupId)) + ended.get(agentKey(ctx.orgId, ctx.agentId)) === ctx.wakeupId ? Effect.void : writer.appendJournal(ctx.orgId, ctx.wakeupId, [`Summary: ${input.summary}`]), ); - ended.add(wakeKey(ctx.orgId, ctx.wakeupId)); + ended.set(agentKey(ctx.orgId, ctx.agentId), ctx.wakeupId); }), ); }, Effect.mapError(failure)), diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 9ca21aaa50a58..ce45c373371a6 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -141,6 +141,103 @@ const sessions = Effect.fn("test.sessions")(function* () { }); describe("WakeupRunner", () => { + it.effect("does not spin the org loop while a lead with queued events is still finishing", () => + Effect.gen(function* () { + const commitEntered = Deferred.makeUnsafe(); + const commitRelease = Deferred.makeUnsafe(); + let commits = 0; + let readyCalls = 0; + const writer = Layer.succeed( + Writer.MemoryWriter, + Writer.MemoryWriter.of({ + dreamBase: () => Effect.die("unused"), + commitDream: () => Effect.die("unused"), + deleteDreamBranch: () => Effect.die("unused"), + settleShadow: () => Effect.die("unused"), + undoChange: () => Effect.die("unused"), + sweepPullRequests: () => Effect.die("unused"), + stage: () => Effect.die("unused"), + appendJournal: () => Effect.void, + discard: () => Effect.void, + commitWakeup: () => + Effect.gen(function* () { + commits++; + if (commits === 1) { + yield* Deferred.succeed(commitEntered, undefined); + yield* Deferred.await(commitRelease); + } + return { commit: `fixture-${commits}` }; + }), + }), + ); + yield* Effect.scoped( + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const runner = yield* WakeupRunner.WakeupRunner; + yield* queue.enqueue(laneMessage("loop-first")); + yield* runner.start; + yield* TestClock.adjust("20 seconds"); + yield* Deferred.await(commitEntered); + // The lead is still finishing its first wake-up, with more work queued. + yield* queue.enqueue(event("loop-second")); + yield* TestClock.adjust("5 seconds"); + const settled = readyCalls; + yield* TestClock.adjust("30 seconds"); + assert.strictEqual(readyCalls, settled); + yield* Deferred.succeed(commitRelease, undefined); + yield* TestClock.adjust("20 seconds"); + // The finish wakes the loop at once: the queued event is claimed long before + // the 60-second cap a missed wake would wait for. + const states = yield* sql<{ + event_id: string; + state: string; + }>`SELECT event_id, state FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`; + assert.strictEqual(states[0]?.state, "done"); + assert.notStrictEqual(states[1]?.state, "queued"); + assert.deepStrictEqual( + yield* sql`SELECT memory_commit FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'completed' ORDER BY started_at`, + [{ memory_commit: "fixture-1" }], + ); + }), + ).pipe( + Effect.provide( + harness({ + writer, + onReadyLeads: () => { + readyCalls++; + }, + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + it.effect("moves a lead's thread off the shared project into its own checkout's", () => + Effect.gen(function* () { + let projectId = ProjectId.make("kin:org"); + yield* Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const threads = yield* ThreadManagement.ThreadManagementService; + const leadThread = Effect.map( + sql<{ threadId: string }>`SELECT thread_id AS "threadId" FROM kin_leads`, + ([row]) => ThreadId.make(row!.threadId), + ); + yield* wake("on-shared", "owner-chat"); + const shared = yield* leadThread; + projectId = ProjectId.make("kin:org:lead"); + yield* wake("on-own", "owner-chat"); + const own = yield* leadThread; + assert.notStrictEqual(own, shared); + assert.strictEqual((yield* threads.getThreadProjection(own)).thread.projectId, projectId); + assert.deepStrictEqual( + yield* sql`SELECT thread_id FROM kin_agent_threads WHERE ended_at IS NULL`, + [{ thread_id: own }], + ); + }).pipe(Effect.provide(harness({ projectId: () => projectId }))); + }).pipe(provideDeterministicTestRuntime), + ); it.effect("handles the owner check in code without starting a provider turn", () => Effect.gen(function* () { yield* seed; @@ -583,14 +680,12 @@ describe("WakeupRunner", () => { let journals = 0; const prompts: Array = []; const opened: Array = []; - let prepares = 0; - // The 5th prepare finds a new checkout directory, as after a purge. const prepare = () => - Effect.sync(() => ({ + Effect.succeed({ head: "abc", indexAsOf: DateTime.makeUnsafe("2026-11-08T00:00:00.000Z"), - ...(++prepares === 5 ? { replaced: true as const } : {}), - })); + read: "/private/tmp", + }); const writer = Layer.mock(Writer.MemoryWriter)({ appendJournal: () => Effect.sync(() => void journals++), commitWakeup: () => Effect.sync(() => (commits++, { commit: "fixture" })), @@ -607,9 +702,9 @@ describe("WakeupRunner", () => { yield* wake("late"); // The 11th starts fresh at the turn cap, and the 12th after 10 idle minutes. assert.deepStrictEqual(yield* sessions(), [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3]); - // Warm wake-ups keep the idle process, since the read checkout is refilled in place; - // a new directory resumes in a new one. (The fake also keeps it across fresh contexts.) - assert.lengthOf(opened, 2); + // Each warm wake-up resumes in a new process, since prepare swapped its cwd for a new + // checkout. (The fake keeps the last one across the two fresh contexts.) + assert.lengthOf(opened, 10); assert.deepStrictEqual( prompts.map((prompt) => prompt.since !== undefined), [false, ...Array.from({ length: 9 }, () => true), false, false], @@ -1435,6 +1530,64 @@ describe("WakeupRunner", () => { }).pipe(provideDeterministicTestRuntime), ); + it.effect("launches other leads while the org loop still waits on a recovered run", () => + Effect.gen(function* () { + const holdRunning = yield* Deferred.make(); + const enteredThreads = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const wakeupId = KinWakeupId.make("left-running"); + yield* queue.enqueue(event("recovered")); + yield* TestClock.adjust("20 seconds"); + // A wake-up a previous process left running, its provider run still going. + yield* Effect.scoped(queue.claim({ orgId, agentId, wakeupId, model: "test" })); + const sent = yield* (yield* ThreadLaunch.ThreadLaunchService).launch({ + commandId: CommandId.make(`kin-wakeup:${wakeupId}`), + projectId: ProjectId.make("kin:org"), + title: "Lead", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-opus-4-6", + }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { + messageId: MessageId.make(`kin-wakeup-msg:${wakeupId}`), + text: "Wake up", + attachments: [], + }, + createdBy: "system", + creationSource: "server", + }); + yield* sql`UPDATE kin_leads SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* sql`UPDATE kin_wakeups SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId}`; + assert.strictEqual(yield* Queue.take(enteredThreads), sent.threadId); + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + SELECT 'lead-z', org_id, display_name, role, owner_person_id, created_at FROM kin_agents WHERE agent_id = 'lead'`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + SELECT 'lead-z', org_id, name, level_since, models_json, worker_projects_json, owner_chat_id FROM kin_leads WHERE agent_id = 'lead'`; + yield* queue.enqueue({ ...event("other"), agentId: KinAgentId.make("lead-z") }); + yield* (yield* WakeupRunner.WakeupRunner).start; + yield* TestClock.adjust("20 seconds"); + assert.notStrictEqual(yield* Queue.take(enteredThreads), sent.threadId); + assert.deepStrictEqual( + yield* sql`SELECT agent_id FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY agent_id`, + [{ agent_id: "lead" }, { agent_id: "lead-z" }], + ); + }).pipe( + Effect.provide( + harness({ + holdRunning, + onTurn: ({ threadId }) => Queue.offer(enteredThreads, threadId).pipe(Effect.asVoid), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + it.effect( "releases an organization paused during the quiet window without consuming an attempt", () => diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts index 2c0af8b538619..20a05702a63da 100644 --- a/apps/server/src/kin/lead/WakeupRunner.testkit.ts +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -24,6 +24,7 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; @@ -149,6 +150,10 @@ type HarnessOptions = { readonly snapshot?: Memory.Memory["Service"]["readPromptSnapshot"]; /** Every provider process the fake adapter opens. */ readonly opened?: Array; + /** Called on every `readyLeads` pass the runner makes. */ + readonly onReadyLeads?: () => void; + /** The project id bootstrap returns, read on each call. */ + readonly projectId?: () => ProjectId; }; export function harness(options: HarnessOptions = {}) { @@ -425,10 +430,12 @@ function parts(options: HarnessOptions) { })), Layer.provide(Layer.mergeAll(orchestration, settings, receipts)), ); + const current = () => + options.projectId === undefined ? project : { ...project, id: options.projectId() }; const projectLayer = Layer.mock(ProjectService.ProjectService)({ - bootstrap: () => Effect.succeed({ project, created: true }), - getById: () => Effect.succeedSome(project), - getByWorkspaceRoot: () => Effect.succeedSome(project), + bootstrap: () => Effect.sync(() => ({ project: current(), created: true })), + getById: () => Effect.sync(() => Option.some(current())), + getByWorkspaceRoot: () => Effect.sync(() => Option.some(current())), }); const external = Layer.mergeAll( projectLayer, @@ -451,8 +458,21 @@ function parts(options: HarnessOptions) { const launch = ThreadLaunch.layer.pipe( Layer.provide(Layer.mergeAll(external, management, receipts, IdAllocator.layer)), ); + const queueLayer = + options.onReadyLeads === undefined + ? KinEventQueue.layer + : Layer.effect( + KinEventQueue.KinEventQueue, + Effect.map(KinEventQueue.KinEventQueue, (queue) => ({ + ...queue, + readyLeads: (orgId: Parameters[0]) => + Effect.sync(() => options.onReadyLeads!()).pipe( + Effect.andThen(queue.readyLeads(orgId)), + ), + })), + ).pipe(Layer.provide(KinEventQueue.layer)); const kin = Layer.mergeAll( - KinEventQueue.layer, + queueLayer, KinStop.layer.pipe( Layer.provide( Layer.mergeAll( @@ -495,7 +515,12 @@ function parts(options: HarnessOptions) { Layer.mock(Memory.Memory)({ prepareWakeup: options.prepare ?? - (() => Effect.succeed({ head: "abc", indexAsOf: DateTime.makeUnsafe(timestamp) })), + (() => + Effect.succeed({ + head: "abc", + indexAsOf: DateTime.makeUnsafe(timestamp), + read: "/private/tmp", + })), ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), readPromptSnapshot: options.snapshot ?? (() => Effect.succeed({ rules: "rules", index: "index" })), diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 0d5488ae03d70..bbb4f2b92f9de 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -16,6 +16,7 @@ import { } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; @@ -67,6 +68,15 @@ const isSteerRun = (wakeupId: KinWakeupId, run: { readonly userMessageId: string KinIdentity.isAdoptedRunMessage(wakeupId, run.userMessageId) && run.userMessageId !== KinIdentity.rescueMessageId(wakeupId); const ownerCheckDispatcherReady = false; +/** A running lead never shortens the org loop's wait; its finish wakes the loop instead. */ +const orgLoopWaitMs = ( + ready: ReadonlyArray<{ readonly agentId: KinAgentId; readonly readyAt: number }>, + active: ReadonlySet, + now: number, +) => { + const next = ready.find((candidate) => !active.has(candidate.agentId)); + return next === undefined ? 60_000 : Math.min(60_000, Math.max(1, next.readyAt - now)); +}; /** A direct chat keeps its provider session for this long idle, for at most this many wake-ups. */ const WARM_IDLE_MS = 600_000; const WARM_MAX_WAKEUPS = 10; @@ -476,11 +486,12 @@ const make = Effect.gen(function* () { yield* writer.value.appendJournal(row.orgId, row.wakeupId, [ Writer.LEAD_ENDED_WITHOUT_KIN_END, ]); - yield* writer.value.commitWakeup( + const { commit } = yield* writer.value.commitWakeup( row.orgId, row.wakeupId, status === "completed" ? "completed" : "failed", ); + yield* sql`UPDATE kin_wakeups SET memory_commit = ${commit} WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId}`; } yield* capture(row); if (stopped) @@ -816,13 +827,26 @@ const make = Effect.gen(function* () { return; } } - const paths = yield* memory.ensureCheckouts(orgId); + // A project per lead, since its root is the provider's cwd: the lead's own checkout. const { project } = yield* projects.bootstrap({ - commandId: CommandId.make(`kin-project:${orgId}`), - projectId: ProjectId.make(`kin:${orgId}`), - workspaceRoot: paths.read, - title: `Kin · ${record.orgName}`, + commandId: CommandId.make(`kin-project:${orgId}:${agentId}`), + projectId: ProjectId.make(`kin:${orgId}:${agentId}`), + workspaceRoot: pinned.read, + title: `Kin · ${record.orgName} · ${record.name}`, }); + let existingThreadId = current.threadId; + if (existingThreadId !== null) { + const existing = yield* threads + .getThreadProjection(existingThreadId) + .pipe(Effect.option); + // A thread launched under the old shared-checkout project keeps that cwd; end it + // so this wake-up launches one in this lead's own checkout instead. + if (Option.isSome(existing) && existing.value.thread.projectId !== project.id) { + yield* identity.endAgentThread({ orgId, threadId: existingThreadId }); + yield* sql`UPDATE kin_leads SET thread_id = NULL WHERE org_id = ${orgId} AND agent_id = ${agentId} AND thread_id = ${existingThreadId}`; + existingThreadId = null; + } + } const urgent = yield* sql`SELECT 1 FROM kin_events WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND priority IN ('P0', 'P1') LIMIT 1`; const modelSelection = @@ -841,27 +865,25 @@ const make = Effect.gen(function* () { previous !== undefined && seen !== undefined && previous.chatId === direct && - previous.threadId === current.threadId && + previous.threadId === existingThreadId && previous.level === current.level && modelSelectionsEqual(previous.model, modelSelection) && previous.wakeups < WARM_MAX_WAKEUPS && row.startedAt - previous.endedAt < WARM_IDLE_MS && // New standing rules are a new contract, so they get a fresh session. previous.rules === seen.rules; - // Prepare refills the read checkout in place, so an idle warm process keeps its cwd, - // unless the directory itself is new (a first fill, or after a purge). + // Prepare swapped in a new read checkout, so a warm session resumes in a new process. const resumes = resumable && - (pinned.replaced !== true || - (yield* releaseProcess(previous.threadId).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Kin lead process release failed", { - orgId, - wakeupId, - cause, - }).pipe(Effect.as(false)), - ), - ))); + (yield* releaseProcess(previous.threadId).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Kin lead process release failed", { + orgId, + wakeupId, + cause, + }).pipe(Effect.as(false)), + ), + )); const text = yield* prompt.build({ orgId, agentId, @@ -873,7 +895,7 @@ const make = Effect.gen(function* () { ...(lane === undefined ? {} : { lane }), ...(resumes ? { since: { index: previous.index, chatSeq: previous.chatSeq } } : {}), }); - const threadId = current.threadId ?? ThreadId.make(`kin:${yield* ulid}`); + const threadId = existingThreadId ?? ThreadId.make(`kin:${yield* ulid}`); row = { ...row, threadId }; yield* sql.withTransaction( Effect.gen(function* () { @@ -914,7 +936,7 @@ const make = Effect.gen(function* () { const commandId = CommandId.make(`kin-wakeup:${wakeupId}`); const messageId = MessageId.make(KinIdentity.wakeupMessageId(wakeupId)); const sent = - current.threadId === null + existingThreadId === null ? (yield* launch.launch({ commandId, threadId, @@ -1005,93 +1027,113 @@ const make = Effect.gen(function* () { ), ); const recoverOrg = Effect.fn("WakeupRunner.recoverOrg")( - function* (orgId: KinOrgId, active: ReadonlySet = new Set()) { + function* ( + orgId: KinOrgId, + active: Set = new Set(), + wake: Effect.Effect = Effect.void, + forkRecoveries = false, + ) { const rows = yield* sql<{ wakeupId: string; agentId: string; }>`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY started_at, wakeup_id`; // Rows of leads this process is already driving are not abandoned; the rest recover together. - const pending = rows.filter((row) => !active.has(row.agentId)); - yield* Effect.forEach( - pending, - (raw) => - Effect.scoped( - Effect.gen(function* () { - yield* Effect.acquireRelease(Effect.void, () => - queue - .finish({ orgId, wakeupId: KinWakeupId.make(raw.wakeupId), status: "failed" }) - .pipe(Effect.ignore), - ); - let row = yield* decodeRunning(raw); - yield* Effect.acquireRelease(Effect.void, (_, exit) => - cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), - ); - const existing = - row.threadId === null - ? [] - : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; - const projection = - row.threadId === null || existing.length === 0 - ? undefined - : yield* threads.getThreadProjection(row.threadId); - const run = projection?.runs.find((run) => - row.runId === null - ? run.userMessageId === KinIdentity.wakeupMessageId(row.wakeupId) - : run.id === row.runId, - ); - if (projection === undefined || run === undefined) { - const receipts = - yield* sql`SELECT 1 FROM kin_lead_tool_calls WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId} LIMIT 1`; - if (receipts.length > 0 && Option.isSome(writer)) { - yield* writer.value - .commitWakeup(row.orgId, row.wakeupId, "failed") - .pipe( - Effect.catchCause((cause) => - Effect.logError( - "Kin recovery commit failed; requeueing unfinished wake-up", - cause, - ), - ), - ); - } - if (projection === undefined) yield* clearMissingThread(row); - yield* queue.requeueUnfinished(row.orgId, row.wakeupId); - return; - } - row = { ...row, runId: run.id }; - yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; - const record = yield* lead(row.orgId, row.agentId); - if (record.retiredAt !== null) { - yield* cleanup(row); - return yield* retire(record); + const pending = rows.filter((row) => !active.has(KinAgentId.make(row.agentId))); + const recoverRow = (raw: (typeof pending)[number]) => + Effect.scoped( + Effect.gen(function* () { + yield* Effect.acquireRelease(Effect.void, () => + queue + .finish({ orgId, wakeupId: KinWakeupId.make(raw.wakeupId), status: "failed" }) + .pipe(Effect.ignore), + ); + let row = yield* decodeRunning(raw); + yield* Effect.acquireRelease(Effect.void, (_, exit) => + cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), + ); + const existing = + row.threadId === null + ? [] + : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; + const projection = + row.threadId === null || existing.length === 0 + ? undefined + : yield* threads.getThreadProjection(row.threadId); + const run = projection?.runs.find((run) => + row.runId === null + ? run.userMessageId === KinIdentity.wakeupMessageId(row.wakeupId) + : run.id === row.runId, + ); + if (projection === undefined || run === undefined) { + const receipts = + yield* sql`SELECT 1 FROM kin_lead_tool_calls WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId} LIMIT 1`; + if (receipts.length > 0 && Option.isSome(writer)) { + yield* writer.value.commitWakeup(row.orgId, row.wakeupId, "failed").pipe( + Effect.tap( + ({ commit }) => + sql`UPDATE kin_wakeups SET memory_commit = ${commit} WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId}`, + ), + Effect.catchCause((cause) => + Effect.logError( + "Kin recovery commit failed; requeueing unfinished wake-up", + cause, + ), + ), + ); } - yield* identity.registerAgentThread({ - orgId: row.orgId, - agentId: row.agentId, - threadId: projection.thread.id, - }); - yield* registry.register(projection.thread.id, { ...row, level: record.level }); - if (Option.isSome(bridge)) - yield* bridge.value.begin({ ...row, threadId: projection.thread.id }); - yield* wait(row, projection.thread.projectId).pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) - ? cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore) - : Effect.void, - ), - ); + if (projection === undefined) yield* clearMissingThread(row); + yield* queue.requeueUnfinished(row.orgId, row.wakeupId); + return; + } + row = { ...row, runId: run.id }; + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + const record = yield* lead(row.orgId, row.agentId); + if (record.retiredAt !== null) { + yield* cleanup(row); + return yield* retire(record); + } + yield* identity.registerAgentThread({ + orgId: row.orgId, + agentId: row.agentId, + threadId: projection.thread.id, + }); + yield* registry.register(projection.thread.id, { ...row, level: record.level }); + if (Option.isSome(bridge)) + yield* bridge.value.begin({ ...row, threadId: projection.thread.id }); + yield* wait(row, projection.thread.projectId).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore) + : Effect.void, + ), + ); + }), + ).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin wake-up recovery failed", { + orgId, + wakeupId: raw.wakeupId, + cause, }), - ).pipe( - Effect.catchCause((cause) => - Effect.logError("Kin wake-up recovery failed", { - orgId, - wakeupId: raw.wakeupId, - cause, - }), - ), ), - { concurrency: "unbounded", discard: true }, - ); + ); + // In the org loop each recovery runs beside the loop, marked active like a fresh run, + // so one slow recovery never stalls the leads behind it. + if (forkRecoveries) { + for (const raw of pending) { + const agentId = KinAgentId.make(raw.agentId); + active.add(agentId); + yield* recoverRow(raw).pipe( + Effect.ensuring( + Effect.sync(() => { + active.delete(agentId); + }).pipe(Effect.andThen(wake)), + ), + Effect.forkScoped, + ); + } + } else + yield* Effect.forEach(pending, recoverRow, { concurrency: "unbounded", discard: true }); }, Effect.mapError((cause) => isWakeupError(cause) @@ -1139,11 +1181,11 @@ const make = Effect.gen(function* () { const start = Effect.gen(function* () { const organizations = new Set(); // Leads with a wake-up in flight, per organization, so each is driven once. - const activeByOrg = new Map>(); + const activeByOrg = new Map>(); const activeOf = (orgId: KinOrgId) => { const existing = activeByOrg.get(orgId); if (existing !== undefined) return existing; - const created = new Set(); + const created = new Set(); activeByOrg.set(orgId, created); return created; }; @@ -1151,10 +1193,19 @@ const make = Effect.gen(function* () { for (const orgId of yield* orgIds()) { if (organizations.has(orgId)) continue; organizations.add(orgId); + // A run leaves `active` before it signals, so the lead it frees launches on the next pass. + let freed = Deferred.makeUnsafe(); + const wake = Effect.suspend(() => { + const signal = freed; + freed = Deferred.makeUnsafe(); + return Deferred.succeed(signal, undefined); + }).pipe(Effect.asVoid); yield* Effect.gen(function* () { const changed = yield* queue.changes(orgId); + // Taken before any read, so a run that finishes mid-pass still ends this pass's wait. + const finished = Deferred.await(freed); const active = activeOf(orgId); - yield* recoverOrg(orgId, active); + yield* recoverOrg(orgId, active, wake, true); const rows = yield* sql<{ agentId: string; }>`SELECT l.agent_id AS "agentId" FROM kin_leads l @@ -1177,14 +1228,14 @@ const make = Effect.gen(function* () { Effect.ensuring( Effect.sync(() => { active.delete(candidate.agentId); - }), + }).pipe(Effect.andThen(wake)), ), Effect.forkScoped, ); } - yield* Effect.raceFirst(changed, threadActivity(orgId)).pipe( + yield* Effect.raceAllFirst([changed, finished, threadActivity(orgId)]).pipe( Effect.timeoutOrElse({ - duration: Math.min(60_000, Math.max(1, (ready[0]?.readyAt ?? Infinity) - now)), + duration: orgLoopWaitMs(ready, active, now), orElse: () => Effect.void, }), ); diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts index 14d0a18b82a43..ac8a7a09b2839 100644 --- a/apps/server/src/kin/lead/layer.test.ts +++ b/apps/server/src/kin/lead/layer.test.ts @@ -60,7 +60,11 @@ vi.mock("../memory/Memory.ts", async (importOriginal) => { prepareWakeup: () => Effect.sync(() => { prepared = true; - return { head: "fixture-head", indexAsOf: DateTime.makeUnsafe(0) }; + return { + head: "fixture-head", + indexAsOf: DateTime.makeUnsafe(0), + read: "/private/tmp", + }; }), readPromptSnapshot: (_orgId, head) => prepared && head === "fixture-head" diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index 390f15d6181a3..a282ca8178267 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -308,8 +308,8 @@ it.live.each([ const readDirectory = (yield* fs.stat(a.read)).ino; yield* fs.writeFileString(`${a.read}/stray.md`, "Planted by hand"); expect((yield* memory.prepareWakeup(orgA)).head).toBe(prepared.head); - // Rebuilt from scratch inside the same directory, so a warm lead's cwd survives. - expect((yield* fs.stat(a.read)).ino).toEqual(readDirectory); + // Swapped in whole: a fresh directory replaces the old one, never a half-filled mix. + expect((yield* fs.stat(a.read)).ino).not.toEqual(readDirectory); expect(yield* fs.exists(`${a.read}/stray.md`)).toBe(false); expect( (yield* vcs.run({ @@ -770,6 +770,78 @@ it.live( ), ); +it.live("gives each lead its own read checkout, swapped in whole on prepare", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const { root, source } = yield* fixtureRepository(); + // What each rename onto lead A's checkout found: a complete source and nothing in the way. + const landings: Array<{ readonly complete: boolean; readonly occupied: boolean }> = []; + const watched = Layer.effect( + FileSystem.FileSystem, + Effect.map(FileSystem.FileSystem, (real) => ({ + ...real, + rename: (from: string, to: string) => + to.endsWith("/memory-read-lead-a") + ? Effect.gen(function* () { + landings.push({ + complete: + (yield* real.exists(`${from}/rules.md`)) && + (yield* real.exists(`${from}/.git`)), + occupied: yield* real.exists(to), + }); + return yield* real.rename(from, to); + }) + : real.rename(from, to), + })), + ).pipe(Layer.provide(NodeServices.layer)); + yield* Effect.gen(function* () { + const memory = yield* Memory.Memory; + const leadA = KinAgentId.make("lead-a"); + const leadB = KinAgentId.make("lead-b"); + const a = yield* memory.prepareWakeup(orgA, "wake-a", { lead: leadA }); + const b = yield* memory.prepareWakeup(orgA, "wake-b", { lead: leadB }); + expect(a.read).toBe(`${root}/state/a/memory-read-lead-a`); + expect(b.read).toBe(`${root}/state/a/memory-read-lead-b`); + for (const read of [a.read, b.read]) { + expect(yield* fs.exists(`${read}/rules.md`)).toBe(true); + expect(yield* fs.exists(`${read}/.git`)).toBe(true); + } + // One lead's refill never touches the other's checkout, stray files and all. + yield* fs.writeFileString(`${a.read}/stray.md`, "A was here"); + expect((yield* memory.prepareWakeup(orgA, "wake-b2", { lead: leadB })).read).toBe(b.read); + expect(yield* fs.readFileString(`${a.read}/stray.md`)).toBe("A was here"); + // Its own refill swaps in a new directory: the stray is gone, every file is present. + const before = (yield* fs.stat(a.read)).ino; + yield* memory.prepareWakeup(orgA, "wake-a2", { lead: leadA }); + expect((yield* fs.stat(a.read)).ino).not.toEqual(before); + expect(yield* fs.exists(`${a.read}/stray.md`)).toBe(false); + expect(yield* fs.exists(`${a.read}/rules.md`)).toBe(true); + expect(yield* fs.exists(`${a.read}/.git`)).toBe(true); + expect(landings).toEqual([ + { complete: true, occupied: false }, + { complete: true, occupied: false }, + ]); + expect( + (yield* fs.readDirectory(`${root}/state/a`)).filter((entry) => + entry.startsWith("memory-read.next-"), + ), + ).toEqual([]); + }).pipe( + Effect.provide( + memoryTestLayer.pipe( + Layer.provide(configLayer(`${root}/state`, source)), + Layer.provide(gitLayer), + Layer.provide(watched), + Layer.provide(NodeServices.layer), + ), + ), + ); + }).pipe( + Effect.provide(Layer.mergeAll(gitLayer, NodeServices.layer, databaseLayer)), + Effect.scoped, + ), +); + it.live.each([ "missing HEAD", "missing objects", diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 0d39e67dbc708..59ba6dd84af66 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- a synchronous SHA-256 names an unusual lead's read checkout. +import * as NodeCrypto from "node:crypto"; import * as SecretScan from "./secretScan.ts"; import { readTree, @@ -193,15 +195,15 @@ export class Memory extends Context.Service< readonly historyGeneration: (orgId: KinOrgId) => Effect.Effect; readonly ensureCheckouts: (orgId: KinOrgId) => Effect.Effect; /** - * Validates the writer and fills the read checkout with the waking lead's view. The first - * wake-up after a start, and every one after a failed `syncCheckout`, syncs inline first. - * The checkout is refilled inside the same directory; `replaced` says it is a new one. + * Validates the writer and swaps in a fresh read checkout, `read`: the waking lead's own + * (`memory-read-`) with its view. No other lead's prepare touches it. The first wake-up + * after a start, and every one after a failed `syncCheckout`, syncs inline first. */ readonly prepareWakeup: ( orgId: KinOrgId, wakeupId?: string, view?: LeadView, - ) => Effect.Effect<{ head: string; indexAsOf: DateTime.Utc; replaced?: true }, MemoryFailure>; + ) => Effect.Effect<{ head: string; indexAsOf: DateTime.Utc; read: string }, MemoryFailure>; /** * The writer's integrity check (`git fsck`) and default-branch sync with the remote, off the * reply path. Skips an organization with no clone yet, or whose last sync failed: its next @@ -376,8 +378,9 @@ const make = Effect.gen(function* () { const blobCaches = new Map(); const snapshotKey = (orgId: KinOrgId, branch: string) => `${orgId}\u0000${branch}`; const triggerViews = new Map(); - // The branch the read checkout holds; its INDEX.md is only ever that branch's index. - const checkoutBranches = new Map(); + // The branch each read checkout holds, keyed by org and lead; its INDEX.md is only that branch's. + const checkoutBranches = new Map(); + const checkoutKey = (orgId: KinOrgId, view?: LeadView) => `${orgId}\u0000${view?.lead ?? ""}`; const forgetSnapshots = (orgId: KinOrgId) => { for (const cache of [snapshots, blobCaches, duplicateWarnings]) for (const key of cache.keys()) if (key.startsWith(`${orgId}\u0000`)) cache.delete(key); @@ -504,6 +507,15 @@ const make = Effect.gen(function* () { return { repo, root, writer, read }; }); + /** Each lead reads from its own checkout, a sibling of the shared one no-view prepare fills. */ + const readDirFor = (paths: { read: string }, view?: LeadView) => { + if (view === undefined) return paths.read; + const name = /^[A-Za-z0-9_-]{1,80}$/.test(view.lead) + ? view.lead + : NodeCrypto.createHash("sha256").update(view.lead).digest("hex").slice(0, 32); + return path.join(path.dirname(paths.read), `memory-read-${name}`); + }; + const markerPath = (root: string) => path.join(root, "purge-in-flight"); /** Half-finished clones from a crash; after a purge they can hold the old history. */ const removeStaleClones = Effect.fn("Memory.removeStaleClones")(function* ( @@ -641,11 +653,22 @@ const make = Effect.gen(function* () { orgId: KinOrgId, force: boolean, branch: string = mainBranch, + explicitHead?: string, ) { const paths = yield* pathsFor(orgId); - const key = snapshotKey(orgId, branch); + // An explicit head reads that commit as pinned at prepare time, even after another lead's + // commit moved the shared pin; it is returned uncached so branch snapshots stay current. + const key = + explicitHead === undefined + ? snapshotKey(orgId, branch) + : snapshotKey(orgId, `${branch}@${explicitHead}`); let head: string; - if (branch === mainBranch) { + if (explicitHead !== undefined) { + const cached = snapshots.get(snapshotKey(orgId, branch)); + if (!force && cached?.head === explicitHead) return cached; + if (!initializedCheckouts.has(orgId)) yield* ensureCheckouts(orgId); + head = explicitHead; + } else if (branch === mainBranch) { const pinned = pinnedHeads.get(orgId); // Bootstraps a missing writer, so reads before the first wake-up don't fail validation. if (pinned === undefined) yield* ensureCheckouts(orgId); @@ -804,7 +827,7 @@ const make = Effect.gen(function* () { if (!duplicateWarnings.get(key)?.has(id)) yield* Effect.logWarning("Duplicate memory id", { orgId, branch, id }); } - duplicateWarnings.set(key, duplicateIds); + if (explicitHead === undefined) duplicateWarnings.set(key, duplicateIds); const snapshot: Snapshot = { head, branch, @@ -814,7 +837,8 @@ const make = Effect.gen(function* () { duplicateIds, ...(previous?.head === head && previous.index !== undefined ? { index: previous.index } : {}), }; - snapshots.set(key, snapshot); + if (explicitHead === undefined) snapshots.set(key, snapshot); + else blobCaches.delete(key); const reachable = new Set(entries.map((entry) => entry.sha)); for (const cache of [blobs, parsedBlobs, mismatchWarnings]) { for (const sha of cache.keys()) if (!reachable.has(sha)) cache.delete(sha); @@ -937,6 +961,27 @@ const make = Effect.gen(function* () { return branch === mainBranch ? main : yield* loadSnapshot(orgId, false, branch); }); + /** The branch as a wake-up pinned it, after another lead's commit moved the branch on. */ + const pinnedSnapshot = Effect.fn("Memory.pinnedSnapshot")(function* ( + orgId: KinOrgId, + branch: string, + head: string, + ) { + const mismatch = new MemoryError({ + orgId, + operation: "promptSnapshot", + reason: "Memory head does not match wake-up", + }); + if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(head)) return yield* mismatch; + const { writer } = yield* pathsFor(orgId); + const known = yield* git(orgId, writer, ["cat-file", "-e", `${head}^{commit}`]).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (!known) return yield* mismatch; + return yield* loadSnapshot(orgId, false, branch, head); + }); + const query = Effect.fn("Memory.query")(function* ( orgId: KinOrgId, q: MemoryQuery, @@ -1018,18 +1063,19 @@ const make = Effect.gen(function* () { now, ownerTimeZone, ); - if ((checkoutBranches.get(orgId) ?? mainBranch) !== snapshot.branch) return; - const paths = yield* pathsFor(orgId); + // Only a checkout this process filled; a lead's next prepare refills its own anyway. + if (checkoutBranches.get(checkoutKey(orgId, view)) !== snapshot.branch) return; + const read = readDirFor(yield* pathsFor(orgId), view); yield* io( orgId, "writeIndex", Effect.gen(function* () { const temporary = yield* fs.makeTempFileScoped({ - directory: paths.read, + directory: read, prefix: ".kin-index-", }); yield* fs.writeFileString(temporary, snapshot.index ?? ""); - yield* fs.rename(temporary, path.join(paths.read, "INDEX.md")); + yield* fs.rename(temporary, path.join(read, "INDEX.md")); }).pipe(Effect.scoped), ).pipe( Effect.catchTags({ @@ -1613,19 +1659,18 @@ const make = Effect.gen(function* () { Effect.provideService(VcsProcess, vcs), )), ); - const replaced = yield* io( + const readTarget = readDirFor(paths, view); + yield* io( orgId, "readSnapshot", Effect.scoped( Effect.gen(function* () { // Staged beside the target: the system temp dir can be another filesystem (EXDEV). - const read = path.join( - yield* fs.makeTempDirectoryScoped({ - directory: path.dirname(paths.read), - prefix: "memory-read.next-", - }), - "read", - ); + const staging = yield* fs.makeTempDirectoryScoped({ + directory: path.dirname(readTarget), + prefix: "memory-read.next-", + }); + const read = path.join(staging, "read"); yield* fs.makeDirectory(read); for (const [index, { relative }] of kept.entries()) { const target = path.join(read, relative); @@ -1640,26 +1685,19 @@ const make = Effect.gen(function* () { args: ["init", "--quiet", "--", read], env: { GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TEMPLATE_DIR: "" }, }); - // Swapped in place: an idle warm lead process keeps this directory as its cwd. - const inPlace = yield* fs.lstat(paths.read).pipe( - Effect.map((info) => info.type === "Directory"), + // Two renames, so a reader sees the old checkout, the new one or none, never a mix. + // The old one leaves with the staging directory; a process still in it must restart. + const existing = yield* fs.lstat(readTarget).pipe( + Effect.as(true), Effect.orElseSucceed(() => false), ); - if (!inPlace) { - yield* fs.remove(paths.read, { recursive: true, force: true }); - yield* fs.rename(read, paths.read); - return true; - } - for (const entry of yield* fs.readDirectory(paths.read)) - yield* fs.remove(path.join(paths.read, entry), { recursive: true, force: true }); - for (const entry of yield* fs.readDirectory(read)) - yield* fs.rename(path.join(read, entry), path.join(paths.read, entry)); - return false; + if (existing) yield* fs.rename(readTarget, path.join(staging, "old")); + yield* fs.rename(read, readTarget); }), ), ); pinnedHeads.set(orgId, head); - checkoutBranches.set(orgId, checkoutBranch); + checkoutBranches.set(checkoutKey(orgId, view), checkoutBranch); const index = yield* readFile(orgId, "INDEX.md", view).pipe( Effect.catchTags({ MemoryError: () => Effect.succeed("") }), ); @@ -1671,7 +1709,7 @@ const make = Effect.gen(function* () { return { head: checkoutHead, indexAsOf: DateTime.toUtc(indexAsOf), - ...(replaced ? { replaced: true as const } : {}), + read: readTarget, }; }); @@ -1681,13 +1719,22 @@ const make = Effect.gen(function* () { // `dreams/` holds plain copies of memory files for a running dream. for (const directory of [paths.writer, paths.read, path.join(paths.root, "dreams")]) yield* io(orgId, "quarantine", fs.remove(directory, { recursive: true, force: true })); - if (yield* io(orgId, "exists", fs.exists(paths.root))) + if (yield* io(orgId, "exists", fs.exists(paths.root))) { + for (const entry of yield* io(orgId, "quarantine", fs.readDirectory(paths.root))) + if (/^memory-read-[A-Za-z0-9_-]+$/.test(entry)) + yield* io( + orgId, + "quarantine", + fs.remove(path.join(paths.root, entry), { recursive: true, force: true }), + ); yield* removeStaleClones(orgId, paths.root); + } initializedCheckouts.delete(orgId); pinnedHeads.delete(orgId); synced.delete(orgId); skipWarnedHeads.delete(orgId); - checkoutBranches.delete(orgId); + for (const key of checkoutBranches.keys()) + if (key.startsWith(`${orgId}\u0000`)) checkoutBranches.delete(key); forgetSnapshots(orgId); return paths; }); @@ -1763,13 +1810,11 @@ const make = Effect.gen(function* () { locked( orgId, Effect.gen(function* () { - const snapshot = yield* load(orgId, false, view); - if (snapshot.head !== expectedHead) - return yield* new MemoryError({ - orgId, - operation: "promptSnapshot", - reason: "Memory head does not match wake-up", - }); + const current = yield* load(orgId, false, view); + const snapshot = + current.head === expectedHead + ? current + : yield* pinnedSnapshot(orgId, current.branch, expectedHead); const rules = snapshot.files.get("rules.md"); const index = snapshot.index ?? snapshot.files.get("INDEX.md") ?? ""; if (rules === undefined) diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 466d7d4ecc17a..7e13ea71bb1db 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -34,6 +34,10 @@ it.live.each([ "recovered-no-end", "lost-session-retry", "two-leads", + "two-leads-same-path", + "two-leads-same-id", + "recovery-after-other-commit", + "prompt-snapshot-stale-head", "completed", "failed", "blocked", @@ -190,6 +194,7 @@ it.live.each([ >) | undefined; let failBranch = false; + let restarted = false; const memoryRuns: Array = []; const interceptedGit = Layer.succeed( Vcs.VcsProcess, @@ -267,11 +272,16 @@ it.live.each([ orgId, wakeupId, agentId: - scenario === "two-leads" && wakeupId === "wake2" + wakeupId === "wake2" && + (scenario.startsWith("two-leads") || + scenario === "recovery-after-other-commit" || + scenario === "prompt-snapshot-stale-head") ? KinAgentId.make("fixture-lead-2") : agentId, level: - scenario === "two-leads" || + scenario.startsWith("two-leads") || + scenario === "recovery-after-other-commit" || + scenario === "prompt-snapshot-stale-head" || scenario === "conflict" || scenario === "B5-journal" || scenario === "read-during-record" || @@ -282,7 +292,9 @@ it.live.each([ startedAt: "2026-11-08T00:00:00.000Z", eventIds: [], stagedReceipts: - scenario === "lost-session-retry" || scenario === "recovered-no-end", + scenario === "lost-session-retry" || + scenario === "recovered-no-end" || + (scenario === "recovery-after-other-commit" && restarted), sources: new Map< string, Writer.WakeupContext["sources"] extends ReadonlyMap ? S : never @@ -382,6 +394,14 @@ it.live.each([ expect((yield* writer.stage(orgId, "wake2", two).pipe(Effect.result))._tag).toBe("Failure"); yield* fs.rename(`${paths.writer}/.git/refs-moved`, `${paths.writer}/.git/refs`); const first = yield* writer.commitWakeup(orgId, "wake1", "completed"); + // The first lead's commit holds none of the second lead's staged work. + expect( + (yield* git(paths.writer, ["show", `${first.commit}:claims/two.md`]).pipe(Effect.result)) + ._tag, + ).toBe("Failure"); + expect( + yield* git(paths.writer, ["show", `${first.commit}:journal/2026/11-08.md`]), + ).not.toContain("Second lead note."); const second = yield* writer.commitWakeup(orgId, "wake2", "completed"); expect(yield* git(paths.writer, ["rev-parse", `${second.commit}^`])).toBe(first.commit); expect(yield* git(paths.writer, ["show", `${second.commit}:claims/one.md`])).toContain( @@ -399,6 +419,71 @@ it.live.each([ expect(cards.map((card) => card.wakeupId)).toEqual(["wake1", "wake2"]); return; } + if (scenario === "two-leads-same-path" || scenario === "two-leads-same-id") { + const first = { + path: scenario === "two-leads-same-id" ? "claims/first.md" : "claims/shared.md", + sourceEventId: "owner", + content: fixtureItem({ id: "shared", kind: "claim", title: "First title", links: [] }), + }; + const second = { + path: "claims/shared.md", + sourceEventId: "owner", + content: fixtureItem({ id: "shared", kind: "claim", title: "Second title", links: [] }), + }; + yield* writer.stage(orgId, "wake1", first); + yield* writer.stage(orgId, "wake2", second); + const committed = yield* writer.commitWakeup(orgId, "wake2", "completed"); + // The first lead validated against the old head: at this one its write conflicts, so + // its commit refuses rather than erasing the other lead's or duplicating its id. + const result = yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result); + expect(result._tag).toBe("Failure"); + if (result._tag === "Failure") expect(result.failure._tag).toBe("MemorySyncConflict"); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(committed.commit); + expect( + yield* git(paths.writer, ["show", `${committed.commit}:claims/shared.md`]), + ).toContain("Second title"); + return; + } + if (scenario === "recovery-after-other-commit") { + yield* writer.stage(orgId, "wake1", { + path: "claims/one.md", + sourceEventId: "owner", + content: fixtureItem({ id: "one", kind: "claim", title: "First claim", links: [] }), + }); + const first = yield* writer.commitWakeup(orgId, "wake1", "completed"); + yield* writer.stage(orgId, "wake2", { + path: "claims/two.md", + sourceEventId: "owner", + content: fixtureItem({ id: "two", kind: "claim", title: "Second claim", links: [] }), + }); + const second = yield* writer.commitWakeup(orgId, "wake2", "completed"); + // A restart now recovers wake1 with only its receipts: its commit is in history, + // just no longer at the branch head, so nothing is recommitted or reported lost. + restarted = true; + yield* writer.appendJournal(orgId, "wake1", ["Lead ended without calling kin_end."]); + expect(yield* writer.commitWakeup(orgId, "wake1", "completed")).toEqual(first); + expect(yield* git(paths.writer, ["rev-parse", "HEAD"])).toBe(second.commit); + const journal = yield* git(paths.writer, [ + "show", + `${second.commit}:journal/2026/11-08.md`, + ]); + expect(journal).not.toContain("lost on restart"); + expect(journal).not.toContain("Lead ended without calling kin_end."); + return; + } + if (scenario === "prompt-snapshot-stale-head") { + const pinned = yield* memory.prepareWakeup(orgId, "wake1"); + yield* writer.stage(orgId, "wake2", { + path: "claims/two.md", + sourceEventId: "owner", + content: fixtureItem({ id: "two", kind: "claim", title: "Second claim", links: [] }), + }); + yield* writer.commitWakeup(orgId, "wake2", "completed"); + // The other lead's commit moved the head; wake1's prompt still reads its own pin. + const snapshot = yield* memory.readPromptSnapshot(orgId, pinned.head); + expect(snapshot.rules).toBe("Invented rules\n"); + return; + } if (scenario === "lost-session-retry") { failBranch = true; expect((yield* writer.appendJournal(orgId, "wake1", []).pipe(Effect.result))._tag).toBe( diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index a030e52dce9ee..2bf22723bbabf 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -294,16 +294,18 @@ const make = Effect.gen(function* () { const locked = memory.withWriterLock; // Staged changes are per wake-up, so concurrent leads never share a session; the // git commit and push themselves stay serialized per repository under `locked`. - const sessions = new Map< - string, - { - wakeupId: string; - generation: number; - files: Map; - lines: string[]; - supersessions: Map; - } - >(); + // `changes` and `heads` keep each staged change and the branch head it was validated at, + // so a commit after another lead's re-validates them instead of laying stale files over it. + type SessionState = { + wakeupId: string; + generation: number; + files: Map; + lines: string[]; + supersessions: Map; + changes: Map; + heads: Map; + }; + const sessions = new Map(); const sessionKey = (orgId: KinOrgId, wakeupId: string) => `${orgId}\u0000${wakeupId}`; const io = (orgId: KinOrgId, effect: Effect.Effect) => effect.pipe(Effect.mapError((cause) => new MemoryError({ orgId, operation: "writer", cause }))); @@ -397,6 +399,34 @@ const make = Effect.gen(function* () { ]); return recorded; }); + const wakeupSubject = (ctx: WakeupContext) => `kin(${ctx.agentId}): wake-up ${ctx.wakeupId}`; + // The wake-up's commit, by the subject that names it: once another lead commits on the same + // branch, the head is no longer this wake-up's commit even though it committed. + const ownCommit = Effect.fn("MemoryWriter.ownCommit")(function* ( + orgId: KinOrgId, + writer: string, + head: string, + ctx: WakeupContext, + ) { + const subject = wakeupSubject(ctx); + // A day of slack for clock changes; the walk stops at older commits, so it stays short. + const since = Date.parse(ctx.startedAt) - 86_400_000; + const bound = since > 0 ? [`--since=${DateTime.formatIso(DateTime.makeUnsafe(since))}`] : []; + const log = yield* git(orgId, writer, [ + "log", + ...bound, + "--fixed-strings", + `--grep=${subject}`, + "--format=%H%x09%s", + head, + "--", + ]); + for (const line of log.split("\n")) { + const [sha, text] = line.split("\t"); + if (sha && text === subject) return sha; + } + return undefined; + }); const session = Effect.fn("MemoryWriter.session")(function* (orgId: KinOrgId, wakeupId: string) { const ctx = yield* authority.resolve(orgId, wakeupId); if ( @@ -420,20 +450,21 @@ const make = Effect.gen(function* () { reason: "Memory history was rewritten during this wake-up; its staged changes were dropped", }); } - const state = previous ?? { + const state: SessionState = previous ?? { wakeupId, generation, files: new Map(), lines: [], supersessions: new Map(), + changes: new Map(), + heads: new Map(), }; if (!previous && ctx.stagedReceipts) { const { writer } = yield* validate(orgId); const branch = ctx.level === "L0" ? shadowBranch(ctx.agentId) : (config.defaultBranch ?? "main"); const head = yield* branchHead(orgId, writer, branch); - const { message } = yield* commitAt(orgId, writer, head); - if (message.split("\n")[0] !== `kin(${ctx.agentId}): wake-up ${wakeupId}`) { + if ((yield* ownCommit(orgId, writer, head, ctx)) === undefined) { const reason = "Staged changes and kin_end summary were lost on restart; tool receipts do not prove those changes were committed."; state.lines.push(reason); @@ -796,6 +827,221 @@ const make = Effect.gen(function* () { .pipe( Effect.catchCause((cause) => Effect.logWarning("Memory change card failed", cause)), ); + const stageChange = Effect.fn("MemoryWriter.stageChange")(function* ( + orgId: KinOrgId, + wakeupId: string, + ctx: WakeupContext, + state: SessionState, + change: MemoryChange, + ) { + const { writer } = yield* validate(orgId); + yield* safePath(orgId, writer, change.path); + if (change.path.startsWith("journal/")) + return yield* new MemoryWriteRejected({ + reason: "Use appendJournal for journal additions", + }); + const parsed = yield* parseFrontMatter(orgId, change.path, change.content, true); + const source = + change.sourceEventId === undefined + ? { + saidBy: ctx.agentId, + trust: "agent", + source: `wakeup:${wakeupId}`, + kind: undefined, + } + : ctx.sources.get(change.sourceEventId); + if (!source) + return yield* new MemoryWriteRejected({ + reason: "Source is not in the current batch", + }); + const fm = parsed.frontMatter; + const folder = fm.kind === "person" ? "people" : `${fm.kind}s`; + if (!change.path.startsWith(`${folder}/`)) + return yield* new MemoryWriteRejected({ reason: "Kind does not match path" }); + const branch = + ctx.level === "L0" ? shadowBranch(ctx.agentId) : (config.defaultBranch ?? "main"); + const recorded = yield* branchHead(orgId, writer, branch); + const recordedFiles = yield* filesAt(orgId, writer, recorded); + const items = []; + const bodies = new Map(); + const regular = new Set( + (yield* readTree(orgId, writer, recorded).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(VcsProcess, vcs), + )) + .filter((entry) => ["100644", "100755"].includes(entry.mode)) + .map((entry) => entry.relative), + ); + for (const [file, bytes] of recordedFiles) { + if (!regular.has(file)) continue; + if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; + const content = bytes.toString("utf8"); + const parsedItem = yield* parseFrontMatter(orgId, file, content).pipe(Effect.result); + if (parsedItem._tag === "Failure") { + if (file === change.path) + return yield* new MemoryWriteRejected({ + reason: "Cannot overwrite malformed recorded item", + }); + continue; + } + if (file === change.path && parsedItem.success.frontMatter.kind !== fm.kind) + return yield* new MemoryWriteRejected({ + reason: "Cannot change recorded item kind", + }); + items.push({ + ...parsedItem.success.frontMatter, + orgId, + path: file, + duplicate: false, + }); + bodies.set(file, parsedItem.success.body); + } + for (const [file, text] of state.files) { + if (file.startsWith("journal/") || file === "INDEX.md") continue; + const staged = yield* parseFrontMatter(orgId, file, text); + const index = items.findIndex((item) => item.path === file); + const head = { ...staged.frontMatter, orgId, path: file, duplicate: false }; + if (index < 0) items.push(head); + else items[index] = head; + bodies.set(file, staged.body); + } + const superseded = + change.supersedes === undefined + ? undefined + : items.find((item) => item.id === change.supersedes); + const supersessionHash = + change.supersedes === undefined + ? undefined + : NodeCrypto.createHash("sha256").update(stringify(change)).digest("hex"); + if ( + superseded?.status === "superseded" && + superseded.superseded_by === fm.id && + state.supersessions.get(change.path) === supersessionHash + ) + return; + if ( + change.supersedes !== undefined && + (!superseded || + superseded.status !== "open" || + superseded.kind !== fm.kind || + superseded.source !== source.source || + Date.parse(fm.recorded_at) <= Date.parse(superseded.recorded_at) || + superseded.id === fm.id) + ) + return yield* new MemoryWriteRejected({ + reason: "Supersession requires the same source and kind", + }); + for (const item of items) { + if ( + (item.id === fm.id || item.status === "open") && + item.kind === fm.kind && + (item.id === fm.id || item.links.some((link) => fm.links.includes(link))) && + item.id !== change.supersedes + ) { + const body = bodies.get(item.path) ?? (yield* memory.readItemQuietly(orgId, item.id)).body; + const rank = (trust: string) => ["external", "agent", "member", "owner"].indexOf(trust); + if ( + item.id === fm.id && + (rank(source.trust) < rank(item.trust) || + (fm.status !== item.status && + !(item.status === "open" && ["done", "cancelled"].includes(fm.status)))) + ) + return yield* new MemoryConflict({ existingId: item.id }); + if (body !== parsed.body || item.title !== fm.title) + return yield* new MemoryConflict({ existingId: item.id }); + if (item.id === fm.id) { + const proposed = new Map(Object.entries(fm)); + const mutable = new Set([ + "orgId", + "path", + "duplicate", + "status", + "invalid_at", + "trust", + "said_by", + "source", + "source_event", + ]); + if ( + source.kind === "owner_check" && + source.itemId === item.id && + source.action === "keep" + ) + mutable.add("review_by"); + if ( + source.kind === "deferral_card" && + source.action === "remind" && + source.itemId === item.id && + item.kind === "deferral" && + fm.on_due === "remind" + ) + mutable.add("on_due"); + if ( + source.kind === "deferral_card" && + source.action === "monday" && + source.itemId === item.id && + item.kind === "deferral" && + fm.due === source.due + ) + mutable.add("due"); + const existing = new Map(Object.entries(item)); + if ( + [...new Set([...existing.keys(), ...proposed.keys()])].some( + (key) => + !mutable.has(key) && + JSON.stringify(existing.get(key)) !== JSON.stringify(proposed.get(key)), + ) + ) + return yield* new MemoryConflict({ existingId: item.id }); + } + } + if ( + (item.path === change.path && item.id !== fm.id) || + (item.id === fm.id && item.path !== change.path) + ) + return yield* new MemoryWriteRejected({ + reason: "Cannot replace a different item", + }); + } + const fields = { + ...fm, + said_by: source.saidBy, + source: source.source, + source_event: change.sourceEventId, + trust: + fm.trust === "external" || + (fm.trust === "agent" && source.trust !== "external") || + (fm.trust === "member" && source.trust === "owner") + ? fm.trust + : source.trust, + review_by: fm.review_by, + }; + const existing = items.find((item) => item.id === fm.id); + const text = render( + existing + ? { + ...fields, + trust: existing.trust, + said_by: existing.said_by, + source: existing.source, + source_event: existing.source_event, + } + : fields, + parsed.body, + ); + yield* parseFrontMatter(orgId, change.path, text); + if (superseded) { + const oldBody = + bodies.get(superseded.path) ?? (yield* memory.readItemQuietly(orgId, superseded.id)).body; + const { orgId: _org, path: oldPath, duplicate: _duplicate, ...oldFields } = superseded; + const oldText = render({ ...oldFields, status: "superseded", superseded_by: fm.id }, oldBody); + state.files.set(oldPath, oldText); + } + state.files.set(change.path, text); + if (supersessionHash !== undefined) state.supersessions.set(change.path, supersessionHash); + state.changes.set(change.path, change); + state.heads.set(change.path, recorded); + }); return MemoryWriter.of({ stage: Effect.fn("MemoryWriter.stage")((orgId, wakeupId, change) => report( @@ -805,223 +1051,7 @@ const make = Effect.gen(function* () { orgId, Effect.gen(function* () { const { ctx, state } = yield* session(orgId, wakeupId); - const { writer } = yield* validate(orgId); - yield* safePath(orgId, writer, change.path); - if (change.path.startsWith("journal/")) - return yield* new MemoryWriteRejected({ - reason: "Use appendJournal for journal additions", - }); - const parsed = yield* parseFrontMatter(orgId, change.path, change.content, true); - const source = - change.sourceEventId === undefined - ? { - saidBy: ctx.agentId, - trust: "agent", - source: `wakeup:${wakeupId}`, - kind: undefined, - } - : ctx.sources.get(change.sourceEventId); - if (!source) - return yield* new MemoryWriteRejected({ - reason: "Source is not in the current batch", - }); - const fm = parsed.frontMatter; - const folder = fm.kind === "person" ? "people" : `${fm.kind}s`; - if (!change.path.startsWith(`${folder}/`)) - return yield* new MemoryWriteRejected({ reason: "Kind does not match path" }); - const branch = - ctx.level === "L0" ? shadowBranch(ctx.agentId) : (config.defaultBranch ?? "main"); - const recorded = yield* branchHead(orgId, writer, branch); - const recordedFiles = yield* filesAt(orgId, writer, recorded); - const items = []; - const bodies = new Map(); - const regular = new Set( - (yield* readTree(orgId, writer, recorded).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(VcsProcess, vcs), - )) - .filter((entry) => ["100644", "100755"].includes(entry.mode)) - .map((entry) => entry.relative), - ); - for (const [file, bytes] of recordedFiles) { - if (!regular.has(file)) continue; - if (!folders.some((folder) => file.startsWith(`${folder}/`))) continue; - const content = bytes.toString("utf8"); - const parsedItem = yield* parseFrontMatter(orgId, file, content).pipe(Effect.result); - if (parsedItem._tag === "Failure") { - if (file === change.path) - return yield* new MemoryWriteRejected({ - reason: "Cannot overwrite malformed recorded item", - }); - continue; - } - if (file === change.path && parsedItem.success.frontMatter.kind !== fm.kind) - return yield* new MemoryWriteRejected({ - reason: "Cannot change recorded item kind", - }); - items.push({ - ...parsedItem.success.frontMatter, - orgId, - path: file, - duplicate: false, - }); - bodies.set(file, parsedItem.success.body); - } - for (const [file, text] of state.files) { - if (file.startsWith("journal/") || file === "INDEX.md") continue; - const staged = yield* parseFrontMatter(orgId, file, text); - const index = items.findIndex((item) => item.path === file); - const head = { ...staged.frontMatter, orgId, path: file, duplicate: false }; - if (index < 0) items.push(head); - else items[index] = head; - bodies.set(file, staged.body); - } - const superseded = - change.supersedes === undefined - ? undefined - : items.find((item) => item.id === change.supersedes); - const supersessionHash = - change.supersedes === undefined - ? undefined - : NodeCrypto.createHash("sha256").update(stringify(change)).digest("hex"); - if ( - superseded?.status === "superseded" && - superseded.superseded_by === fm.id && - state.supersessions.get(change.path) === supersessionHash - ) - return; - if ( - change.supersedes !== undefined && - (!superseded || - superseded.status !== "open" || - superseded.kind !== fm.kind || - superseded.source !== source.source || - Date.parse(fm.recorded_at) <= Date.parse(superseded.recorded_at) || - superseded.id === fm.id) - ) - return yield* new MemoryWriteRejected({ - reason: "Supersession requires the same source and kind", - }); - for (const item of items) { - if ( - (item.id === fm.id || item.status === "open") && - item.kind === fm.kind && - (item.id === fm.id || item.links.some((link) => fm.links.includes(link))) && - item.id !== change.supersedes - ) { - const body = - bodies.get(item.path) ?? (yield* memory.readItemQuietly(orgId, item.id)).body; - const rank = (trust: string) => - ["external", "agent", "member", "owner"].indexOf(trust); - if ( - item.id === fm.id && - (rank(source.trust) < rank(item.trust) || - (fm.status !== item.status && - !(item.status === "open" && ["done", "cancelled"].includes(fm.status)))) - ) - return yield* new MemoryConflict({ existingId: item.id }); - if (body !== parsed.body || item.title !== fm.title) - return yield* new MemoryConflict({ existingId: item.id }); - if (item.id === fm.id) { - const proposed = new Map(Object.entries(fm)); - const mutable = new Set([ - "orgId", - "path", - "duplicate", - "status", - "invalid_at", - "trust", - "said_by", - "source", - "source_event", - ]); - if ( - source.kind === "owner_check" && - source.itemId === item.id && - source.action === "keep" - ) - mutable.add("review_by"); - if ( - source.kind === "deferral_card" && - source.action === "remind" && - source.itemId === item.id && - item.kind === "deferral" && - fm.on_due === "remind" - ) - mutable.add("on_due"); - if ( - source.kind === "deferral_card" && - source.action === "monday" && - source.itemId === item.id && - item.kind === "deferral" && - fm.due === source.due - ) - mutable.add("due"); - const existing = new Map(Object.entries(item)); - if ( - [...new Set([...existing.keys(), ...proposed.keys()])].some( - (key) => - !mutable.has(key) && - JSON.stringify(existing.get(key)) !== JSON.stringify(proposed.get(key)), - ) - ) - return yield* new MemoryConflict({ existingId: item.id }); - } - } - if ( - (item.path === change.path && item.id !== fm.id) || - (item.id === fm.id && item.path !== change.path) - ) - return yield* new MemoryWriteRejected({ - reason: "Cannot replace a different item", - }); - } - const fields = { - ...fm, - said_by: source.saidBy, - source: source.source, - source_event: change.sourceEventId, - trust: - fm.trust === "external" || - (fm.trust === "agent" && source.trust !== "external") || - (fm.trust === "member" && source.trust === "owner") - ? fm.trust - : source.trust, - review_by: fm.review_by, - }; - const existing = items.find((item) => item.id === fm.id); - const text = render( - existing - ? { - ...fields, - trust: existing.trust, - said_by: existing.said_by, - source: existing.source, - source_event: existing.source_event, - } - : fields, - parsed.body, - ); - yield* parseFrontMatter(orgId, change.path, text); - if (superseded) { - const oldBody = - bodies.get(superseded.path) ?? - (yield* memory.readItemQuietly(orgId, superseded.id)).body; - const { - orgId: _org, - path: oldPath, - duplicate: _duplicate, - ...oldFields - } = superseded; - const oldText = render( - { ...oldFields, status: "superseded", superseded_by: fm.id }, - oldBody, - ); - state.files.set(oldPath, oldText); - } - state.files.set(change.path, text); - if (supersessionHash !== undefined) - state.supersessions.set(change.path, supersessionHash); + yield* stageChange(orgId, wakeupId, ctx, state, change); }), ), ), @@ -1036,8 +1066,7 @@ const make = Effect.gen(function* () { const branch = ctx.level === "L0" ? shadowBranch(ctx.agentId) : (config.defaultBranch ?? "main"); const head = yield* branchHead(orgId, writer, branch); - const { message } = yield* commitAt(orgId, writer, head); - if (message.split("\n")[0] === `kin(${ctx.agentId}): wake-up ${wakeupId}`) return; + if ((yield* ownCommit(orgId, writer, head, ctx)) !== undefined) return; } state.lines.push(...lines); }), @@ -1103,20 +1132,23 @@ const make = Effect.gen(function* () { ), ); return yield* Effect.gen(function* () { - const subject = `kin(${ctx.agentId}): wake-up ${wakeupId}`; - const fingerprint = yield* git( - orgId, - writer, - ["hash-object", "--no-filters", "--stdin"], - {}, - stringify({ - outcome, - files: [...state.files].sort(([a], [b]) => a.localeCompare(b)), - lines: state.lines, - }), - ); - const { message: body } = yield* commitAt(orgId, writer, recorded); - if (body.split("\n")[0] === subject) { + const subject = wakeupSubject(ctx); + const fingerprintOf = () => + git( + orgId, + writer, + ["hash-object", "--no-filters", "--stdin"], + {}, + stringify({ + outcome, + files: [...state.files].sort(([a], [b]) => a.localeCompare(b)), + lines: state.lines, + }), + ); + let fingerprint = yield* fingerprintOf(); + const own = yield* ownCommit(orgId, writer, recorded, ctx); + if (own !== undefined) { + const { message: body } = yield* commitAt(orgId, writer, own); if ( !body.includes(`Kin-Outcome: ${outcome}`) || (state.files.size + state.lines.length > 0 && @@ -1125,9 +1157,37 @@ const make = Effect.gen(function* () { return yield* new MemoryWriteRejected({ reason: "Different wake-up retry" }); // Through PRs the default branch only ever holds what GitHub already merged. if (!viaPullRequest) yield* push(recorded); - return { commit: recorded }; + return { commit: own }; } if (outcome === "failed") state.files.clear(); + // Staged files are whole files: after another lead's commit, every change is + // validated again at this head, supersession and duplicate ids included. + if ( + outcome !== "failed" && + [...state.heads.values()].some((head) => head !== recorded) + ) { + const original = [...state.changes.values()]; + state.files.clear(); + state.supersessions.clear(); + state.changes.clear(); + state.heads.clear(); + for (const change of original) + yield* stageChange(orgId, wakeupId, ctx, state, change).pipe( + Effect.mapError((error) => + error._tag === "MemoryConflict" || + error._tag === "MemoryWriteRejected" || + error._tag === "MemoryDuplicateId" + ? new MemorySyncConflict({ + orgId, + operation: "push", + reason: `Memory changed since this wake-up staged ${change.path}; its change no longer applies cleanly and was not committed`, + cause: error, + }) + : error, + ), + ); + fingerprint = yield* fingerprintOf(); + } const entry = yield* journalEntry(ctx, outcome, state.lines); yield* safePath(orgId, writer, entry.path); const recordedFiles = yield* filesAt(orgId, writer, recorded); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index a80c29649876e..bcbd86ecc18e8 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -222,15 +222,15 @@ Table `kin_wakeups` (**new**): `org_id, wakeup_id, agent_id, thread_id, run_id, Setup once per organization (`Memory.ensureCheckouts` owns the checkouts; the wake-up runner owns project bootstrap): - Writer checkout: `${stateDir}/kin//memory`, a clone of the organization's memory repo. -- Read checkout: `${stateDir}/kin//memory-read`, a disposable independent clone, detached at the SQLite-pinned commit. This is the lead's working directory. -- Lead project: `ProjectService.bootstrap({ workspaceRoot: , title: "Kin · " })`. +- Read checkout: `${stateDir}/kin//memory-read-` (a hash for an id outside `[A-Za-z0-9_-]`), one disposable checkout per lead at the SQLite-pinned commit. This is that lead's working directory; only that lead's prepare replaces it. +- Lead project: `ProjectService.bootstrap({ workspaceRoot: , title: "Kin · · " })`, one per lead, since a project's root is the provider's cwd. A lead thread still in the older shared project is ended at its next wake-up, which launches a new thread in the lead's own project. Each wake-up: 1. `KinStop.isPaused(orgId)` → leave events queued and return. Before claiming on an existing thread, defer while any provider turn remains unfinished. 2. `KinBudget.admit(orgId, batchPriority)` → if refused, leave lower-priority events `queued` with `not_before` = next budget reset and run what remains. 3. Pick the model: `models.strong` if the batch holds a P0 or P1 event, a contradiction question or a `routine`; otherwise `models.cheap`. -4. `Memory.prepareWakeup(orgId)`: validate the writer (its tip matches the SQLite anchor, no unfinished git operation), rebuild the read checkout from scratch at the anchored head inside the same directory, so an idle lead process keeps its cwd, and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. The integrity check and the default-branch sync with the remote (`Memory.syncCheckout`: `fsck`, then `ls-remote`, fetch and merge as below) run off the reply path, at start and every five minutes. The first wake-up after a start, or after a failed sync, runs that sync inline first and refuses to start while it fails, so a bad checkout is refused exactly as before; only the timing of the check moved. Network git (`ls-remote`, `fetch`, `push`) times out after 30 s, since it holds the writer lock. +4. `Memory.prepareWakeup(orgId)`: validate the writer (its tip matches the SQLite anchor, no unfinished git operation), build that lead's read checkout from scratch beside the old one at the anchored head and swap it in with two renames (old aside, new in), so the directory is the old checkout, the new one or briefly absent, never a mix, and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. The integrity check and the default-branch sync with the remote (`Memory.syncCheckout`: `fsck`, then `ls-remote`, fetch and merge as below) run off the reply path, at start and every five minutes. The first wake-up after a start, or after a failed sync, runs that sync inline first and refuses to start while it fails, so a bad checkout is refused exactly as before; only the timing of the check moved. Network git (`ls-remote`, `fetch`, `push`) times out after 30 s, since it holds the writer lock. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`, without `freshContext` for a warm wake-up. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. 7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. The wake-up also adopts later runs whose root message is `kin-steer-msg::` (a late steer) or `kin-rescue-msg:` (the reply rescue), with an ordinal above the recorded `run_id`; the helpers live in `KinIdentity.ts`. Its persisted message, including an adopted run's, must belong to that root and have `createdBy=system` and `creationSource=server`, and every continuation's message must have `createdBy=agent` and `creationSource=server`. Other paths can produce that provenance (server notification wakes, and client-written scheduled tasks, whose upsert accepts `createdBy`/`creationSource`), but none of them sets a restart or usage-limit continuation link: client dispatches that set one are forced to `createdBy=user`, so only the server's recovery workers produce both. The id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. @@ -245,7 +245,7 @@ Known input-integrity gap: clients can steer an active wake-up with `steer_activ A wake-up whose events all come from one `direct` chat may continue the previous wake-up's provider conversation instead of starting a fresh one, which saves reloading rules, the index and recent chat. It continues only when the previous wake-up was for the same organization, lead, thread and chat, at the same level and model selection, completed, ended less than 10 minutes before this one started, and the session has had fewer than 10 wake-ups. Anything else starts fresh: another chat or any non-chat event in the batch, a level or model change, changed `rules.md`, a failed, timed-out or stopped wake-up, or a server restart (the session record lives only in the runner's memory). Every wake-up still claims, admits, renews floor grants, commits memory and the journal, charges its own runs and finishes on its own. -A warm wake-up sends the run without `freshContext`, so the orchestrator stays on the same provider thread. The runner keeps the lead thread's idle provider process between warm wake-ups, since `prepareWakeup` refills the read checkout in place. Only when the directory itself is new (its first fill, or after a purge) does it release the process before the warm send (`ProviderSessionManagerV2.releaseUnlessBusy`); the next run then opens a new process that resumes the conversation: the Claude adapter passes the SDK's `resume: ` once the native thread has a persisted turn. The message omits the rules (unchanged by definition), says the index is unchanged or quotes only the added and removed index lines, and lists only chat messages after the last seq the session saw, excluding the lead's own. The divider and everything below it are as in a cold wake-up. +A warm wake-up sends the run without `freshContext`, so the orchestrator stays on the same provider thread. Since `prepareWakeup` swapped in a new read checkout, the runner releases the idle provider process before the warm send (`ProviderSessionManagerV2.releaseUnlessBusy`), and stays cold when it can't; the next run then opens a new process in the new checkout that resumes the conversation: the Claude adapter passes the SDK's `resume: ` once the native thread has a persisted turn. The message omits the rules (unchanged by definition), says the index is unchanged or quotes only the added and removed index lines, and lists only chat messages after the last seq the session saw, excluding the lead's own. The divider and everything below it are as in a cold wake-up. If the lead writes files by hand in the read checkout, step 9's reset throws the changes away. A non-empty `git status --porcelain` before the reset is logged in the journal as "Lead edited memory files directly; discarded." @@ -253,7 +253,7 @@ Kin boot is optional: configuration, migrations, identity bootstrap and runner s The scheduled-task dispatch wrapper also contains Kin migration failures. When Kin persistence is unavailable, launch refusal is limited to `scheduled-task::webhook:` command ids. Sends classify the task using upstream `scheduled_tasks.schedule_json` and refuse only webhooks; an unreadable upstream row fails closed for that send only. Cron, interval, manual and ordinary dispatch remain available. Restoring the Kin database and restarting restores webhook dispatch. -A `MemorySyncConflict` fails the wake-up and leaves the events queued with the queue's five-minute counted-failure backoff. `MemoryCheckoutInvalid` does the same and logs an error; `KinWakeupHooks.memoryCheckoutInvalid` is the typed notification hook for LEAD-3. Writer commits/journal/reset and budget charging remain LEAD-3 and LEAD-10 integrations, not fabricated runner implementations. Until the writer lands, `memory_commit` remains null rather than claiming a writer commit. `tokens_json` contains each provider turn's cumulative `turnTokenUsage` ledger and fallback `usedTokens` and provider-turn id, limited to the wake-up run's attempts, on success and failure. +A `MemorySyncConflict` fails the wake-up and leaves the events queued with the queue's five-minute counted-failure backoff. `MemoryCheckoutInvalid` does the same and logs an error; `KinWakeupHooks.memoryCheckoutInvalid` is the typed notification hook for LEAD-3. Writer commits/journal/reset and budget charging remain LEAD-3 and LEAD-10 integrations, not fabricated runner implementations. The runner records the writer's commit in `memory_commit` after each wake-up's commit. `tokens_json` contains each provider turn's cumulative `turnTokenUsage` ledger and fallback `usedTokens` and provider-turn id, limited to the wake-up run's attempts, on success and failure. ### The prompt and caching (gap 2) @@ -467,7 +467,7 @@ Front matter matches the kin-memory README, including snake_case keys. Date-only LEAD-2: the front matter parser must use YAML’s core or JSON schema; YAML 1.1 turns `review_by: 2026-11-13` into a Date. -The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and `memory-read` (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K011 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them is rolled back to the recorded anchor with a compare-and-swap and an operator notice. Prepare and commit share a per-org lock. Pinned reads and writer operations skip re-bootstrap; writer operations still validate the writer checkout every time, and reads validate when they set a new pin. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks and grafts; merge plumbing uses verified bytes and a private index, never git merge-tree or a writer work tree, with system/global config disabled and hooks/fsmonitor disabled. +The production `MemoryConfig` layer derives its server-owned base from `ServerConfig.stateDir`, using `${stateDir}/kin//memory` and the `memory-read-` checkouts beside it (organization ids are path-encoded). Tests may supply an isolated base and relative paths. Paths must not overlap or escape that base. `ensureCheckouts` bootstraps a missing writer from a fresh fetch of the configured URL and configured default branch (`main` by default). K011 stores `(org_id, branch, head)` in `kin_memory_heads`; no writable file or local remote ref can bootstrap an existing checkout. Before committing or pushing, each local branch tip must match its anchor. Compare-and-swap branch updates and SQLite recording happen together in the writer operation; a crash between them is rolled back to the recorded anchor with a compare-and-swap and an operator notice. Prepare and commit share a per-org lock. Pinned reads and writer operations skip re-bootstrap; writer operations still validate the writer checkout every time, and reads validate when they set a new pin. Read clones are replaced from scratch on prepare: git is never run in an existing lead read clone. Every git invocation uses a private git directory excluding repository config, hooks and grafts; merge plumbing uses verified bytes and a private index, never git merge-tree or a writer work tree, with system/global config disabled and hooks/fsmonitor disabled. **Threat model.** Leads currently run full-access as the same OS user (`kin`) as the server. Assume leads can modify everything under `${stateDir}/kin/*/memory*`, including `.git` and file anchors. Code trusts only SQLite and server configuration, checks recorded branch tips, and builds commits from server-validated bytes rather than ambient working-tree/index content. These are not an OS sandbox: a same-user lead can still attack server processes or SQLite itself. Separating leads into an OS user unable to write `${stateDir}/kin` remains Nishchay's operations decision and is required before production use. @@ -532,7 +532,7 @@ T3 runs on the owner's subscriptions, so budget is counted in tokens, not money. - `units = (inputTokens − cachedInputTokens) + 0.1 × cachedInputTokens + 4 × outputTokens`, summed over the run's provider turns' cumulative `turnTokenUsage`, including `otherUsage` at input/output weights. Incomplete or missing field counts fall back to `usedTokens` at weight 1 and are logged; absent usage is recorded and logged. - `day` is the owner-local `YYYY-MM-DD`. Units are finite non-negative real numbers because cached input has fractional weight; limits are non-negative integers. Lasting limits live in `kin_budget_limits (org_id PRIMARY KEY, limits_json)`, where `limits_json` is `{ leadUnits, workerUnits }`; day rows hold counters only. `exhausted_notified_at` records the once-per-day exhaustion notice. - Defaults: 2,000,000 lead units and 6,000,000 worker units per day, reset at 00:00 in the owner's timezone from `KinTriggerSettings.timeZone` (triggers spec). The owner sets them in settings. -- `admit(orgId, priority)` by share of the lead budget left: above 40% everything runs; 20–40% defers P3 to tomorrow; under 20% only P0–P1; at 0 only P0, on the cheap model, and the owner is told once. Dreaming (P4) runs only above 30%. +- `admit(orgId, priority)` by share of the lead budget left: above 40% everything runs; 20–40% defers P3 to tomorrow; under 20% only P0–P1; at 0 only P0, on the cheap model, and the owner is told once. Dreaming (P4) runs only above 30%. Admission does not reserve units, so N leads waking at once can overshoot the daily limit by up to N wake-ups before the first charge lands. - Deferred P2/P3 events wait until the next owner-local reset unless limits change. - A wake-up spanning midnight bills the owner-local day on which the charge is made. - `admitWorker(orgId)` refuses at 0 worker units left. The lead gets an error and tells the owner. LEAD-9 owns charging `worker_units` from worker runs (`KinBudget.chargeWorker`, keyed by run in `kin_worker_charges`); it must not also count the lead turn's Claude `otherUsage`, which is already lead units. @@ -722,7 +722,7 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record - The unit weights (0.1 cached, 4 output) and the default limits are guesses. Set them after one pilot week. - Which privileged kinds besides merge may an L3 lead request (deploy, release)? This spec allows only low-risk merges. - Task ids for dependencies in the chat, approvals and triggers specs are named by subject here. Swap in the real ids once those specs are written. -- The lead's thread shows in T3's thread list under the "Kin · " project. Hide it, or leave it as the lead's readable log? +- The lead's thread shows in T3's thread list under its "Kin · · " project. Hide it, or leave it as the lead's readable log? ### LEAD-3 integration boundaries @@ -730,7 +730,7 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record `KinSecretScan` is deliberately supplied separately. `secretScan.layer` supplies the TRIG-16 scanner and reads the owner-signed allowlist; there is no no-findings production stub. Any finding or scanner failure blocks the whole commit. TRIG-16 must preserve this fail-closed interface: `scan({ orgId, files: ReadonlyArray<{ path: string; content: Uint8Array }> })` returns `{ path, ruleId, line }[]`, with typed `KinSecretScanError`. -`commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. K011 adds the SQLite branch anchors required by the writer. The runner serializes prepare/stage/commit and dreaming per organization. Identical commit retries find the existing wake-up commit; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. +`commitWakeup` builds TRIG-12's sectioned `INDEX.md` from the hash-verified recorded tree plus staged files, read history, the server clock and the owner's time zone, in the same wake-up commit. The read-side rebuild is display-only; no read-checkout text is trusted. K011 adds the SQLite branch anchors required by the writer. Prepare, stage and commit hold the per-organization writer lock one call at a time, so leads' wake-ups interleave between calls. A staged change keeps the branch head it was validated at; when another commit landed since, `commitWakeup` validates every change again at the new head, supersession and duplicate ids included, and refuses with `MemorySyncConflict` rather than overwrite. Retries and recovery find the wake-up's existing commit by its subject (`kin(): wake-up `) in history since the wake-up started, not at the branch head, which another lead may have moved; failed pushes retain local commits, and failures discard uncommitted files. Journal additions use `appendJournal`, never replacement through `stage`. ### Memory writer operations @@ -740,7 +740,7 @@ LEAD-14. **End-to-end shadow week dry run.** Script that replays a day of record Operators or owners can call server-only `Memory.recover(orgId)` after losing a writer or to reconcile its anchors. It fetches remote main and shadow heads, keeps and pushes an anchored shadow head the remote lacks or that fast-forwards the remote's (a refused push leaves the lead's only copy here), preserves other surviving unpushed anchored work on a conflict branch, recreates a missing writer, records remote anchors and notifies. This method must never be registered as an agent tool. For writers predating K011 anchors, stop the runner and delete only that organization's writer directory, then bootstrap again. Do not delete anchored writers to recover unpushed work; call `recover` first. Recovery cannot restore commits lost with a deleted writer before they reached any remote. -The read checkout is refilled in place, so a provider process's cwd survives `prepareWakeup`. A cold wake-up starts fresh context; a warm direct-chat wake-up continues in the lead's idle process, and only a new checkout directory (a first fill, or after a purge) replaces that process with one that resumes the conversation. +Each prepare swaps in a new read checkout, so no provider process outlives it in its cwd. A cold wake-up starts fresh context; a warm direct-chat wake-up releases the lead's idle process and resumes the conversation in a new one. **Running on the Kin host.** System, global and repository git config are excluded, so memory git never sees the host's credentials. `MemoryConfig.layer` reads `KIN_MEMORY_CREDENTIAL_HELPER` (a helper name or absolute path, for HTTPS) and `KIN_MEMORY_SSH_COMMAND` (an ssh command naming its key file, for SSH) and every memory clone, fetch and push uses them; a value that fails `purgePush.ts`'s reference checks fails Kin startup, naming the variable but not the value. They go to memory git only, as `-c credential.helper` and `GIT_SSH_COMMAND`; a server-wide `GIT_SSH_COMMAND` would reach every agent's git. Repository credential helpers cannot override them. Configure the Kin host's dedicated server user and credentials before enabling writes; never put credentials in memory, repository config, or lead instructions. See also “Running on the Kin host” in `triggers.md`. diff --git a/docs/operations/kin-memory-purge.md b/docs/operations/kin-memory-purge.md index e42ff8ae18d86..2f957a27d1a96 100644 --- a/docs/operations/kin-memory-purge.md +++ b/docs/operations/kin-memory-purge.md @@ -14,7 +14,7 @@ The marker and the `purge-plans/*.tombstones.json` files live beside the lead's Delete the marker by hand only when `reconcile` keeps answering `unknown` and you have checked the remote yourself. First stop the T3 server, or confirm that no purge or `reconcile` for the org is running: both hold the memory writer lock and write these paths. Deleting the marker tells Kin that no local copy of the old history will be pushed again, so first delete everything under `` that can hold it: -- `memory` and `memory-read` (the writer and read checkouts); +- `memory`, `memory-read` and each lead's `memory-read-` (the writer and read checkouts); - `dreams` (dream scratch copies); - any `memory.clone-*` (a clone a crash left half-finished); - `purge-scratch` and any `purge-scratch/mirror-*` inside it (the purge's rewritten bare clone, which also holds old objects). From 9f8806919601cbd11d2ad803af615ebf88a2d4b8 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:05:30 +0530 Subject: [PATCH 6/6] fix(kin): replay staged memory at commit and bound the memory-check wait An identical retry re-staged a change over the lead's own stale overlay and moved its validation head forward, so its commit skipped revalidation and overwrote another lead's file. Every commit now replays all staged changes from a clean state at its own head. The memory PR poll could hold the writer lock for hours through slow GitHub calls. One 15-minute deadline (the memory-check job timeout) now covers the whole wait, replacing the poll count. Co-Authored-By: Claude Opus 5.5 --- .../src/kin/memory/MemoryPullRequests.test.ts | 26 +++-- .../src/kin/memory/MemoryPullRequests.ts | 94 ++++++++++--------- .../src/kin/memory/MemoryWriter.test.ts | 3 + apps/server/src/kin/memory/MemoryWriter.ts | 17 +--- 4 files changed, 79 insertions(+), 61 deletions(-) diff --git a/apps/server/src/kin/memory/MemoryPullRequests.test.ts b/apps/server/src/kin/memory/MemoryPullRequests.test.ts index 44dc0a0daa023..c7372bdaa95d2 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.test.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.test.ts @@ -276,10 +276,10 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => const ids = jobs[1] !== "10" ? [200] : scenario === "rerun" ? [100, 101] : [100]; // A re-run's earlier attempts are listed only with `filter=all`. const listed = query.get("filter") === "all" ? ids : ids.slice(-1); - return { - status: 200, - body: { total_count: listed.length, jobs: yield* Effect.forEach(listed, job) }, - }; + const answered = yield* Effect.forEach(listed, job); + // The poll's git reads are done, so moving the test clock times out no subprocess. + if (pending) yield* Deferred.succeed(silenced, undefined); + return { status: 200, body: { total_count: listed.length, jobs: answered } }; } const checks = /^\/commits\/([a-f0-9]+)\/check-runs\?(.*)$/.exec(path); if (checks) { @@ -344,7 +344,7 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => ); const pullRequests = Layer.effect( PullRequests.MemoryPullRequests, - PullRequests.make({ pollInterval: 0, polls: 2 }), + PullRequests.make({ pollInterval: scenario === "check-pending" ? "20 minutes" : 0 }), ).pipe(Layer.provide(Layer.mergeAll(app, github.layer))); const notices: Array<{ branch: string; reason: string }> = []; @@ -497,7 +497,21 @@ it.effect.each(scenarios)("memory writes through PRs: %s", (scenario) => scenario !== "records-broken" ) { const foreign = "did not come from .github/workflows/memory-check.yml"; - const error = yield* refused(committing); + const error = yield* refused( + scenario === "check-pending" + ? Effect.gen(function* () { + const running = yield* Effect.forkChild(committing); + yield* Deferred.await(silenced); + // The 15-minute deadline ends the wait; the second poll is 5 minutes later. + yield* TestClock.adjust("15 minutes"); + return yield* Fiber.join(running); + }) + : committing, + ); + if (scenario === "check-pending") + expect( + github.requests.filter((request) => request.path.includes("/check-runs?")), + ).toHaveLength(1); yield* unchanged; expect(merges.length).toBe(scenario === "merge-rejected" ? 1 : 0); expect(pulls.get(1)?.state).toBe("closed"); diff --git a/apps/server/src/kin/memory/MemoryPullRequests.ts b/apps/server/src/kin/memory/MemoryPullRequests.ts index fa35ed0da2e20..9e43dad47c104 100644 --- a/apps/server/src/kin/memory/MemoryPullRequests.ts +++ b/apps/server/src/kin/memory/MemoryPullRequests.ts @@ -208,6 +208,8 @@ const PAGE = 100; const MAX_RUNS = 10; // Every call runs under the writer lock; GitHub's own edge gives up after about 10 seconds. const CALL_TIMEOUT: Duration.Input = "30 seconds"; +/** `memory-check.yml`'s job timeout: past it the check can't pass. */ +const CHECK_DEADLINE: Duration.Input = "15 minutes"; /** A full page, or a `next` link, may hide more; lists that decide anything fail closed on it. */ const pastOnePage = (answer: GitHubAnswer, length: number) => @@ -215,7 +217,6 @@ const pastOnePage = (answer: GitHubAnswer, length: number) => export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { readonly pollInterval: Duration.Input; - readonly polls: number; }) { const app = yield* KinGitHubApp; const client = yield* HttpClient.HttpClient; @@ -380,44 +381,54 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { } yield* refuseUnbound; - let verdict: Verdict = { _tag: "pending" }; - for (let poll = 0; poll < options.polls && verdict._tag === "pending"; poll++) { - if (poll > 0) yield* Effect.sleep(options.pollInterval); - // Check runs first: a real job listed there is already listed by its run. - const checks = yield* call( - "checks.list", - "GET", - `/commits/${input.sha}/check-runs?check_name=${MEMORY_CHECK_NAME}&filter=all&per_page=${PAGE}`, - ); - const runs = yield* call( - "actions.runs", - "GET", - `/actions/runs?head_sha=${input.sha}&event=pull_request&per_page=${PAGE}`, - ); - const decodedChecks = checks.status === 200 ? decodeChecks(checks.body) : Option.none(); - const decodedRuns = runs.status === 200 ? decodeRuns(runs.body) : Option.none(); - if (Option.isNone(decodedChecks) || Option.isNone(decodedRuns)) continue; - if (decodedRuns.value.total_count > decodedRuns.value.workflow_runs.length) { - verdict = truncated("workflow runs"); - break; - } - const ours = decodedRuns.value.workflow_runs.filter((run) => isOurRun(input, number, run)); - if (ours.length > MAX_RUNS) { - verdict = truncated("memory-check runs"); - break; - } - const jobs = yield* Effect.forEach(ours, (run) => - call( - "actions.jobs", + const awaitCheck = Effect.gen(function* () { + let verdict: Verdict = { _tag: "pending" }; + for (let poll = 0; verdict._tag === "pending"; poll++) { + if (poll > 0) yield* Effect.sleep(options.pollInterval); + // Check runs first: a real job listed there is already listed by its run. + const checks = yield* call( + "checks.list", "GET", - `/actions/runs/${run.id}/jobs?filter=all&per_page=${PAGE}`, - ).pipe( - Effect.map((answer) => (answer.status === 200 ? decodeJobs(answer.body) : Option.none())), - ), - ); - if (jobs.some(Option.isNone)) continue; - verdict = memoryCheckVerdict(input.sha, decodedChecks.value, jobs.flatMap(Option.toArray)); - } + `/commits/${input.sha}/check-runs?check_name=${MEMORY_CHECK_NAME}&filter=all&per_page=${PAGE}`, + ); + const runs = yield* call( + "actions.runs", + "GET", + `/actions/runs?head_sha=${input.sha}&event=pull_request&per_page=${PAGE}`, + ); + const decodedChecks = checks.status === 200 ? decodeChecks(checks.body) : Option.none(); + const decodedRuns = runs.status === 200 ? decodeRuns(runs.body) : Option.none(); + if (Option.isNone(decodedChecks) || Option.isNone(decodedRuns)) continue; + if (decodedRuns.value.total_count > decodedRuns.value.workflow_runs.length) { + verdict = truncated("workflow runs"); + break; + } + const ours = decodedRuns.value.workflow_runs.filter((run) => isOurRun(input, number, run)); + if (ours.length > MAX_RUNS) { + verdict = truncated("memory-check runs"); + break; + } + const jobs = yield* Effect.forEach(ours, (run) => + call( + "actions.jobs", + "GET", + `/actions/runs/${run.id}/jobs?filter=all&per_page=${PAGE}`, + ).pipe( + Effect.map((answer) => + answer.status === 200 ? decodeJobs(answer.body) : Option.none(), + ), + ), + ); + if (jobs.some(Option.isNone)) continue; + verdict = memoryCheckVerdict(input.sha, decodedChecks.value, jobs.flatMap(Option.toArray)); + } + return verdict; + }); + // The writer lock is held while this waits, so the deadline bounds slow calls too. + const verdict = Option.getOrElse( + yield* awaitCheck.pipe(Effect.timeoutOption(CHECK_DEADLINE)), + (): Verdict => ({ _tag: "pending" }), + ); if (verdict._tag === "pending") return yield* refuseAndClose(`${MEMORY_CHECK_NAME} did not finish in time`); if (verdict._tag === "fail") return yield* refuseAndClose(verdict.reason); @@ -537,8 +548,5 @@ export const make = Effect.fn("MemoryPullRequests.make")(function* (options: { return MemoryPullRequests.of({ land, sweep }); }); -/** Polls `memory-check` every 15 seconds for up to the job's 15-minute timeout. */ -export const layer = Layer.effect( - MemoryPullRequests, - make({ pollInterval: Duration.seconds(15), polls: 60 }), -); +/** Polls `memory-check` every 15 seconds until the job's 15-minute timeout. */ +export const layer = Layer.effect(MemoryPullRequests, make({ pollInterval: Duration.seconds(15) })); diff --git a/apps/server/src/kin/memory/MemoryWriter.test.ts b/apps/server/src/kin/memory/MemoryWriter.test.ts index 7e13ea71bb1db..b6bcf84b2775f 100644 --- a/apps/server/src/kin/memory/MemoryWriter.test.ts +++ b/apps/server/src/kin/memory/MemoryWriter.test.ts @@ -433,6 +433,9 @@ it.live.each([ yield* writer.stage(orgId, "wake1", first); yield* writer.stage(orgId, "wake2", second); const committed = yield* writer.commitWakeup(orgId, "wake2", "completed"); + // An identical retry at the new head must not hide the other lead's version. + if (scenario === "two-leads-same-path") + yield* writer.stage(orgId, "wake1", first).pipe(Effect.result); // The first lead validated against the old head: at this one its write conflicts, so // its commit refuses rather than erasing the other lead's or duplicating its id. const result = yield* writer.commitWakeup(orgId, "wake1", "completed").pipe(Effect.result); diff --git a/apps/server/src/kin/memory/MemoryWriter.ts b/apps/server/src/kin/memory/MemoryWriter.ts index 2bf22723bbabf..c8c46d99332ac 100644 --- a/apps/server/src/kin/memory/MemoryWriter.ts +++ b/apps/server/src/kin/memory/MemoryWriter.ts @@ -294,8 +294,8 @@ const make = Effect.gen(function* () { const locked = memory.withWriterLock; // Staged changes are per wake-up, so concurrent leads never share a session; the // git commit and push themselves stay serialized per repository under `locked`. - // `changes` and `heads` keep each staged change and the branch head it was validated at, - // so a commit after another lead's re-validates them instead of laying stale files over it. + // `changes` keeps each staged change so the commit replays them all against its own head + // instead of laying files validated at an older one over another lead's commit. type SessionState = { wakeupId: string; generation: number; @@ -303,7 +303,6 @@ const make = Effect.gen(function* () { lines: string[]; supersessions: Map; changes: Map; - heads: Map; }; const sessions = new Map(); const sessionKey = (orgId: KinOrgId, wakeupId: string) => `${orgId}\u0000${wakeupId}`; @@ -457,7 +456,6 @@ const make = Effect.gen(function* () { lines: [], supersessions: new Map(), changes: new Map(), - heads: new Map(), }; if (!previous && ctx.stagedReceipts) { const { writer } = yield* validate(orgId); @@ -1040,7 +1038,6 @@ const make = Effect.gen(function* () { state.files.set(change.path, text); if (supersessionHash !== undefined) state.supersessions.set(change.path, supersessionHash); state.changes.set(change.path, change); - state.heads.set(change.path, recorded); }); return MemoryWriter.of({ stage: Effect.fn("MemoryWriter.stage")((orgId, wakeupId, change) => @@ -1160,17 +1157,13 @@ const make = Effect.gen(function* () { return { commit: own }; } if (outcome === "failed") state.files.clear(); - // Staged files are whole files: after another lead's commit, every change is - // validated again at this head, supersession and duplicate ids included. - if ( - outcome !== "failed" && - [...state.heads.values()].some((head) => head !== recorded) - ) { + // Staged files are whole files, validated against whatever head and overlay they + // were staged on, so every change is replayed from a clean state at this head. + if (outcome !== "failed" && state.changes.size > 0) { const original = [...state.changes.values()]; state.files.clear(); state.supersessions.clear(); state.changes.clear(); - state.heads.clear(); for (const change of original) yield* stageChange(orgId, wakeupId, ctx, state, change).pipe( Effect.mapError((error) =>