From 5d5307db7194f854d31daf2643c9d70c2842d7d6 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:36:19 +0530 Subject: [PATCH 01/12] feat(kin): add lead wake-up runner (LEAD-5) --- .../src/kin/chat/KinChatRepository.test.ts | 15 +- apps/server/src/kin/chat/KinChatRepository.ts | 7 + .../server/src/kin/lead/KinEventQueue.test.ts | 32 +- apps/server/src/kin/lead/KinEventQueue.ts | 181 ++++--- apps/server/src/kin/lead/KinLeadRegistry.ts | 51 ++ apps/server/src/kin/lead/LeadPrompt.ts | 29 + apps/server/src/kin/lead/WakeupRunner.test.ts | 454 ++++++++++++++++ .../src/kin/lead/WakeupRunner.testkit.ts | 259 +++++++++ apps/server/src/kin/lead/WakeupRunner.ts | 504 ++++++++++++++++++ apps/server/src/kin/lead/layer.ts | 85 +++ apps/server/src/kin/memory/Memory.test.ts | 4 +- apps/server/src/kin/memory/Memory.ts | 13 +- .../src/kin/triggers/KinTriggerSink.test.ts | 67 +++ .../server/src/kin/triggers/KinTriggerSink.ts | 51 ++ apps/server/src/server.ts | 2 + docs/kin/specs/chat.md | 4 +- docs/kin/specs/lead.md | 14 +- 17 files changed, 1694 insertions(+), 78 deletions(-) create mode 100644 apps/server/src/kin/lead/KinLeadRegistry.ts create mode 100644 apps/server/src/kin/lead/LeadPrompt.ts create mode 100644 apps/server/src/kin/lead/WakeupRunner.test.ts create mode 100644 apps/server/src/kin/lead/WakeupRunner.testkit.ts create mode 100644 apps/server/src/kin/lead/WakeupRunner.ts create mode 100644 apps/server/src/kin/lead/layer.ts diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index f6b81bae6a600..028cebc444417 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -205,6 +205,20 @@ describe("KinChatRepository", () => { }), { orgId, agentId, threadId, startedAt: timestamp, endedAt: null }, ); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_webhook_sources (source_id, org_id, repo, scheduled_task_id, inbox_thread_id, bot_login, status) VALUES ('source', ${orgId}, 'owner/repo', 'task', 'webhook-inbox', 'bot', 'active')`; + const inboxError = yield* repo + .registerAgentThread({ + orgId, + agentId, + threadId: decodeThreadId("webhook-inbox"), + startedAt: timestamp, + endedAt: null, + }) + .pipe(Effect.flip); + assert.strictEqual(inboxError._tag, "KinChatError"); + if (inboxError._tag === "KinChatError") + assert.strictEqual(inboxError.reason, "not_allowed"); const secondPerson = decodePersonId("second-person"); yield* repo.insertPerson({ orgId, @@ -432,7 +446,6 @@ describe("KinChatRepository", () => { assert.strictEqual(endedRetry._tag, "KinChatError"); if (endedRetry._tag === "KinChatError") assert.strictEqual(endedRetry.reason, "not_allowed"); - const sql = yield* SqlClient.SqlClient; assert.deepStrictEqual( yield* sql`SELECT typeof(seq) AS type FROM kin_messages WHERE org_id = ${orgId}`, [{ type: "integer" }, { type: "integer" }], diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index 63443d4c2e517..a4e021ee0c8e2 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -700,6 +700,13 @@ const make = Effect.gen(function* () { reason: "not_allowed", message: "Retired Kin agents cannot register threads.", }); + const inbox = + yield* sql`SELECT 1 FROM kin_webhook_sources WHERE inbox_thread_id = ${input.threadId}`; + if (inbox.length > 0) + return yield* new KinChatError({ + reason: "not_allowed", + message: "Webhook inbox threads cannot be Kin agents.", + }); const existing = yield* SqlSchema.findOneOption({ Request: Schema.Unknown, Result: AgentThreadRow, diff --git a/apps/server/src/kin/lead/KinEventQueue.test.ts b/apps/server/src/kin/lead/KinEventQueue.test.ts index 0c4f25ca81355..79eff4375b21c 100644 --- a/apps/server/src/kin/lead/KinEventQueue.test.ts +++ b/apps/server/src/kin/lead/KinEventQueue.test.ts @@ -108,6 +108,12 @@ describe("KinEventQueue", () => { yield* seed(); const queue = yield* KinEventQueue.KinEventQueue; const sql = yield* SqlClient.SqlClient; + for (const agentId of ["unknown", "foreign-lead"]) { + const error = yield* queue + .enqueue({ ...(yield* event("missing")), agentId: KinAgentId.make(agentId) }) + .pipe(Effect.flip); + assert.strictEqual(error._tag, "KinEventQueueUnknownAgent"); + } yield* queue.enqueue({ ...(yield* event("first")), receivedAt: DateTime.makeUnsafe(-600_000), @@ -123,6 +129,11 @@ describe("KinEventQueue", () => { ["first", "second"], ); yield* finish("first-batch"); + yield* queue.enqueue(yield* event("colliding")); + yield* TestClock.adjust("20 seconds"); + const collision = yield* Effect.scoped(queue.claim(input("first-batch"))).pipe(Effect.flip); + assert.strictEqual(collision._tag, "KinEventQueueWakeupConflict"); + yield* sql`UPDATE kin_events SET state = 'expired' WHERE event_id = 'colliding'`; yield* queue.enqueue(yield* event("later")); const second = yield* Effect.scoped(queue.claim(input("second-batch"))).pipe( Effect.forkChild, @@ -272,7 +283,10 @@ describe("KinEventQueue", () => { }, ]) { const conflict = yield* queue.enqueue(changed).pipe(Effect.flip); - assert.instanceOf(conflict, KinEventQueue.KinEventQueueConflict); + assert.strictEqual( + conflict._tag, + changed.agentId === "lead-b" ? "KinEventQueueUnknownAgent" : "KinEventQueueConflict", + ); } for (const redelivery of [ { @@ -397,20 +411,20 @@ describe("KinEventQueue", () => { assert.strictEqual(rejected._tag, "PersistenceDecodeError"); } assert.strictEqual( - yield* queue.release({ + (yield* queue.release({ ...input("admission", "b"), countAttempt: false, notBefore: 60_000, - }), + })).released, 0, ); assert.strictEqual( - yield* queue.release({ + (yield* queue.release({ ...input("admission"), eventIds: [KinEventId.make("deferred")], notBefore: 60_000, countAttempt: false, - }), + })).released, 1, ); assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY wakeup_id`, [ @@ -468,8 +482,12 @@ describe("KinEventQueue", () => { mode === "release" ? queue.release({ ...claimInput, countAttempt: true }) : queue.finish({ ...claimInput, status: mode }); - yield* failed; - yield* failed; + const terminal = yield* failed; + assert.deepStrictEqual( + terminal.failedEventIds, + attempt === 3 ? [KinEventId.make(id)] : [], + ); + assert.deepStrictEqual((yield* failed).failedEventIds, []); const now = yield* Clock.currentTimeMillis; assert.deepStrictEqual( yield* sql`SELECT state, attempts, not_before FROM kin_events WHERE event_id = ${id}`, diff --git a/apps/server/src/kin/lead/KinEventQueue.ts b/apps/server/src/kin/lead/KinEventQueue.ts index 259e9c3b5c043..fa303b416eb09 100644 --- a/apps/server/src/kin/lead/KinEventQueue.ts +++ b/apps/server/src/kin/lead/KinEventQueue.ts @@ -74,10 +74,20 @@ export class KinEventQueueUnknownEvent extends Schema.TaggedError()( + "KinEventQueueUnknownAgent", + { orgId: KinOrgId, agentId: KinAgentId }, +) {} +export class KinEventQueueWakeupConflict extends Schema.TaggedError()( + "KinEventQueueWakeupConflict", + { orgId: KinOrgId, wakeupId: KinWakeupId }, +) {} export class KinEventQueueBusy extends Schema.TaggedError()( "KinEventQueueBusy", { orgId: KinOrgId }, ) {} +const isUnknownAgent = Schema.is(KinEventQueueUnknownAgent); +const isWakeupConflict = Schema.is(KinEventQueueWakeupConflict); const isConflict = Schema.is(KinEventQueueConflict); const isUnknownEvent = Schema.is(KinEventQueueUnknownEvent); const isBusy = Schema.is(KinEventQueueBusy); @@ -101,21 +111,31 @@ export class KinEventQueue extends Context.Service< | PersistenceDecodeError | KinEventQueueConflict | KinEventQueueUnknownEvent + | KinEventQueueUnknownAgent >; // Scope closure releases only the permit; LEAD-5 must finish(timed_out | failed) in its own onExit. readonly claim: ( input: ClaimInput, ) => Effect.Effect< ReadonlyArray, - PersistenceSqlError | PersistenceDecodeError | KinEventQueueBusy, + | PersistenceSqlError + | PersistenceDecodeError + | KinEventQueueBusy + | KinEventQueueWakeupConflict, Scope.Scope >; readonly release: ( input: typeof ReleaseInput.Type, - ) => Effect.Effect; + ) => Effect.Effect< + { readonly released: number; readonly failedEventIds: ReadonlyArray }, + PersistenceSqlError | PersistenceDecodeError + >; readonly finish: ( input: typeof FinishInput.Type, - ) => Effect.Effect; + ) => Effect.Effect< + { readonly failedEventIds: ReadonlyArray }, + PersistenceSqlError | PersistenceDecodeError + >; // LEAD-5 startup recovery calls this only after confirming the provider run stopped. readonly requeueUnfinished: ( orgId: KinOrgId, @@ -157,24 +177,33 @@ const make = Effect.gen(function* () { ) { return yield* new KinEventQueueConflict({ orgId: input.orgId, eventId: input.eventId }); } - const event = input._tag === "github" ? { ...input, dedupeKey: input.deliveryId } : input; - let priority: "P0" | "P1" | "P2" | "P3" = "P2"; - switch (event._tag) { - case "resume": - priority = "P0"; - break; - case "deferral.due": - case "action.resolved": - case "worker.report": - priority = "P1"; - break; - case "watchdog": - priority = "P3"; - break; - case "chat.message": - case "widget.action": - if (event.provenance._tag === "human") { - const owners = yield* sql` + return yield* sql.withTransaction( + Effect.gen(function* () { + const agents = + yield* sql`SELECT 1 FROM kin_leads WHERE org_id = ${input.orgId} AND agent_id = ${input.agentId}`; + if (agents.length === 0) + return yield* new KinEventQueueUnknownAgent({ + orgId: input.orgId, + agentId: input.agentId, + }); + const event = input._tag === "github" ? { ...input, dedupeKey: input.deliveryId } : input; + let priority: "P0" | "P1" | "P2" | "P3" = "P2"; + switch (event._tag) { + case "resume": + priority = "P0"; + break; + case "deferral.due": + case "action.resolved": + case "worker.report": + priority = "P1"; + break; + case "watchdog": + priority = "P3"; + break; + case "chat.message": + case "widget.action": + if (event.provenance._tag === "human") { + const owners = yield* sql` SELECT 1 FROM kin_chat_members WHERE org_id = ${event.orgId} AND chat_id = ${event.chatId} AND member_id = ${event.provenance.memberId} AND person_id = ${event.provenance.personId} @@ -186,46 +215,54 @@ const make = Effect.gen(function* () { WHERE org_id = ${event.orgId} AND chat_id = ${event.chatId} AND principal_kind = 'agent' AND agent_id = ${event.agentId} AND left_at IS NULL) `; - if (owners.length > 0) priority = "P0"; + if (owners.length > 0) priority = "P0"; + } + break; } - break; - } - const json = yield* encodeEvent(event); - const now = yield* Clock.currentTimeMillis; - const inserted = yield* sql` + const json = yield* encodeEvent(event); + const now = yield* Clock.currentTimeMillis; + const inserted = yield* sql` INSERT INTO kin_events (event_id, org_id, agent_id, tag, priority, json, dedupe_key, received_at, queued_at) VALUES (${event.eventId}, ${event.orgId}, ${event.agentId}, ${event._tag}, ${priority}, ${json}, ${event.dedupeKey}, ${DateTime.toEpochMillis(event.receivedAt)}, ${now}) ON CONFLICT DO NOTHING RETURNING event_id `; - if (inserted.length > 0) { - yield* notify(event.orgId); - return { event, duplicate: false }; - } - const rows = yield* sql<{ json: string }>`SELECT json FROM kin_events + if (inserted.length > 0) { + yield* notify(event.orgId); + return { event, duplicate: false }; + } + const rows = yield* sql<{ json: string }>`SELECT json FROM kin_events WHERE org_id = ${event.orgId} AND (event_id = ${event.eventId} OR (dedupe_key IS NOT NULL AND dedupe_key = ${event.dedupeKey})) ORDER BY event_id`; - let stored: KinEvent | undefined; - for (const row of rows) { - const candidate = yield* decodeEvent(row.json); - if ( - candidate.eventId === event.eventId - ? (yield* encodeEvent(candidate)) !== json - : candidate.agentId !== event.agentId || candidate._tag !== event._tag - ) { - return yield* new KinEventQueueConflict({ orgId: event.orgId, eventId: event.eventId }); - } - stored = candidate; - } - if (stored === undefined) { - return yield* new KinEventQueueUnknownEvent({ orgId: event.orgId, eventId: event.eventId }); - } - return { event: stored, duplicate: true }; + let stored: KinEvent | undefined; + for (const row of rows) { + const candidate = yield* decodeEvent(row.json); + if ( + candidate.eventId === event.eventId + ? (yield* encodeEvent(candidate)) !== json + : candidate.agentId !== event.agentId || candidate._tag !== event._tag + ) { + return yield* new KinEventQueueConflict({ + orgId: event.orgId, + eventId: event.eventId, + }); + } + stored = candidate; + } + if (stored === undefined) { + return yield* new KinEventQueueUnknownEvent({ + orgId: event.orgId, + eventId: event.eventId, + }); + } + return { event: stored, duplicate: true }; + }), + ); }, Effect.mapError((cause) => - isConflict(cause) || isUnknownEvent(cause) + isConflict(cause) || isUnknownEvent(cause) || isUnknownAgent(cause) ? cause : toQueuePersistenceError("KinEventQueue.enqueue")(cause), ), @@ -301,6 +338,13 @@ const make = Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; if ((yield* readiness(input, now)).readyAt > now) return yield* new NoClaimableEvents({}); + const collision = + yield* sql`SELECT 1 FROM kin_wakeups WHERE wakeup_id = ${input.wakeupId}`; + if (collision.length > 0) + return yield* new KinEventQueueWakeupConflict({ + orgId: input.orgId, + wakeupId: input.wakeupId, + }); // Cross-process exclusion rests on idx_kin_wakeups_running_org. const started = yield* sql` INSERT INTO kin_wakeups (org_id, wakeup_id, agent_id, model, started_at, status, event_ids) @@ -349,20 +393,22 @@ const make = Effect.gen(function* () { ); }, Effect.mapError((cause) => - isBusy(cause) ? cause : toQueuePersistenceError("KinEventQueue.claim")(cause), + isBusy(cause) || isWakeupConflict(cause) + ? cause + : toQueuePersistenceError("KinEventQueue.claim")(cause), ), ); const release = Effect.fn("KinEventQueue.release")( function* (input: typeof ReleaseInput.Type) { yield* decodeReleaseInput(input); - const count = yield* sql.withTransaction( + const result = yield* sql.withTransaction( Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; const notBefore = input.notBefore ?? now + 300_000; const ids = input.eventIds === undefined ? undefined : yield* encodeEventIds(input.eventIds); - const rows = yield* sql` + const rows = yield* sql<{ event_id: string; state: string }>` UPDATE kin_events SET attempts = attempts + ${input.countAttempt ? 1 : 0}, state = CASE WHEN attempts + ${input.countAttempt ? 1 : 0} >= 3 THEN 'failed' ELSE 'queued' END, wakeup_id = NULL, not_before = ${notBefore} @@ -370,16 +416,22 @@ const make = Effect.gen(function* () { AND (${ids ?? null} IS NULL OR event_id IN (SELECT value FROM json_each(${ids ?? null}))) AND EXISTS (SELECT 1 FROM kin_wakeups WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND status = 'running') - RETURNING event_id`; + RETURNING event_id, state`; yield* sql`UPDATE kin_wakeups SET status = ${input.countAttempt ? "failed" : "released"}, ended_at = ${now} WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND status = 'running' AND NOT EXISTS (SELECT 1 FROM kin_events WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed')`; - return rows.length; + return { + released: rows.length, + failedEventIds: rows + .filter((row) => row.state === "failed") + .map((row) => KinEventId.make(row.event_id)) + .sort(), + }; }), ); - if (count > 0) yield* notify(input.orgId); - return count; + if (result.released > 0) yield* notify(input.orgId); + return result; }, Effect.mapError(toQueuePersistenceError("KinEventQueue.release")), Effect.uninterruptible, @@ -388,26 +440,35 @@ const make = Effect.gen(function* () { const finish = Effect.fn("KinEventQueue.finish")( function* (input: typeof FinishInput.Type) { yield* decodeFinishInput(input); - yield* sql.withTransaction( + const failedEventIds = yield* sql.withTransaction( Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; const closed = yield* sql`UPDATE kin_wakeups SET status = ${input.status}, ended_at = ${now} WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND status = 'running' RETURNING wakeup_id`; - if (closed.length === 0) return; + if (closed.length === 0) return []; if (input.status === "completed") { yield* sql`UPDATE kin_events SET state = 'done' WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed'`; } else { - yield* sql`UPDATE kin_events SET attempts = attempts + 1, + const rows = yield* sql<{ + event_id: string; + state: string; + }>`UPDATE kin_events SET attempts = attempts + 1, state = CASE WHEN attempts + 1 >= 3 THEN 'failed' ELSE 'queued' END, wakeup_id = NULL, not_before = ${now + 300_000} - WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed'`; + WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed' RETURNING event_id, state`; + return rows + .filter((row) => row.state === "failed") + .map((row) => KinEventId.make(row.event_id)) + .sort(); } + return []; }), ); yield* notify(input.orgId); + return { failedEventIds }; }, Effect.mapError(toQueuePersistenceError("KinEventQueue.finish")), Effect.uninterruptible, diff --git a/apps/server/src/kin/lead/KinLeadRegistry.ts b/apps/server/src/kin/lead/KinLeadRegistry.ts new file mode 100644 index 0000000000000..df066194e521a --- /dev/null +++ b/apps/server/src/kin/lead/KinLeadRegistry.ts @@ -0,0 +1,51 @@ +import { KinAgentId, KinLevel, KinOrgId, KinWakeupId, ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; + +const Caller = Schema.Struct({ + orgId: KinOrgId, + agentId: KinAgentId, + wakeupId: KinWakeupId, + level: KinLevel, +}); +export class KinLeadRegistryConflict extends Schema.TaggedError()( + "KinLeadRegistryConflict", + { threadId: ThreadId }, +) {} + +export class KinLeadRegistry extends Context.Service< + KinLeadRegistry, + { + readonly register: ( + threadId: ThreadId, + caller: typeof Caller.Type, + ) => Effect.Effect; + readonly resolve: (threadId: ThreadId) => Option.Option; + readonly isLeadCaller: (threadId: ThreadId) => boolean; + } +>()("t3/kin/lead/KinLeadRegistry") {} + +export const layer = Layer.sync(KinLeadRegistry, () => { + const callers = new Map(); + return KinLeadRegistry.of({ + register: Effect.fn("KinLeadRegistry.register")((threadId, caller) => + Effect.acquireRelease( + Effect.suspend(() => { + if (callers.has(threadId)) return Effect.fail(new KinLeadRegistryConflict({ threadId })); + callers.set(threadId, caller); + return Effect.void; + }), + () => + Effect.sync(() => { + callers.delete(threadId); + }), + ), + ), + resolve: (threadId) => Option.fromUndefinedOr(callers.get(threadId)), + isLeadCaller: (threadId) => callers.has(threadId), + }); +}); diff --git a/apps/server/src/kin/lead/LeadPrompt.ts b/apps/server/src/kin/lead/LeadPrompt.ts new file mode 100644 index 0000000000000..71e73840d3741 --- /dev/null +++ b/apps/server/src/kin/lead/LeadPrompt.ts @@ -0,0 +1,29 @@ +import { KinAgentId, KinEvent, KinOrgId, KinWakeupId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +export class KinLeadPromptError extends Schema.TaggedError()( + "KinLeadPromptError", + { + orgId: KinOrgId, + reason: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) {} +export interface KinLeadPromptInput { + readonly orgId: KinOrgId; + readonly agentId: KinAgentId; + readonly wakeupId: KinWakeupId; + readonly events: ReadonlyArray; + readonly memoryHead: string; +} +export class KinLeadPrompt extends Context.Service< + KinLeadPrompt, + { + readonly build: (input: KinLeadPromptInput) => Effect.Effect; + } +>()( + // @effect-diagnostics-next-line deterministicKeys:off -- LEAD-6 shares this exact dependency key. + "t3/kin/lead/LeadPrompt", +) {} diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts new file mode 100644 index 0000000000000..e93681eed8074 --- /dev/null +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -0,0 +1,454 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + CommandId, + MessageId, + KinWakeupId, + ThreadId, + RunId, + RunAttemptId, + ProviderTurnId, + ProjectId, + ProviderInstanceId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as KinIdentity from "../chat/KinIdentity.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryConfig from "../memory/MemoryConfig.ts"; +import * as VcsProcess from "../../vcs/VcsProcess.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as KinLead from "./layer.ts"; +import * as KinLeadRegistry from "./KinLeadRegistry.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as WakeupRunner from "./WakeupRunner.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; +import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; +import { harness, seed, orgId, agentId, event } from "./WakeupRunner.testkit.ts"; + +const decodeUsage = Schema.decodeEffect( + Schema.fromJsonString( + Schema.Array( + Schema.Struct({ + inputTokens: Schema.Number, + cachedInputTokens: Schema.Number, + outputTokens: Schema.Number, + }), + ), + ), +); +const run = Effect.fn("test.run")(function* () { + const runner = yield* WakeupRunner.WakeupRunner; + const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); + return fiber; +}); + +describe("WakeupRunner", () => { + it.effect( + "runs two wake-ups on one T3 thread with distinct provider threads and captures only each run's usage", + () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const registry = yield* KinLeadRegistry.KinLeadRegistry; + for (const id of ["first", "second"]) { + yield* queue.enqueue(event(id)); + yield* Fiber.join(yield* run()); + } + const rows = yield* sql<{ + thread_id: string; + tokens_json: string; + wakeup_id: string; + }>`SELECT thread_id, tokens_json, wakeup_id FROM kin_wakeups ORDER BY started_at`; + assert.strictEqual(rows.length, 2); + assert.strictEqual(rows[0]?.thread_id, rows[1]?.thread_id); + assert.match(rows[0]!.wakeup_id, /^[0-7][0-9A-HJKMNP-TV-Z]{25}$/); + const threads = yield* ThreadManagement.ThreadManagementService; + const projection = yield* threads.getThreadProjection(ThreadId.make(rows[0]!.thread_id)); + assert.strictEqual(projection.runs.length, 2); + assert.notEqual(projection.runs[0]?.providerThreadId, projection.runs[1]?.providerThreadId); + assert.strictEqual(projection.providerThreads.length, 2); + for (const row of rows) { + const usage = yield* decodeUsage(row.tokens_json); + assert.deepStrictEqual(usage, [ + { inputTokens: 20, cachedInputTokens: 5, outputTokens: 10 }, + ]); + } + assert.isFalse(registry.isLeadCaller(projection.thread.id)); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events ORDER BY event_id`, [ + { state: "done" }, + { state: "done" }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + it.effect( + "interrupts mid-wake-up, finishes failed, clears the registry and admits the next claim", + () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("interrupt")); + const fiber = yield* run(); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY wakeup_id`, [ + { status: "failed" }, + ]); + yield* queue.enqueue(event("next")); + yield* TestClock.adjust("20 seconds"); + const events = yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("next"), model: "test" }), + ); + assert.deepStrictEqual( + events.map((item) => item.eventId), + [event("next").eventId], + ); + yield* queue.finish({ orgId, wakeupId: KinWakeupId.make("next"), status: "completed" }); + assert.isDefined(runner); + }).pipe( + Effect.provide( + harness({ + prepare: () => + Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect("backs off a cross-process busy claim on TestClock instead of spinning", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("blocker")); + yield* TestClock.adjust("20 seconds"); + yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("blocker"), model: "test" }), + ); + yield* queue.enqueue(event("waiting")); + const fiber = yield* run(); + yield* queue.finish({ orgId, wakeupId: KinWakeupId.make("blocker"), status: "completed" }); + yield* TestClock.adjust("4 seconds"); + assert.isUndefined(fiber.pollUnsafe()); + assert.strictEqual((yield* sql`SELECT * FROM kin_wakeups`).length, 1); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(fiber); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events ORDER BY event_id`, [ + { state: "done" }, + { state: "done" }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect( + "times out and interrupts a held provider run without leaving the organization busy", + () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("timeout")); + const fiber = yield* run(); + yield* Deferred.await(entered); + yield* TestClock.adjust("15 minutes"); + yield* Fiber.join(fiber); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [ + { status: "timed_out" }, + ]); + yield* Deferred.succeed(hold, undefined); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + }).pipe(Effect.provide(harness({ entered, hold }))); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect( + "recovers abandoned and completed wake-ups once after restart without relaunching", + () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("abandoned")); + yield* TestClock.adjust("20 seconds"); + yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("abandoned"), model: "test" }), + ); + const missingThread = ThreadId.make("provisional-thread"); + yield* (yield* KinIdentity.KinIdentity).registerAgentThread({ + orgId, + agentId, + threadId: missingThread, + }); + yield* sql`UPDATE kin_leads SET thread_id = ${missingThread} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* sql`UPDATE kin_wakeups SET thread_id = ${missingThread} WHERE org_id = ${orgId} AND wakeup_id = 'abandoned'`; + yield* runner.recover; + assert.deepStrictEqual(yield* sql`SELECT thread_id FROM kin_leads`, [{ thread_id: null }]); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [ + { status: "interrupted" }, + ]); + yield* Fiber.join(yield* run()); + yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL WHERE status = 'completed'`; + yield* sql`UPDATE kin_events SET state = 'claimed' WHERE state = 'done'`; + const threads = yield* ThreadManagement.ThreadManagementService; + const recoveredRunner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + getThreadProjection: (threadId) => + threads.getThreadProjection(threadId).pipe( + Effect.map((projection) => { + const run = projection.runs[0]; + const attempt = projection.attempts[0]; + const turn = projection.providerTurns[0]; + assert.isDefined(run); + assert.isDefined(attempt); + assert.isDefined(turn); + const continuedRunId = RunId.make("restart-continuation"); + const continuedAttemptId = RunAttemptId.make("restart-attempt"); + return { + ...projection, + runs: [ + ...projection.runs, + { + ...run!, + id: continuedRunId, + ordinal: run!.ordinal + 1, + restartContinuationOfRunId: run!.id, + }, + ], + attempts: [ + ...projection.attempts, + { ...attempt!, id: continuedAttemptId, runId: continuedRunId }, + ], + providerTurns: [ + ...projection.providerTurns, + { + ...turn!, + id: ProviderTurnId.make("restart-turn"), + runAttemptId: continuedAttemptId, + }, + ], + }; + }), + ), + }), + ); + yield* recoveredRunner.recover; + yield* recoveredRunner.recover; + const [usageRow] = yield* sql<{ + tokens_json: string; + }>`SELECT tokens_json FROM kin_wakeups WHERE status = 'completed'`; + assert.strictEqual((yield* decodeUsage(usageRow!.tokens_json)).length, 2); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY status`, [ + { status: "completed" }, + { status: "interrupted" }, + ]); + const [row] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const projection = + yield* (yield* ThreadManagement.ThreadManagementService).getThreadProjection( + ThreadId.make(row!.thread_id), + ); + assert.strictEqual(projection.runs.length, 1); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "done", attempts: 0 }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect.each(["MemorySyncConflict", "MemoryCheckoutInvalid"] as const)( + "fails and backs off %s without launching a provider", + (tag) => + Effect.gen(function* () { + const error = + tag === "MemorySyncConflict" + ? new Memory.MemorySyncConflict({ orgId, operation: "sync", reason: "fixture" }) + : new Memory.MemoryCheckoutInvalid({ orgId, operation: "checkout", reason: "fixture" }); + let notified = 0; + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + yield* queue.enqueue(event("memory-failure")); + const result = yield* Fiber.await(yield* run()); + assert.strictEqual(result._tag, "Failure"); + const sql = yield* SqlClient.SqlClient; + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 1 }, + ]); + assert.deepStrictEqual(yield* sql`SELECT status, thread_id FROM kin_wakeups`, [ + { status: "failed", thread_id: null }, + ]); + assert.strictEqual(notified, tag === "MemoryCheckoutInvalid" ? 1 : 0); + }).pipe( + Effect.provide( + harness({ prepare: () => Effect.fail(error) }).pipe( + Layer.provide( + Layer.succeed(WakeupRunner.KinWakeupHooks, { + memoryCheckoutInvalid: () => + Effect.sync(() => { + notified++; + }), + }), + ), + ), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect("expires retired leads' queued events and ends their durable thread registration", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + yield* Fiber.join(yield* run()); + yield* queue.enqueue(event("retired")); + yield* sql`UPDATE kin_agents SET retired_at = '2026-11-08T00:00:00.000Z' WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* runner.runOnce(orgId, agentId); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'retired'`, [ + { state: "expired" }, + ]); + assert.strictEqual( + (yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL AND org_id = ${orgId}`) + .length, + 0, + ); + assert.strictEqual((yield* sql`SELECT 1 FROM kin_wakeups`).length, 1); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect( + "keeps a recovered active run claimed until it completes rather than requeueing it", + () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const wakeupId = KinWakeupId.make("recovery-active"); + yield* queue.enqueue(event("active")); + yield* TestClock.adjust("20 seconds"); + yield* Effect.scoped(queue.claim({ orgId, agentId, wakeupId, model: "test" })); + const launch = yield* ThreadLaunch.ThreadLaunchService; + const sent = yield* launch.launch({ + commandId: CommandId.make(`kin-wakeup:${wakeupId}`), + projectId: ProjectId.make("kin:org"), + title: "Lead", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5.1-codex", + }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { + messageId: MessageId.make(`kin-wakeup-msg:${wakeupId}`), + text: "Wake up", + attachments: [], + }, + createdBy: "system", + creationSource: "server", + }); + yield* sql`UPDATE kin_leads SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* sql`UPDATE kin_wakeups SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId}`; + yield* Deferred.await(entered); + const runner = yield* WakeupRunner.WakeupRunner; + const recovered = yield* runner.recover.pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events`, [{ state: "claimed" }]); + assert.isUndefined(recovered.pollUnsafe()); + yield* Deferred.succeed(hold, undefined); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + yield* Fiber.join(recovered); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [ + { status: "completed" }, + ]); + yield* runner.recover; + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events`, [{ state: "done" }]); + }).pipe(Effect.provide(harness({ entered, hold }))); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect( + "releases an organization paused during the quiet window without consuming an attempt", + () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("paused")); + const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* sql`INSERT INTO kin_org_state (org_id, paused, paused_at, paused_by) VALUES (${orgId}, 1, 1000, 'owner')`; + yield* TestClock.adjust("19 seconds"); + yield* Fiber.join(fiber); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 0 }, + ]); + assert.deepStrictEqual(yield* sql`SELECT status, thread_id FROM kin_wakeups`, [ + { status: "released", thread_id: null }, + ]); + yield* sql`UPDATE kin_org_state SET paused = 0, paused_at = NULL, paused_by = NULL WHERE org_id = ${orgId}`; + yield* Fiber.join(yield* run()); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events`, [{ state: "done" }]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect("contains Kin startup failures while building the non-Kin server runtime", () => { + class Core extends Context.Service()( + "t3/kin/lead/WakeupRunner.test/Core", + ) {} + return Effect.gen(function* () { + yield* seed; + const failing = KinLead.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadLaunch.ThreadLaunchService)({}), + Layer.mock(ThreadManagement.ThreadManagementService)({}), + Layer.mock(ProjectService.ProjectService)({}), + Layer.succeed(KinLeadPrompt, { build: () => Effect.succeed("unused") }), + Layer.succeed(MemoryConfig.MemoryConfig, { + baseDirectory: "/private/tmp", + repositories: [], + }), + Layer.mock(VcsProcess.VcsProcess)({}), + NodeServices.layer, + ), + ), + ); + assert.strictEqual((yield* Layer.build(failing).pipe(Effect.exit))._tag, "Failure"); + const runtime = Layer.mergeAll( + Layer.succeed(Core, { ready: true }), + KinLead.containStartup(failing), + ); + const context = yield* Layer.build(runtime); + assert.isTrue(Context.get(context, Core).ready); + }).pipe(Effect.provide(KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)))); + }); +}); diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts new file mode 100644 index 0000000000000..42c54597814f0 --- /dev/null +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -0,0 +1,259 @@ +import { + KinAgentId, + KinEvent, + KinOrgId, + ProjectId, + ProviderInstanceId, + ProviderDriverKind, + ProviderThreadId, + ProviderTurnId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; + +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as KinChatRepository from "../chat/KinChatRepository.ts"; +import * as KinIdentity from "../chat/KinIdentity.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as WakeupRunner from "./WakeupRunner.ts"; +import * as KinLeadRegistry from "./KinLeadRegistry.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; +import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as ProjectSetupScriptRunner from "../../project/ProjectSetupScriptRunner.ts"; +import * as ProjectCloneTracker from "../../project/ProjectCloneTracker.ts"; +import * as WorktreeSetupTracker from "../../project/WorktreeSetupTracker.ts"; +import * as ManagedProjectFolders from "../../project/ManagedProjectFolders.ts"; +import * as TerminalManager from "../../terminal/Manager.ts"; +import * as GitWorkflow from "../../git/GitWorkflowService.ts"; +import * as TextGeneration from "../../textGeneration/TextGeneration.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import * as ProviderRegistryMock from "../../provider/testUtils/providerRegistryMock.ts"; +import * as CommandReceiptStore from "../../orchestration-v2/CommandReceiptStore.ts"; +import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "../../orchestration-v2/testkit/ProviderReplayHarness.ts"; +import { CodexProviderCapabilitiesV2 } from "../../orchestration-v2/Adapters/CodexAdapterV2.ts"; + +export const orgId = KinOrgId.make("org"); +export const agentId = KinAgentId.make("lead"); +const timestamp = "2026-11-08T00:00:00.000Z"; +const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const decodeEvent = Schema.decodeSync(KinEvent); +const selection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.1-codex" }; +const project = { + id: ProjectId.make("kin:org"), + title: "Kin · Org", + workspaceRoot: "/private/tmp", + repositoryIdentity: null, + faviconPath: null, + defaultModelSelection: selection, + defaultThreadEnvMode: null, + scripts: [], + pinnedAt: null, + archivedAt: null, + createdAt: timestamp, + updatedAt: timestamp, + deletedAt: null, +}; +const database = KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); +export const seed = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_orgs VALUES ('org', 'Org', ${timestamp})`; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) VALUES ('owner', 'org', 'owner', 'Owner', ${timestamp})`; + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) VALUES ('lead', 'org', 'Lead', 'lead', 'owner', ${timestamp})`; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) VALUES ('owner-chat', 'org', 'team_group', 0, ${timestamp})`; + const models = yield* encodeJson({ + strong: selection, + cheap: selection, + }); + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) VALUES ('lead', 'org', 'Lead', 0, ${models}, '[]', 'owner-chat')`; +}); +export const event = (id: string) => + decodeEvent({ + _tag: "routine", + orgId, + agentId, + eventId: id, + receivedAt: timestamp, + dedupeKey: id, + provenance: { _tag: "system" }, + routine: "owner_check", + }); + +export function harness( + options: { + readonly hold?: Deferred.Deferred; + readonly entered?: Deferred.Deferred; + readonly prepare?: Memory.Memory["Service"]["prepareWakeup"]; + } = {}, +) { + let nativeOrdinal = 0; + const driver = ProviderDriverKind.make("codex"); + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId: selection.instanceId, + driver, + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (session) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const events = yield* PubSub.unbounded(); + const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { + instanceId: selection.instanceId, + driver, + providerSessionId: session.providerSessionId, + providerSession: { + id: session.providerSessionId, + driver, + providerInstanceId: selection.instanceId, + status: "ready", + cwd: "/private/tmp", + model: selection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromPubSub(events), + ensureThread: (input) => + Effect.sync(() => ({ + id: ProviderThreadId.make(`native-${++nativeOrdinal}`), + driver, + providerInstanceId: selection.instanceId, + providerSessionId: session.providerSessionId, + appThreadId: input.threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: `native-${nativeOrdinal}`, strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + contextUsage: null, + forkedFrom: null, + createdAt: now, + updatedAt: now, + })), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: Effect.fn("test.startTurn")(function* (input) { + if (options.entered !== undefined) yield* Deferred.succeed(options.entered, undefined); + if (options.hold !== undefined) yield* Deferred.await(options.hold); + const time = yield* DateTime.now; + const id = ProviderTurnId.make(`turn-${input.runOrdinal}`); + yield* PubSub.publish(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id, + providerThreadId: input.providerThread.id, + nodeId: input.rootNodeId, + runAttemptId: input.attemptId, + nativeTurnRef: null, + ordinal: input.runOrdinal, + status: "completed", + startedAt: time, + completedAt: time, + tokenUsage: { + usedTokens: 30, + inputTokens: 20, + cachedInputTokens: 5, + outputTokens: 10, + updatedAt: DateTime.formatIso(time), + }, + }, + }); + yield* PubSub.publish(events, { + type: "turn.terminal", + driver, + providerThreadId: input.providerThread.id, + providerTurnId: id, + runOrdinal: input.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + }), + compactThread: (input) => runtime.startTurn(input), + steerTurn: () => Effect.void, + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused snapshot"), + rollbackThread: () => Effect.die("unused rollback"), + forkThread: () => Effect.die("unused fork"), + }; + return runtime; + }), + }; + const orchestration = ProviderReplayHarness.layerWithRegistry( + { name: "kin-wakeup" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { databaseLayer: database }, + ); + const settings = ServerSettings.layerTest(); + const receipts = CommandReceiptStore.layer.pipe(Layer.provide(database)); + const management = ThreadManagement.layer.pipe( + Layer.provide(Layer.mergeAll(orchestration, settings, receipts)), + ); + const projectLayer = Layer.mock(ProjectService.ProjectService)({ + bootstrap: () => Effect.succeed({ project, created: true }), + getById: () => Effect.succeedSome(project), + getByWorkspaceRoot: () => Effect.succeedSome(project), + }); + const external = Layer.mergeAll( + projectLayer, + WorktreeSetupTracker.layer, + Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({ get: () => Effect.succeed(null) }), + Layer.mock(TerminalManager.TerminalManager)({ close: () => Effect.void }), + Layer.mock(GitWorkflow.GitWorkflowService)({}), + Layer.mock(TextGeneration.TextGeneration)({}), + Layer.mock(ProjectSetupScriptRunner.ProjectSetupScriptRunner)({ + runForThread: () => Effect.succeed({ status: "no-script" }), + }), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/private/tmp", + isScratchProject: () => Effect.succeed(false), + folderForThread: () => Effect.succeedNone, + }), + ProviderRegistryMock.layer(), + settings, + ); + const launch = ThreadLaunch.layer.pipe( + Layer.provide(Layer.mergeAll(external, management, receipts, IdAllocator.layer)), + ); + const kin = Layer.mergeAll( + KinEventQueue.layer, + KinLeadRegistry.layer, + KinIdentity.KinIdentity.layer.pipe( + Layer.provide(KinChatRepository.layer), + Layer.provide( + Layer.succeed(KinIdentity.KinIdentityConfig, { + orgName: "Org", + ownerHandle: "owner", + ownerDisplayName: "Owner", + }), + ), + ), + Layer.succeed(KinLeadPrompt, { build: () => Effect.succeed("Wake up") }), + Layer.mock(Memory.Memory)({ + prepareWakeup: + options.prepare ?? + (() => Effect.succeed({ head: "abc", indexAsOf: DateTime.makeUnsafe(timestamp) })), + ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), + }), + ); + const services = Layer.mergeAll(kin, launch, management, projectLayer, orchestration).pipe( + Layer.provideMerge(database), + ); + return WakeupRunner.layer.pipe(Layer.provideMerge(services)); +} diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts new file mode 100644 index 0000000000000..adf32ec49ce9b --- /dev/null +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -0,0 +1,504 @@ +import { + CommandId, + KinAgentId, + KinLead, + KinOrgId, + KinWakeupId, + MessageId, + ModelSelection, + ProjectId, + RunId, + ThreadId, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Random from "effect/Random"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as SqlClient from "effect/sql/SqlClient"; +import { isSqlError } from "effect/sql/SqlError"; + +import * as ThreadLaunchService from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagementService from "../../orchestration-v2/ThreadManagementService.ts"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as KinIdentity from "../chat/KinIdentity.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinLeadRegistry from "./KinLeadRegistry.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; + +export class KinWakeupError extends Schema.TaggedError()("KinWakeupError", { + orgId: KinOrgId, + agentId: KinAgentId, + cause: Schema.Defect(), +}) {} + +export class KinWakeupHooks extends Context.Service< + KinWakeupHooks, + { + readonly memoryCheckoutInvalid: (error: Memory.MemoryCheckoutInvalid) => Effect.Effect; + } +>()("t3/kin/lead/WakeupRunner/KinWakeupHooks") {} + +export class WakeupRunner extends Context.Service< + WakeupRunner, + { + readonly runOnce: (orgId: KinOrgId, agentId: KinAgentId) => Effect.Effect; + readonly recover: Effect.Effect; + readonly start: Effect.Effect; + } +>()("t3/kin/lead/WakeupRunner") {} + +const runnerCause = (cause: unknown) => + isSqlError(cause) ? toPersistenceSqlError("WakeupRunner")(cause) : cause; +const isWakeupError = Schema.is(KinWakeupError); +const LeadRow = Schema.Struct({ + orgId: KinOrgId, + agentId: KinAgentId, + name: Schema.String, + level: KinLead.fields.level, + threadId: Schema.NullOr(ThreadId), + models: Schema.fromJsonString(Schema.Struct({ strong: ModelSelection, cheap: ModelSelection })), + orgName: Schema.String, + retiredAt: Schema.NullOr(Schema.String), +}); +const RunningRow = Schema.Struct({ + orgId: KinOrgId, + agentId: KinAgentId, + wakeupId: KinWakeupId, + threadId: Schema.NullOr(ThreadId), + runId: Schema.NullOr(RunId), + startedAt: Schema.Number, +}); +const encodeUsage = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const decodeLead = Schema.decodeUnknownEffect(LeadRow); +const decodeRunning = Schema.decodeUnknownEffect(Schema.Array(RunningRow)); +const ulid = Effect.gen(function* () { + const alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + let timestamp = yield* Clock.currentTimeMillis; + let value = ""; + for (let i = 0; i < 10; i++) { + value = alphabet.charAt(timestamp % 32) + value; + timestamp = Math.floor(timestamp / 32); + } + for (let i = 0; i < 16; i++) value += alphabet.charAt(yield* Random.nextIntBetween(0, 31)); + return value; +}); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const queue = yield* KinEventQueue.KinEventQueue; + const memory = yield* Memory.Memory; + const hooks = yield* Effect.serviceOption(KinWakeupHooks); + const prompt = yield* KinLeadPrompt; + const identity = yield* KinIdentity.KinIdentity; + const registry = yield* KinLeadRegistry.KinLeadRegistry; + const projects = yield* ProjectService.ProjectService; + const launch = yield* ThreadLaunchService.ThreadLaunchService; + const threads = yield* ThreadManagementService.ThreadManagementService; + + const lead = Effect.fn("WakeupRunner.lead")(function* (orgId: KinOrgId, agentId: KinAgentId) { + const [row] = yield* sql`SELECT l.org_id AS "orgId", l.agent_id AS "agentId", l.name, + l.level, l.thread_id AS "threadId", l.models_json AS models, o.name AS "orgName", a.retired_at AS "retiredAt" + FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + JOIN kin_orgs o ON o.org_id = l.org_id WHERE l.org_id = ${orgId} AND l.agent_id = ${agentId}`; + return yield* decodeLead(row); + }); + const retire = Effect.fn("WakeupRunner.retire")(function* (record: typeof LeadRow.Type) { + yield* sql`UPDATE kin_events SET state = 'expired' WHERE org_id = ${record.orgId} AND agent_id = ${record.agentId} AND state = 'queued'`; + if (record.threadId !== null) + yield* identity.endAgentThread({ orgId: record.orgId, threadId: record.threadId }); + }); + const capture = Effect.fn("WakeupRunner.capture")(function* (row: typeof RunningRow.Type) { + if (row.threadId === null || row.runId === null) return; + const projection = yield* threads.getThreadProjection(row.threadId); + const runIds = new Set([row.runId]); + for (const run of projection.runs.toSorted((a, b) => a.ordinal - b.ordinal)) { + if ( + (run.restartContinuationOfRunId !== undefined && + runIds.has(run.restartContinuationOfRunId)) || + (run.usageLimitContinuationOfRunId !== undefined && + runIds.has(run.usageLimitContinuationOfRunId)) + ) + runIds.add(run.id); + } + const attempts = new Set( + projection.attempts + .filter((attempt) => runIds.has(attempt.runId)) + .map((attempt) => attempt.id), + ); + const turns = projection.providerTurns.filter( + (turn) => turn.runAttemptId !== null && attempts.has(turn.runAttemptId), + ); + const usage = turns.flatMap((turn) => + turn.tokenUsage === undefined ? [] : [{ providerTurnId: turn.id, ...turn.tokenUsage }], + ); + const tokens = yield* encodeUsage(usage); + yield* sql`UPDATE kin_wakeups SET tokens_json = ${tokens} + WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + }); + const wait = Effect.fn("WakeupRunner.wait")(function* ( + row: typeof RunningRow.Type, + projectId: ProjectId, + ) { + if (row.threadId === null || row.runId === null) return; + const threadId = row.threadId; + const runId = row.runId; + const remaining = Math.max(0, row.startedAt + 900_000 - (yield* Clock.currentTimeMillis)); + const result = yield* threads.waitForThread({ + projectId, + threadId: row.threadId, + runId: row.runId, + timeoutMs: remaining, + }); + const status = result.timedOut + ? "timed_out" + : result.run?.status === "completed" + ? "completed" + : "failed"; + yield* Effect.gen(function* () { + if (result.timedOut) + yield* threads.interruptThread({ + projectId, + threadId, + runId: result.run?.id ?? runId, + commandId: CommandId.make(`kin-timeout:${row.wakeupId}`), + }); + yield* capture(row); + yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status }); + }).pipe( + Effect.onExit((exit) => + result.timedOut && Exit.isFailure(exit) + ? queue + .finish({ orgId: row.orgId, wakeupId: row.wakeupId, status: "timed_out" }) + .pipe(Effect.ignore) + : Effect.void, + ), + ); + if (status === "completed") + yield* threads.settleThread({ + threadId: row.threadId, + commandId: CommandId.make(`kin-settle:${row.wakeupId}`), + byOwnAgent: true, + }); + }); + const clearMissingThread = Effect.fn("WakeupRunner.clearMissingThread")(function* ( + row: typeof RunningRow.Type, + ) { + if (row.threadId === null) return; + const threadId = row.threadId; + yield* sql.withTransaction( + Effect.gen(function* () { + const existing = + yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${threadId}`; + if (existing.length > 0) return; + const registered = + yield* sql`SELECT 1 FROM kin_agent_threads WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND thread_id = ${threadId} AND ended_at IS NULL`; + if (registered.length > 0) yield* identity.endAgentThread({ orgId: row.orgId, threadId }); + yield* sql`UPDATE kin_leads SET thread_id = NULL WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND thread_id = ${threadId}`; + }), + ); + }); + const cleanup = Effect.fn("WakeupRunner.cleanup")(function* (row: typeof RunningRow.Type) { + const [running] = + yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + if (running === undefined) return; + if (row.threadId !== null) { + const projection = yield* threads.getThreadProjection(row.threadId).pipe(Effect.option); + if (Option.isSome(projection)) + yield* threads + .interruptThread({ + projectId: projection.value.thread.projectId, + threadId: row.threadId, + commandId: CommandId.make(`kin-exit:${row.wakeupId}`), + }) + .pipe(Effect.ignore); + } + yield* clearMissingThread(row).pipe(Effect.ignore); + yield* capture(row).pipe(Effect.ignore); + yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status: "failed" }); + }); + + const runOnce = Effect.fn("WakeupRunner.runOnce")( + function* (orgId: KinOrgId, agentId: KinAgentId) { + const record = yield* lead(orgId, agentId); + if (record.retiredAt !== null) return yield* retire(record); + const paused = yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; + if (paused.length > 0) return; + const wakeupId = KinWakeupId.make(yield* ulid); + const startedAt = yield* Clock.currentTimeMillis; + let row: typeof RunningRow.Type = { + orgId, + agentId, + wakeupId, + threadId: null, + runId: null, + startedAt, + }; + yield* Effect.scoped( + Effect.gen(function* () { + const events = yield* Effect.acquireRelease( + queue.claim({ orgId, agentId, wakeupId, model: record.models.strong.model }), + () => + cleanup(row).pipe( + Effect.catch((error) => + Effect.logError("Kin wake-up finalizer failed", { + orgId, + wakeupId, + errorTag: error._tag, + }), + ), + ), + ); + row = { ...row, startedAt: yield* Clock.currentTimeMillis }; + yield* Effect.gen(function* () { + const paused = + yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; + if (paused.length > 0) { + yield* queue.release({ + orgId, + wakeupId, + countAttempt: false, + notBefore: (yield* Clock.currentTimeMillis) + 5_000, + }); + return; + } + const current = yield* lead(orgId, agentId); + if (current.retiredAt !== null) { + yield* queue.finish({ orgId, wakeupId, status: "failed" }); + return yield* retire(current); + } + const pinned = yield* memory.prepareWakeup(orgId).pipe( + Effect.tapError((error) => + Effect.logError("Kin memory preparation failed", { + orgId, + wakeupId, + errorTag: error._tag, + }), + ), + Effect.tapError((error) => + error._tag === "MemoryCheckoutInvalid" && Option.isSome(hooks) + ? hooks.value.memoryCheckoutInvalid(error) + : Effect.void, + ), + ); + const paths = yield* memory.ensureCheckouts(orgId); + const { project } = yield* projects.bootstrap({ + commandId: CommandId.make(`kin-project:${orgId}`), + projectId: ProjectId.make(`kin:${orgId}`), + workspaceRoot: paths.read, + title: `Kin · ${record.orgName}`, + }); + const text = yield* prompt.build({ + orgId, + agentId, + wakeupId, + events, + memoryHead: pinned.head, + }); + const urgent = + yield* sql`SELECT 1 FROM kin_events WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND priority IN ('P0', 'P1') LIMIT 1`; + const modelSelection = + urgent.length > 0 || events.some((event) => event._tag === "routine") + ? current.models.strong + : current.models.cheap; + const threadId = current.threadId ?? ThreadId.make(`kin:${yield* ulid}`); + row = { ...row, threadId }; + yield* sql.withTransaction( + Effect.gen(function* () { + const [active] = + yield* sql`SELECT 1 FROM kin_agents WHERE org_id = ${orgId} AND agent_id = ${agentId} AND retired_at IS NULL`; + if (active === undefined) + return yield* new KinWakeupError({ orgId, agentId, cause: "retired" }); + yield* identity.registerAgentThread({ orgId, agentId, threadId }); + yield* sql`UPDATE kin_leads SET thread_id = ${threadId} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* sql`UPDATE kin_wakeups SET thread_id = ${threadId}, model = ${modelSelection.model} + WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND agent_id = ${agentId} AND status = 'running'`; + }), + ); + yield* registry.register(threadId, { orgId, agentId, wakeupId, level: current.level }); + const commandId = CommandId.make(`kin-wakeup:${wakeupId}`); + const messageId = MessageId.make(`kin-wakeup-msg:${wakeupId}`); + const sent = + current.threadId === null + ? (yield* launch.launch({ + commandId, + threadId, + projectId: project.id, + title: current.name, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { messageId, text, attachments: [] }, + createdBy: "system", + creationSource: "server", + })).projection + : (yield* threads.sendToThread({ + commandId, + projectId: project.id, + threadId, + messageId, + text, + attachments: [], + mode: "queue", + freshContext: true, + modelSelection, + createdBy: "system", + creationSource: "server", + })).projection; + const run = sent.runs.find((run) => run.userMessageId === messageId); + if (run === undefined) + return yield* new KinWakeupError({ orgId, agentId, cause: "missing_run" }); + row = { ...row, runId: run.id }; + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND agent_id = ${agentId} AND status = 'running'`; + yield* wait(row, project.id); + }).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? cleanup(row).pipe( + Effect.catch((error) => + Effect.logError("Kin wake-up cleanup failed", { + orgId, + wakeupId, + errorTag: error._tag, + }), + ), + ) + : Effect.void, + ), + ); + }), + ).pipe( + Effect.retry({ + while: (error) => error._tag === "KinEventQueueBusy", + schedule: Schedule.spaced("5 seconds"), + }), + ); + }, + (effect, orgId, agentId) => + effect.pipe( + Effect.mapError( + (cause) => new KinWakeupError({ orgId, agentId, cause: runnerCause(cause) }), + ), + ), + ); + const recover = Effect.gen(function* () { + const rows = + yield* sql`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE status = 'running' ORDER BY started_at, wakeup_id`; + for (const original of yield* decodeRunning(rows)) { + yield* Effect.scoped( + Effect.gen(function* () { + let row = original; + const existing = + row.threadId === null + ? [] + : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; + const projection = + row.threadId === null || existing.length === 0 + ? undefined + : yield* threads.getThreadProjection(row.threadId); + const run = projection?.runs.find((run) => + row.runId === null + ? run.userMessageId === `kin-wakeup-msg:${row.wakeupId}` + : run.id === row.runId, + ); + if (projection === undefined || run === undefined) { + if (projection === undefined) yield* clearMissingThread(row); + yield* queue.requeueUnfinished(row.orgId, row.wakeupId); + return; + } + row = { ...row, runId: run.id }; + yield* Effect.acquireRelease(Effect.void, () => cleanup(row).pipe(Effect.ignore)); + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + const record = yield* lead(row.orgId, row.agentId); + if (record.retiredAt !== null) { + yield* cleanup(row); + return yield* retire(record); + } + yield* identity.registerAgentThread({ + orgId: row.orgId, + agentId: row.agentId, + threadId: projection.thread.id, + }); + yield* registry.register(projection.thread.id, { ...row, level: record.level }); + yield* wait(row, projection.thread.projectId).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) ? cleanup(row).pipe(Effect.ignore) : Effect.void, + ), + ); + }), + ).pipe( + Effect.mapError( + (cause) => + new KinWakeupError({ + orgId: original.orgId, + agentId: original.agentId, + cause: runnerCause(cause), + }), + ), + ); + } + }).pipe( + Effect.mapError((cause) => + isWakeupError(cause) + ? cause + : new KinWakeupError({ + orgId: KinOrgId.make("startup"), + agentId: KinAgentId.make("startup"), + cause: runnerCause(cause), + }), + ), + ); + + const start = Effect.gen(function* () { + yield* recover; + const organizations = new Set(); + yield* Effect.gen(function* () { + const rows = yield* sql<{ + orgId: string; + }>`SELECT org_id AS "orgId" FROM kin_leads GROUP BY org_id ORDER BY org_id`; + for (const raw of rows) { + const orgId = KinOrgId.make(raw.orgId); + if (organizations.has(orgId)) continue; + organizations.add(orgId); + yield* Effect.gen(function* () { + const retired = yield* sql<{ + agentId: string; + }>`SELECT l.agent_id AS "agentId" FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NOT NULL ORDER BY l.agent_id`; + for (const record of retired) + yield* retire(yield* lead(orgId, KinAgentId.make(record.agentId))); + const [next] = yield* sql<{ + agentId: string; + }>`SELECT e.agent_id AS "agentId" FROM kin_events e JOIN kin_agents a ON a.org_id = e.org_id AND a.agent_id = e.agent_id WHERE e.org_id = ${orgId} AND e.state = 'queued' AND a.retired_at IS NULL ORDER BY e.priority, e.received_at, e.event_id LIMIT 1`; + if (next !== undefined) yield* runOnce(orgId, KinAgentId.make(next.agentId)); + }).pipe( + Effect.catch((error) => + Effect.logError("Kin runner failed", { orgId, errorTag: error._tag }), + ), + Effect.repeat(Schedule.spaced("5 seconds")), + Effect.forkScoped, + ); + } + }).pipe( + Effect.catch((error) => + Effect.logError("Kin runner discovery failed", { errorTag: error._tag }), + ), + Effect.repeat(Schedule.spaced("5 seconds")), + Effect.forkScoped, + ); + }).pipe( + Effect.catch((error) => + Effect.logError("Kin runner recovery failed; disabled", { errorTag: error._tag }), + ), + ); + return WakeupRunner.of({ runOnce, recover, start }); +}); + +export const layer = Layer.effect(WakeupRunner, make); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts new file mode 100644 index 0000000000000..76d91efe99210 --- /dev/null +++ b/apps/server/src/kin/lead/layer.ts @@ -0,0 +1,85 @@ +import { KinOrgId } from "@t3tools/contracts"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import * as KinIdentity from "../chat/KinIdentity.ts"; +import * as KinChatRepository from "../chat/KinChatRepository.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryConfig from "../memory/MemoryConfig.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinLeadRegistry from "./KinLeadRegistry.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; +import * as WakeupRunner from "./WakeupRunner.ts"; + +export class KinLeadStartupError extends Schema.TaggedError()( + "KinLeadStartupError", + { + stage: Schema.Literals(["dependencies", "startup"]), + cause: Schema.optional(Schema.Defect()), + }, +) {} + +export function containStartup(layer: Layer.Layer) { + return Layer.effectDiscard(Effect.void).pipe( + Layer.provide(layer), + Layer.catch((cause) => + Layer.effectDiscard( + Effect.logError( + "Kin startup failed; Kin disabled", + new KinLeadStartupError({ stage: "startup", cause }), + ), + ), + ), + ); +} + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const runner = yield* WakeupRunner.WakeupRunner; + yield* runner.start.pipe(Effect.forkScoped); + }), +).pipe( + Layer.provide(WakeupRunner.layer), + Layer.provide( + Layer.mergeAll( + KinLeadRegistry.layer, + KinEventQueue.layer, + Memory.layer, + KinIdentity.KinIdentity.layer.pipe( + Layer.provide(KinIdentity.KinIdentityConfig.layer), + Layer.provide(KinChatRepository.layer), + ), + ), + ), + Layer.provide(KinPersistence.layer), +); + +const Repositories = Schema.fromJsonString( + Schema.Array(Schema.Struct({ orgId: KinOrgId, repoUrl: Schema.String })), +); +export const layerServer = containStartup( + Layer.unwrap( + Effect.gen(function* () { + const prompt = yield* Effect.serviceOption(KinLeadPrompt); + if (Option.isNone(prompt)) { + yield* Effect.logWarning( + "Kin runner disabled: LeadPrompt implementation is not installed", + new KinLeadStartupError({ stage: "dependencies" }), + ); + return Layer.empty; + } + const repositories = yield* Config.schema(Repositories, "KIN_MEMORY_REPOSITORIES").pipe( + Config.withDefault([]), + ); + if (repositories.length === 0) return Layer.empty; + return layer.pipe( + Layer.provide(Layer.succeed(KinLeadPrompt, prompt.value)), + Layer.provide(MemoryConfig.layer(repositories)), + ); + }), + ), +); diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index cfb90d839e580..43f161ef4df77 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -42,6 +42,7 @@ const fakeRepository = () => { } return sha; }; + let validations = 0; let treeReads = 0; let blobReads = 0; let revision = 0; @@ -60,6 +61,7 @@ const fakeRepository = () => { : `fixture-head-${revision}`; break; case "remote": + validations++; stdout = "unused"; break; case "ls-tree": { @@ -115,7 +117,7 @@ const fakeRepository = () => { commit: () => { revision++; }, - counts: () => ({ treeReads, blobReads }), + counts: () => ({ treeReads, blobReads, validations }), corruptBatch: () => { corruptNextBatch = true; }, diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 2a81aa46fe482..26c7d11d68336 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -173,6 +173,7 @@ const make = Effect.gen(function* () { const cache = yield* MemoryCache.MemoryCache; const locks = new Map(); const pinnedHeads = new Map(); + const initializedCheckouts = new Set(); const duplicateWarnings = new Map>(); const snapshots = new Map(); const blobCaches = new Map(); @@ -336,8 +337,13 @@ const make = Effect.gen(function* () { ); }); - const ensureCheckouts = Effect.fn("Memory.ensureCheckouts")(function* (orgId: KinOrgId) { + const ensureCheckouts = Effect.fn("Memory.ensureCheckouts")(function* ( + orgId: KinOrgId, + reusePin = false, + ) { const paths = yield* pathsFor(orgId); + if (reusePin && initializedCheckouts.has(orgId) && pinnedHeads.has(orgId)) + return { writer: paths.writer, read: paths.read }; yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); const gitDirectory = path.join(paths.writer, ".git"); if (yield* io(orgId, "exists", fs.exists(paths.writer))) { @@ -393,14 +399,15 @@ const make = Effect.gen(function* () { yield* git(orgId, paths.writer, ["worktree", "prune"]); yield* git(orgId, paths.writer, ["worktree", "add", "--detach", paths.read, "HEAD"]); } + initializedCheckouts.add(orgId); return { writer: paths.writer, read: paths.read }; }); const load = Effect.fn("Memory.load")(function* (orgId: KinOrgId, force = false) { const paths = yield* pathsFor(orgId); - yield* validateWriter(orgId, paths); const head = pinnedHeads.get(orgId) ?? (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); + if (pinnedHeads.get(orgId) !== head) yield* validateWriter(orgId, paths); pinnedHeads.set(orgId, head); const previous = snapshots.get(orgId); if (!force && previous?.head === head) return previous; @@ -793,7 +800,7 @@ const make = Effect.gen(function* () { return Memory.of({ ensureCheckouts: Effect.fn("Memory.ensureCheckoutsLocked")((orgId) => - locked(orgId, ensureCheckouts(orgId)), + locked(orgId, ensureCheckouts(orgId, true)), ), prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId) => locked(orgId, prepare(orgId))), readIndex: Effect.fn("Memory.readIndex")((orgId) => locked(orgId, readFile(orgId, "INDEX.md"))), diff --git a/apps/server/src/kin/triggers/KinTriggerSink.test.ts b/apps/server/src/kin/triggers/KinTriggerSink.test.ts index 4120cc9aac88c..2ee9f763bbdd8 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.test.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.test.ts @@ -11,6 +11,9 @@ import { KinWebhookSource, OrchestrationV2ThreadProjectionJson, ScheduledTaskUpsertInput, + ProjectId, + ProviderInstanceId, + ScheduledTaskId, } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -549,3 +552,67 @@ describe("KinTriggerSink", () => { }).pipe(Effect.provide(services.pipe(Layer.provideMerge(database)))), ); }); + +it.effect( + "builds the runtime after a Kin migration failure and refuses scheduled turns without blocking ordinary launches", + () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_orgs (broken TEXT)`; + const launches: ThreadLaunchService.ThreadLaunchInput[] = []; + const inner = Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: (input) => + Effect.sync(() => { + launches.push(input); + return { + threadId: projection(orgA).thread.id, + projection: projection(orgA), + resumed: false, + }; + }), + }); + yield* Effect.gen(function* () { + const launch = yield* ThreadLaunchService.ThreadLaunchService; + const input: ThreadLaunchService.ThreadLaunchInput = { + commandId: CommandId.make("ordinary"), + projectId: ProjectId.make("project"), + title: "Ordinary", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "test" }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + createdBy: "system", + creationSource: "server", + }; + yield* launch.launch(input); + const blocked = yield* launch + .launch({ + ...input, + initialMessage: { + text: "Scheduled", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("task"), + }, + }) + .pipe(Effect.flip); + assert.strictEqual(blocked._tag, "ThreadLaunchError"); + const management = yield* ThreadManagementService.ThreadManagementService; + const bound = yield* management + .sendToThread({ + projectId: input.projectId, + commandId: input.commandId, + threadId: projection(orgA).thread.id, + messageId: MessageId.make("message"), + text: "Bound", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("task"), + mode: "queue", + createdBy: "system", + creationSource: "server", + }) + .pipe(Effect.flip); + assert.strictEqual(bound._tag, "OrchestratorCommandRejectedError"); + assert.strictEqual(launches.length, 1); + }).pipe(Effect.provide(KinTriggerSink.wrapTaskDispatch(inner, threads))); + }).pipe(Effect.provide(Sqlite.layerMemory)), +); diff --git a/apps/server/src/kin/triggers/KinTriggerSink.ts b/apps/server/src/kin/triggers/KinTriggerSink.ts index 9d48324d6d4d6..a23d0adc43c5b 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.ts @@ -25,6 +25,11 @@ const decodeDelivery = Schema.decodeUnknownEffect(Delivery); const decodeFire = Schema.decodeEffect(KinTriggerFired); const decodeDeliveryId = Schema.decodeEffect(ScheduledTaskWebhookDeliveryId); +export class KinTriggerStartupError extends Schema.TaggedError()( + "KinTriggerStartupError", + { cause: Schema.Defect() }, +) {} + export class KinTriggerSink extends Context.Service< KinTriggerSink, { @@ -172,5 +177,51 @@ export function wrapTaskDispatch( ).pipe( Layer.provide(Layer.mergeAll(inner, threads)), Layer.provide(Layer.mergeAll(layer, KinTriggerStore.layer).pipe(Layer.provide(threads))), + Layer.catch((cause) => + Layer.mergeAll( + Layer.effectDiscard( + Effect.logError( + "Kin trigger startup failed; scheduled dispatch disabled", + new KinTriggerStartupError({ cause }), + ), + ), + Layer.effect( + ThreadLaunchService.ThreadLaunchService, + Effect.gen(function* () { + const service = yield* ThreadLaunchService.ThreadLaunchService; + return ThreadLaunchService.ThreadLaunchService.of({ + ...service, + launch: Effect.fn("KinTriggerSink.disabledLaunch")(function* (input) { + if (input.initialMessage?.scheduledTaskId === undefined) + return yield* service.launch(input); + return yield* new ThreadLaunchService.ThreadLaunchError({ + operation: "dispatch-message", + commandId: input.commandId, + projectId: input.projectId, + cause, + }); + }), + }); + }), + ), + Layer.effect( + ThreadManagementService.ThreadManagementService, + Effect.gen(function* () { + const service = yield* ThreadManagementService.ThreadManagementService; + return ThreadManagementService.ThreadManagementService.of({ + ...service, + sendToThread: Effect.fn("KinTriggerSink.disabledSend")(function* (input) { + if (input.scheduledTaskId === undefined) return yield* service.sendToThread(input); + return yield* new OrchestratorCommandRejectedError({ + commandId: input.commandId, + commandType: "kin.trigger.disabled", + cause, + }); + }), + }); + }), + ), + ).pipe(Layer.provide(Layer.mergeAll(inner, threads))), + ), ); } diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index fe2740a8c7a2d..78db66f822982 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,3 +1,4 @@ +import * as KinLead from "./kin/lead/layer.ts"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as Random from "effect/Random"; @@ -668,6 +669,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( ReplayMarkers.layer, Layer.effectDiscard(ProjectSettingsOverrideSweep.sweep), ).pipe( + Layer.provide(KinLead.layerServer), Layer.provideMerge(layerOrchestrationApplication), Layer.provideMerge(RuntimeLayer.layerEventInfrastructure), Layer.provideMerge(Layer.merge(ProjectStore.layer, ThreadSearch.layer)), diff --git a/docs/kin/specs/chat.md b/docs/kin/specs/chat.md index b98dc0e07ccd3..963fd9dd8b7d3 100644 --- a/docs/kin/specs/chat.md +++ b/docs/kin/specs/chat.md @@ -46,7 +46,7 @@ Every row carries `orgId`. Phase 1 has one org, created on first start. Bootstra - **Session to person** (`KinIdentity.resolve(session)`): - Phase 1: after the RPC scope check, a `/ws` session holding every scope in `AuthStandardClientScopes` resolves to the default org owner. A narrower session resolves to an empty list, never to a person inferred from `subject`. The result is a list of `{ orgId, personId }` entries so phase-2 bindings do not change callers. Only the owner may pair devices before CHAT-11. - Phase 2: a row in `kin_session_bindings (session_id, org_id, person_id)` decides. A session without a binding is the owner only if it holds `AuthStandardClientScopes`. Members get sessions with the new `kin:chat` scope only (CHAT-11), so their device never gets the owner's terminal or files. -- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead session's thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` when the session ends; an ended thread cannot be registered again. +- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead's durable thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` on retirement, not between fresh provider sessions on the same T3 thread; an ended thread cannot be registered again. ### Chats, members, roles @@ -467,7 +467,7 @@ Upstream files touched, hooks only: ## Interfaces to other specs -- **Lead spec:** calls `KinIdentity.registerAgentThread` / `endAgentThread` around each lead session. Subscribes to `KinChatHub` for `message.posted` and `widget_action` to build `KinEvent`s. Uses the Kin MCP tools to post. Owns `KinAgent` config beyond identity, the `deferral` card state, levels per chat (its own table keyed by `chatId`), and the brief builder that quotes non-owner text. Receives `floor.offer` wake requests from the floor manager. +- **Lead spec:** calls `KinIdentity.registerAgentThread` before the first lead session and `endAgentThread` on retirement. Subscribes to `KinChatHub` for `message.posted` and `widget_action` to build `KinEvent`s. Uses the Kin MCP tools to post. Owns `KinAgent` config beyond identity, the `deferral` card state, levels per chat (its own table keyed by `chatId`), and the brief builder that quotes non-owner text. Receives `floor.offer` wake requests from the floor manager. - **Approvals spec:** creates `approval` widgets server-side from `KinPendingAction` and calls `addWaiting` / `resolveWaiting`. Reads `KinMessage.author` and `authorRole` as provenance for ceilings. Its signed-grant RPC is the only way to approve; `kin.widgets.act` refuses approvals. Shows "Waiting on Nishchay" to members from `KinWidget.awaiting`. - **Triggers spec:** deferral and routine posts arrive as lead notices; nothing here. Its upstream contract tests should cover what this spec depends on: `Migrator` `table` option, `WsRpcGroup` and `RPC_REQUIRED_SCOPES` shapes, `layerWsRpc` session argument, `McpInvocationScope.thread`, `toolkitRegistration`, `getThreadShell` and `streamDomainEvents`, `OrchestratorMcpDelegateTaskResult`. - **Mobile spec:** consumes `kin.*` RPCs and both streams through the normal WS client; draws native cards from `KinWidget.type` and `state`; renders generated widgets with `ThreadMcpApp`; opens task threads with `formatThreadLink`; folds runs by `floorRunId`; avatars from `KinAuthor._tag` (rule 7). diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index cd1b7c86552a8..5074266df5648 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -32,7 +32,7 @@ Only one wake-up runs per organization at a time. Every wake-up counts against a | Test harness | `DeterministicRuntime`, `OrchestratorScenario`, replay fixtures | `apps/server/src/orchestration-v2/testkit/` | | Advertised tool budget | Measures every tool in `layerTools` | `apps/server/src/provider/AgentContextBudget.test.ts` | -`apps/server/src/kin/` does not exist yet. +Kin services live in `apps/server/src/kin/`. ## Design @@ -179,9 +179,9 @@ Coalescing: - Events that arrive while a wake-up runs wait for the next one. The runner never starts a second session for the same organization. This is the one-writer rule: memory is written only inside a wake-up, and wake-ups are serial. - Implementation: an organization permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 must call `finish(timed_out | failed)` in its own `onExit`. Idle semaphore entries are evicted. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. -The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval`, so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. +The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval`, so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. `release` returns `{ released, failedEventIds }` and `finish` returns `{ failedEventIds }`; the ids are the events that transitioned to terminal `failed` in that call, ordered by id. Unknown or foreign lead ids fail with `KinEventQueueUnknownAgent`, and reused wake-up ids fail with `KinEventQueueWakeupConflict`, not SQL constraint errors. Enqueue validates the lead and computes owner priority inside the insert transaction. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. -Restart: the runner first reconciles each unfinished wake-up with its provider run (see failure cases). It must not requeue events belonging to a still-active run. For an abandoned run only, `requeueUnfinished(orgId, wakeupId)` atomically marks the running wake-up interrupted and returns its claimed events to `queued`, clearing their `wakeup_id` while preserving `not_before` and `attempts`. It is idempotent and organization-scoped; `idx_kin_events_wakeup` supports the update. Completed runs are finished by the runner instead. Layer construction never blindly requeues active work. +Restart: the runner first reconciles each unfinished wake-up with its provider run (see failure cases). It must not requeue events belonging to a still-active run. For an abandoned run only, `requeueUnfinished(orgId, wakeupId)` atomically marks the running wake-up interrupted and returns its claimed events to `queued`, clearing their `wakeup_id` while preserving `not_before` and `attempts`. It is idempotent and organization-scoped; `idx_kin_events_wakeup` supports the update. Completed runs are finished by the runner instead. Layer construction never blindly requeues active work. Recovery also checks the deterministic first-message id if launch committed before `run_id` was saved. A provisional lead thread with no upstream thread is ended and cleared before requeue, so the next attempt launches rather than sending to a nonexistent thread. ### A wake-up (one fresh lead session) @@ -203,12 +203,18 @@ Each wake-up: 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every run is a new provider session. -7. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, an in-memory map the tools use. It is filled before the send and cleared after the run ends. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, an in-memory map the tools use. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. 8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread` and mark `timed_out`. 9. Finish: `Memory.commitWakeup` (one commit holding the staged item writes and the journal entry), reset the read checkout, record token usage from the run's provider turns into `kin_wakeups.tokens_json` and `KinBudget`, mark events `done`, and `settleThread`. If the lead writes files by hand in the read checkout, step 9's reset throws the changes away. A non-empty `git status --porcelain` before the reset is logged in the journal as "Lead edited memory files directly; discarded." +Kin boot is optional: configuration, migrations, identity bootstrap and runner startup failures are logged with typed errors and disable Kin, never the core T3 runtime. The runner requires the real `KinLeadPrompt` implementation; while LEAD-6 is absent, the server hook does not construct the runner. Memory repositories are supplied through `KIN_MEMORY_REPOSITORIES` (a JSON array of `{ orgId, repoUrl }`); no private repository is assumed by source code. + +The scheduled-task dispatch wrapper also contains Kin migration failures. While it cannot identify Kin webhook sources, it refuses **all scheduled launches and sends** rather than risk executing an inbox as a provider prompt; ordinary launches and sends and the rest of T3 remain available. Restoring the Kin database and restarting restores scheduled dispatch. + +A `MemorySyncConflict` fails the wake-up and leaves the events queued with the queue's five-minute counted-failure backoff. `MemoryCheckoutInvalid` does the same and logs an error; `KinWakeupHooks.memoryCheckoutInvalid` is the typed notification hook for LEAD-3. Writer commits/journal/reset and budget charging remain LEAD-3 and LEAD-10 integrations, not fabricated runner implementations. Until the writer lands, `memory_commit` remains null rather than claiming a writer commit. `tokens_json` contains each provider turn's recorded token usage and provider-turn id, limited to the wake-up run's attempts, on success and failure. + ### The prompt and caching (gap 2) The CLIs own their system prompt and tool list, and Kin cannot set cache breakpoints through them. Kin can keep the start of the first message byte-identical between wake-ups, so the providers' automatic prefix caching hits during bursts. Order, most stable first: From 10137bde984276767867362aca93be6cbc977e0e Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:12:49 +0530 Subject: [PATCH 02/12] fix(kin): enforce wake-up authority and isolate runner recovery --- .../src/kin/chat/KinChatRepository.test.ts | 18 ++ apps/server/src/kin/chat/KinChatRepository.ts | 24 ++ apps/server/src/kin/chat/KinIdentity.test.ts | 12 + apps/server/src/kin/chat/KinIdentity.ts | 5 + .../server/src/kin/lead/KinEventQueue.test.ts | 33 ++- apps/server/src/kin/lead/KinEventQueue.ts | 112 ++++++-- .../src/kin/lead/RunnerOwnership.test.ts | 65 +++++ apps/server/src/kin/lead/RunnerOwnership.ts | 47 +++ apps/server/src/kin/lead/WakeupRunner.test.ts | 175 +++++++++--- apps/server/src/kin/lead/WakeupRunner.ts | 270 ++++++++++++------ apps/server/src/kin/lead/layer.ts | 10 +- .../src/kin/triggers/KinTriggerSink.test.ts | 180 ++++++++---- .../server/src/kin/triggers/KinTriggerSink.ts | 24 +- .../src/kin/triggers/KinTriggerStore.ts | 1 + docs/kin/specs/chat.md | 2 +- docs/kin/specs/lead.md | 12 +- 16 files changed, 765 insertions(+), 225 deletions(-) create mode 100644 apps/server/src/kin/lead/RunnerOwnership.test.ts create mode 100644 apps/server/src/kin/lead/RunnerOwnership.ts diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index 028cebc444417..0fe3f76a541d6 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -873,6 +873,11 @@ describe("KinChatRepository", () => { it.effect("advances read state monotonically and counts only others' posted messages", () => Effect.gen(function* () { const repo = yield* seed; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('chat-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; const agentAuthor = { _tag: "agent", agentId, @@ -889,6 +894,19 @@ describe("KinChatRepository", () => { ), ]) yield* repo.insertMessage(input); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'chat-wakeup'`; + const rejected = yield* repo + .insertMessage( + message("outside-wakeup", { + author: agentAuthor, + authorRole: "member", + intent: "answer", + }), + ) + .pipe(Effect.flip); + assert.include(rejected, { reason: "not_allowed" }); + assert.strictEqual(Option.getOrThrow(yield* repo.getChat(scope)).lastSeq, 6); + yield* sql`UPDATE kin_wakeups SET status = 'running' WHERE wakeup_id = 'chat-wakeup'`; const resumedThread = decodeThreadId("resumed-thread"); yield* repo.registerAgentThread({ orgId, diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index a4e021ee0c8e2..46770beed66a5 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -173,6 +173,9 @@ export class KinChatRepository extends Context.Service< readonly ownerDisplayName: string; readonly createdAt: IsoDateTime; }) => Effect.Effect; + readonly hasRunningWakeup: ( + input: Scope & { readonly agentId: KinAgentId; readonly threadId: ThreadId }, + ) => Effect.Effect; readonly findAgentThread: ( threadId: ThreadId, ) => Effect.Effect, RepositoryError>; @@ -362,6 +365,14 @@ const make = Effect.gen(function* () { }, Effect.mapError(toRepositoryError("KinChatRepository.getAgentThread")), ); + const hasRunningWakeup = Effect.fn("KinChatRepository.hasRunningWakeup")( + function* (input: Scope & { readonly agentId: KinAgentId; readonly threadId: ThreadId }) { + const rows = yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${input.orgId} + AND agent_id = ${input.agentId} AND thread_id = ${input.threadId} AND status = 'running'`; + return rows.length > 0; + }, + Effect.mapError(toRepositoryError("KinChatRepository.hasRunningWakeup")), + ); const validateWidgetReferences = Effect.fn("KinChatRepository.validateWidgetReferences")( function* (input: Pick) { for (const personId of input.awaiting) @@ -373,6 +384,7 @@ const make = Effect.gen(function* () { }, ); return KinChatRepository.of({ + hasRunningWakeup, getOrg: orgOne, getPerson: personOne, getAgent: agentOne, @@ -840,6 +852,18 @@ const make = Effect.gen(function* () { message: "Kin agent author not found.", }); } + if ( + author._tag === "agent" && + !(yield* hasRunningWakeup({ + ...input, + agentId: author.agentId, + threadId: author.threadId, + })) + ) + return yield* new KinChatError({ + reason: "not_allowed", + message: "Kin wake-up is not running.", + }); if (input.replyTo !== null) { const reply = yield* requireRecord( messageOne({ orgId: input.orgId, messageId: input.replyTo }), diff --git a/apps/server/src/kin/chat/KinIdentity.test.ts b/apps/server/src/kin/chat/KinIdentity.test.ts index ab1c7ea10d94f..71b56c6b07919 100644 --- a/apps/server/src/kin/chat/KinIdentity.test.ts +++ b/apps/server/src/kin/chat/KinIdentity.test.ts @@ -308,6 +308,12 @@ describe("KinIdentity", () => { yield* identity.registerAgentThread(input); assert.deepStrictEqual(yield* repository.getAgentThread(input), original); assert.isTrue(identity.isAgentThread(threadId)); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) VALUES ('identity-chat', ${owner.orgId}, 'team_group', 0, '1970-01-01T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${owner.orgId}, 'Lead', 0, '{}', '[]', 'identity-chat')`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('identity-wakeup', ${owner.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; assert.deepStrictEqual(yield* identity.resolveAgent(caller()), { ...input, ownerPersonId: owner.personId, @@ -384,6 +390,11 @@ describe("KinIdentity", () => { ...input, ownerPersonId: owner.personId, }); + yield* sql`UPDATE kin_wakeups SET status = 'completed', ended_at = 1000 WHERE wakeup_id = 'identity-wakeup'`; + assert.include(yield* restarted.resolveAgent(caller()).pipe(Effect.flip), { + reason: "not_allowed", + }); + assert.isTrue(restarted.isAgentThread(threadId)); assert.deepStrictEqual(yield* restarted.resolve(session), [owner]); assert.isTrue(yield* restarted.endAgentThread(input)); assert.isFalse(restarted.isAgentThread(threadId)); @@ -404,6 +415,7 @@ describe("KinIdentity", () => { const afterEnd = yield* start; assert.isFalse(afterEnd.isAgentThread(threadId)); assert.isTrue(afterEnd.isAgentThread(secondThreadId)); + yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL, thread_id = ${secondThreadId} WHERE wakeup_id = 'identity-wakeup'`; assert.deepStrictEqual(yield* afterEnd.resolveAgent(caller(secondThreadId)), { ...input, threadId: secondThreadId, diff --git a/apps/server/src/kin/chat/KinIdentity.ts b/apps/server/src/kin/chat/KinIdentity.ts index 8ca9e73cc07c1..3b85ffb56634d 100644 --- a/apps/server/src/kin/chat/KinIdentity.ts +++ b/apps/server/src/kin/chat/KinIdentity.ts @@ -111,6 +111,11 @@ export class KinIdentity extends Context.Service< message: "Kin agent thread is not active.", }); } + if (!(yield* repository.hasRunningWakeup(row.value))) + return yield* new KinChatError({ + reason: "not_allowed", + message: "Kin wake-up is not running.", + }); const agent = yield* repository.getAgent(row.value); if (Option.isNone(agent) || agent.value.retiredAt !== null) return yield* new KinChatError({ diff --git a/apps/server/src/kin/lead/KinEventQueue.test.ts b/apps/server/src/kin/lead/KinEventQueue.test.ts index 79eff4375b21c..497cd5b5b1105 100644 --- a/apps/server/src/kin/lead/KinEventQueue.test.ts +++ b/apps/server/src/kin/lead/KinEventQueue.test.ts @@ -220,7 +220,13 @@ describe("KinEventQueue", () => { WHERE member_id = 'agent-member-lead-a'`; yield* sql`UPDATE kin_agents SET retired_at = ${id === "retired" ? "2026-11-07T00:00:00.000Z" : null} WHERE agent_id = 'lead-a'`; - yield* queue.enqueue({ ...owner, eventId: KinEventId.make(id), dedupeKey: id }); + const enqueue = queue.enqueue({ ...owner, eventId: KinEventId.make(id), dedupeKey: id }); + if (id === "retired") + assert.strictEqual( + (yield* enqueue.pipe(Effect.flip))._tag, + "KinEventQueueRetiredAgent", + ); + else yield* enqueue; } yield* sql`UPDATE kin_agents SET retired_at = NULL WHERE agent_id = 'lead-a'`; yield* sql`UPDATE kin_chat_members SET left_at = NULL WHERE member_id = 'agent-member-lead-a'`; @@ -248,6 +254,31 @@ describe("KinEventQueue", () => { }).pipe(Effect.provide(layer)), ); + it.effect("selects a ready lead ahead of a deferred lead and notifies retirement", () => + Effect.gen(function* () { + yield* seed(); + yield* seed("a", "lead-a2"); + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const orgId = KinOrgId.make("a"); + yield* queue.enqueue(yield* event("deferred")); + yield* queue.enqueue(yield* event("ready", "routine", "a", "lead-a2")); + yield* sql`UPDATE kin_events SET not_before = 300000 WHERE event_id = 'deferred'`; + yield* TestClock.adjust("20 seconds"); + assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a2"); + const changed = yield* queue.changes(orgId); + const signal = yield* changed.pipe(Effect.forkChild); + yield* sql`UPDATE kin_agents SET retired_at = '2026-11-07T00:00:00.000Z' WHERE agent_id = 'lead-a2'`; + yield* queue.expireAgent(orgId, KinAgentId.make("lead-a2")); + yield* Fiber.join(signal); + yield* queue.expireAgent(orgId, KinAgentId.make("lead-a2")); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'ready'`, [ + { state: "expired" }, + ]); + assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a"); + }).pipe(Effect.provide(layer)), + ); + it.effect( "dedupes per organization, normalizes github deliveries, and surfaces other constraints", () => diff --git a/apps/server/src/kin/lead/KinEventQueue.ts b/apps/server/src/kin/lead/KinEventQueue.ts index fa303b416eb09..bc97bd2ec1613 100644 --- a/apps/server/src/kin/lead/KinEventQueue.ts +++ b/apps/server/src/kin/lead/KinEventQueue.ts @@ -78,6 +78,11 @@ export class KinEventQueueUnknownAgent extends Schema.TaggedError()( + "KinEventQueueRetiredAgent", + { orgId: KinOrgId, agentId: KinAgentId }, +) {} +const isRetiredAgent = Schema.is(KinEventQueueRetiredAgent); export class KinEventQueueWakeupConflict extends Schema.TaggedError()( "KinEventQueueWakeupConflict", { orgId: KinOrgId, wakeupId: KinWakeupId }, @@ -112,8 +117,23 @@ export class KinEventQueue extends Context.Service< | KinEventQueueConflict | KinEventQueueUnknownEvent | KinEventQueueUnknownAgent + | KinEventQueueRetiredAgent + >; + readonly changes: (orgId: KinOrgId) => Effect.Effect>; + readonly nextReady: ( + orgId: KinOrgId, + ) => Effect.Effect< + { readonly agentId: KinAgentId; readonly readyAt: number } | undefined, + PersistenceSqlError | PersistenceDecodeError + >; + readonly expireAgent: ( + orgId: KinOrgId, + agentId: KinAgentId, + ) => Effect.Effect< + void, + PersistenceSqlError | PersistenceDecodeError | KinEventQueueUnknownAgent >; - // Scope closure releases only the permit; LEAD-5 must finish(timed_out | failed) in its own onExit. + // Scope closure releases only the permit; the runner owns wake-up finalization. readonly claim: ( input: ClaimInput, ) => Effect.Effect< @@ -148,13 +168,13 @@ const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const organizations = new Map< KinOrgId, - { gate: Semaphore.Semaphore; changed: Deferred.Deferred; users: number } + { gate: Semaphore.Semaphore; changed: Deferred.Deferred } >(); const organization = (orgId: KinOrgId) => { const existing = organizations.get(orgId); if (existing !== undefined) return existing; - const entry = { gate: Semaphore.makeUnsafe(1), changed: Deferred.makeUnsafe(), users: 0 }; + const entry = { gate: Semaphore.makeUnsafe(1), changed: Deferred.makeUnsafe() }; organizations.set(orgId, entry); return entry; }; @@ -179,13 +199,21 @@ const make = Effect.gen(function* () { } return yield* sql.withTransaction( Effect.gen(function* () { - const agents = - yield* sql`SELECT 1 FROM kin_leads WHERE org_id = ${input.orgId} AND agent_id = ${input.agentId}`; + const agents = yield* sql<{ + retired_at: string | null; + }>`SELECT a.retired_at FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${input.orgId} AND l.agent_id = ${input.agentId}`; if (agents.length === 0) return yield* new KinEventQueueUnknownAgent({ orgId: input.orgId, agentId: input.agentId, }); + if (agents[0]?.retired_at !== null) + return yield* new KinEventQueueRetiredAgent({ + orgId: input.orgId, + agentId: input.agentId, + }); const event = input._tag === "github" ? { ...input, dedupeKey: input.deliveryId } : input; let priority: "P0" | "P1" | "P2" | "P3" = "P2"; switch (event._tag) { @@ -229,7 +257,7 @@ const make = Effect.gen(function* () { RETURNING event_id `; if (inserted.length > 0) { - yield* notify(event.orgId); + yield* sql.afterCommit(notify(event.orgId)); return { event, duplicate: false }; } const rows = yield* sql<{ json: string }>`SELECT json FROM kin_events @@ -262,7 +290,7 @@ const make = Effect.gen(function* () { ); }, Effect.mapError((cause) => - isConflict(cause) || isUnknownEvent(cause) || isUnknownAgent(cause) + isConflict(cause) || isUnknownEvent(cause) || isUnknownAgent(cause) || isRetiredAgent(cause) ? cause : toQueuePersistenceError("KinEventQueue.enqueue")(cause), ), @@ -270,7 +298,7 @@ const make = Effect.gen(function* () { ); const readiness = Effect.fn("KinEventQueue.readiness")(function* ( - input: ClaimInput, + input: Pick, now: number, ) { const [window] = yield* sql<{ @@ -302,17 +330,7 @@ const make = Effect.gen(function* () { const claim = Effect.fn("KinEventQueue.claim")( function* (input: ClaimInput) { yield* decodeClaimInput(input); - const entry = yield* Effect.acquireRelease( - Effect.sync(() => { - const entry = organization(input.orgId); - entry.users++; - return entry; - }), - (entry) => - Effect.sync(() => { - if (--entry.users === 0) organizations.delete(input.orgId); - }), - ); + const entry = organization(input.orgId); return yield* Effect.acquireRelease( Effect.interruptible( Effect.uninterruptibleMask((restore) => @@ -501,7 +519,61 @@ const make = Effect.gen(function* () { Effect.mapError(toQueuePersistenceError("KinEventQueue.requeueUnfinished")), ); - return KinEventQueue.of({ enqueue, claim, release, finish, requeueUnfinished }); + const changes = Effect.fn("KinEventQueue.changes")((orgId: KinOrgId) => + Effect.sync(() => Deferred.await(organization(orgId).changed)), + ); + const expireAgent = Effect.fn("KinEventQueue.expireAgent")( + function* (orgId: KinOrgId, agentId: KinAgentId) { + yield* sql.withTransaction( + Effect.gen(function* () { + const leads = + yield* sql`SELECT 1 FROM kin_leads WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + if (leads.length === 0) return yield* new KinEventQueueUnknownAgent({ orgId, agentId }); + yield* sql`UPDATE kin_events SET state = 'expired' WHERE org_id = ${orgId} AND agent_id = ${agentId} AND state = 'queued'`; + }), + ); + yield* notify(orgId); + }, + Effect.mapError((cause) => + isUnknownAgent(cause) ? cause : toQueuePersistenceError("KinEventQueue.expireAgent")(cause), + ), + ); + const nextReady = Effect.fn("KinEventQueue.nextReady")( + function* (orgId: KinOrgId) { + const now = yield* Clock.currentTimeMillis; + const rows = yield* sql<{ agent_id: string }>`SELECT e.agent_id + FROM kin_events e JOIN kin_agents a ON a.org_id = e.org_id AND a.agent_id = e.agent_id + WHERE e.org_id = ${orgId} AND e.state = 'queued' AND a.retired_at IS NULL + GROUP BY e.agent_id ORDER BY e.agent_id`; + const candidates = yield* Effect.forEach(rows, (row) => + Effect.gen(function* () { + const input = { + orgId, + agentId: KinAgentId.make(row.agent_id), + }; + const window = yield* readiness(input, now); + return { + agentId: input.agentId, + readyAt: Math.min(window.readyAt, window.next > now ? window.next : Infinity), + }; + }), + ); + return candidates.toSorted( + (a, b) => a.readyAt - b.readyAt || a.agentId.localeCompare(b.agentId), + )[0]; + }, + Effect.mapError(toQueuePersistenceError("KinEventQueue.nextReady")), + ); + return KinEventQueue.of({ + enqueue, + claim, + release, + finish, + requeueUnfinished, + changes, + expireAgent, + nextReady, + }); }); export const layer = Layer.effect(KinEventQueue, make); diff --git a/apps/server/src/kin/lead/RunnerOwnership.test.ts b/apps/server/src/kin/lead/RunnerOwnership.test.ts new file mode 100644 index 0000000000000..d8aa0bea70451 --- /dev/null +++ b/apps/server/src/kin/lead/RunnerOwnership.test.ts @@ -0,0 +1,65 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Scope from "effect/Scope"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as ProcessRunner from "../../processRunner.ts"; +import * as RunnerOwnership from "./RunnerOwnership.ts"; + +it.live("excludes another process without locking data writes and releases on scope close", () => + Effect.gen(function* () { + const directory = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped(); + const filename = `${directory}/state.sqlite`; + const context = yield* Layer.build(NodeSqliteClient.layer({ filename })); + const sql = Context.get(context, SqlClient.SqlClient); + const scope = yield* Effect.acquireRelease(Scope.make(), (scope) => + Scope.close(scope, Exit.void), + ); + yield* RunnerOwnership.acquire().pipe( + Effect.provideService(SqlClient.SqlClient, sql), + Effect.provideService(Scope.Scope, scope), + ); + const contender = yield* Effect.scoped(RunnerOwnership.acquire()).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + Effect.flip, + ); + assert.strictEqual(contender._tag, "KinRunnerOwned"); + yield* sql`CREATE TABLE normal_writes (id INTEGER PRIMARY KEY)`; + yield* sql`INSERT INTO normal_writes VALUES (1)`; + const runner = yield* ProcessRunner.ProcessRunner; + const probe = Effect.fn("test.probe")(function* () { + return yield* runner.run({ + command: "node", + args: [ + "-e", + ` + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(process.argv[1]); + try { db.exec('BEGIN IMMEDIATE'); process.stdout.write('acquired'); } + catch (error) { if (error.errcode !== 5) throw error; process.stdout.write('owned'); } + finally { db.close(); } + `, + `${filename}.kin-runner.sqlite`, + ], + }); + }); + const blocked = yield* probe(); + assert.strictEqual(blocked.code, 0); + assert.strictEqual(blocked.stdout, "owned"); + yield* Scope.close(scope, Exit.void); + const acquired = yield* probe(); + assert.strictEqual(acquired.code, 0); + assert.strictEqual(acquired.stdout, "acquired"); + yield* Effect.scoped(RunnerOwnership.acquire()).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + }).pipe( + Effect.scoped, + Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer))), + ), +); diff --git a/apps/server/src/kin/lead/RunnerOwnership.ts b/apps/server/src/kin/lead/RunnerOwnership.ts new file mode 100644 index 0000000000000..7addec74a3aa5 --- /dev/null +++ b/apps/server/src/kin/lead/RunnerOwnership.ts @@ -0,0 +1,47 @@ +// @effect-diagnostics nodeBuiltinImport:off -- SQLite owns the process-lifetime advisory lock. +import * as NodeSqlite from "node:sqlite"; + +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; + +export class KinRunnerOwned extends Schema.TaggedError()("KinRunnerOwned", {}) {} + +// Do not open a second descriptor: closing it can release this process's POSIX locks. +const owners = new Set(); +export const acquire = Effect.fn("RunnerOwnership.acquire")(function* () { + const sql = yield* SqlClient.SqlClient; + const databases = yield* sql<{ name: string; file: string }>`PRAGMA database_list`.pipe( + Effect.mapError(toPersistenceSqlError("RunnerOwnership.database")), + ); + const filename = databases.find((row) => row.name === "main")?.file; + const key = filename ? `${filename}.kin-runner.sqlite` : sql; + yield* Effect.acquireRelease( + Effect.gen(function* () { + if (owners.has(key)) return yield* new KinRunnerOwned({}); + owners.add(key); + return yield* Effect.try({ + try: () => { + if (typeof key !== "string") return undefined; + const db = new NodeSqlite.DatabaseSync(key); + try { + db.exec( + "PRAGMA busy_timeout = 0; CREATE TABLE IF NOT EXISTS kin_runner_lock (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE;", + ); + return db; + } catch (cause) { + db.close(); + throw cause; + } + }, + catch: toPersistenceSqlError("RunnerOwnership.acquire"), + }).pipe(Effect.tapError(() => Effect.sync(() => owners.delete(key)))); + }), + (db) => + Effect.sync(() => { + db?.close(); + owners.delete(key); + }), + ); +}); diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index e93681eed8074..db58207e6e437 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -1,6 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; import { CommandId, + EnvironmentId, MessageId, KinWakeupId, ThreadId, @@ -48,6 +49,22 @@ const decodeUsage = Schema.decodeEffect( ), ), ); +const resolveAgent = Effect.fn("test.resolveAgent")(function* (threadId: ThreadId) { + const identity = yield* KinIdentity.KinIdentity; + return yield* identity.resolveAgent({ + environmentId: EnvironmentId.make("test"), + capabilities: new Set(), + issuedAt: 0, + requestNamespace: "test", + client: undefined, + thread: { + threadId, + providerSessionId: "test", + providerInstanceId: ProviderInstanceId.make("codex"), + subagent: false, + }, + }); +}); const run = Effect.fn("test.run")(function* () { const runner = yield* WakeupRunner.WakeupRunner; const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); @@ -88,49 +105,53 @@ describe("WakeupRunner", () => { ]); } assert.isFalse(registry.isLeadCaller(projection.thread.id)); + assert.include(yield* resolveAgent(projection.thread.id).pipe(Effect.flip), { + reason: "not_allowed", + }); + assert.isTrue((yield* KinIdentity.KinIdentity).isAgentThread(projection.thread.id)); assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events ORDER BY event_id`, [ { state: "done" }, { state: "done" }, ]); }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), ); - it.effect( - "interrupts mid-wake-up, finishes failed, clears the registry and admits the next claim", - () => - Effect.gen(function* () { - const entered = yield* Deferred.make(); - yield* Effect.gen(function* () { - yield* seed; - const queue = yield* KinEventQueue.KinEventQueue; - const runner = yield* WakeupRunner.WakeupRunner; - const sql = yield* SqlClient.SqlClient; - yield* queue.enqueue(event("interrupt")); - const fiber = yield* run(); - yield* Deferred.await(entered); - yield* Fiber.interrupt(fiber); - assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY wakeup_id`, [ - { status: "failed" }, - ]); - yield* queue.enqueue(event("next")); - yield* TestClock.adjust("20 seconds"); - const events = yield* Effect.scoped( - queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("next"), model: "test" }), - ); - assert.deepStrictEqual( - events.map((item) => item.eventId), - [event("next").eventId], - ); - yield* queue.finish({ orgId, wakeupId: KinWakeupId.make("next"), status: "completed" }); - assert.isDefined(runner); - }).pipe( - Effect.provide( - harness({ - prepare: () => - Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), - }), - ), + it.effect("interrupts mid-wake-up without counting an attempt and admits the next claim", () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("interrupt")); + const fiber = yield* run(); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY wakeup_id`, [ + { status: "interrupted" }, + ]); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 0 }, + ]); + yield* queue.enqueue(event("next")); + yield* TestClock.adjust("20 seconds"); + const events = yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("next"), model: "test" }), ); - }).pipe(provideDeterministicTestRuntime), + assert.deepStrictEqual( + events.map((item) => item.eventId), + [event("interrupt").eventId, event("next").eventId], + ); + yield* queue.finish({ orgId, wakeupId: KinWakeupId.make("next"), status: "completed" }); + assert.isDefined(runner); + }).pipe( + Effect.provide( + harness({ + prepare: () => Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), ); it.effect("backs off a cross-process busy claim on TestClock instead of spinning", () => @@ -171,8 +192,17 @@ describe("WakeupRunner", () => { yield* queue.enqueue(event("timeout")); const fiber = yield* run(); yield* Deferred.await(entered); + const [lead] = yield* sql<{ + thread_id: string; + }>`SELECT thread_id FROM kin_leads WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + const threadId = ThreadId.make(lead!.thread_id); + assert.strictEqual((yield* resolveAgent(threadId)).agentId, agentId); yield* TestClock.adjust("15 minutes"); + yield* TestClock.adjust("30 seconds"); yield* Fiber.join(fiber); + assert.include(yield* resolveAgent(threadId).pipe(Effect.flip), { + reason: "not_allowed", + }); assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [ { status: "timed_out" }, ]); @@ -341,7 +371,7 @@ describe("WakeupRunner", () => { ); it.effect( - "keeps a recovered active run claimed until it completes rather than requeueing it", + "recovers other organizations while keeping a recovered active run claimed until completion", () => Effect.gen(function* () { const entered = yield* Deferred.make(); @@ -377,9 +407,36 @@ describe("WakeupRunner", () => { yield* sql`UPDATE kin_leads SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; yield* sql`UPDATE kin_wakeups SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId}`; yield* Deferred.await(entered); - const runner = yield* WakeupRunner.WakeupRunner; + yield* sql`INSERT INTO kin_orgs VALUES ('other-org', 'Other', '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) + VALUES ('other-owner', 'other-org', 'owner', 'Owner', '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + VALUES ('other-lead', 'other-org', 'Lead', 'lead', 'other-owner', '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) + VALUES ('other-chat', 'other-org', 'team_group', 0, '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + SELECT 'other-lead', 'other-org', 'Lead', 0, models_json, '[]', 'other-chat' FROM kin_leads WHERE agent_id = ${agentId}`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, model, started_at, status, event_ids) + VALUES ('other-abandoned', 'other-org', 'other-lead', 'test', 0, 'running', '[]')`; + const reconciled = yield* Deferred.make(); + const runner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(KinEventQueue.KinEventQueue, { + ...queue, + requeueUnfinished: Effect.fn("test.requeueUnfinished")(function* (org, wakeup) { + const count = yield* queue.requeueUnfinished(org, wakeup); + yield* Deferred.succeed(reconciled, undefined); + return count; + }), + }), + ); const recovered = yield* runner.recover.pipe(Effect.forkChild); - yield* TestClock.adjust("1 second"); + yield* Deferred.await(reconciled); + assert.deepStrictEqual( + yield* sql`SELECT status FROM kin_wakeups WHERE wakeup_id = 'other-abandoned'`, + [{ status: "interrupted" }], + ); + yield* sql`DELETE FROM kin_wakeups WHERE wakeup_id = 'other-abandoned'`; assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events`, [{ state: "claimed" }]); assert.isUndefined(recovered.pollUnsafe()); yield* Deferred.succeed(hold, undefined); @@ -420,6 +477,41 @@ describe("WakeupRunner", () => { }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), ); + it.effect("keeps an org loop alive after a one-shot defect", () => + Effect.gen(function* () { + const failed = yield* Deferred.make(); + const continued = yield* Deferred.make(); + let calls = 0; + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + yield* runner.start; + yield* queue.enqueue(event("defect")); + yield* TestClock.adjust("20 seconds"); + yield* Deferred.await(failed); + yield* queue.enqueue(event("after-defect")); + yield* TestClock.adjust("21 seconds"); + yield* Deferred.await(continued); + assert.strictEqual(calls, 2); + }).pipe( + Effect.provide( + harness({ + prepare: () => + Effect.gen(function* () { + if (++calls === 1) { + yield* Deferred.succeed(failed, undefined); + return yield* Effect.die("one-shot defect"); + } + yield* Deferred.succeed(continued, undefined); + return yield* Effect.never; + }), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + it.effect("contains Kin startup failures while building the non-Kin server runtime", () => { class Core extends Context.Service()( "t3/kin/lead/WakeupRunner.test/Core", @@ -449,6 +541,11 @@ describe("WakeupRunner", () => { ); const context = yield* Layer.build(runtime); assert.isTrue(Context.get(context, Core).ready); + const defect = Layer.mergeAll( + Layer.succeed(Core, { ready: true }), + KinLead.containStartup(Layer.effectDiscard(Effect.die("Kin layer defect"))), + ); + assert.isTrue(Context.get(yield* Layer.build(defect), Core).ready); }).pipe(Effect.provide(KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)))); }); }); diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index adf32ec49ce9b..4488640e0ba0b 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -32,6 +32,7 @@ import * as Memory from "../memory/Memory.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; +import * as RunnerOwnership from "./RunnerOwnership.ts"; export class KinWakeupError extends Schema.TaggedError()("KinWakeupError", { orgId: KinOrgId, @@ -78,7 +79,7 @@ const RunningRow = Schema.Struct({ }); const encodeUsage = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); const decodeLead = Schema.decodeUnknownEffect(LeadRow); -const decodeRunning = Schema.decodeUnknownEffect(Schema.Array(RunningRow)); +const decodeRunning = Schema.decodeUnknownEffect(RunningRow); const ulid = Effect.gen(function* () { const alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; let timestamp = yield* Clock.currentTimeMillis; @@ -94,6 +95,7 @@ const ulid = Effect.gen(function* () { const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const queue = yield* KinEventQueue.KinEventQueue; + const ownership = RunnerOwnership.acquire().pipe(Effect.provideService(SqlClient.SqlClient, sql)); const memory = yield* Memory.Memory; const hooks = yield* Effect.serviceOption(KinWakeupHooks); const prompt = yield* KinLeadPrompt; @@ -110,10 +112,13 @@ const make = Effect.gen(function* () { JOIN kin_orgs o ON o.org_id = l.org_id WHERE l.org_id = ${orgId} AND l.agent_id = ${agentId}`; return yield* decodeLead(row); }); + const retired = new Set(); const retire = Effect.fn("WakeupRunner.retire")(function* (record: typeof LeadRow.Type) { - yield* sql`UPDATE kin_events SET state = 'expired' WHERE org_id = ${record.orgId} AND agent_id = ${record.agentId} AND state = 'queued'`; + if (retired.has(record.agentId)) return; + yield* queue.expireAgent(record.orgId, record.agentId); if (record.threadId !== null) yield* identity.endAgentThread({ orgId: record.orgId, threadId: record.threadId }); + retired.add(record.agentId); }); const capture = Effect.fn("WakeupRunner.capture")(function* (row: typeof RunningRow.Type) { if (row.threadId === null || row.runId === null) return; @@ -163,13 +168,26 @@ const make = Effect.gen(function* () { ? "completed" : "failed"; yield* Effect.gen(function* () { - if (result.timedOut) + if (result.timedOut) { yield* threads.interruptThread({ projectId, threadId, runId: result.run?.id ?? runId, commandId: CommandId.make(`kin-timeout:${row.wakeupId}`), }); + const settled = yield* threads.waitForThread({ + projectId, + threadId, + runId, + timeoutMs: 30_000, + }); + if (settled.timedOut) + yield* Effect.logWarning("Kin timed-out run may continue after interrupt", { + orgId: row.orgId, + wakeupId: row.wakeupId, + runId, + }); + } yield* capture(row); yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status }); }).pipe( @@ -205,7 +223,10 @@ const make = Effect.gen(function* () { }), ); }); - const cleanup = Effect.fn("WakeupRunner.cleanup")(function* (row: typeof RunningRow.Type) { + const cleanup = Effect.fn("WakeupRunner.cleanup")(function* ( + row: typeof RunningRow.Type, + interrupted = false, + ) { const [running] = yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; if (running === undefined) return; @@ -222,7 +243,8 @@ const make = Effect.gen(function* () { } yield* clearMissingThread(row).pipe(Effect.ignore); yield* capture(row).pipe(Effect.ignore); - yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status: "failed" }); + if (interrupted) yield* queue.requeueUnfinished(row.orgId, row.wakeupId); + else yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status: "failed" }); }); const runOnce = Effect.fn("WakeupRunner.runOnce")( @@ -245,8 +267,8 @@ const make = Effect.gen(function* () { Effect.gen(function* () { const events = yield* Effect.acquireRelease( queue.claim({ orgId, agentId, wakeupId, model: record.models.strong.model }), - () => - cleanup(row).pipe( + (_, exit) => + cleanup(row, Exit.hasInterrupts(exit)).pipe( Effect.catch((error) => Effect.logError("Kin wake-up finalizer failed", { orgId, @@ -362,7 +384,7 @@ const make = Effect.gen(function* () { }).pipe( Effect.onExit((exit) => Exit.isFailure(exit) - ? cleanup(row).pipe( + ? cleanup(row, Exit.hasInterrupts(exit)).pipe( Effect.catch((error) => Effect.logError("Kin wake-up cleanup failed", { orgId, @@ -389,63 +411,67 @@ const make = Effect.gen(function* () { ), ), ); - const recover = Effect.gen(function* () { - const rows = - yield* sql`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE status = 'running' ORDER BY started_at, wakeup_id`; - for (const original of yield* decodeRunning(rows)) { - yield* Effect.scoped( - Effect.gen(function* () { - let row = original; - const existing = - row.threadId === null - ? [] - : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; - const projection = - row.threadId === null || existing.length === 0 - ? undefined - : yield* threads.getThreadProjection(row.threadId); - const run = projection?.runs.find((run) => - row.runId === null - ? run.userMessageId === `kin-wakeup-msg:${row.wakeupId}` - : run.id === row.runId, - ); - if (projection === undefined || run === undefined) { - if (projection === undefined) yield* clearMissingThread(row); - yield* queue.requeueUnfinished(row.orgId, row.wakeupId); - return; - } - row = { ...row, runId: run.id }; - yield* Effect.acquireRelease(Effect.void, () => cleanup(row).pipe(Effect.ignore)); - yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; - const record = yield* lead(row.orgId, row.agentId); - if (record.retiredAt !== null) { - yield* cleanup(row); - return yield* retire(record); - } - yield* identity.registerAgentThread({ - orgId: row.orgId, - agentId: row.agentId, - threadId: projection.thread.id, - }); - yield* registry.register(projection.thread.id, { ...row, level: record.level }); - yield* wait(row, projection.thread.projectId).pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) ? cleanup(row).pipe(Effect.ignore) : Effect.void, - ), - ); - }), - ).pipe( - Effect.mapError( - (cause) => - new KinWakeupError({ - orgId: original.orgId, - agentId: original.agentId, - cause: runnerCause(cause), + const recoverOrg = Effect.fn("WakeupRunner.recoverOrg")( + function* (orgId: KinOrgId) { + const rows = + yield* sql`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY started_at, wakeup_id`; + for (const raw of rows) { + yield* Effect.scoped( + Effect.gen(function* () { + let row = yield* decodeRunning(raw); + const existing = + row.threadId === null + ? [] + : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; + const projection = + row.threadId === null || existing.length === 0 + ? undefined + : yield* threads.getThreadProjection(row.threadId); + const run = projection?.runs.find((run) => + row.runId === null + ? run.userMessageId === `kin-wakeup-msg:${row.wakeupId}` + : run.id === row.runId, + ); + if (projection === undefined || run === undefined) { + if (projection === undefined) yield* clearMissingThread(row); + yield* queue.requeueUnfinished(row.orgId, row.wakeupId); + return; + } + row = { ...row, runId: run.id }; + yield* Effect.acquireRelease(Effect.void, (_, exit) => + cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), + ); + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + const record = yield* lead(row.orgId, row.agentId); + if (record.retiredAt !== null) { + yield* cleanup(row); + return yield* retire(record); + } + yield* identity.registerAgentThread({ + orgId: row.orgId, + agentId: row.agentId, + threadId: projection.thread.id, + }); + yield* registry.register(projection.thread.id, { ...row, level: record.level }); + yield* wait(row, projection.thread.projectId).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore) + : Effect.void, + ), + ); + }), + ).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin wake-up recovery failed", { + orgId, + wakeupId: raw.wakeupId, + cause, }), - ), - ); - } - }).pipe( + ), + ); + } + }, Effect.mapError((cause) => isWakeupError(cause) ? cause @@ -457,48 +483,112 @@ const make = Effect.gen(function* () { ), ); + const orgIds = Effect.fn("WakeupRunner.orgIds")(function* () { + const rows = yield* sql<{ + orgId: string; + }>`SELECT org_id AS "orgId" FROM kin_leads GROUP BY org_id ORDER BY org_id`; + return rows.map((row) => KinOrgId.make(row.orgId)); + }); + const recover = Effect.scoped( + Effect.gen(function* () { + yield* ownership; + yield* Effect.forEach( + yield* orgIds(), + (orgId) => + recoverOrg(orgId).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin org recovery failed", { orgId, cause }), + ), + ), + { + concurrency: "unbounded", + discard: true, + }, + ); + }), + ).pipe( + Effect.mapError( + (cause) => + new KinWakeupError({ + orgId: KinOrgId.make("startup"), + agentId: KinAgentId.make("startup"), + cause: runnerCause(cause), + }), + ), + ); const start = Effect.gen(function* () { - yield* recover; + yield* ownership; const organizations = new Set(); yield* Effect.gen(function* () { - const rows = yield* sql<{ - orgId: string; - }>`SELECT org_id AS "orgId" FROM kin_leads GROUP BY org_id ORDER BY org_id`; - for (const raw of rows) { - const orgId = KinOrgId.make(raw.orgId); + for (const orgId of yield* orgIds()) { if (organizations.has(orgId)) continue; organizations.add(orgId); yield* Effect.gen(function* () { - const retired = yield* sql<{ - agentId: string; - }>`SELECT l.agent_id AS "agentId" FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id WHERE l.org_id = ${orgId} AND a.retired_at IS NOT NULL ORDER BY l.agent_id`; - for (const record of retired) - yield* retire(yield* lead(orgId, KinAgentId.make(record.agentId))); - const [next] = yield* sql<{ + const changed = yield* queue.changes(orgId); + yield* recoverOrg(orgId); + const unfinished = + yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running'`; + if (unfinished.length > 0) { + yield* changed.pipe( + Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), + ); + return; + } + const rows = yield* sql<{ agentId: string; - }>`SELECT e.agent_id AS "agentId" FROM kin_events e JOIN kin_agents a ON a.org_id = e.org_id AND a.agent_id = e.agent_id WHERE e.org_id = ${orgId} AND e.state = 'queued' AND a.retired_at IS NULL ORDER BY e.priority, e.received_at, e.event_id LIMIT 1`; - if (next !== undefined) yield* runOnce(orgId, KinAgentId.make(next.agentId)); + }>`SELECT l.agent_id AS "agentId" FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NOT NULL ORDER BY l.agent_id`; + for (const row of rows) yield* retire(yield* lead(orgId, KinAgentId.make(row.agentId))); + const paused = + yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; + const next = paused.length === 0 ? yield* queue.nextReady(orgId) : undefined; + const now = yield* Clock.currentTimeMillis; + if (next !== undefined && next.readyAt <= now) { + yield* runOnce(orgId, next.agentId); + return; + } + yield* changed.pipe( + Effect.timeoutOrElse({ + duration: Math.min(60_000, Math.max(1, (next?.readyAt ?? Infinity) - now)), + orElse: () => Effect.void, + }), + ); }).pipe( - Effect.catch((error) => - Effect.logError("Kin runner failed", { orgId, errorTag: error._tag }), + Effect.catchCause((cause) => + Effect.logError("Kin runner iteration failed", { orgId, cause }).pipe( + Effect.andThen(Effect.sleep("1 second")), + ), ), - Effect.repeat(Schedule.spaced("5 seconds")), + Effect.forever, + Effect.catchCause((cause) => Effect.logError("Kin org loop failed", { orgId, cause })), + Effect.ensuring(Effect.sync(() => organizations.delete(orgId))), Effect.forkScoped, ); } }).pipe( - Effect.catch((error) => - Effect.logError("Kin runner discovery failed", { errorTag: error._tag }), - ), - Effect.repeat(Schedule.spaced("5 seconds")), + Effect.catchCause((cause) => Effect.logError("Kin runner discovery failed", { cause })), + Effect.repeat(Schedule.spaced("60 seconds")), Effect.forkScoped, ); - }).pipe( - Effect.catch((error) => - Effect.logError("Kin runner recovery failed; disabled", { errorTag: error._tag }), - ), + }).pipe(Effect.catchCause((cause) => Effect.logError("Kin runner unavailable", { cause }))); + const guardedRunOnce = Effect.fn("WakeupRunner.guardedRunOnce")( + function* (orgId: KinOrgId, agentId: KinAgentId) { + return yield* Effect.scoped( + Effect.gen(function* () { + yield* ownership; + yield* runOnce(orgId, agentId); + }), + ); + }, + (effect, orgId, agentId) => + effect.pipe( + Effect.mapError( + (cause) => new KinWakeupError({ orgId, agentId, cause: runnerCause(cause) }), + ), + ), ); - return WakeupRunner.of({ runOnce, recover, start }); + return WakeupRunner.of({ runOnce: guardedRunOnce, recover, start }); }); export const layer = Layer.effect(WakeupRunner, make); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 76d91efe99210..751ebe71dfe52 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -26,7 +26,7 @@ export class KinLeadStartupError extends Schema.TaggedError export function containStartup(layer: Layer.Layer) { return Layer.effectDiscard(Effect.void).pipe( Layer.provide(layer), - Layer.catch((cause) => + Layer.catchCause((cause) => Layer.effectDiscard( Effect.logError( "Kin startup failed; Kin disabled", @@ -64,6 +64,10 @@ const Repositories = Schema.fromJsonString( export const layerServer = containStartup( Layer.unwrap( Effect.gen(function* () { + const repositories = yield* Config.schema(Repositories, "KIN_MEMORY_REPOSITORIES").pipe( + Config.withDefault([]), + ); + if (repositories.length === 0) return Layer.empty; const prompt = yield* Effect.serviceOption(KinLeadPrompt); if (Option.isNone(prompt)) { yield* Effect.logWarning( @@ -72,10 +76,6 @@ export const layerServer = containStartup( ); return Layer.empty; } - const repositories = yield* Config.schema(Repositories, "KIN_MEMORY_REPOSITORIES").pipe( - Config.withDefault([]), - ); - if (repositories.length === 0) return Layer.empty; return layer.pipe( Layer.provide(Layer.succeed(KinLeadPrompt, prompt.value)), Layer.provide(MemoryConfig.layer(repositories)), diff --git a/apps/server/src/kin/triggers/KinTriggerSink.test.ts b/apps/server/src/kin/triggers/KinTriggerSink.test.ts index 2ee9f763bbdd8..23ac5ff8368cf 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.test.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.test.ts @@ -486,6 +486,14 @@ describe("KinTriggerSink", () => { Effect.gen(function* () { yield* service.upsert(yield* upsert("task-org-a")); const a = source(orgA); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_agent_threads (org_id, agent_id, thread_id, started_at) + VALUES (${orgA}, ${`agent-${orgA}`}, ${a.inboxThreadId}, ${timestamp})`; + assert.strictEqual( + (yield* store.registerSource(a).pipe(Effect.flip))._tag, + "KinWebhookSourceConflict", + ); + yield* sql`DELETE FROM kin_agent_threads WHERE thread_id = ${a.inboxThreadId}`; yield* store.registerSource(a); for (const candidate of [ { ...a, orgId: orgB }, @@ -553,66 +561,122 @@ describe("KinTriggerSink", () => { ); }); -it.effect( - "builds the runtime after a Kin migration failure and refuses scheduled turns without blocking ordinary launches", - () => - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql`CREATE TABLE kin_orgs (broken TEXT)`; - const launches: ThreadLaunchService.ThreadLaunchInput[] = []; - const inner = Layer.mock(ThreadLaunchService.ThreadLaunchService)({ - launch: (input) => - Effect.sync(() => { - launches.push(input); - return { - threadId: projection(orgA).thread.id, - projection: projection(orgA), - resumed: false, - }; - }), - }); - yield* Effect.gen(function* () { - const launch = yield* ThreadLaunchService.ThreadLaunchService; - const input: ThreadLaunchService.ThreadLaunchInput = { - commandId: CommandId.make("ordinary"), - projectId: ProjectId.make("project"), - title: "Ordinary", - modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "test" }, - runtimeMode: "full-access", - interactionMode: "default", - workspaceStrategy: { type: "root" }, - createdBy: "system", - creationSource: "server", - }; - yield* launch.launch(input); - const blocked = yield* launch - .launch({ - ...input, - initialMessage: { - text: "Scheduled", - attachments: [], - scheduledTaskId: ScheduledTaskId.make("task"), - }, - }) - .pipe(Effect.flip); - assert.strictEqual(blocked._tag, "ThreadLaunchError"); - const management = yield* ThreadManagementService.ThreadManagementService; - const bound = yield* management - .sendToThread({ - projectId: input.projectId, +it.effect("builds the runtime after a Kin migration failure and refuses only webhook turns", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_orgs (broken TEXT)`; + for (const [id, type] of [ + ["task", "webhook"], + ["interval", "interval"], + ]) { + yield* sql`INSERT INTO scheduled_tasks (task_id, title, prompt, enabled, schedule_json, project_id, + workspace_strategy_json, model_selection_json, runtime_mode, interaction_mode, created_by, + creation_source, created_at, updated_at, last_run_status, run_count) + VALUES (${id}, 'Task', 'Prompt', 1, ${JSON.stringify({ type })}, 'project', '{}', '{}', + 'full-access', 'default', 'system', 'server', ${timestamp}, ${timestamp}, 'never', 0)`; + } + const sends: string[] = []; + const managementLayer = Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Effect.gen(function* () { + sends.push(input.scheduledTaskId ?? "ordinary"); + return yield* new OrchestratorCommandRejectedError({ commandId: input.commandId, + commandType: "test.inner_send", + }); + }), + }); + const launches: ThreadLaunchService.ThreadLaunchInput[] = []; + const inner = Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: (input) => + Effect.sync(() => { + launches.push(input); + return { threadId: projection(orgA).thread.id, - messageId: MessageId.make("message"), - text: "Bound", + projection: projection(orgA), + resumed: false, + }; + }), + }); + yield* Effect.gen(function* () { + const launch = yield* ThreadLaunchService.ThreadLaunchService; + const input: ThreadLaunchService.ThreadLaunchInput = { + commandId: CommandId.make("ordinary"), + projectId: ProjectId.make("project"), + title: "Ordinary", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "test" }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + createdBy: "system", + creationSource: "server", + }; + yield* launch.launch(input); + yield* launch.launch({ + ...input, + commandId: CommandId.make("scheduled-task:interval:interval:1"), + initialMessage: { + text: "Interval", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("interval"), + }, + }); + const blocked = yield* launch + .launch({ + ...input, + commandId: CommandId.make("scheduled-task:task:webhook:delivery"), + initialMessage: { + text: "Scheduled", attachments: [], scheduledTaskId: ScheduledTaskId.make("task"), - mode: "queue", - createdBy: "system", - creationSource: "server", - }) - .pipe(Effect.flip); - assert.strictEqual(bound._tag, "OrchestratorCommandRejectedError"); - assert.strictEqual(launches.length, 1); - }).pipe(Effect.provide(KinTriggerSink.wrapTaskDispatch(inner, threads))); - }).pipe(Effect.provide(Sqlite.layerMemory)), + }, + }) + .pipe(Effect.flip); + assert.strictEqual(blocked._tag, "ThreadLaunchError"); + const management = yield* ThreadManagementService.ThreadManagementService; + const passed = yield* management + .sendToThread({ + projectId: input.projectId, + commandId: input.commandId, + threadId: projection(orgA).thread.id, + messageId: MessageId.make("interval-message"), + text: "Interval", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("interval"), + mode: "queue", + createdBy: "system", + creationSource: "server", + }) + .pipe(Effect.flip); + assert.include(passed, { commandType: "test.inner_send" }); + assert.deepStrictEqual(sends, ["interval"]); + const bound = yield* management + .sendToThread({ + projectId: input.projectId, + commandId: input.commandId, + threadId: projection(orgA).thread.id, + messageId: MessageId.make("message"), + text: "Bound", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("task"), + mode: "queue", + createdBy: "system", + creationSource: "server", + }) + .pipe(Effect.flip); + assert.strictEqual(bound._tag, "OrchestratorCommandRejectedError"); + assert.strictEqual(launches.length, 2); + }).pipe(Effect.provide(KinTriggerSink.wrapTaskDispatch(inner, managementLayer))); + }).pipe(Effect.provide(Sqlite.layerMemory)), +); + +it.effect("does not relabel a core dispatch layer failure as a Kin startup failure", () => + Effect.gen(function* () { + const failure = new Error("core dispatch unavailable"); + const inner = Layer.effect(ThreadLaunchService.ThreadLaunchService, Effect.fail(failure)); + const result = yield* Layer.build(KinTriggerSink.wrapTaskDispatch(inner, threads)).pipe( + Effect.flip, + ); + assert.strictEqual(result, failure); + }).pipe(Effect.provide(Sqlite.layerMemory)), ); diff --git a/apps/server/src/kin/triggers/KinTriggerSink.ts b/apps/server/src/kin/triggers/KinTriggerSink.ts index a23d0adc43c5b..ae0d4ca088001 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.ts @@ -175,13 +175,12 @@ export function wrapTaskDispatch( }), ), ).pipe( - Layer.provide(Layer.mergeAll(inner, threads)), - Layer.provide(Layer.mergeAll(layer, KinTriggerStore.layer).pipe(Layer.provide(threads))), - Layer.catch((cause) => + Layer.provide(Layer.mergeAll(layer, KinTriggerStore.layer)), + Layer.catchCause((cause) => Layer.mergeAll( Layer.effectDiscard( Effect.logError( - "Kin trigger startup failed; scheduled dispatch disabled", + "Kin trigger startup failed; webhook dispatch disabled", new KinTriggerStartupError({ cause }), ), ), @@ -192,7 +191,11 @@ export function wrapTaskDispatch( return ThreadLaunchService.ThreadLaunchService.of({ ...service, launch: Effect.fn("KinTriggerSink.disabledLaunch")(function* (input) { - if (input.initialMessage?.scheduledTaskId === undefined) + const taskId = input.initialMessage?.scheduledTaskId; + if ( + taskId === undefined || + !input.commandId.startsWith(`scheduled-task:${taskId}:webhook:`) + ) return yield* service.launch(input); return yield* new ThreadLaunchService.ThreadLaunchError({ operation: "dispatch-message", @@ -208,10 +211,18 @@ export function wrapTaskDispatch( ThreadManagementService.ThreadManagementService, Effect.gen(function* () { const service = yield* ThreadManagementService.ThreadManagementService; + const sql = yield* SqlClient.SqlClient; return ThreadManagementService.ThreadManagementService.of({ ...service, sendToThread: Effect.fn("KinTriggerSink.disabledSend")(function* (input) { if (input.scheduledTaskId === undefined) return yield* service.sendToThread(input); + const webhook = + yield* sql`SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.scheduledTaskId} + AND json_extract(schedule_json, '$.type') = 'webhook'`.pipe( + Effect.map((rows) => rows.length > 0), + Effect.catchCause(() => Effect.succeed(true)), + ); + if (!webhook) return yield* service.sendToThread(input); return yield* new OrchestratorCommandRejectedError({ commandId: input.commandId, commandType: "kin.trigger.disabled", @@ -221,7 +232,8 @@ export function wrapTaskDispatch( }); }), ), - ).pipe(Layer.provide(Layer.mergeAll(inner, threads))), + ), ), + Layer.provide(Layer.mergeAll(inner, threads)), ); } diff --git a/apps/server/src/kin/triggers/KinTriggerStore.ts b/apps/server/src/kin/triggers/KinTriggerStore.ts index c8c57f0b5c3cc..efa4c8be7028a 100644 --- a/apps/server/src/kin/triggers/KinTriggerStore.ts +++ b/apps/server/src/kin/triggers/KinTriggerStore.ts @@ -195,6 +195,7 @@ const make = Effect.gen(function* () { AND t.thread_id IS NULL AND t.thread_mode = 'new_thread' AND json_extract(t.schedule_json, '$.type') = 'webhook' AND EXISTS (SELECT 1 FROM kin_orgs WHERE org_id = ${value.orgId}) + AND NOT EXISTS (SELECT 1 FROM kin_agent_threads WHERE thread_id = ${value.inboxThreadId}) AND NOT EXISTS ( SELECT 1 FROM kin_webhook_sources WHERE org_id <> ${value.orgId} diff --git a/docs/kin/specs/chat.md b/docs/kin/specs/chat.md index 963fd9dd8b7d3..d67116b9d2c27 100644 --- a/docs/kin/specs/chat.md +++ b/docs/kin/specs/chat.md @@ -46,7 +46,7 @@ Every row carries `orgId`. Phase 1 has one org, created on first start. Bootstra - **Session to person** (`KinIdentity.resolve(session)`): - Phase 1: after the RPC scope check, a `/ws` session holding every scope in `AuthStandardClientScopes` resolves to the default org owner. A narrower session resolves to an empty list, never to a person inferred from `subject`. The result is a list of `{ orgId, personId }` entries so phase-2 bindings do not change callers. Only the owner may pair devices before CHAT-11. - Phase 2: a row in `kin_session_bindings (session_id, org_id, person_id)` decides. A session without a binding is the owner only if it holds `AuthStandardClientScopes`. Members get sessions with the new `kin:chat` scope only (CHAT-11), so their device never gets the owner's terminal or files. -- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead's durable thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` on retirement, not between fresh provider sessions on the same T3 thread; an ended thread cannot be registered again. +- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead's durable thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution also requires a persisted `kin_wakeups` row with `status = running` and the same org, agent and thread. Durable registration alone grants no authority between wake-ups; `insertMessage` rechecks the running row inside its write transaction. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` on retirement, not between fresh provider sessions on the same T3 thread; an ended thread cannot be registered again. ### Chats, members, roles diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 5074266df5648..0b84f3f246223 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -177,11 +177,13 @@ Coalescing: - It then claims every queued event for that lead whose `not_before` has passed, ordered by `priority, received_at`, in one transaction, and starts one wake-up for the batch. The claim index is `(org_id, agent_id, state, priority, received_at, not_before)`. Twenty webhook events from one push become one wake-up. - A batch holds at most 50 events. Overflow stays queued for the next batch; no delivery references are dropped or marked handled. Same-repo summaries belong to the prompt builder, not the durable queue. - Events that arrive while a wake-up runs wait for the next one. The runner never starts a second session for the same organization. This is the one-writer rule: memory is written only inside a wake-up, and wake-ups are serial. -- Implementation: an organization permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 must call `finish(timed_out | failed)` in its own `onExit`. Idle semaphore entries are evicted. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. +- Implementation: an organization permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 calls `finish(timed_out | failed)` for real failures and timeouts; graceful scope interruption uses `requeueUnfinished` without consuming an attempt. Organization signal entries stay available for idle runner subscriptions. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval`, so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. `release` returns `{ released, failedEventIds }` and `finish` returns `{ failedEventIds }`; the ids are the events that transitioned to terminal `failed` in that call, ordered by id. Unknown or foreign lead ids fail with `KinEventQueueUnknownAgent`, and reused wake-up ids fail with `KinEventQueueWakeupConflict`, not SQL constraint errors. Enqueue validates the lead and computes owner priority inside the insert transaction. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. -Restart: the runner first reconciles each unfinished wake-up with its provider run (see failure cases). It must not requeue events belonging to a still-active run. For an abandoned run only, `requeueUnfinished(orgId, wakeupId)` atomically marks the running wake-up interrupted and returns its claimed events to `queued`, clearing their `wakeup_id` while preserving `not_before` and `attempts`. It is idempotent and organization-scoped; `idx_kin_events_wakeup` supports the update. Completed runs are finished by the runner instead. Layer construction never blindly requeues active work. Recovery also checks the deterministic first-message id if launch committed before `run_id` was saved. A provisional lead thread with no upstream thread is ended and cleared before requeue, so the next attempt launches rather than sending to a nonexistent thread. +Ownership: the runner holds `BEGIN IMMEDIATE` on a lock table in `.kin-runner.sqlite` for its scope, separate from the data database so normal writes continue. Never delete that sidecar while running. SQLite releases ownership on process death; another live process cannot run recovery or claims. No Kin migration is required. + +Restart: recovery runs independently in each organization loop and logs failures per row. The runner first reconciles each unfinished wake-up with its provider run (see failure cases). It must not requeue events belonging to a still-active run. For an abandoned run only, `requeueUnfinished(orgId, wakeupId)` atomically marks the running wake-up interrupted and returns its claimed events to `queued`, clearing their `wakeup_id` while preserving `not_before` and `attempts`. It is idempotent and organization-scoped; `idx_kin_events_wakeup` supports the update. Completed runs are finished by the runner instead. Layer construction never blindly requeues active work. Recovery also checks the deterministic first-message id if launch committed before `run_id` was saved. A provisional lead thread with no upstream thread is ended and cleared before requeue, so the next attempt launches rather than sending to a nonexistent thread. ### A wake-up (one fresh lead session) @@ -203,15 +205,15 @@ Each wake-up: 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every run is a new provider session. -7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, an in-memory map the tools use. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. -8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread` and mark `timed_out`. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and thread; durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, an in-memory map the tools use. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. +8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. 9. Finish: `Memory.commitWakeup` (one commit holding the staged item writes and the journal entry), reset the read checkout, record token usage from the run's provider turns into `kin_wakeups.tokens_json` and `KinBudget`, mark events `done`, and `settleThread`. If the lead writes files by hand in the read checkout, step 9's reset throws the changes away. A non-empty `git status --porcelain` before the reset is logged in the journal as "Lead edited memory files directly; discarded." Kin boot is optional: configuration, migrations, identity bootstrap and runner startup failures are logged with typed errors and disable Kin, never the core T3 runtime. The runner requires the real `KinLeadPrompt` implementation; while LEAD-6 is absent, the server hook does not construct the runner. Memory repositories are supplied through `KIN_MEMORY_REPOSITORIES` (a JSON array of `{ orgId, repoUrl }`); no private repository is assumed by source code. -The scheduled-task dispatch wrapper also contains Kin migration failures. While it cannot identify Kin webhook sources, it refuses **all scheduled launches and sends** rather than risk executing an inbox as a provider prompt; ordinary launches and sends and the rest of T3 remain available. Restoring the Kin database and restarting restores scheduled dispatch. +The scheduled-task dispatch wrapper also contains Kin migration failures. When Kin persistence is unavailable, launch refusal is limited to `scheduled-task::webhook:` command ids. Sends classify the task using upstream `scheduled_tasks.schedule_json` and refuse only webhooks; an unreadable upstream row fails closed for that send only. Cron, interval, manual and ordinary dispatch remain available. Restoring the Kin database and restarting restores webhook dispatch. A `MemorySyncConflict` fails the wake-up and leaves the events queued with the queue's five-minute counted-failure backoff. `MemoryCheckoutInvalid` does the same and logs an error; `KinWakeupHooks.memoryCheckoutInvalid` is the typed notification hook for LEAD-3. Writer commits/journal/reset and budget charging remain LEAD-3 and LEAD-10 integrations, not fabricated runner implementations. Until the writer lands, `memory_commit` remains null rather than claiming a writer commit. `tokens_json` contains each provider turn's recorded token usage and provider-turn id, limited to the wake-up run's attempts, on success and failure. From 7079c22bace475186cb4f37c3ac6ce586339a435 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:21:27 +0530 Subject: [PATCH 03/12] fix(kin): recheck wake-up authority for chat mutations --- .../src/kin/chat/KinChatService.test.ts | 19 +++++++++++++++++++ apps/server/src/kin/chat/KinChatService.ts | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index aff95aaa3e34c..b256f61959fe2 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -209,6 +209,25 @@ describe("KinChatService", () => { .messages, [posted], ); + const widget = yield* service.upsertWidget(agent, { + chatId: chat.chatId, + type: "task", + state, + }); + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; + assert.include( + yield* service + .upsertWidget(agent, { + chatId: chat.chatId, + widgetId: widget.widgetId, + revision: widget.revision, + type: "task", + state, + }) + .pipe(Effect.flip), + { reason: "not_allowed" }, + ); }).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/kin/chat/KinChatService.ts b/apps/server/src/kin/chat/KinChatService.ts index 970b125421c54..7720680951451 100644 --- a/apps/server/src/kin/chat/KinChatService.ts +++ b/apps/server/src/kin/chat/KinChatService.ts @@ -150,7 +150,8 @@ const make = Effect.gen(function* () { agent.retiredAt !== null || thread.endedAt !== null || thread.agentId !== agent.agentId || - agent.ownerPersonId !== identity.ownerPersonId + agent.ownerPersonId !== identity.ownerPersonId || + !(yield* repo.hasRunningWakeup(identity)) ) return yield* notAllowed; } From 847c21320791fa66009185a480ef2dc404b9859d Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:01:48 +0530 Subject: [PATCH 04/12] fix(kin): bind wake-up authority to its provider session --- .../src/kin/chat/KinChatRepository.test.ts | 35 +++++- apps/server/src/kin/chat/KinChatRepository.ts | 37 ++++-- .../src/kin/chat/KinChatService.test.ts | 15 +++ apps/server/src/kin/chat/KinChatService.ts | 1 + apps/server/src/kin/chat/KinIdentity.test.ts | 35 +++++- apps/server/src/kin/chat/KinIdentity.ts | 6 +- .../src/kin/chat/WakeupAuthority.testkit.ts | 17 +++ .../scheduledTask.kin-contract.test.ts | 14 +++ apps/server/src/kin/lead/KinLeadRegistry.ts | 5 - .../src/kin/lead/RunnerOwnership.test.ts | 65 ----------- apps/server/src/kin/lead/RunnerOwnership.ts | 47 -------- apps/server/src/kin/lead/WakeupRunner.test.ts | 108 +++++++++++++++++- .../src/kin/lead/WakeupRunner.testkit.ts | 12 +- apps/server/src/kin/lead/WakeupRunner.ts | 59 +++++----- apps/server/src/kin/lead/layer.ts | 3 +- .../server/src/kin/triggers/KinTriggerSink.ts | 8 +- docs/kin/specs/lead.md | 12 +- 17 files changed, 300 insertions(+), 179 deletions(-) create mode 100644 apps/server/src/kin/chat/WakeupAuthority.testkit.ts delete mode 100644 apps/server/src/kin/lead/RunnerOwnership.test.ts delete mode 100644 apps/server/src/kin/lead/RunnerOwnership.ts diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index 636e5d601a30b..76f122ace726b 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -1,3 +1,5 @@ +import { seedWakeupRun } from "./WakeupAuthority.testkit.ts"; +import type { McpThreadCaller } from "../../mcp/McpInvocationContext.ts"; import { assert, describe, it } from "@effect/vitest"; import { KinOrgId, @@ -14,6 +16,7 @@ import { KinWidget, ThreadId, TurnItemId, + ProviderInstanceId, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -75,8 +78,14 @@ const agentMember: KinChatMember = { principal: { _tag: "agent", agentId }, role: "member", }; +const caller: McpThreadCaller = { + threadId, + providerSessionId: "provider-session", + providerInstanceId: ProviderInstanceId.make("codex"), +}; const message = (id: string, overrides: Partial> = {}) => ({ + caller, orgId, chatId, messageId: decodeMessageId(id), @@ -95,7 +104,10 @@ const message = (id: string, overrides: Partial> = {}) = editedAt: null, ...overrides, clientMessageId: { kind: "send", id }, - }) satisfies Omit & { clientMessageId: { kind: "send" | "act"; id: string } }; + }) satisfies Omit & { + caller: McpThreadCaller; + clientMessageId: { kind: "send" | "act"; id: string }; + }; const widget: KinWidget = { orgId, chatId, @@ -973,6 +985,27 @@ describe("KinChatRepository", () => { VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('chat-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + yield* seedWakeupRun(threadId); + yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'chat-wakeup'`; + for (const providerSessionId of ["leftover-session", "user-session"]) { + assert.include( + yield* repo + .insertMessage({ + ...message("unauthorized-session", { + author: { + _tag: "agent", + agentId, + memberId: agentMemberId, + ownerPersonId: personId, + threadId, + }, + }), + caller: { ...caller, providerSessionId }, + }) + .pipe(Effect.flip), + { reason: "not_allowed" }, + ); + } const agentAuthor = { _tag: "agent", agentId, diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index c5cfd46433cf2..eadb2b3c42857 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -1,3 +1,4 @@ +import type { McpThreadCaller } from "../../mcp/McpInvocationContext.ts"; import { KinOrg, KinOrgId, @@ -58,6 +59,7 @@ type Scope = { readonly orgId: KinOrgId }; type ChatScope = Scope & { readonly chatId: KinChatId }; type MemberScope = ChatScope & { readonly memberId: KinMemberId }; type InsertMessage = Omit & { + readonly caller?: McpThreadCaller | undefined; readonly clientMessageId: { readonly kind: "send" | "act"; readonly id: typeof TrimmedNonEmptyString.Type; @@ -174,7 +176,11 @@ export class KinChatRepository extends Context.Service< readonly createdAt: IsoDateTime; }) => Effect.Effect; readonly hasRunningWakeup: ( - input: Scope & { readonly agentId: KinAgentId; readonly threadId: ThreadId }, + input: Scope & { + readonly agentId: KinAgentId; + readonly threadId: ThreadId; + readonly caller: McpThreadCaller; + }, ) => Effect.Effect; readonly findAgentThread: ( threadId: ThreadId, @@ -371,9 +377,20 @@ const make = Effect.gen(function* () { Effect.mapError(toRepositoryError("KinChatRepository.getAgentThread")), ); const hasRunningWakeup = Effect.fn("KinChatRepository.hasRunningWakeup")( - function* (input: Scope & { readonly agentId: KinAgentId; readonly threadId: ThreadId }) { - const rows = yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${input.orgId} - AND agent_id = ${input.agentId} AND thread_id = ${input.threadId} AND status = 'running'`; + function* ( + input: Scope & { + readonly agentId: KinAgentId; + readonly threadId: ThreadId; + readonly caller: McpThreadCaller; + }, + ) { + if (input.caller.subagent === true || input.caller.threadId !== input.threadId) return false; + const rows = yield* sql`SELECT 1 FROM kin_wakeups w + JOIN orchestration_v2_projection_runs r ON r.run_id = w.run_id AND r.thread_id = w.thread_id + JOIN orchestration_v2_projection_provider_threads p ON p.provider_thread_id = r.provider_thread_id + WHERE w.org_id = ${input.orgId} AND w.agent_id = ${input.agentId} + AND w.thread_id = ${input.threadId} AND w.status = 'running' AND w.run_id IS NOT NULL + AND p.provider_session_id = ${input.caller.providerSessionId}`; return rows.length > 0; }, Effect.mapError(toRepositoryError("KinChatRepository.hasRunningWakeup")), @@ -929,11 +946,13 @@ const make = Effect.gen(function* () { } if ( author._tag === "agent" && - !(yield* hasRunningWakeup({ - ...input, - agentId: author.agentId, - threadId: author.threadId, - })) + (input.caller === undefined || + !(yield* hasRunningWakeup({ + ...input, + agentId: author.agentId, + threadId: author.threadId, + caller: input.caller, + }))) ) return yield* new KinChatError({ reason: "not_allowed", diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index b256f61959fe2..787e783276245 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -1,3 +1,4 @@ +import { seedWakeupRun } from "./WakeupAuthority.testkit.ts"; import { assert, describe, it } from "@effect/vitest"; import { AuthSessionId, @@ -104,6 +105,8 @@ const seed = Effect.gen(function* () { VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('service-wakeup', ${person.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + yield* seedWakeupRun(threadId); + yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'service-wakeup'`; const agent = yield* identity.resolveAgent(caller); return { repo, service, identity, person, agent, chat }; }); @@ -215,6 +218,18 @@ describe("KinChatService", () => { state, }); const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE orchestration_v2_projection_provider_threads SET provider_session_id = 'next-wakeup-session' WHERE provider_thread_id = 'authority-run'`; + assert.include( + yield* service + .upsertWidget(agent, { + chatId: chat.chatId, + widgetId: widget.widgetId, + type: "task", + state, + }) + .pipe(Effect.flip), + { reason: "not_allowed" }, + ); yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; assert.include( yield* service diff --git a/apps/server/src/kin/chat/KinChatService.ts b/apps/server/src/kin/chat/KinChatService.ts index 7720680951451..45e21d3daaaf4 100644 --- a/apps/server/src/kin/chat/KinChatService.ts +++ b/apps/server/src/kin/chat/KinChatService.ts @@ -231,6 +231,7 @@ const make = Effect.gen(function* () { const messageId = yield* decodeMessageId(SecureRandom.uuidV4Unsafe()).pipe(Effect.orDie); const message = yield* repo.insertMessage({ orgId: identity.orgId, + caller: identity._tag === "agent" ? identity.caller : undefined, chatId: input.chatId, messageId, author: authorFor(identity, member), diff --git a/apps/server/src/kin/chat/KinIdentity.test.ts b/apps/server/src/kin/chat/KinIdentity.test.ts index 8a0a238d69f55..b6fb6a09da8f5 100644 --- a/apps/server/src/kin/chat/KinIdentity.test.ts +++ b/apps/server/src/kin/chat/KinIdentity.test.ts @@ -23,11 +23,12 @@ import * as Scope from "effect/Scope"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; -import type { McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; +import type { McpThreadInvocationScope } from "../../mcp/McpInvocationContext.ts"; import * as Sqlite from "../../persistence/Sqlite.ts"; import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Repository from "./KinChatRepository.ts"; import * as Identity from "./KinIdentity.ts"; +import { seedWakeupRun } from "./WakeupAuthority.testkit.ts"; const database = Layer.provideMerge(KinPersistence.layer, Sqlite.layerMemory); const repositoryLayer = Layer.provideMerge(Repository.layer, database); @@ -54,7 +55,7 @@ const session: AuthenticatedSession = { }; const environmentId = Schema.decodeSync(EnvironmentId)("test-environment"); const providerInstanceId = Schema.decodeSync(ProviderInstanceId)("provider"); -const caller = (thread = threadId, subagent = false): McpInvocationScope => ({ +const caller = (thread = threadId, subagent = false): McpThreadInvocationScope => ({ environmentId, capabilities: new Set(), issuedAt: 0, @@ -314,8 +315,35 @@ describe("KinIdentity", () => { VALUES (${agentId}, ${owner.orgId}, 'Lead', 0, '{}', '[]', 'identity-chat')`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('identity-wakeup', ${owner.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + yield* seedWakeupRun(threadId, "leftover-run", "leftover-wakeup-session"); + yield* sql`UPDATE kin_wakeups SET run_id = 'leftover-run', status = 'timed_out' WHERE wakeup_id = 'identity-wakeup'`; + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'interrupted' WHERE run_id = 'leftover-run'`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('next-wakeup', ${owner.orgId}, ${agentId}, ${threadId}, 'test', 1000, 'running', '[]')`; + const allowed = [yield* Effect.isSuccess(identity.resolveAgent(caller()))]; + yield* seedWakeupRun(threadId); + yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'next-wakeup'`; + yield* seedWakeupRun(threadId, "user-run", "user-turn-session"); + for (const providerSessionId of ["leftover-wakeup-session", "user-turn-session"]) { + allowed.push( + yield* Effect.isSuccess( + identity.resolveAgent({ + ...caller(), + thread: { threadId, providerInstanceId, providerSessionId }, + }), + ), + ); + } + assert.deepStrictEqual( + allowed, + [false, false, false], + "unbound run, leftover wake-up, user turn", + ); + yield* sql`DELETE FROM kin_wakeups WHERE wakeup_id = 'next-wakeup'`; + yield* sql`UPDATE kin_wakeups SET status = 'running', run_id = 'authority-run' WHERE wakeup_id = 'identity-wakeup'`; assert.deepStrictEqual(yield* identity.resolveAgent(caller()), { _tag: "agent", + caller: caller().thread, ...input, ownerPersonId: owner.personId, }); @@ -389,6 +417,7 @@ describe("KinIdentity", () => { assert.isTrue(restarted.isAgentThread(threadId)); assert.deepStrictEqual(yield* restarted.resolveAgent(caller()), { _tag: "agent", + caller: caller().thread, ...input, ownerPersonId: owner.personId, }); @@ -417,9 +446,11 @@ describe("KinIdentity", () => { const afterEnd = yield* start; assert.isFalse(afterEnd.isAgentThread(threadId)); assert.isTrue(afterEnd.isAgentThread(secondThreadId)); + yield* sql`UPDATE orchestration_v2_projection_runs SET thread_id = ${secondThreadId} WHERE run_id = 'authority-run'`; yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL, thread_id = ${secondThreadId} WHERE wakeup_id = 'identity-wakeup'`; assert.deepStrictEqual(yield* afterEnd.resolveAgent(caller(secondThreadId)), { _tag: "agent", + caller: caller(secondThreadId).thread, ...input, threadId: secondThreadId, ownerPersonId: owner.personId, diff --git a/apps/server/src/kin/chat/KinIdentity.ts b/apps/server/src/kin/chat/KinIdentity.ts index e0a4bb1617ec3..1dd1f4977cad4 100644 --- a/apps/server/src/kin/chat/KinIdentity.ts +++ b/apps/server/src/kin/chat/KinIdentity.ts @@ -16,7 +16,7 @@ import * as Option from "effect/Option"; import * as Semaphore from "effect/Semaphore"; import * as SqlClient from "effect/sql/SqlClient"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; -import type { McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; +import type { McpThreadCaller, McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; import * as KinChatRepository from "./KinChatRepository.ts"; export class KinIdentityConfig extends Context.Service< @@ -48,6 +48,7 @@ export type PersonIdentity = { readonly personId: KinPersonId; }; export type AgentIdentity = { + readonly caller: McpThreadCaller; readonly _tag: "agent"; readonly orgId: KinOrgId; readonly agentId: KinAgentId; @@ -116,7 +117,7 @@ export class KinIdentity extends Context.Service< message: "Kin agent thread is not active.", }); } - if (!(yield* repository.hasRunningWakeup(row.value))) + if (!(yield* repository.hasRunningWakeup({ ...row.value, caller }))) return yield* new KinChatError({ reason: "not_allowed", message: "Kin wake-up is not running.", @@ -129,6 +130,7 @@ export class KinIdentity extends Context.Service< }); return { _tag: "agent", + caller, orgId: row.value.orgId, agentId: agent.value.agentId, ownerPersonId: agent.value.ownerPersonId, diff --git a/apps/server/src/kin/chat/WakeupAuthority.testkit.ts b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts new file mode 100644 index 0000000000000..defecc7ee62c7 --- /dev/null +++ b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts @@ -0,0 +1,17 @@ +import { ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export const seedWakeupRun = Effect.fn("test.seedWakeupRun")(function* ( + threadId: ThreadId, + runId = "authority-run", + providerSessionId = "provider-session", +) { + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO orchestration_v2_projection_provider_threads + (provider_thread_id, thread_id, provider, provider_session_id, status, updated_at, payload_json) + VALUES (${runId}, ${threadId}, 'codex', ${providerSessionId}, 'running', '1970-01-01T00:00:00.000Z', '{}')`; + yield* sql`INSERT INTO orchestration_v2_projection_runs + (run_id, thread_id, ordinal, provider, provider_thread_id, status, requested_at, payload_json) + SELECT ${runId}, ${threadId}, COALESCE(MAX(ordinal), 0) + 1, 'codex', ${runId}, 'running', '1970-01-01T00:00:00.000Z', '{}' FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId}`; +}); diff --git a/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts b/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts index 3f806ad84f3cd..37b0c9b3e2db5 100644 --- a/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts +++ b/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts @@ -64,6 +64,20 @@ it.each([ it.effect("preserves the upstream database columns Kin reads", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + for (const [table, columns] of [ + ["orchestration_v2_projection_runs", ["run_id", "thread_id", "provider_thread_id"]], + [ + "orchestration_v2_projection_provider_threads", + ["provider_thread_id", "provider_session_id"], + ], + ["orchestration_v2_projection_provider_turns", ["thread_id", "completed_at"]], + ] as const) { + const rows = yield* sql`PRAGMA table_info(${sql.literal(table)})`; + assert.includeMembers( + rows.map((row) => row.name), + [...columns], + ); + } const tasks = yield* sql`PRAGMA table_info(scheduled_tasks)`; const deliveries = yield* sql`PRAGMA table_info(scheduled_task_webhook_deliveries)`; assert.includeMembers( diff --git a/apps/server/src/kin/lead/KinLeadRegistry.ts b/apps/server/src/kin/lead/KinLeadRegistry.ts index df066194e521a..7167e40dc2e86 100644 --- a/apps/server/src/kin/lead/KinLeadRegistry.ts +++ b/apps/server/src/kin/lead/KinLeadRegistry.ts @@ -2,7 +2,6 @@ import { KinAgentId, KinLevel, KinOrgId, KinWakeupId, ThreadId } from "@t3tools/ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import type * as Scope from "effect/Scope"; @@ -24,8 +23,6 @@ export class KinLeadRegistry extends Context.Service< threadId: ThreadId, caller: typeof Caller.Type, ) => Effect.Effect; - readonly resolve: (threadId: ThreadId) => Option.Option; - readonly isLeadCaller: (threadId: ThreadId) => boolean; } >()("t3/kin/lead/KinLeadRegistry") {} @@ -45,7 +42,5 @@ export const layer = Layer.sync(KinLeadRegistry, () => { }), ), ), - resolve: (threadId) => Option.fromUndefinedOr(callers.get(threadId)), - isLeadCaller: (threadId) => callers.has(threadId), }); }); diff --git a/apps/server/src/kin/lead/RunnerOwnership.test.ts b/apps/server/src/kin/lead/RunnerOwnership.test.ts deleted file mode 100644 index d8aa0bea70451..0000000000000 --- a/apps/server/src/kin/lead/RunnerOwnership.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; -import * as Context from "effect/Context"; -import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; -import * as FileSystem from "effect/FileSystem"; -import * as Layer from "effect/Layer"; -import * as Scope from "effect/Scope"; -import * as SqlClient from "effect/sql/SqlClient"; -import * as ProcessRunner from "../../processRunner.ts"; -import * as RunnerOwnership from "./RunnerOwnership.ts"; - -it.live("excludes another process without locking data writes and releases on scope close", () => - Effect.gen(function* () { - const directory = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped(); - const filename = `${directory}/state.sqlite`; - const context = yield* Layer.build(NodeSqliteClient.layer({ filename })); - const sql = Context.get(context, SqlClient.SqlClient); - const scope = yield* Effect.acquireRelease(Scope.make(), (scope) => - Scope.close(scope, Exit.void), - ); - yield* RunnerOwnership.acquire().pipe( - Effect.provideService(SqlClient.SqlClient, sql), - Effect.provideService(Scope.Scope, scope), - ); - const contender = yield* Effect.scoped(RunnerOwnership.acquire()).pipe( - Effect.provideService(SqlClient.SqlClient, sql), - Effect.flip, - ); - assert.strictEqual(contender._tag, "KinRunnerOwned"); - yield* sql`CREATE TABLE normal_writes (id INTEGER PRIMARY KEY)`; - yield* sql`INSERT INTO normal_writes VALUES (1)`; - const runner = yield* ProcessRunner.ProcessRunner; - const probe = Effect.fn("test.probe")(function* () { - return yield* runner.run({ - command: "node", - args: [ - "-e", - ` - const { DatabaseSync } = require('node:sqlite'); - const db = new DatabaseSync(process.argv[1]); - try { db.exec('BEGIN IMMEDIATE'); process.stdout.write('acquired'); } - catch (error) { if (error.errcode !== 5) throw error; process.stdout.write('owned'); } - finally { db.close(); } - `, - `${filename}.kin-runner.sqlite`, - ], - }); - }); - const blocked = yield* probe(); - assert.strictEqual(blocked.code, 0); - assert.strictEqual(blocked.stdout, "owned"); - yield* Scope.close(scope, Exit.void); - const acquired = yield* probe(); - assert.strictEqual(acquired.code, 0); - assert.strictEqual(acquired.stdout, "acquired"); - yield* Effect.scoped(RunnerOwnership.acquire()).pipe( - Effect.provideService(SqlClient.SqlClient, sql), - ); - }).pipe( - Effect.scoped, - Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer))), - ), -); diff --git a/apps/server/src/kin/lead/RunnerOwnership.ts b/apps/server/src/kin/lead/RunnerOwnership.ts deleted file mode 100644 index 7addec74a3aa5..0000000000000 --- a/apps/server/src/kin/lead/RunnerOwnership.ts +++ /dev/null @@ -1,47 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off -- SQLite owns the process-lifetime advisory lock. -import * as NodeSqlite from "node:sqlite"; - -import * as Effect from "effect/Effect"; -import * as Schema from "effect/Schema"; -import * as SqlClient from "effect/sql/SqlClient"; -import { toPersistenceSqlError } from "../../persistence/Errors.ts"; - -export class KinRunnerOwned extends Schema.TaggedError()("KinRunnerOwned", {}) {} - -// Do not open a second descriptor: closing it can release this process's POSIX locks. -const owners = new Set(); -export const acquire = Effect.fn("RunnerOwnership.acquire")(function* () { - const sql = yield* SqlClient.SqlClient; - const databases = yield* sql<{ name: string; file: string }>`PRAGMA database_list`.pipe( - Effect.mapError(toPersistenceSqlError("RunnerOwnership.database")), - ); - const filename = databases.find((row) => row.name === "main")?.file; - const key = filename ? `${filename}.kin-runner.sqlite` : sql; - yield* Effect.acquireRelease( - Effect.gen(function* () { - if (owners.has(key)) return yield* new KinRunnerOwned({}); - owners.add(key); - return yield* Effect.try({ - try: () => { - if (typeof key !== "string") return undefined; - const db = new NodeSqlite.DatabaseSync(key); - try { - db.exec( - "PRAGMA busy_timeout = 0; CREATE TABLE IF NOT EXISTS kin_runner_lock (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE;", - ); - return db; - } catch (cause) { - db.close(); - throw cause; - } - }, - catch: toPersistenceSqlError("RunnerOwnership.acquire"), - }).pipe(Effect.tapError(() => Effect.sync(() => owners.delete(key)))); - }), - (db) => - Effect.sync(() => { - db?.close(); - owners.delete(key); - }), - ); -}); diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index db58207e6e437..2127d0746dd48 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -1,3 +1,9 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { HttpServer } from "effect/http"; +import * as NetAddress from "effect/net/NetAddress"; +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; +import * as McpSessionRegistryTestkit from "../../mcp/McpSessionRegistry.testkit.ts"; import { assert, describe, it } from "@effect/vitest"; import { CommandId, @@ -26,7 +32,6 @@ import * as MemoryConfig from "../memory/MemoryConfig.ts"; import * as VcsProcess from "../../vcs/VcsProcess.ts"; import * as ProjectService from "../../project/ProjectService.ts"; import * as KinLead from "./layer.ts"; -import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as WakeupRunner from "./WakeupRunner.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; @@ -51,6 +56,11 @@ const decodeUsage = Schema.decodeEffect( ); const resolveAgent = Effect.fn("test.resolveAgent")(function* (threadId: ThreadId) { const identity = yield* KinIdentity.KinIdentity; + const sql = yield* SqlClient.SqlClient; + const [provider] = yield* sql<{ + session: string; + }>`SELECT provider_session_id AS session FROM orchestration_v2_projection_provider_threads + WHERE thread_id = ${threadId} ORDER BY updated_at DESC, provider_thread_id LIMIT 1`; return yield* identity.resolveAgent({ environmentId: EnvironmentId.make("test"), capabilities: new Set(), @@ -59,7 +69,7 @@ const resolveAgent = Effect.fn("test.resolveAgent")(function* (threadId: ThreadI client: undefined, thread: { threadId, - providerSessionId: "test", + providerSessionId: provider?.session ?? "missing", providerInstanceId: ProviderInstanceId.make("codex"), subagent: false, }, @@ -80,7 +90,6 @@ describe("WakeupRunner", () => { yield* seed; const queue = yield* KinEventQueue.KinEventQueue; const sql = yield* SqlClient.SqlClient; - const registry = yield* KinLeadRegistry.KinLeadRegistry; for (const id of ["first", "second"]) { yield* queue.enqueue(event(id)); yield* Fiber.join(yield* run()); @@ -104,7 +113,6 @@ describe("WakeupRunner", () => { { inputTokens: 20, cachedInputTokens: 5, outputTokens: 10 }, ]); } - assert.isFalse(registry.isLeadCaller(projection.thread.id)); assert.include(yield* resolveAgent(projection.thread.id).pipe(Effect.flip), { reason: "not_allowed", }); @@ -189,17 +197,44 @@ describe("WakeupRunner", () => { yield* seed; const queue = yield* KinEventQueue.KinEventQueue; const sql = yield* SqlClient.SqlClient; + const sessions = yield* McpSessionRegistry.__testing.make().pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("test")), + }), + ), + ), + ); + const runner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(McpSessionRegistry.McpSessionRegistry, sessions), + ); yield* queue.enqueue(event("timeout")); - const fiber = yield* run(); + const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); yield* Deferred.await(entered); const [lead] = yield* sql<{ thread_id: string; }>`SELECT thread_id FROM kin_leads WHERE org_id = ${orgId} AND agent_id = ${agentId}`; const threadId = ThreadId.make(lead!.thread_id); + const issued = yield* sessions.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), + }); + const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); + yield* sql`UPDATE orchestration_v2_projection_provider_threads SET provider_session_id = ${issued.config.providerSessionId} WHERE thread_id = ${threadId}`; + assert.isDefined(yield* sessions.resolve(token)); assert.strictEqual((yield* resolveAgent(threadId)).agentId, agentId); yield* TestClock.adjust("15 minutes"); yield* TestClock.adjust("30 seconds"); yield* Fiber.join(fiber); + assert.isUndefined(yield* sessions.resolve(token)); assert.include(yield* resolveAgent(threadId).pipe(Effect.flip), { reason: "not_allowed", }); @@ -307,6 +342,60 @@ describe("WakeupRunner", () => { }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), ); + it.effect.each(["projection", "decode"] as const)( + "finalizes recovery after a %s failure instead of retrying forever", + (failure) => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("recovery-failure")); + yield* Fiber.join(yield* run()); + yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL`; + yield* sql`UPDATE kin_events SET state = 'claimed'`; + if (failure === "decode") yield* sql`UPDATE kin_wakeups SET run_id = ''`; + const threads = yield* ThreadManagement.ThreadManagementService; + const runner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + getThreadProjection: () => Effect.die("projection unavailable"), + }), + ); + yield* runner.recover; + yield* runner.recover; + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [{ status: "failed" }]); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 1 }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect("defers claims while an interrupted run's provider turn is unfinished", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + yield* Fiber.join(yield* run()); + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'interrupted'`; + yield* sql`UPDATE orchestration_v2_projection_provider_turns SET completed_at = NULL`; + yield* queue.enqueue(event("next")); + const deferred = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("60 seconds"); + yield* Fiber.join(deferred); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [{ status: "completed" }]); + assert.deepStrictEqual( + yield* sql`SELECT state, attempts FROM kin_events WHERE event_id = 'next'`, + [{ state: "queued", attempts: 0 }], + ); + yield* sql`UPDATE orchestration_v2_projection_provider_turns SET completed_at = '2026-11-08T00:00:00.000Z'`; + yield* Fiber.join(yield* run()); + assert.lengthOf(yield* sql`SELECT 1 FROM kin_wakeups`, 2); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + it.effect.each(["MemorySyncConflict", "MemoryCheckoutInvalid"] as const)( "fails and backs off %s without launching a provider", (tag) => @@ -546,6 +635,13 @@ describe("WakeupRunner", () => { KinLead.containStartup(Layer.effectDiscard(Effect.die("Kin layer defect"))), ); assert.isTrue(Context.get(yield* Layer.build(defect), Core).ready); - }).pipe(Effect.provide(KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)))); + }).pipe( + Effect.provide( + Layer.mergeAll( + McpSessionRegistryTestkit.layer, + KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), + ), + ), + ); }); }); diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts index 42c54597814f0..f4f23f86f461f 100644 --- a/apps/server/src/kin/lead/WakeupRunner.testkit.ts +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -1,3 +1,4 @@ +import * as McpSessionRegistryTestkit from "../../mcp/McpSessionRegistry.testkit.ts"; import { KinAgentId, KinEvent, @@ -252,8 +253,13 @@ export function harness( ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), }), ); - const services = Layer.mergeAll(kin, launch, management, projectLayer, orchestration).pipe( - Layer.provideMerge(database), - ); + const services = Layer.mergeAll( + kin, + launch, + management, + projectLayer, + orchestration, + McpSessionRegistryTestkit.layer, + ).pipe(Layer.provideMerge(database)); return WakeupRunner.layer.pipe(Layer.provideMerge(services)); } diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 4488640e0ba0b..b59cbff2c5149 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -32,7 +32,7 @@ import * as Memory from "../memory/Memory.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; -import * as RunnerOwnership from "./RunnerOwnership.ts"; +import { McpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; export class KinWakeupError extends Schema.TaggedError()("KinWakeupError", { orgId: KinOrgId, @@ -95,7 +95,7 @@ const ulid = Effect.gen(function* () { const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const queue = yield* KinEventQueue.KinEventQueue; - const ownership = RunnerOwnership.acquire().pipe(Effect.provideService(SqlClient.SqlClient, sql)); + const sessions = yield* McpSessionRegistry; const memory = yield* Memory.Memory; const hooks = yield* Effect.serviceOption(KinWakeupHooks); const prompt = yield* KinLeadPrompt; @@ -169,6 +169,15 @@ const make = Effect.gen(function* () { : "failed"; yield* Effect.gen(function* () { if (result.timedOut) { + const providers = yield* sql<{ + providerSessionId: string; + }>`SELECT p.provider_session_id AS "providerSessionId" + FROM orchestration_v2_projection_runs r + JOIN orchestration_v2_projection_provider_threads p ON p.provider_thread_id = r.provider_thread_id + WHERE r.run_id IN (${runId}, ${result.run?.id ?? runId}) AND r.thread_id = ${threadId} AND p.provider_session_id IS NOT NULL + ORDER BY p.provider_thread_id`; + for (const provider of providers) + yield* sessions.revokeProviderSession(provider.providerSessionId); yield* threads.interruptThread({ projectId, threadId, @@ -253,6 +262,16 @@ const make = Effect.gen(function* () { if (record.retiredAt !== null) return yield* retire(record); const paused = yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; if (paused.length > 0) return; + if (record.threadId !== null) { + const unfinished = yield* sql`SELECT 1 FROM orchestration_v2_projection_provider_turns + WHERE thread_id = ${record.threadId} AND completed_at IS NULL LIMIT 1`; + if (unfinished.length > 0) { + yield* (yield* queue.changes(orgId)).pipe( + Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), + ); + return; + } + } const wakeupId = KinWakeupId.make(yield* ulid); const startedAt = yield* Clock.currentTimeMillis; let row: typeof RunningRow.Type = { @@ -413,12 +432,21 @@ const make = Effect.gen(function* () { ); const recoverOrg = Effect.fn("WakeupRunner.recoverOrg")( function* (orgId: KinOrgId) { - const rows = - yield* sql`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY started_at, wakeup_id`; + const rows = yield* sql<{ + wakeupId: string; + }>`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY started_at, wakeup_id`; for (const raw of rows) { yield* Effect.scoped( Effect.gen(function* () { + yield* Effect.acquireRelease(Effect.void, () => + queue + .finish({ orgId, wakeupId: KinWakeupId.make(raw.wakeupId), status: "failed" }) + .pipe(Effect.ignore), + ); let row = yield* decodeRunning(raw); + yield* Effect.acquireRelease(Effect.void, (_, exit) => + cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), + ); const existing = row.threadId === null ? [] @@ -438,9 +466,6 @@ const make = Effect.gen(function* () { return; } row = { ...row, runId: run.id }; - yield* Effect.acquireRelease(Effect.void, (_, exit) => - cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), - ); yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; const record = yield* lead(row.orgId, row.agentId); if (record.retiredAt !== null) { @@ -491,7 +516,6 @@ const make = Effect.gen(function* () { }); const recover = Effect.scoped( Effect.gen(function* () { - yield* ownership; yield* Effect.forEach( yield* orgIds(), (orgId) => @@ -517,7 +541,6 @@ const make = Effect.gen(function* () { ), ); const start = Effect.gen(function* () { - yield* ownership; const organizations = new Set(); yield* Effect.gen(function* () { for (const orgId of yield* orgIds()) { @@ -572,23 +595,7 @@ const make = Effect.gen(function* () { Effect.forkScoped, ); }).pipe(Effect.catchCause((cause) => Effect.logError("Kin runner unavailable", { cause }))); - const guardedRunOnce = Effect.fn("WakeupRunner.guardedRunOnce")( - function* (orgId: KinOrgId, agentId: KinAgentId) { - return yield* Effect.scoped( - Effect.gen(function* () { - yield* ownership; - yield* runOnce(orgId, agentId); - }), - ); - }, - (effect, orgId, agentId) => - effect.pipe( - Effect.mapError( - (cause) => new KinWakeupError({ orgId, agentId, cause: runnerCause(cause) }), - ), - ), - ); - return WakeupRunner.of({ runOnce: guardedRunOnce, recover, start }); + return WakeupRunner.of({ runOnce, recover, start }); }); export const layer = Layer.effect(WakeupRunner, make); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 751ebe71dfe52..ff19bc353ca27 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -1,3 +1,4 @@ +import { forkParked } from "../../serverActivation.ts"; import { KinOrgId } from "@t3tools/contracts"; import * as Config from "effect/Config"; import * as Effect from "effect/Effect"; @@ -40,7 +41,7 @@ export function containStartup(layer: Layer.Layer) { export const layer = Layer.effectDiscard( Effect.gen(function* () { const runner = yield* WakeupRunner.WakeupRunner; - yield* runner.start.pipe(Effect.forkScoped); + yield* runner.start.pipe(forkParked); }), ).pipe( Layer.provide(WakeupRunner.layer), diff --git a/apps/server/src/kin/triggers/KinTriggerSink.ts b/apps/server/src/kin/triggers/KinTriggerSink.ts index ceefd6ab5ec54..a33e8fad5086e 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.ts @@ -1,3 +1,4 @@ +import * as Cause from "effect/Cause"; import { CanonicalUtcTimestamp, KinTriggerFireId, @@ -25,11 +26,6 @@ const decodeDelivery = Schema.decodeUnknownEffect(Delivery); const decodeFire = Schema.decodeEffect(KinTriggerFired); const decodeDeliveryId = Schema.decodeEffect(ScheduledTaskWebhookDeliveryId); -export class KinTriggerStartupError extends Schema.TaggedError()( - "KinTriggerStartupError", - { cause: Schema.Defect() }, -) {} - // Shared with the upstream contract test so the tripwire pins exactly what the sink reads. export const findLaunchDelivery = ( sql: SqlClient.SqlClient, @@ -191,7 +187,7 @@ export function wrapTaskDispatch( Layer.effectDiscard( Effect.logError( "Kin trigger startup failed; webhook dispatch disabled", - new KinTriggerStartupError({ cause }), + Cause.pretty(cause), ), ), Layer.effect( diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 01fd353601cbd..ddec92773b474 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -181,7 +181,7 @@ Coalescing: The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval`, so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. `release` returns `{ released, failedEventIds }` and `finish` returns `{ failedEventIds }`; the ids are the events that transitioned to terminal `failed` in that call, ordered by id. Unknown or foreign lead ids fail with `KinEventQueueUnknownAgent`, and reused wake-up ids fail with `KinEventQueueWakeupConflict`, not SQL constraint errors. Enqueue validates the lead and computes owner priority inside the insert transaction. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. -Ownership: the runner holds `BEGIN IMMEDIATE` on a lock table in `.kin-runner.sqlite` for its scope, separate from the data database so normal writes continue. Never delete that sidecar while running. SQLite releases ownership on process death; another live process cannot run recovery or claims. No Kin migration is required. +Ownership: T3's `serverOwnership.ts` takes `server-owner.sqlite` before opening the application database, guaranteeing one server per T3 home. Kin relies on that ownership and has no sidecar lock. Restart: recovery runs independently in each organization loop and logs failures per row. The runner first reconciles each unfinished wake-up with its provider run (see failure cases). It must not requeue events belonging to a still-active run. For an abandoned run only, `requeueUnfinished(orgId, wakeupId)` atomically marks the running wake-up interrupted and returns its claimed events to `queued`, clearing their `wakeup_id` while preserving `not_before` and `attempts`. It is idempotent and organization-scoped; `idx_kin_events_wakeup` supports the update. Completed runs are finished by the runner instead. Layer construction never blindly requeues active work. Recovery also checks the deterministic first-message id if launch committed before `run_id` was saved. A provisional lead thread with no upstream thread is ended and cleared before requeue, so the next attempt launches rather than sending to a nonexistent thread. @@ -199,14 +199,14 @@ Setup once per organization (`Memory.ensureCheckouts` owns the checkouts; the wa Each wake-up: -1. `KinStop.isPaused(orgId)` → leave events queued and return. +1. `KinStop.isPaused(orgId)` → leave events queued and return. Before claiming on an existing thread, defer while any provider turn remains unfinished. 2. `KinBudget.admit(orgId, batchPriority)` → if refused, leave lower-priority events `queued` with `not_before` = next budget reset and run what remains. 3. Pick the model: `models.strong` if the batch holds a P0 or P1 event, a contradiction question or a `routine`; otherwise `models.cheap`. 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). -6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every run is a new provider session. -7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and thread; durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, an in-memory map the tools use. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. -8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. +6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent, thread and run's provider session; an unset `run_id` never grants authority, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. +8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, revoke the run's MCP provider session, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. 9. Finish: `Memory.commitWakeup` (one commit holding the staged item writes and the journal entry), reset the read checkout, record token usage from the run's provider turns into `kin_wakeups.tokens_json` and `KinBudget`, mark events `done`, and `settleThread`. If the lead writes files by hand in the read checkout, step 9's reset throws the changes away. A non-empty `git status --porcelain` before the reset is logged in the journal as "Lead edited memory files directly; discarded." @@ -246,7 +246,7 @@ No timestamps or ids go above the divider. A test asserts that two prompts built ### Lead tools (`KinLeadToolkit`) -**New** `apps/server/src/kin/lead/mcp/tools.ts` and `handlers.ts`, served on Kin's own `/kin/mcp` endpoint (set up in CHAT-7 and shared with the approvals spec), not in upstream's `layerTools`. Only lead threads are given that endpoint, and each call is checked with `KinLeadRegistry.isLeadCaller`, which reads `McpInvocationContext.thread.threadId`. Other threads never see these tools, so `AgentContextBudget.test.ts` stays unchanged. Each handler resolves `{ orgId, agentId, wakeupId }` from the registry, never from tool input, so a lead can't act for another organization. +**New** `apps/server/src/kin/lead/mcp/tools.ts` and `handlers.ts`, served on Kin's own `/kin/mcp` endpoint (set up in CHAT-7 and shared with the approvals spec), not in upstream's `layerTools`. Only lead threads are given that endpoint, and each call is checked with `KinIdentity.resolveAgent`, binding `McpInvocationContext.thread` to the persisted running wake-up's run and provider session. Other threads never see these tools, so `AgentContextBudget.test.ts` stays unchanged. Each handler derives `{ orgId, agentId }` from that identity and `wakeupId` from its matching running database row, never from tool input, so a lead can't act for another organization. | Tool | Input (sketch) | What it does | | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | From 501c75f000b0ad1117626310daa34e6b0a2507f1 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:13:47 +0530 Subject: [PATCH 05/12] test(kin): preserve terminal wake-up authority coverage --- apps/server/src/kin/chat/KinChatService.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index 787e783276245..69f4e7e44e1fd 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -230,6 +230,7 @@ describe("KinChatService", () => { .pipe(Effect.flip), { reason: "not_allowed" }, ); + yield* sql`UPDATE orchestration_v2_projection_provider_threads SET provider_session_id = 'provider-session' WHERE provider_thread_id = 'authority-run'`; yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; assert.include( yield* service From 853ea8408499cf6435f27643e3d1e7ec355f1b65 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:21:20 +0530 Subject: [PATCH 06/12] fix(kin): bind wake-up authority to the current run --- .../src/kin/chat/KinChatRepository.test.ts | 7 +++- apps/server/src/kin/chat/KinChatRepository.ts | 4 +- .../src/kin/chat/KinChatService.test.ts | 4 +- apps/server/src/kin/chat/KinIdentity.test.ts | 33 ++++++++-------- apps/server/src/kin/lead/WakeupRunner.test.ts | 38 +++++++++++++------ apps/server/src/kin/lead/WakeupRunner.ts | 12 +----- docs/kin/specs/lead.md | 8 ++-- 7 files changed, 57 insertions(+), 49 deletions(-) diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index 255eb5fb27207..05abb27896830 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -995,7 +995,10 @@ describe("KinChatRepository", () => { VALUES ('chat-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; yield* seedWakeupRun(threadId); yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'chat-wakeup'`; - for (const providerSessionId of ["leftover-session", "user-session"]) { + for (const rejectedCaller of [ + { ...caller, threadId: ThreadId.make("other-thread") }, + { ...caller, subagent: true }, + ]) { assert.include( yield* repo .insertMessage({ @@ -1008,7 +1011,7 @@ describe("KinChatRepository", () => { threadId, }, }), - caller: { ...caller, providerSessionId }, + caller: rejectedCaller, }) .pipe(Effect.flip), { reason: "not_allowed" }, diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index dc395be1d7453..879755d24c54b 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -388,10 +388,10 @@ const make = Effect.gen(function* () { if (input.caller.subagent === true || input.caller.threadId !== input.threadId) return false; const rows = yield* sql`SELECT 1 FROM kin_wakeups w JOIN orchestration_v2_projection_runs r ON r.run_id = w.run_id AND r.thread_id = w.thread_id - JOIN orchestration_v2_projection_provider_threads p ON p.provider_thread_id = r.provider_thread_id WHERE w.org_id = ${input.orgId} AND w.agent_id = ${input.agentId} AND w.thread_id = ${input.threadId} AND w.status = 'running' AND w.run_id IS NOT NULL - AND p.provider_session_id = ${input.caller.providerSessionId}`; + AND r.ordinal = (SELECT MAX(latest.ordinal) FROM orchestration_v2_projection_runs latest WHERE latest.thread_id = w.thread_id) + AND r.status NOT IN ('completed', 'failed', 'cancelled', 'interrupted', 'rolled_back')`; return rows.length > 0; }, Effect.mapError(toRepositoryError("KinChatRepository.hasRunningWakeup")), diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index 69f4e7e44e1fd..b1e12bd287bb0 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -218,7 +218,7 @@ describe("KinChatService", () => { state, }); const sql = yield* SqlClient.SqlClient; - yield* sql`UPDATE orchestration_v2_projection_provider_threads SET provider_session_id = 'next-wakeup-session' WHERE provider_thread_id = 'authority-run'`; + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'completed' WHERE run_id = 'authority-run'`; assert.include( yield* service .upsertWidget(agent, { @@ -230,7 +230,7 @@ describe("KinChatService", () => { .pipe(Effect.flip), { reason: "not_allowed" }, ); - yield* sql`UPDATE orchestration_v2_projection_provider_threads SET provider_session_id = 'provider-session' WHERE provider_thread_id = 'authority-run'`; + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'running' WHERE run_id = 'authority-run'`; yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; assert.include( yield* service diff --git a/apps/server/src/kin/chat/KinIdentity.test.ts b/apps/server/src/kin/chat/KinIdentity.test.ts index b6fb6a09da8f5..1f2f72ef7b7a3 100644 --- a/apps/server/src/kin/chat/KinIdentity.test.ts +++ b/apps/server/src/kin/chat/KinIdentity.test.ts @@ -320,27 +320,23 @@ describe("KinIdentity", () => { yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'interrupted' WHERE run_id = 'leftover-run'`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('next-wakeup', ${owner.orgId}, ${agentId}, ${threadId}, 'test', 1000, 'running', '[]')`; - const allowed = [yield* Effect.isSuccess(identity.resolveAgent(caller()))]; + assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller()))); yield* seedWakeupRun(threadId); - yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'next-wakeup'`; - yield* seedWakeupRun(threadId, "user-run", "user-turn-session"); - for (const providerSessionId of ["leftover-wakeup-session", "user-turn-session"]) { - allowed.push( - yield* Effect.isSuccess( - identity.resolveAgent({ - ...caller(), - thread: { threadId, providerInstanceId, providerSessionId }, - }), - ), - ); + yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE org_id = ${owner.orgId} AND wakeup_id = 'next-wakeup'`; + assert.isTrue(yield* Effect.isSuccess(identity.resolveAgent(caller()))); + for (const status of ["completed", "failed", "cancelled", "interrupted", "rolled_back"]) { + yield* sql`UPDATE orchestration_v2_projection_runs SET status = ${status} WHERE run_id = 'authority-run'`; + assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller()))); } - assert.deepStrictEqual( - allowed, - [false, false, false], - "unbound run, leftover wake-up, user turn", + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'running' WHERE run_id = 'authority-run'`; + yield* seedWakeupRun(threadId, "user-run", "user-turn-session"); + assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller()))); + yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = 'user-run'`; + yield* sql`DELETE FROM kin_wakeups WHERE org_id = ${owner.orgId} AND wakeup_id = 'next-wakeup'`; + yield* sql`UPDATE kin_wakeups SET status = 'running', run_id = 'authority-run' WHERE org_id = ${owner.orgId} AND wakeup_id = 'identity-wakeup'`; + assert.isFalse( + yield* Effect.isSuccess(identity.resolveAgent(caller(decodeThreadId("other-thread")))), ); - yield* sql`DELETE FROM kin_wakeups WHERE wakeup_id = 'next-wakeup'`; - yield* sql`UPDATE kin_wakeups SET status = 'running', run_id = 'authority-run' WHERE wakeup_id = 'identity-wakeup'`; assert.deepStrictEqual(yield* identity.resolveAgent(caller()), { _tag: "agent", caller: caller().thread, @@ -372,6 +368,7 @@ describe("KinIdentity", () => { ); const secondThreadId = decodeThreadId("second-thread"); yield* identity.registerAgentThread({ ...input, threadId: secondThreadId }); + assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller(secondThreadId)))); const otherOrgId = decodeOrgId("other-org"); const otherPersonId = decodePersonId("other-owner"); const otherAgentId = decodeAgentId("other-agent"); diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 2127d0746dd48..c3bfe45611783 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -54,13 +54,11 @@ const decodeUsage = Schema.decodeEffect( ), ), ); -const resolveAgent = Effect.fn("test.resolveAgent")(function* (threadId: ThreadId) { +const resolveAgent = Effect.fn("test.resolveAgent")(function* ( + threadId: ThreadId, + providerSessionId = "mcp-credential-session", +) { const identity = yield* KinIdentity.KinIdentity; - const sql = yield* SqlClient.SqlClient; - const [provider] = yield* sql<{ - session: string; - }>`SELECT provider_session_id AS session FROM orchestration_v2_projection_provider_threads - WHERE thread_id = ${threadId} ORDER BY updated_at DESC, provider_thread_id LIMIT 1`; return yield* identity.resolveAgent({ environmentId: EnvironmentId.make("test"), capabilities: new Set(), @@ -69,7 +67,7 @@ const resolveAgent = Effect.fn("test.resolveAgent")(function* (threadId: ThreadI client: undefined, thread: { threadId, - providerSessionId: provider?.session ?? "missing", + providerSessionId, providerInstanceId: ProviderInstanceId.make("codex"), subagent: false, }, @@ -211,9 +209,25 @@ describe("WakeupRunner", () => { ), ), ); + const launch = yield* ThreadLaunch.ThreadLaunchService; const runner = yield* WakeupRunner.WakeupRunner.pipe( Effect.provide(Layer.fresh(WakeupRunner.layer)), - Effect.provideService(McpSessionRegistry.McpSessionRegistry, sessions), + Effect.provideService(ThreadLaunch.ThreadLaunchService, { + ...launch, + launch: (input) => + launch.launch(input).pipe( + Effect.tap((result) => + Effect.gen(function* () { + assert.include( + yield* resolveAgent(result.projection.thread.id).pipe(Effect.flip), + { + reason: "not_allowed", + }, + ); + }), + ), + ), + }), ); yield* queue.enqueue(event("timeout")); const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); @@ -228,13 +242,15 @@ describe("WakeupRunner", () => { providerInstanceId: ProviderInstanceId.make("codex"), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); - yield* sql`UPDATE orchestration_v2_projection_provider_threads SET provider_session_id = ${issued.config.providerSessionId} WHERE thread_id = ${threadId}`; assert.isDefined(yield* sessions.resolve(token)); - assert.strictEqual((yield* resolveAgent(threadId)).agentId, agentId); + assert.strictEqual( + (yield* resolveAgent(threadId, issued.config.providerSessionId)).agentId, + agentId, + ); yield* TestClock.adjust("15 minutes"); yield* TestClock.adjust("30 seconds"); yield* Fiber.join(fiber); - assert.isUndefined(yield* sessions.resolve(token)); + assert.isDefined(yield* sessions.resolve(token)); assert.include(yield* resolveAgent(threadId).pipe(Effect.flip), { reason: "not_allowed", }); diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index b59cbff2c5149..fd9bcfc8551a5 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -32,7 +32,6 @@ import * as Memory from "../memory/Memory.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; -import { McpSessionRegistry } from "../../mcp/McpSessionRegistry.ts"; export class KinWakeupError extends Schema.TaggedError()("KinWakeupError", { orgId: KinOrgId, @@ -95,7 +94,6 @@ const ulid = Effect.gen(function* () { const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const queue = yield* KinEventQueue.KinEventQueue; - const sessions = yield* McpSessionRegistry; const memory = yield* Memory.Memory; const hooks = yield* Effect.serviceOption(KinWakeupHooks); const prompt = yield* KinLeadPrompt; @@ -169,15 +167,6 @@ const make = Effect.gen(function* () { : "failed"; yield* Effect.gen(function* () { if (result.timedOut) { - const providers = yield* sql<{ - providerSessionId: string; - }>`SELECT p.provider_session_id AS "providerSessionId" - FROM orchestration_v2_projection_runs r - JOIN orchestration_v2_projection_provider_threads p ON p.provider_thread_id = r.provider_thread_id - WHERE r.run_id IN (${runId}, ${result.run?.id ?? runId}) AND r.thread_id = ${threadId} AND p.provider_session_id IS NOT NULL - ORDER BY p.provider_thread_id`; - for (const provider of providers) - yield* sessions.revokeProviderSession(provider.providerSessionId); yield* threads.interruptThread({ projectId, threadId, @@ -366,6 +355,7 @@ const make = Effect.gen(function* () { yield* registry.register(threadId, { orgId, agentId, wakeupId, level: current.level }); const commandId = CommandId.make(`kin-wakeup:${wakeupId}`); const messageId = MessageId.make(`kin-wakeup-msg:${wakeupId}`); + // Launch/send expose no preassigned run id or pre-execution callback; early tools fail closed until this run is recorded. const sent = current.threadId === null ? (yield* launch.launch({ diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index ece61eac3f995..2ed54a8ab26e7 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -205,8 +205,10 @@ Each wake-up: 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. -7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent, thread and run's provider session; an unset `run_id` never grants authority, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. -8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, revoke the run's MCP provider session, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose non-null `run_id` is the highest-ordinal, non-terminal run on that thread; an unset `run_id` never grants authority, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. + Launch/send do not expose a preassigned run id or pre-execution callback; early tool calls before the runner records `run_id` fail closed. + +8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. 9. Finish: `Memory.commitWakeup` (one commit holding the staged item writes and the journal entry), reset the read checkout, record token usage from the run's provider turns into `kin_wakeups.tokens_json` and `KinBudget`, mark events `done`, and `settleThread`. If the lead writes files by hand in the read checkout, step 9's reset throws the changes away. A non-empty `git status --porcelain` before the reset is logged in the journal as "Lead edited memory files directly; discarded." @@ -246,7 +248,7 @@ No wake-up timestamps or ids go above the divider (dates and item ids already in ### Lead tools (`KinLeadToolkit`) -**New** `apps/server/src/kin/lead/mcp/tools.ts` and `handlers.ts`, served on Kin's own `/kin/mcp` endpoint (set up in CHAT-7 and shared with the approvals spec), not in upstream's `layerTools`. Only lead threads are given that endpoint, and each call is checked with `KinIdentity.resolveAgent`, binding `McpInvocationContext.thread` to the persisted running wake-up's run and provider session. Other threads never see these tools, so `AgentContextBudget.test.ts` stays unchanged. Each handler derives `{ orgId, agentId }` from that identity and `wakeupId` from its matching running database row, never from tool input, so a lead can't act for another organization. +**New** `apps/server/src/kin/lead/mcp/tools.ts` and `handlers.ts`, served on Kin's own `/kin/mcp` endpoint (set up in CHAT-7 and shared with the approvals spec), not in upstream's `layerTools`. Only lead threads are given that endpoint, and each call is checked with `KinIdentity.resolveAgent`, binding the server-resolved `McpInvocationContext.thread.threadId` to the persisted running wake-up's current non-terminal run, rejecting subagents. Other threads never see these tools, so `AgentContextBudget.test.ts` stays unchanged. Each handler derives `{ orgId, agentId }` from that identity and `wakeupId` from its matching running database row, never from tool input, so a lead can't act for another organization. | Tool | Input (sketch) | What it does | | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | From 467a30e62a952d54ef6a42aa36b6648fc802d923 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:47:06 +0530 Subject: [PATCH 07/12] fix(kin): bind wake-up authority to its active run and continuations --- .../src/kin/chat/KinChatRepository.test.ts | 126 +++++++++++++++- apps/server/src/kin/chat/KinChatRepository.ts | 19 ++- .../src/kin/chat/KinChatService.test.ts | 19 ++- apps/server/src/kin/chat/KinIdentity.test.ts | 26 +--- .../src/kin/chat/WakeupAuthority.testkit.ts | 74 ++++++++-- apps/server/src/kin/chat/rpcHandlers.test.ts | 7 +- .../scheduledTask.kin-contract.test.ts | 6 +- apps/server/src/kin/layer.test.ts | 4 +- apps/server/src/kin/layer.ts | 4 +- apps/server/src/kin/lead/LeadPrompt.test.ts | 8 +- apps/server/src/kin/lead/WakeupRunner.test.ts | 43 +++--- .../src/kin/lead/WakeupRunner.testkit.ts | 23 ++- apps/server/src/kin/lead/WakeupRunner.ts | 6 +- apps/server/src/kin/lead/layer.testkit.ts | 16 ++ apps/server/src/kin/lead/layer.ts | 10 +- .../src/kin/triggers/KinTriggerSink.test.ts | 8 +- .../server/src/kin/triggers/KinTriggerSink.ts | 7 +- .../ThreadManagementService.test.ts | 139 +++++++++--------- .../ThreadManagementService.ts | 15 +- docs/kin/specs/chat.md | 2 +- docs/kin/specs/lead.md | 2 +- 21 files changed, 381 insertions(+), 183 deletions(-) create mode 100644 apps/server/src/kin/lead/layer.testkit.ts diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index ccc27703769e2..b2556939c389c 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -1,4 +1,6 @@ -import { seedWakeupRun } from "./WakeupAuthority.testkit.ts"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import { CommandId, MessageId } from "@t3tools/contracts"; +import { startWakeupRun, authorityLayer } from "./WakeupAuthority.testkit.ts"; import type { McpThreadCaller } from "../../mcp/McpInvocationContext.ts"; import { assert, describe, it } from "@effect/vitest"; import { @@ -23,8 +25,6 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; -import * as Sqlite from "../../persistence/Sqlite.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Repository from "./KinChatRepository.ts"; const decodeMessage = Schema.decodeSync(KinMessage); @@ -124,7 +124,7 @@ const widget: KinWidget = { closedAt: null, updatedAt: timestamp, }; -const database = Layer.provideMerge(KinPersistence.layer, Sqlite.layerMemory); +const database = authorityLayer; const layer = Layer.provideMerge(Repository.layer, database); const seed = Effect.gen(function* () { const repo = yield* Repository.KinChatRepository; @@ -1002,6 +1002,121 @@ describe("KinChatRepository", () => { }).pipe(Effect.provide(layer)), ); + it.effect("authorizes only the wake-up active run across queued messages and continuations", () => + Effect.gen(function* () { + const repo = yield* seed; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('active-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const wakeup = yield* startWakeupRun(threadId, "active-wakeup"); + const scope = { orgId, agentId, threadId, caller }; + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + const dispatch = (id: string) => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(id), + threadId, + messageId: MessageId.make(id), + text: "Queued user message", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + yield* dispatch("queued-user"); + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + yield* repo.insertMessage({ + ...message("during-queue", { + author: { + _tag: "agent", + agentId, + ownerPersonId: personId, + threadId, + memberId: agentMemberId, + }, + authorRole: "member", + intent: "answer", + }), + caller, + }); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-wakeup"), + threadId, + runId: wakeup.id, + holdQueue: true, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* orchestrator.dispatch({ + type: "queued-run.start", + commandId: CommandId.make("release-user"), + threadId, + runId: queued.id, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-user"), + threadId, + runId: queued.id, + holdQueue: true, + }); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-thread"), + threadId, + }); + yield* dispatch("held-before-wakeup"); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* dispatch("held-queued-ahead"); + const next = yield* startWakeupRun(threadId, "active-wakeup-next"); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-existing-queue"), + threadId, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET wakeup_id = 'active-wakeup-next' WHERE org_id = ${orgId} AND wakeup_id = 'active-wakeup'`; + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + const held = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (r) => r.userMessageId === "held-queued-ahead", + )!; + assert.strictEqual(held.status, "queued"); + yield* orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("cancel-wakeup"), + threadId, + runId: next.id, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart-wakeup"), + threadId, + messageId: MessageId.make("restart-message"), + text: "Continue", + attachments: [], + dispatchMode: { type: "start_immediately" }, + restartContinuationOfRunId: next.id, + createdBy: "system", + creationSource: "server", + }); + const continuation = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + assert.strictEqual(continuation.restartContinuationOfRunId, next.id); + if (continuation.status === "queued") + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("start-continuation"), + threadId, + }); + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + assert.notEqual(next.id, wakeup.id); + }).pipe(Effect.provide(layer)), + ); + it.effect("advances read state monotonically and counts only others' posted messages", () => Effect.gen(function* () { const repo = yield* seed; @@ -1010,8 +1125,7 @@ describe("KinChatRepository", () => { VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('chat-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; - yield* seedWakeupRun(threadId); - yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'chat-wakeup'`; + yield* startWakeupRun(threadId, "chat-wakeup"); for (const rejectedCaller of [ { ...caller, threadId: ThreadId.make("other-thread") }, { ...caller, subagent: true }, diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index d24731d6e18a0..787de94a62cca 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -421,12 +421,21 @@ const make = Effect.gen(function* () { }, ) { if (input.caller.subagent === true || input.caller.threadId !== input.threadId) return false; - const rows = yield* sql`SELECT 1 FROM kin_wakeups w - JOIN orchestration_v2_projection_runs r ON r.run_id = w.run_id AND r.thread_id = w.thread_id + const rows = yield* sql`WITH RECURSIVE wakeup_runs(run_id, thread_id) AS ( + SELECT r.run_id, r.thread_id FROM kin_wakeups w + JOIN orchestration_v2_projection_runs r ON r.thread_id = w.thread_id + AND json_extract(r.payload_json, '$.userMessageId') = 'kin-wakeup-msg:' || w.wakeup_id WHERE w.org_id = ${input.orgId} AND w.agent_id = ${input.agentId} - AND w.thread_id = ${input.threadId} AND w.status = 'running' AND w.run_id IS NOT NULL - AND r.ordinal = (SELECT MAX(latest.ordinal) FROM orchestration_v2_projection_runs latest WHERE latest.thread_id = w.thread_id) - AND r.status NOT IN ('completed', 'failed', 'cancelled', 'interrupted', 'rolled_back')`; + AND w.thread_id = ${input.threadId} AND w.status = 'running' + UNION + SELECT r.run_id, r.thread_id FROM orchestration_v2_projection_runs r + JOIN wakeup_runs parent ON r.thread_id = parent.thread_id + AND (json_extract(r.payload_json, '$.restartContinuationOfRunId') = parent.run_id + OR json_extract(r.payload_json, '$.usageLimitContinuationOfRunId') = parent.run_id) + ) + SELECT 1 FROM wakeup_runs w JOIN orchestration_v2_projection_runs r + ON r.run_id = w.run_id AND r.thread_id = w.thread_id + WHERE r.status IN ('preparing', 'starting', 'running', 'waiting') LIMIT 1`; return rows.length > 0; }, Effect.mapError(toRepositoryError("KinChatRepository.hasRunningWakeup")), diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index ef2326416a252..6206317f404cb 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -1,4 +1,6 @@ -import { seedWakeupRun } from "./WakeupAuthority.testkit.ts"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import { CommandId } from "@t3tools/contracts"; +import { startWakeupRun, authorityLayer } from "./WakeupAuthority.testkit.ts"; import { assert, describe, it } from "@effect/vitest"; import { AuthSessionId, @@ -26,8 +28,6 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; import type { McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; -import * as Sqlite from "../../persistence/Sqlite.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Hub from "./KinChatHub.ts"; import * as Repository from "./KinChatRepository.ts"; import * as Identity from "./KinIdentity.ts"; @@ -40,7 +40,7 @@ const decodeKinChatId = Schema.decodeSync(KinChatId); const decodeKinMemberId = Schema.decodeSync(KinMemberId); const decodeThreadId = Schema.decodeSync(ThreadId); const decodeSend = Schema.decodeUnknownEffect(KinMessagesSendInput); -const database = Layer.provideMerge(KinPersistence.layer, Sqlite.layerMemory); +const database = authorityLayer; const repositories = Layer.provideMerge(Repository.layer, database); const dependencies = Layer.provideMerge(Hub.layer, repositories); const layer = Layer.provideMerge(Service.layer, dependencies); @@ -107,7 +107,7 @@ const seed = Effect.gen(function* () { VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('service-wakeup', ${person.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; - yield* seedWakeupRun(threadId); + yield* startWakeupRun(threadId, "service-wakeup"); yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'service-wakeup'`; const agent = yield* identity.resolveAgent(caller); return { repo, service, identity, person, agent, chat }; @@ -220,7 +220,14 @@ describe("KinChatService", () => { state, }); const sql = yield* SqlClient.SqlClient; - yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'completed' WHERE run_id = 'authority-run'`; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const run = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-service-wakeup"), + threadId, + runId: run.id, + }); assert.include( yield* service .upsertWidget(agent, { diff --git a/apps/server/src/kin/chat/KinIdentity.test.ts b/apps/server/src/kin/chat/KinIdentity.test.ts index 1f2f72ef7b7a3..1ec0016613c4e 100644 --- a/apps/server/src/kin/chat/KinIdentity.test.ts +++ b/apps/server/src/kin/chat/KinIdentity.test.ts @@ -24,13 +24,11 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; import type { McpThreadInvocationScope } from "../../mcp/McpInvocationContext.ts"; -import * as Sqlite from "../../persistence/Sqlite.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Repository from "./KinChatRepository.ts"; import * as Identity from "./KinIdentity.ts"; -import { seedWakeupRun } from "./WakeupAuthority.testkit.ts"; +import { startWakeupRun, authorityLayer } from "./WakeupAuthority.testkit.ts"; -const database = Layer.provideMerge(KinPersistence.layer, Sqlite.layerMemory); +const database = authorityLayer; const repositoryLayer = Layer.provideMerge(Repository.layer, database); const config = { orgName: "Example", ownerHandle: "owner", ownerDisplayName: "Example Owner" }; const identityLayer = Identity.KinIdentity.layer.pipe( @@ -315,25 +313,9 @@ describe("KinIdentity", () => { VALUES (${agentId}, ${owner.orgId}, 'Lead', 0, '{}', '[]', 'identity-chat')`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('identity-wakeup', ${owner.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; - yield* seedWakeupRun(threadId, "leftover-run", "leftover-wakeup-session"); - yield* sql`UPDATE kin_wakeups SET run_id = 'leftover-run', status = 'timed_out' WHERE wakeup_id = 'identity-wakeup'`; - yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'interrupted' WHERE run_id = 'leftover-run'`; - yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) - VALUES ('next-wakeup', ${owner.orgId}, ${agentId}, ${threadId}, 'test', 1000, 'running', '[]')`; assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller()))); - yield* seedWakeupRun(threadId); - yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE org_id = ${owner.orgId} AND wakeup_id = 'next-wakeup'`; + yield* startWakeupRun(threadId, "identity-wakeup"); assert.isTrue(yield* Effect.isSuccess(identity.resolveAgent(caller()))); - for (const status of ["completed", "failed", "cancelled", "interrupted", "rolled_back"]) { - yield* sql`UPDATE orchestration_v2_projection_runs SET status = ${status} WHERE run_id = 'authority-run'`; - assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller()))); - } - yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'running' WHERE run_id = 'authority-run'`; - yield* seedWakeupRun(threadId, "user-run", "user-turn-session"); - assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller()))); - yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = 'user-run'`; - yield* sql`DELETE FROM kin_wakeups WHERE org_id = ${owner.orgId} AND wakeup_id = 'next-wakeup'`; - yield* sql`UPDATE kin_wakeups SET status = 'running', run_id = 'authority-run' WHERE org_id = ${owner.orgId} AND wakeup_id = 'identity-wakeup'`; assert.isFalse( yield* Effect.isSuccess(identity.resolveAgent(caller(decodeThreadId("other-thread")))), ); @@ -443,7 +425,7 @@ describe("KinIdentity", () => { const afterEnd = yield* start; assert.isFalse(afterEnd.isAgentThread(threadId)); assert.isTrue(afterEnd.isAgentThread(secondThreadId)); - yield* sql`UPDATE orchestration_v2_projection_runs SET thread_id = ${secondThreadId} WHERE run_id = 'authority-run'`; + yield* startWakeupRun(secondThreadId, "identity-wakeup"); yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL, thread_id = ${secondThreadId} WHERE wakeup_id = 'identity-wakeup'`; assert.deepStrictEqual(yield* afterEnd.resolveAgent(caller(secondThreadId)), { _tag: "agent", diff --git a/apps/server/src/kin/chat/WakeupAuthority.testkit.ts b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts index defecc7ee62c7..78a23a3e2013e 100644 --- a/apps/server/src/kin/chat/WakeupAuthority.testkit.ts +++ b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts @@ -1,17 +1,67 @@ -import { ThreadId } from "@t3tools/contracts"; +import { + CommandId, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; -import * as SqlClient from "effect/sql/SqlClient"; +import * as Layer from "effect/Layer"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "../../orchestration-v2/testkit/ProviderReplayHarness.ts"; +import { CodexProviderCapabilitiesV2 } from "../../orchestration-v2/Adapters/CodexAdapterV2.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as Sqlite from "../../persistence/Sqlite.ts"; -export const seedWakeupRun = Effect.fn("test.seedWakeupRun")(function* ( +const database = KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); +const instanceId = ProviderInstanceId.make("codex"); +export const authorityLayer = ProviderReplayHarness.layerWithRegistry( + { name: "wakeup-authority" }, + ProviderAdapterRegistry.layerFromAdapters([ + { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: () => Effect.die("Authority tests do not start provider processes"), + }, + ]), + { databaseLayer: database, runEffectWorker: false }, +).pipe(Layer.provideMerge(database)); + +export const startWakeupRun = Effect.fn("test.startWakeupRun")(function* ( threadId: ThreadId, - runId = "authority-run", - providerSessionId = "provider-session", + wakeupId: string, ) { - const sql = yield* SqlClient.SqlClient; - yield* sql`INSERT INTO orchestration_v2_projection_provider_threads - (provider_thread_id, thread_id, provider, provider_session_id, status, updated_at, payload_json) - VALUES (${runId}, ${threadId}, 'codex', ${providerSessionId}, 'running', '1970-01-01T00:00:00.000Z', '{}')`; - yield* sql`INSERT INTO orchestration_v2_projection_runs - (run_id, thread_id, ordinal, provider, provider_thread_id, status, requested_at, payload_json) - SELECT ${runId}, ${threadId}, COALESCE(MAX(ordinal), 0) + 1, 'codex', ${runId}, 'running', '1970-01-01T00:00:00.000Z', '{}' FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId}`; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projection = yield* orchestrator.getThreadProjection(threadId).pipe(Effect.option); + if (projection._tag === "None") + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create:${threadId}`), + threadId, + projectId: ProjectId.make("authority-project"), + title: "Lead", + modelSelection: { instanceId, model: "gpt-5.1-codex" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "system", + creationSource: "server", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`dispatch:${threadId}:${wakeupId}`), + threadId, + messageId: MessageId.make(`kin-wakeup-msg:${wakeupId}`), + text: "Wake up", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "system", + creationSource: "server", + }); + return (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; }); diff --git a/apps/server/src/kin/chat/rpcHandlers.test.ts b/apps/server/src/kin/chat/rpcHandlers.test.ts index ed3944a4e798b..b96e43938ca82 100644 --- a/apps/server/src/kin/chat/rpcHandlers.test.ts +++ b/apps/server/src/kin/chat/rpcHandlers.test.ts @@ -1,3 +1,4 @@ +import { dormantLeadDependencies } from "../lead/layer.testkit.ts"; import { assert, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Hub from "./KinChatHub.ts"; @@ -59,7 +60,11 @@ const session = { scopes: AuthStandardClientScopes, }; const configured = ConfigProvider.layer(ConfigProvider.fromUnknown({ KIN_ENABLED: true })); -const root = Kin.layer.pipe(Layer.provideMerge(Chat.layer), Layer.provide(configured)); +const root = Kin.layer.pipe( + Layer.provide(dormantLeadDependencies), + Layer.provideMerge(Chat.layer), + Layer.provide(configured), +); const probe = WsRpcGroup.omit( ...[...WsRpcGroup.requests.keys()].filter( ( diff --git a/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts b/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts index 37b0c9b3e2db5..ef3f262fab51b 100644 --- a/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts +++ b/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts @@ -65,11 +65,7 @@ it.effect("preserves the upstream database columns Kin reads", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; for (const [table, columns] of [ - ["orchestration_v2_projection_runs", ["run_id", "thread_id", "provider_thread_id"]], - [ - "orchestration_v2_projection_provider_threads", - ["provider_thread_id", "provider_session_id"], - ], + ["orchestration_v2_projection_runs", ["run_id", "thread_id", "status", "payload_json"]], ["orchestration_v2_projection_provider_turns", ["thread_id", "completed_at"]], ] as const) { const rows = yield* sql`PRAGMA table_info(${sql.literal(table)})`; diff --git a/apps/server/src/kin/layer.test.ts b/apps/server/src/kin/layer.test.ts index 87db879d28828..bcc9d489269a6 100644 --- a/apps/server/src/kin/layer.test.ts +++ b/apps/server/src/kin/layer.test.ts @@ -1,3 +1,4 @@ +import { dormantLeadDependencies } from "./lead/layer.testkit.ts"; import * as ThreadLaunch from "../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as KinTriggerSink from "./triggers/KinTriggerSink.ts"; @@ -66,6 +67,7 @@ it.effect.each(["identity", "defect", "unconfigured"] as const)( const kin = yield* Kin.KinServer.pipe( Effect.provide( Kin.layer.pipe( + Layer.provide(dormantLeadDependencies), Layer.provide( ConfigProvider.layer( ConfigProvider.fromUnknown({ KIN_ENABLED: mode !== "unconfigured" }), @@ -155,7 +157,7 @@ it.effect.each(["disabled", "invalid", "failed"] as const)( }, }); const runtime = Layer.mergeAll( - Kin.layer, + Kin.layer.pipe(Layer.provide(dormantLeadDependencies)), KinTriggerSink.wrapTaskDispatch( Layer.mock(ThreadLaunch.ThreadLaunchService)({ launch }), Layer.mock(ThreadManagement.ThreadManagementService)({ sendToThread }), diff --git a/apps/server/src/kin/layer.ts b/apps/server/src/kin/layer.ts index 6f529da783146..b9ad750d08cf4 100644 --- a/apps/server/src/kin/layer.ts +++ b/apps/server/src/kin/layer.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as HttpRouter from "effect/http/HttpRouter"; import * as Option from "effect/Option"; +import * as KinLead from "./lead/layer.ts"; import * as KinChat from "./chat/layer.ts"; import { KinIdentity } from "./chat/KinIdentity.ts"; import { KinChatService } from "./chat/KinChatService.ts"; @@ -20,8 +21,7 @@ export class KinServer extends Context.Service< } >()("t3/kin/layer/KinServer") {} -// LEAD-5 supplies its runner here, using this root's identity rather than building another. -const lead: Layer.Layer = Layer.empty; +const lead = KinLead.layerServer; const enabled = Layer.effect( KinServer, Effect.gen(function* () { diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index 2c35e15af3975..26c83ebb54998 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -19,10 +19,8 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; -import * as Sqlite from "../../persistence/Sqlite.ts"; import * as Chat from "../chat/KinChatRepository.ts"; -import { seedWakeupRun } from "../chat/WakeupAuthority.testkit.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; +import { startWakeupRun, authorityLayer } from "../chat/WakeupAuthority.testkit.ts"; import * as Memory from "../memory/Memory.ts"; import { KinLeadPrompt, type KinLeadPromptInput } from "./LeadPrompt.ts"; @@ -56,7 +54,7 @@ const event = (messageId: string) => dedupeKey: null, provenance: { _tag: "system" }, }); -const database = KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); +const database = authorityLayer; const repository = Chat.layer.pipe(Layer.provide(database)); const memory = Layer.succeed( Memory.Memory, @@ -190,7 +188,7 @@ it.effect.each([ yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) VALUES (${agentId}, ${orgId}, 'Fixture lead', 0, '{}', '[]', ${chatId})`; - yield* seedWakeupRun(threadId); + yield* startWakeupRun(threadId, input.wakeupId); yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids, run_id) VALUES (${input.wakeupId}, ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]', 'authority-run')`; diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index c3bfe45611783..58992c7b71ec5 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -54,24 +54,17 @@ const decodeUsage = Schema.decodeEffect( ), ), ); -const resolveAgent = Effect.fn("test.resolveAgent")(function* ( - threadId: ThreadId, - providerSessionId = "mcp-credential-session", -) { +const resolveAgent = Effect.fn("test.resolveAgent")(function* (threadId: ThreadId) { const identity = yield* KinIdentity.KinIdentity; - return yield* identity.resolveAgent({ - environmentId: EnvironmentId.make("test"), - capabilities: new Set(), - issuedAt: 0, - requestNamespace: "test", - client: undefined, - thread: { - threadId, - providerSessionId, - providerInstanceId: ProviderInstanceId.make("codex"), - subagent: false, - }, + const sessions = yield* McpSessionRegistry.McpSessionRegistry; + const issued = yield* sessions.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), }); + const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); + const scope = yield* sessions.resolve(token); + assert.isDefined(scope); + return yield* identity.resolveAgent(scope!); }); const run = Effect.fn("test.run")(function* () { const runner = yield* WakeupRunner.WakeupRunner; @@ -218,11 +211,9 @@ describe("WakeupRunner", () => { launch.launch(input).pipe( Effect.tap((result) => Effect.gen(function* () { - assert.include( - yield* resolveAgent(result.projection.thread.id).pipe(Effect.flip), - { - reason: "not_allowed", - }, + assert.strictEqual( + (yield* resolveAgent(result.projection.thread.id)).agentId, + agentId, ); }), ), @@ -244,7 +235,9 @@ describe("WakeupRunner", () => { const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); assert.isDefined(yield* sessions.resolve(token)); assert.strictEqual( - (yield* resolveAgent(threadId, issued.config.providerSessionId)).agentId, + (yield* (yield* KinIdentity.KinIdentity).resolveAgent( + (yield* sessions.resolve(token))!, + )).agentId, agentId, ); yield* TestClock.adjust("15 minutes"); @@ -624,16 +617,14 @@ describe("WakeupRunner", () => { return Effect.gen(function* () { yield* seed; const failing = KinLead.layer.pipe( + Layer.provide(Layer.mock(KinIdentity.KinIdentity)({ isAgentThread: () => false })), Layer.provide( Layer.mergeAll( Layer.mock(ThreadLaunch.ThreadLaunchService)({}), Layer.mock(ThreadManagement.ThreadManagementService)({}), Layer.mock(ProjectService.ProjectService)({}), Layer.succeed(KinLeadPrompt, { build: () => Effect.succeed("unused") }), - Layer.succeed(MemoryConfig.MemoryConfig, { - baseDirectory: "/private/tmp", - repositories: [], - }), + Layer.effect(MemoryConfig.MemoryConfig, Effect.die("invalid memory config")), Layer.mock(VcsProcess.VcsProcess)({}), NodeServices.layer, ), diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts index f4f23f86f461f..df4baa1205da3 100644 --- a/apps/server/src/kin/lead/WakeupRunner.testkit.ts +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -1,4 +1,10 @@ -import * as McpSessionRegistryTestkit from "../../mcp/McpSessionRegistry.testkit.ts"; +import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { HttpServer } from "effect/http"; +import * as NetAddress from "effect/net/NetAddress"; +import { EnvironmentId } from "@t3tools/contracts"; +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; import { KinAgentId, KinEvent, @@ -259,7 +265,20 @@ export function harness( management, projectLayer, orchestration, - McpSessionRegistryTestkit.layer, + McpSessionRegistry.layer.pipe( + Layer.provide( + Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("test")), + }), + ), + ), + ), ).pipe(Layer.provideMerge(database)); return WakeupRunner.layer.pipe(Layer.provideMerge(services)); } diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index fd9bcfc8551a5..33bdb09864372 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -254,7 +254,10 @@ const make = Effect.gen(function* () { if (record.threadId !== null) { const unfinished = yield* sql`SELECT 1 FROM orchestration_v2_projection_provider_turns WHERE thread_id = ${record.threadId} AND completed_at IS NULL LIMIT 1`; - if (unfinished.length > 0) { + const busy = yield* sql`SELECT 1 FROM orchestration_v2_projection_runs + WHERE thread_id = ${record.threadId} + AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') LIMIT 1`; + if (unfinished.length > 0 || busy.length > 0) { yield* (yield* queue.changes(orgId)).pipe( Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), ); @@ -355,7 +358,6 @@ const make = Effect.gen(function* () { yield* registry.register(threadId, { orgId, agentId, wakeupId, level: current.level }); const commandId = CommandId.make(`kin-wakeup:${wakeupId}`); const messageId = MessageId.make(`kin-wakeup-msg:${wakeupId}`); - // Launch/send expose no preassigned run id or pre-execution callback; early tools fail closed until this run is recorded. const sent = current.threadId === null ? (yield* launch.launch({ diff --git a/apps/server/src/kin/lead/layer.testkit.ts b/apps/server/src/kin/lead/layer.testkit.ts new file mode 100644 index 0000000000000..f93d64ade578e --- /dev/null +++ b/apps/server/src/kin/lead/layer.testkit.ts @@ -0,0 +1,16 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Layer from "effect/Layer"; +import * as ServerConfig from "../../config.ts"; +import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as VcsProcess from "../../vcs/VcsProcess.ts"; + +// Unconfigured lead runners must not call any of these external services. +export const dormantLeadDependencies = Layer.mergeAll( + Layer.mock(ThreadLaunch.ThreadLaunchService)({}), + Layer.mock(ThreadManagement.ThreadManagementService)({}), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(VcsProcess.VcsProcess)({}), + ServerConfig.layerTest(process.cwd(), { prefix: "kin-dormant-lead-" }), +).pipe(Layer.provideMerge(NodeServices.layer)); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index a3c37c5879177..7f01a7671b1c7 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -5,7 +5,6 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; -import * as KinIdentity from "../chat/KinIdentity.ts"; import * as KinChatRepository from "../chat/KinChatRepository.ts"; import * as Memory from "../memory/Memory.ts"; import * as MemoryConfig from "../memory/MemoryConfig.ts"; @@ -45,12 +44,9 @@ export const layer = Layer.effectDiscard( ).pipe( Layer.provide(WakeupRunner.layer), Layer.provide( - Layer.mergeAll( - KinLeadRegistry.layer, - KinEventQueue.layer, - KinLeadPrompt.layer, - KinIdentity.KinIdentity.layer.pipe(Layer.provide(KinIdentity.KinIdentityConfig.layer)), - ).pipe(Layer.provideMerge(Layer.mergeAll(Memory.layer, KinChatRepository.layer))), + Layer.mergeAll(KinLeadRegistry.layer, KinEventQueue.layer, KinLeadPrompt.layer).pipe( + Layer.provideMerge(Layer.mergeAll(Memory.layer, KinChatRepository.layer)), + ), ), Layer.provide(KinPersistence.layer), ); diff --git a/apps/server/src/kin/triggers/KinTriggerSink.test.ts b/apps/server/src/kin/triggers/KinTriggerSink.test.ts index 4206081eace1b..38ec337580ceb 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.test.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.test.ts @@ -671,7 +671,13 @@ it.effect("builds the runtime after a Kin migration failure and refuses only web .pipe(Effect.flip); assert.strictEqual(bound._tag, "OrchestratorCommandRejectedError"); assert.strictEqual(launches.length, 2); - }).pipe(Effect.provide(KinTriggerSink.wrapTaskDispatch(inner, managementLayer))); + }).pipe( + Effect.provide( + KinTriggerSink.wrapTaskDispatch(inner, managementLayer).pipe( + Layer.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({ KIN_ENABLED: true }))), + ), + ), + ); }).pipe(Effect.provide(Sqlite.layerMemory)), ); diff --git a/apps/server/src/kin/triggers/KinTriggerSink.ts b/apps/server/src/kin/triggers/KinTriggerSink.ts index 61f31761173a2..29a8f99a2b32b 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.ts @@ -185,10 +185,7 @@ export function wrapTaskDispatch( Layer.catchCause((cause) => Layer.mergeAll( Layer.effectDiscard( - Effect.logError( - "Kin trigger startup failed; webhook dispatch disabled", - Cause.pretty(cause), - ), + KinEnabled.KinEnabled.pipe(Effect.flatMap((config) => config.reportFailure(cause))), ), Layer.effect( ThreadLaunchService.ThreadLaunchService, @@ -207,7 +204,7 @@ export function wrapTaskDispatch( operation: "dispatch-message", commandId: input.commandId, projectId: input.projectId, - cause, + cause: new Error("Kin is unavailable; webhook dispatch refused.", { cause }), }); }), }); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index 742616f27bd3e..f7c2bb91ad0a9 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -198,75 +198,78 @@ it.effect( }, ); -it.effect("waitForThread follows a restart continuation instead of the cut run", () => - Effect.gen(function* () { - const projectId = ProjectId.make("project:thread-management:wait-restart"); - const threadId = ThreadId.make("thread:thread-management:wait-restart"); - const runId = RunId.make("run:thread-management:wait-restart"); - const continuationId = RunId.make("run:thread-management:wait-restart:2"); - const subscribed = yield* Deferred.make(); - const events = yield* Queue.unbounded(); - let runs: Array<{ - id: RunId; - status: OrchestrationV2Run["status"]; - restartContinuationOfRunId?: RunId; - }> = [{ id: runId, status: "cancelled" }]; - let pending = true; - const stored = (sequence: number, event: object) => - ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; - const layerTest = ThreadManagementService.layer.pipe( - Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), - Layer.provide( - Layer.mock(Orchestrator.OrchestratorV2)({ - getThreadEventSequence: () => Effect.succeed(0), - restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId), - awaitRestartContinuation: () => Effect.never, - streamStoredEventsFrom: (input) => - input?.eventType === "run.updated" - ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( - Stream.drain, - Stream.concat(Stream.fromQueue(events)), - ) - : Stream.never, - getThreadRecords: () => - Effect.sync( - () => - ({ - thread: { id: threadId, projectId, deletedAt: null }, - runs, - }) as unknown as OrchestrationV2ThreadProjection, - ), - }), - ), - Layer.provide(ServerSettings.layerTest()), - Layer.provide(layerReceiptsTest), - ); - const service = yield* ThreadManagementService.ThreadManagementService.pipe( - Effect.provide(layerTest), - ); - const fiber = yield* service - .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) - .pipe(Effect.forkChild); - yield* Deferred.await(subscribed); - const continuation = { id: continuationId, restartContinuationOfRunId: runId }; - runs = [...runs, { ...continuation, status: "running" }]; - pending = false; - yield* Queue.offer( - events, - stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }), - ); - runs = [runs[0]!, { ...continuation, status: "completed" }]; - yield* Queue.offer( - events, - stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }), - ); - const result = yield* Fiber.join(fiber); +it.effect.each(["restartContinuationOfRunId", "usageLimitContinuationOfRunId"] as const)( + "waitForThread follows %s instead of the cut run", + (continuationField) => + Effect.gen(function* () { + const projectId = ProjectId.make("project:thread-management:wait-restart"); + const threadId = ThreadId.make("thread:thread-management:wait-restart"); + const runId = RunId.make("run:thread-management:wait-restart"); + const continuationId = RunId.make("run:thread-management:wait-restart:2"); + const subscribed = yield* Deferred.make(); + const events = yield* Queue.unbounded(); + let runs: Array<{ + id: RunId; + status: OrchestrationV2Run["status"]; + restartContinuationOfRunId?: RunId; + usageLimitContinuationOfRunId?: RunId; + }> = [{ id: runId, status: "cancelled" }]; + let pending = true; + const stored = (sequence: number, event: object) => + ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadEventSequence: () => Effect.succeed(0), + restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId), + awaitRestartContinuation: () => Effect.never, + streamStoredEventsFrom: (input) => + input?.eventType === "run.updated" + ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.fromQueue(events)), + ) + : Stream.never, + getThreadRecords: () => + Effect.sync( + () => + ({ + thread: { id: threadId, projectId, deletedAt: null }, + runs, + }) as unknown as OrchestrationV2ThreadProjection, + ), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + const service = yield* ThreadManagementService.ThreadManagementService.pipe( + Effect.provide(layerTest), + ); + const fiber = yield* service + .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) + .pipe(Effect.forkChild); + yield* Deferred.await(subscribed); + const continuation = { id: continuationId, [continuationField]: runId }; + runs = [...runs, { ...continuation, status: "running" }]; + pending = false; + yield* Queue.offer( + events, + stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }), + ); + runs = [runs[0]!, { ...continuation, status: "completed" }]; + yield* Queue.offer( + events, + stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }), + ); + const result = yield* Fiber.join(fiber); - expect(result).toMatchObject({ - timedOut: false, - run: { id: continuationId, status: "completed" }, - }); - }), + expect(result).toMatchObject({ + timedOut: false, + run: { id: continuationId, status: "completed" }, + }); + }), ); it.effect("waitForThread can wait past an ended run for the thread's delegated work", () => diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 46ea454310554..529cbc04099a5 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -530,13 +530,17 @@ export function latestRun( return projection.runs.toSorted((left, right) => right.ordinal - left.ordinal)[0]; } -function latestRestartContinuation( +function latestContinuation( runs: ReadonlyArray, run: OrchestrationV2Run, ): OrchestrationV2Run { let current = run; for (let hops = 0; hops < runs.length; hops++) { - const next = runs.find((candidate) => candidate.restartContinuationOfRunId === current.id); + const next = runs.find( + (candidate) => + candidate.restartContinuationOfRunId === current.id || + candidate.usageLimitContinuationOfRunId === current.id, + ); if (next === undefined) break; current = next; } @@ -930,7 +934,7 @@ const make = Effect.gen(function* () { if (selectedRun === undefined) { return { threadId: input.threadId, run: null, timedOut: false }; } - // A restart cuts a run and continues it as a new run. Follow that chain, + // Restarts and usage-limit recovery continue as new runs. Follow both chains, // so a cut run is not the answer while its continuation is pending. // Runs per matching event and is already covered by the waitForThread span. const outcome = Effect.fnUntraced(function* (runs: ReadonlyArray) { @@ -941,7 +945,7 @@ const make = Effect.gen(function* () { runId: selectedRun.id, }); } - const run = latestRestartContinuation(runs, current); + const run = latestContinuation(runs, current); const pending = isTerminalRunStatus(run.status) && (yield* orchestrator.restartContinuationPending(run)); return { run, pending, settled: isTerminalRunStatus(run.status) && !pending }; @@ -1015,7 +1019,8 @@ const make = Effect.gen(function* () { untilDelegatedWorkSettled || stored.event.type !== "run.updated" || stored.event.payload.id === selectedRun.id || - stored.event.payload.restartContinuationOfRunId !== undefined, + stored.event.payload.restartContinuationOfRunId !== undefined || + stored.event.payload.usageLimitContinuationOfRunId !== undefined, ), ), initial.pending diff --git a/docs/kin/specs/chat.md b/docs/kin/specs/chat.md index 8755346b059f4..d1f4cdae6810f 100644 --- a/docs/kin/specs/chat.md +++ b/docs/kin/specs/chat.md @@ -48,7 +48,7 @@ Every row carries `orgId`. Phase 1 has one org, created on first start. Bootstra - **Session to person** (`KinIdentity.resolve(session)`): - Phase 1: after the RPC scope check, a `/ws` session holding every scope in `AuthStandardClientScopes` resolves to the default org owner. A narrower session resolves to an empty list, never to a person inferred from `subject`. The result is a list of `{ orgId, personId }` entries so phase-2 bindings do not change callers. Only the owner may pair devices before CHAT-11. - Phase 2: a row in `kin_session_bindings (session_id, org_id, person_id)` decides. A session without a binding is the owner only if it holds `AuthStandardClientScopes`. Members get sessions with the new `kin:chat` scope only (CHAT-11), so their device never gets the owner's terminal or files. -- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead's durable thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution also requires a persisted `kin_wakeups` row with `status = running` and the same org, agent and thread. Durable registration alone grants no authority between wake-ups; `insertMessage` rechecks the running row inside its write transaction. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` on retirement, not between fresh provider sessions on the same T3 thread; an ended thread cannot be registered again. Retiring an agent ends its threads through `KinIdentity`; active-thread discovery also excludes retired agents. +- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead's durable thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution also requires a persisted `kin_wakeups` row with `status = running` and the same org, agent and thread. The thread's active run must belong to the wake-up, including restart and usage-limit continuations. Durable registration alone grants no authority between wake-ups; `insertMessage` rechecks the running row inside its write transaction. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` on retirement, not between fresh provider sessions on the same T3 thread; an ended thread cannot be registered again. Retiring an agent ends its threads through `KinIdentity`; active-thread discovery also excludes retired agents. ### Chats, members, roles diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 2ed54a8ab26e7..87021027ff230 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -205,7 +205,7 @@ Each wake-up: 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. -7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose non-null `run_id` is the highest-ordinal, non-terminal run on that thread; an unset `run_id` never grants authority, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), identified by its server-minted `kin-wakeup-msg:` message id; `run_id` is used only for waiting and accounting, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. Launch/send do not expose a preassigned run id or pre-execution callback; early tool calls before the runner records `run_id` fail closed. 8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. From 6862213f171f9f4c020e667a03c4fe2ff424fcea Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:18:27 +0530 Subject: [PATCH 08/12] fix(kin): anchor wake-up roots and skip busy leads --- .../src/kin/chat/KinChatRepository.test.ts | 260 +++++++++++------- apps/server/src/kin/chat/KinChatRepository.ts | 39 ++- .../src/kin/chat/KinChatService.test.ts | 5 +- apps/server/src/kin/lead/KinEventQueue.ts | 10 + apps/server/src/kin/lead/LeadPrompt.test.ts | 4 +- apps/server/src/kin/lead/WakeupRunner.test.ts | 112 +++++++- apps/server/src/kin/lead/WakeupRunner.ts | 19 +- .../ThreadManagementService.ts | 8 +- docs/kin/specs/lead.md | 8 +- 9 files changed, 337 insertions(+), 128 deletions(-) diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index b2556939c389c..773fce47b512a 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -1,3 +1,6 @@ +import * as EventSink from "../../orchestration-v2/EventSink.ts"; +import * as DateTime from "effect/DateTime"; +import { EventId } from "@t3tools/contracts"; import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; import { CommandId, MessageId } from "@t3tools/contracts"; import { startWakeupRun, authorityLayer } from "./WakeupAuthority.testkit.ts"; @@ -1002,119 +1005,190 @@ describe("KinChatRepository", () => { }).pipe(Effect.provide(layer)), ); - it.effect("authorizes only the wake-up active run across queued messages and continuations", () => + it.effect("rejects a lower-ordinal client forgery before the runner sends", () => Effect.gen(function* () { const repo = yield* seed; const sql = yield* SqlClient.SqlClient; - yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) - VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; - yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) - VALUES ('active-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; const orchestrator = yield* Orchestrator.OrchestratorV2; - const wakeup = yield* startWakeupRun(threadId, "active-wakeup"); - const scope = { orgId, agentId, threadId, caller }; - assert.isTrue(yield* repo.hasRunningWakeup(scope)); - const dispatch = (id: string) => - orchestrator.dispatch({ - type: "message.dispatch", - commandId: CommandId.make(id), - threadId, - messageId: MessageId.make(id), - text: "Queued user message", - attachments: [], - dispatchMode: { type: "queue_after_active" }, - createdBy: "user", - creationSource: "web", - }); - yield* dispatch("queued-user"); - assert.isTrue(yield* repo.hasRunningWakeup(scope)); - yield* repo.insertMessage({ - ...message("during-queue", { - author: { - _tag: "agent", - agentId, - ownerPersonId: personId, - threadId, - memberId: agentMemberId, - }, - authorRole: "member", - intent: "answer", - }), - caller, - }); + const bootstrap = yield* startWakeupRun(threadId, "bootstrap"); yield* orchestrator.dispatch({ type: "run.interrupt", - commandId: CommandId.make("end-wakeup"), + commandId: CommandId.make("end-bootstrap"), threadId, - runId: wakeup.id, - holdQueue: true, + runId: bootstrap.id, }); - assert.isFalse(yield* repo.hasRunningWakeup(scope)); - const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('pre-forged', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; yield* orchestrator.dispatch({ - type: "queued-run.start", - commandId: CommandId.make("release-user"), + type: "message.dispatch", + commandId: CommandId.make("pre-forged"), threadId, - runId: queued.id, + messageId: MessageId.make("kin-wakeup-msg:pre-forged"), + text: "Forged", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", }); + const scope = { orgId, agentId, threadId, caller }; assert.isFalse(yield* repo.hasRunningWakeup(scope)); - yield* orchestrator.dispatch({ - type: "run.interrupt", - commandId: CommandId.make("end-user"), - threadId, - runId: queued.id, - holdQueue: true, - }); - yield* orchestrator.dispatch({ - type: "thread.stop", - commandId: CommandId.make("hold-thread"), - threadId, - }); - yield* dispatch("held-before-wakeup"); + const forged = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* sql`UPDATE kin_wakeups SET run_id = ${forged.id} WHERE wakeup_id = 'pre-forged'`; assert.isFalse(yield* repo.hasRunningWakeup(scope)); - yield* dispatch("held-queued-ahead"); - const next = yield* startWakeupRun(threadId, "active-wakeup-next"); + yield* sql`UPDATE kin_wakeups SET run_id = NULL WHERE wakeup_id = 'pre-forged'`; yield* orchestrator.dispatch({ - type: "thread.stop", - commandId: CommandId.make("hold-existing-queue"), + type: "run.interrupt", + commandId: CommandId.make("end-forged"), threadId, + runId: forged.id, }); + const real = yield* startWakeupRun(threadId, "pre-forged"); + assert.isBelow(forged.ordinal, real.ordinal); assert.isFalse(yield* repo.hasRunningWakeup(scope)); - yield* sql`UPDATE kin_wakeups SET wakeup_id = 'active-wakeup-next' WHERE org_id = ${orgId} AND wakeup_id = 'active-wakeup'`; - assert.isFalse(yield* repo.hasRunningWakeup(scope)); - const held = (yield* orchestrator.getThreadProjection(threadId)).runs.find( - (r) => r.userMessageId === "held-queued-ahead", - )!; - assert.strictEqual(held.status, "queued"); - yield* orchestrator.dispatch({ - type: "queued-run.cancel", - commandId: CommandId.make("cancel-wakeup"), - threadId, - runId: next.id, - }); - yield* orchestrator.dispatch({ - type: "message.dispatch", - commandId: CommandId.make("restart-wakeup"), - threadId, - messageId: MessageId.make("restart-message"), - text: "Continue", - attachments: [], - dispatchMode: { type: "start_immediately" }, - restartContinuationOfRunId: next.id, - createdBy: "system", - creationSource: "server", - }); - const continuation = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; - assert.strictEqual(continuation.restartContinuationOfRunId, next.id); - if (continuation.status === "queued") + yield* sql`UPDATE kin_wakeups SET run_id = ${real.id} WHERE wakeup_id = 'pre-forged'`; + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + }).pipe(Effect.provide(layer)), + ); + + it.effect.each(["restartContinuationOfRunId", "usageLimitContinuationOfRunId"] as const)( + "authorizes only the wake-up active run across reused ids and %s", + (continuationField) => + Effect.gen(function* () { + const repo = yield* seed; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('active-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const wakeup = yield* startWakeupRun(threadId, "active-wakeup"); + const scope = { orgId, agentId, threadId, caller }; + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + const dispatch = (id: string, messageId = id) => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(id), + threadId, + messageId: MessageId.make(messageId), + text: "Queued user message", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + yield* dispatch("queued-user"); + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + yield* repo.insertMessage({ + ...message("during-queue", { + author: { + _tag: "agent", + agentId, + ownerPersonId: personId, + threadId, + memberId: agentMemberId, + }, + authorRole: "member", + intent: "answer", + }), + caller, + }); + yield* dispatch("forged-wakeup", "kin-wakeup-msg:active-wakeup"); yield* orchestrator.dispatch({ - type: "queue.resume", - commandId: CommandId.make("start-continuation"), + type: "run.interrupt", + commandId: CommandId.make("end-wakeup"), threadId, + runId: wakeup.id, + holdQueue: true, }); - assert.isTrue(yield* repo.hasRunningWakeup(scope)); - assert.notEqual(next.id, wakeup.id); - }).pipe(Effect.provide(layer)), + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* orchestrator.dispatch({ + type: "queued-run.start", + commandId: CommandId.make("release-user"), + threadId, + runId: queued.id, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET run_id = ${wakeup.id} WHERE wakeup_id = 'active-wakeup'`; + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET run_id = NULL WHERE wakeup_id = 'active-wakeup'`; + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-user"), + threadId, + runId: queued.id, + holdQueue: true, + }); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-thread"), + threadId, + }); + yield* dispatch("held-before-wakeup"); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* dispatch("held-queued-ahead"); + const next = yield* startWakeupRun(threadId, "active-wakeup-next"); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-existing-queue"), + threadId, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET wakeup_id = 'active-wakeup-next' WHERE org_id = ${orgId} AND wakeup_id = 'active-wakeup'`; + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + const held = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (r) => r.userMessageId === "held-queued-ahead", + )!; + assert.strictEqual(held.status, "queued"); + yield* orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("cancel-wakeup"), + threadId, + runId: next.id, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart-wakeup"), + threadId, + messageId: MessageId.make("restart-message"), + text: "Continue", + attachments: [], + dispatchMode: { type: "start_immediately" }, + restartContinuationOfRunId: next.id, + createdBy: "system", + creationSource: "server", + }); + let continuation = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + if (continuationField === "usageLimitContinuationOfRunId") { + continuation = { + ...continuation, + restartContinuationOfRunId: undefined, + usageLimitContinuationOfRunId: next.id, + }; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("usage-continuation"), + type: "run.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: continuation, + }, + ], + }); + } + assert.strictEqual(continuation[continuationField], next.id); + if (continuation.status === "queued") + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("start-continuation"), + threadId, + }); + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + assert.notEqual(next.id, wakeup.id); + }).pipe(Effect.provide(layer)), ); it.effect("advances read state monotonically and counts only others' posted messages", () => diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index 787de94a62cca..5266c6735e05c 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -1,3 +1,4 @@ +import { ACTIVE_RUN_STATUSES } from "../../orchestration-v2/ThreadManagementService.ts"; import type { McpThreadCaller } from "../../mcp/McpInvocationContext.ts"; import { KinOrg, @@ -421,21 +422,33 @@ const make = Effect.gen(function* () { }, ) { if (input.caller.subagent === true || input.caller.threadId !== input.threadId) return false; - const rows = yield* sql`WITH RECURSIVE wakeup_runs(run_id, thread_id) AS ( - SELECT r.run_id, r.thread_id FROM kin_wakeups w - JOIN orchestration_v2_projection_runs r ON r.thread_id = w.thread_id - AND json_extract(r.payload_json, '$.userMessageId') = 'kin-wakeup-msg:' || w.wakeup_id - WHERE w.org_id = ${input.orgId} AND w.agent_id = ${input.agentId} - AND w.thread_id = ${input.threadId} AND w.status = 'running' + const rows = + yield* sql`WITH RECURSIVE ancestors(run_id, thread_id, ordinal, payload_json) AS ( + SELECT run_id, thread_id, ordinal, payload_json + FROM orchestration_v2_projection_runs INDEXED BY orchestration_v2_projection_runs_thread_status_idx + WHERE thread_id = ${input.threadId} AND status IN ${sql.in(ACTIVE_RUN_STATUSES)} UNION - SELECT r.run_id, r.thread_id FROM orchestration_v2_projection_runs r - JOIN wakeup_runs parent ON r.thread_id = parent.thread_id - AND (json_extract(r.payload_json, '$.restartContinuationOfRunId') = parent.run_id - OR json_extract(r.payload_json, '$.usageLimitContinuationOfRunId') = parent.run_id) + SELECT p.run_id, p.thread_id, p.ordinal, p.payload_json + FROM ancestors child JOIN orchestration_v2_projection_runs p + ON p.thread_id = child.thread_id + AND (p.run_id = json_extract(child.payload_json, '$.restartContinuationOfRunId') + OR p.run_id = json_extract(child.payload_json, '$.usageLimitContinuationOfRunId')) ) - SELECT 1 FROM wakeup_runs w JOIN orchestration_v2_projection_runs r - ON r.run_id = w.run_id AND r.thread_id = w.thread_id - WHERE r.status IN ('preparing', 'starting', 'running', 'waiting') LIMIT 1`; + SELECT 1 FROM kin_wakeups w JOIN ancestors root ON root.thread_id = w.thread_id + JOIN orchestration_v2_projection_messages m + ON m.message_id = json_extract(root.payload_json, '$.userMessageId') + AND m.thread_id = root.thread_id AND m.run_id = root.run_id + WHERE w.org_id = ${input.orgId} AND w.agent_id = ${input.agentId} + AND w.thread_id = ${input.threadId} AND w.status = 'running' + AND json_extract(root.payload_json, '$.userMessageId') = 'kin-wakeup-msg:' || w.wakeup_id + AND json_extract(m.payload_json, '$.createdBy') = 'system' + AND json_extract(m.payload_json, '$.creationSource') = 'server' + AND root.run_id = COALESCE(w.run_id, ( + SELECT first.run_id FROM orchestration_v2_projection_runs first + WHERE first.thread_id = w.thread_id + AND json_extract(first.payload_json, '$.userMessageId') = 'kin-wakeup-msg:' || w.wakeup_id + ORDER BY first.ordinal LIMIT 1 + )) LIMIT 1`; return rows.length > 0; }, Effect.mapError(toRepositoryError("KinChatRepository.hasRunningWakeup")), diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index 6206317f404cb..6f1084255e934 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -107,8 +107,8 @@ const seed = Effect.gen(function* () { VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) VALUES ('service-wakeup', ${person.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; - yield* startWakeupRun(threadId, "service-wakeup"); - yield* sql`UPDATE kin_wakeups SET run_id = 'authority-run' WHERE wakeup_id = 'service-wakeup'`; + const run = yield* startWakeupRun(threadId, "service-wakeup"); + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE wakeup_id = 'service-wakeup'`; const agent = yield* identity.resolveAgent(caller); return { repo, service, identity, person, agent, chat }; }); @@ -239,7 +239,6 @@ describe("KinChatService", () => { .pipe(Effect.flip), { reason: "not_allowed" }, ); - yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'running' WHERE run_id = 'authority-run'`; yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; assert.include( yield* service diff --git a/apps/server/src/kin/lead/KinEventQueue.ts b/apps/server/src/kin/lead/KinEventQueue.ts index bc97bd2ec1613..48443ef05ef7f 100644 --- a/apps/server/src/kin/lead/KinEventQueue.ts +++ b/apps/server/src/kin/lead/KinEventQueue.ts @@ -1,3 +1,4 @@ +import { ACTIVE_RUN_STATUSES } from "../../orchestration-v2/ThreadManagementService.ts"; import { KinEvent, KinEventId, @@ -544,6 +545,15 @@ const make = Effect.gen(function* () { const rows = yield* sql<{ agent_id: string }>`SELECT e.agent_id FROM kin_events e JOIN kin_agents a ON a.org_id = e.org_id AND a.agent_id = e.agent_id WHERE e.org_id = ${orgId} AND e.state = 'queued' AND a.retired_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM kin_leads l + WHERE l.org_id = e.org_id AND l.agent_id = e.agent_id AND ( + EXISTS (SELECT 1 FROM orchestration_v2_projection_runs r + WHERE r.thread_id = l.thread_id AND r.status IN ${sql.in(["queued", ...ACTIVE_RUN_STATUSES])}) + OR EXISTS (SELECT 1 FROM orchestration_v2_projection_provider_turns t + WHERE t.thread_id = l.thread_id AND t.completed_at IS NULL) + ) + ) GROUP BY e.agent_id ORDER BY e.agent_id`; const candidates = yield* Effect.forEach(rows, (row) => Effect.gen(function* () { diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index 26c83ebb54998..1c8c5d3d9a5ab 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -188,10 +188,10 @@ it.effect.each([ yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) VALUES (${agentId}, ${orgId}, 'Fixture lead', 0, '{}', '[]', ${chatId})`; - yield* startWakeupRun(threadId, input.wakeupId); + const run = yield* startWakeupRun(threadId, input.wakeupId); yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids, run_id) - VALUES (${input.wakeupId}, ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]', 'authority-run')`; + VALUES (${input.wakeupId}, ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]', ${run.id})`; } yield* repo.addMember({ orgId, diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 58992c7b71ec5..0a11df76a8cac 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -7,9 +7,11 @@ import * as McpSessionRegistryTestkit from "../../mcp/McpSessionRegistry.testkit import { assert, describe, it } from "@effect/vitest"; import { CommandId, + EventId, EnvironmentId, MessageId, KinWakeupId, + KinAgentId, ThreadId, RunId, RunAttemptId, @@ -39,6 +41,9 @@ import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Sqlite from "../../persistence/Sqlite.ts"; import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as DateTime from "effect/DateTime"; +import * as EventSink from "../../orchestration-v2/EventSink.ts"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; import { harness, seed, orgId, agentId, event } from "./WakeupRunner.testkit.ts"; @@ -380,6 +385,75 @@ describe("WakeupRunner", () => { }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), ); + it.effect("wakes another lead while the first has a held queued run", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + yield* Fiber.join(yield* run()); + const [lead] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const threadId = ThreadId.make(lead!.thread_id); + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold"), + threadId, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("held-message"), + threadId, + messageId: MessageId.make("held-message"), + text: "Held", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("held-second"), + threadId, + messageId: MessageId.make("held-second"), + text: "Held", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-queue"), + threadId, + }); + assert.strictEqual( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!.status, + "queued", + ); + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + SELECT 'lead-z', org_id, display_name, role, owner_person_id, created_at FROM kin_agents WHERE agent_id = 'lead'`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + SELECT 'lead-z', org_id, name, level_since, models_json, worker_projects_json, owner_chat_id FROM kin_leads WHERE agent_id = 'lead'`; + yield* queue.enqueue(event("busy")); + yield* queue.enqueue({ ...event("other"), agentId: KinAgentId.make("lead-z") }); + yield* TestClock.adjust("20 seconds"); + const next = yield* queue.nextReady(orgId); + assert.strictEqual(next?.agentId, "lead-z"); + yield* Fiber.join( + yield* (yield* WakeupRunner.WakeupRunner) + .runOnce(orgId, next!.agentId) + .pipe(Effect.forkChild), + ); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'other'`, [ + { state: "done" }, + ]); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'busy'`, [ + { state: "queued" }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + it.effect("defers claims while an interrupted run's provider turn is unfinished", () => Effect.gen(function* () { yield* seed; @@ -388,8 +462,30 @@ describe("WakeupRunner", () => { const sql = yield* SqlClient.SqlClient; yield* queue.enqueue(event("first")); yield* Fiber.join(yield* run()); - yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'interrupted'`; - yield* sql`UPDATE orchestration_v2_projection_provider_turns SET completed_at = NULL`; + const threads = yield* ThreadManagement.ThreadManagementService; + const [lead] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const projection = yield* threads.getThreadProjection(ThreadId.make(lead!.thread_id)); + const sink = yield* EventSink.EventSinkV2; + const turn = projection.providerTurns[0]!; + const occurredAt = yield* DateTime.now; + yield* sink.write({ + events: [ + { + id: EventId.make("unfinished-run"), + type: "run.updated", + threadId: projection.thread.id, + occurredAt, + payload: { ...projection.runs[0]!, status: "interrupted" }, + }, + { + id: EventId.make("unfinished-turn"), + type: "provider-turn.updated", + threadId: projection.thread.id, + occurredAt, + payload: { ...turn, status: "running", completedAt: null }, + }, + ], + }); yield* queue.enqueue(event("next")); const deferred = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); yield* TestClock.adjust("60 seconds"); @@ -399,7 +495,17 @@ describe("WakeupRunner", () => { yield* sql`SELECT state, attempts FROM kin_events WHERE event_id = 'next'`, [{ state: "queued", attempts: 0 }], ); - yield* sql`UPDATE orchestration_v2_projection_provider_turns SET completed_at = '2026-11-08T00:00:00.000Z'`; + yield* sink.write({ + events: [ + { + id: EventId.make("finished-turn"), + type: "provider-turn.updated", + threadId: projection.thread.id, + occurredAt: yield* DateTime.now, + payload: turn, + }, + ], + }); yield* Fiber.join(yield* run()); assert.lengthOf(yield* sql`SELECT 1 FROM kin_wakeups`, 2); }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 33bdb09864372..656da3401e7f7 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -18,6 +18,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Random from "effect/Random"; import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; import * as Schema from "effect/Schema"; import type * as Scope from "effect/Scope"; import * as SqlClient from "effect/sql/SqlClient"; @@ -102,6 +103,14 @@ const make = Effect.gen(function* () { const projects = yield* ProjectService.ProjectService; const launch = yield* ThreadLaunchService.ThreadLaunchService; const threads = yield* ThreadManagementService.ThreadManagementService; + const threadActivity = threads.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" || + (event.type === "provider-turn.updated" && event.payload.completedAt !== null), + ), + Stream.runHead, + ); const lead = Effect.fn("WakeupRunner.lead")(function* (orgId: KinOrgId, agentId: KinAgentId) { const [row] = yield* sql`SELECT l.org_id AS "orgId", l.agent_id AS "agentId", l.name, @@ -256,9 +265,9 @@ const make = Effect.gen(function* () { WHERE thread_id = ${record.threadId} AND completed_at IS NULL LIMIT 1`; const busy = yield* sql`SELECT 1 FROM orchestration_v2_projection_runs WHERE thread_id = ${record.threadId} - AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') LIMIT 1`; + AND status IN ${sql.in(["queued", ...ThreadManagementService.ACTIVE_RUN_STATUSES])} LIMIT 1`; if (unfinished.length > 0 || busy.length > 0) { - yield* (yield* queue.changes(orgId)).pipe( + yield* Effect.raceFirst(yield* queue.changes(orgId), threadActivity).pipe( Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), ); return; @@ -386,7 +395,7 @@ const make = Effect.gen(function* () { createdBy: "system", creationSource: "server", })).projection; - const run = sent.runs.find((run) => run.userMessageId === messageId); + const run = sent.runs.findLast((run) => run.userMessageId === messageId); if (run === undefined) return yield* new KinWakeupError({ orgId, agentId, cause: "missing_run" }); row = { ...row, runId: run.id }; @@ -544,7 +553,7 @@ const make = Effect.gen(function* () { const unfinished = yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running'`; if (unfinished.length > 0) { - yield* changed.pipe( + yield* Effect.raceFirst(changed, threadActivity).pipe( Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), ); return; @@ -563,7 +572,7 @@ const make = Effect.gen(function* () { yield* runOnce(orgId, next.agentId); return; } - yield* changed.pipe( + yield* Effect.raceFirst(changed, threadActivity).pipe( Effect.timeoutOrElse({ duration: Math.min(60_000, Math.max(1, (next?.readyAt ?? Infinity) - now)), orElse: () => Effect.void, diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 529cbc04099a5..d7ce5554b4605 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -503,13 +503,9 @@ export class ThreadManagementService extends Context.Service< ThreadManagementServiceShape >()("t3/orchestration-v2/ThreadManagementService") {} +export const ACTIVE_RUN_STATUSES = ["preparing", "starting", "running", "waiting"] as const; export function isActiveRun(run: OrchestrationV2Run): boolean { - return ( - run.status === "preparing" || - run.status === "starting" || - run.status === "running" || - run.status === "waiting" - ); + return ACTIVE_RUN_STATUSES.some((status) => status === run.status); } export function isTerminalRunStatus( diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 87021027ff230..073623cae2ef4 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -205,15 +205,17 @@ Each wake-up: 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. -7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), identified by its server-minted `kin-wakeup-msg:` message id; `run_id` is used only for waiting and accounting, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. - Launch/send do not expose a preassigned run id or pre-execution callback; early tool calls before the runner records `run_id` fail closed. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. Its persisted message must belong to that root and have `createdBy=system` and `creationSource=server`; the id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. + Launch/send do not expose a preassigned run id or pre-execution callback; early tool calls before the runner records `run_id` are allowed under the lowest-ordinal root rule. + +Known input-integrity gap: clients can steer an active wake-up with `steer_active`, and other agents can use `t3_thread_send` with `steer` or `auto`. No new run is created, so run authority does not exclude that injected text. Steered text is untrusted input; the lead brief treats non-owner text as quoted, not instructions. CHAT-11 (backend) owns refusal of non-owner steer and restart into lead threads at the orchestration access boundary; Kin chat authorization alone cannot guard these upstream entry points. 8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. 9. Finish: `Memory.commitWakeup` (one commit holding the staged item writes and the journal entry), reset the read checkout, record token usage from the run's provider turns into `kin_wakeups.tokens_json` and `KinBudget`, mark events `done`, and `settleThread`. If the lead writes files by hand in the read checkout, step 9's reset throws the changes away. A non-empty `git status --porcelain` before the reset is logged in the journal as "Lead edited memory files directly; discarded." -Kin boot is optional: configuration, migrations, identity bootstrap and runner startup failures are logged with typed errors and disable Kin, never the core T3 runtime. The runner requires the real `KinLeadPrompt` implementation; while LEAD-6 is absent, the server hook does not construct the runner. Memory repositories are supplied through `KIN_MEMORY_REPOSITORIES` (a JSON array of `{ orgId, repoUrl }`); no private repository is assumed by source code. +Kin boot is optional: configuration, migrations, identity bootstrap and runner startup failures are logged with typed errors and disable Kin, never the core T3 runtime. The runner is constructed in `kin/layer.ts` when `KIN_MEMORY_REPOSITORIES` is non-empty, with the real `KinLeadPrompt` implementation. Memory repositories are supplied through `KIN_MEMORY_REPOSITORIES` (a JSON array of `{ orgId, repoUrl }`); no private repository is assumed by source code. The scheduled-task dispatch wrapper also contains Kin migration failures. When Kin persistence is unavailable, launch refusal is limited to `scheduled-task::webhook:` command ids. Sends classify the task using upstream `scheduled_tasks.schedule_json` and refuse only webhooks; an unreadable upstream row fails closed for that send only. Cron, interval, manual and ordinary dispatch remain available. Restoring the Kin database and restarting restores webhook dispatch. From 45f65278f094c2591e9f946e90c4e2e6be7820fc Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:39:06 +0530 Subject: [PATCH 09/12] fix(kin): only server recovery continuations inherit wake-up authority - Each continuation run's message must be agent/server-created, which only the restart and usage-limit workers produce; a client continuation of a cancelled wake-up run is refused (table-tested, fails before). - Deferral wakes only on the org's lead-thread activity, not every event. - Spec: continuation provenance; message-id reuse is fail-closed denial, reserved at client intake by CHAT-11. Co-Authored-By: Claude Opus 5.5 --- .../src/kin/chat/KinChatRepository.test.ts | 15 ++++++---- apps/server/src/kin/chat/KinChatRepository.ts | 6 ++++ apps/server/src/kin/lead/WakeupRunner.ts | 29 ++++++++++++------- docs/kin/specs/lead.md | 4 +-- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index 773fce47b512a..27e1c6b5652c2 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -1052,9 +1052,14 @@ describe("KinChatRepository", () => { }).pipe(Effect.provide(layer)), ); - it.effect.each(["restartContinuationOfRunId", "usageLimitContinuationOfRunId"] as const)( - "authorizes only the wake-up active run across reused ids and %s", - (continuationField) => + it.effect.each([ + ["restartContinuationOfRunId", "agent"], + ["usageLimitContinuationOfRunId", "agent"], + ["restartContinuationOfRunId", "user"], + ["usageLimitContinuationOfRunId", "user"], + ] as const)( + "authorizes only the wake-up active run across reused ids and %s created by %s", + ([continuationField, continuedBy]) => Effect.gen(function* () { const repo = yield* seed; const sql = yield* SqlClient.SqlClient; @@ -1157,7 +1162,7 @@ describe("KinChatRepository", () => { attachments: [], dispatchMode: { type: "start_immediately" }, restartContinuationOfRunId: next.id, - createdBy: "system", + createdBy: continuedBy, creationSource: "server", }); let continuation = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; @@ -1186,7 +1191,7 @@ describe("KinChatRepository", () => { commandId: CommandId.make("start-continuation"), threadId, }); - assert.isTrue(yield* repo.hasRunningWakeup(scope)); + assert.strictEqual(yield* repo.hasRunningWakeup(scope), continuedBy === "agent"); assert.notEqual(next.id, wakeup.id); }).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index 5266c6735e05c..0e8f92abefaef 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -433,6 +433,12 @@ const make = Effect.gen(function* () { ON p.thread_id = child.thread_id AND (p.run_id = json_extract(child.payload_json, '$.restartContinuationOfRunId') OR p.run_id = json_extract(child.payload_json, '$.usageLimitContinuationOfRunId')) + -- Only server recovery workers (agent/server) may continue a wake-up; clients are forced to user. + JOIN orchestration_v2_projection_messages cm + ON cm.message_id = json_extract(child.payload_json, '$.userMessageId') + AND cm.thread_id = child.thread_id AND cm.run_id = child.run_id + AND json_extract(cm.payload_json, '$.createdBy') = 'agent' + AND json_extract(cm.payload_json, '$.creationSource') = 'server' ) SELECT 1 FROM kin_wakeups w JOIN ancestors root ON root.thread_id = w.thread_id JOIN orchestration_v2_projection_messages m diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 656da3401e7f7..30367709c26ef 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -103,14 +103,21 @@ const make = Effect.gen(function* () { const projects = yield* ProjectService.ProjectService; const launch = yield* ThreadLaunchService.ThreadLaunchService; const threads = yield* ThreadManagementService.ThreadManagementService; - const threadActivity = threads.streamDomainEvents.pipe( - Stream.filter( - (event) => - event.type === "run.updated" || - (event.type === "provider-turn.updated" && event.payload.completedAt !== null), - ), - Stream.runHead, - ); + const threadActivity = Effect.fn("WakeupRunner.threadActivity")(function* (orgId: KinOrgId) { + const rows = yield* sql<{ + threadId: string; + }>`SELECT thread_id AS "threadId" FROM kin_leads WHERE org_id = ${orgId} AND thread_id IS NOT NULL ORDER BY thread_id`; + const leadThreads = new Set(rows.map((row) => row.threadId)); + return yield* threads.streamDomainEvents.pipe( + Stream.filter( + (event) => + leadThreads.has(event.threadId) && + (event.type === "run.updated" || + (event.type === "provider-turn.updated" && event.payload.completedAt !== null)), + ), + Stream.runHead, + ); + }); const lead = Effect.fn("WakeupRunner.lead")(function* (orgId: KinOrgId, agentId: KinAgentId) { const [row] = yield* sql`SELECT l.org_id AS "orgId", l.agent_id AS "agentId", l.name, @@ -267,7 +274,7 @@ const make = Effect.gen(function* () { WHERE thread_id = ${record.threadId} AND status IN ${sql.in(["queued", ...ThreadManagementService.ACTIVE_RUN_STATUSES])} LIMIT 1`; if (unfinished.length > 0 || busy.length > 0) { - yield* Effect.raceFirst(yield* queue.changes(orgId), threadActivity).pipe( + yield* Effect.raceFirst(yield* queue.changes(orgId), threadActivity(orgId)).pipe( Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), ); return; @@ -553,7 +560,7 @@ const make = Effect.gen(function* () { const unfinished = yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running'`; if (unfinished.length > 0) { - yield* Effect.raceFirst(changed, threadActivity).pipe( + yield* Effect.raceFirst(changed, threadActivity(orgId)).pipe( Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), ); return; @@ -572,7 +579,7 @@ const make = Effect.gen(function* () { yield* runOnce(orgId, next.agentId); return; } - yield* Effect.raceFirst(changed, threadActivity).pipe( + yield* Effect.raceFirst(changed, threadActivity(orgId)).pipe( Effect.timeoutOrElse({ duration: Math.min(60_000, Math.max(1, (next?.readyAt ?? Infinity) - now)), orElse: () => Effect.void, diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 073623cae2ef4..b076458f5eb64 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -205,10 +205,10 @@ Each wake-up: 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. -7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. Its persisted message must belong to that root and have `createdBy=system` and `creationSource=server`; the id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. Its persisted message must belong to that root and have `createdBy=system` and `creationSource=server`, and every continuation's message must have `createdBy=agent` and `creationSource=server` (only the server's restart and usage-limit workers produce that; clients are forced to `user`). The id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. Launch/send do not expose a preassigned run id or pre-execution callback; early tool calls before the runner records `run_id` are allowed under the lowest-ordinal root rule. -Known input-integrity gap: clients can steer an active wake-up with `steer_active`, and other agents can use `t3_thread_send` with `steer` or `auto`. No new run is created, so run authority does not exclude that injected text. Steered text is untrusted input; the lead brief treats non-owner text as quoted, not instructions. CHAT-11 (backend) owns refusal of non-owner steer and restart into lead threads at the orchestration access boundary; Kin chat authorization alone cannot guard these upstream entry points. +Known input-integrity gap: clients can steer an active wake-up with `steer_active`, and other agents can use `t3_thread_send` with `steer` or `auto`. No new run is created, so run authority does not exclude that injected text. Steered text is untrusted input; the lead brief treats non-owner text as quoted, not instructions. CHAT-11 (backend) owns refusal of non-owner steer and restart into lead threads at the orchestration access boundary; Kin chat authorization alone cannot guard these upstream entry points. Clients can also reuse a visible `kin-wakeup-msg:` message id, which overwrites the projected message row and makes the running wake-up fail closed (denial, not authority); CHAT-11 also reserves that prefix at client message intake. 8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. 9. Finish: `Memory.commitWakeup` (one commit holding the staged item writes and the journal entry), reset the read checkout, record token usage from the run's provider turns into `kin_wakeups.tokens_json` and `KinBudget`, mark events `done`, and `settleThread`. From b74c2d98a7536ad573315f079d0499c24676fba6 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:49:45 +0530 Subject: [PATCH 10/12] docs(kin): accurate continuation provenance invariant Co-Authored-By: Claude Opus 5.5 --- apps/server/src/kin/chat/KinChatRepository.ts | 2 +- docs/kin/specs/lead.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index 0e8f92abefaef..87c5009b39e81 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -433,7 +433,7 @@ const make = Effect.gen(function* () { ON p.thread_id = child.thread_id AND (p.run_id = json_extract(child.payload_json, '$.restartContinuationOfRunId') OR p.run_id = json_extract(child.payload_json, '$.usageLimitContinuationOfRunId')) - -- Only server recovery workers (agent/server) may continue a wake-up; clients are forced to user. + -- Continuation links plus agent/server provenance come only from the restart and usage-limit workers. JOIN orchestration_v2_projection_messages cm ON cm.message_id = json_extract(child.payload_json, '$.userMessageId') AND cm.thread_id = child.thread_id AND cm.run_id = child.run_id diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index b076458f5eb64..762499473c67d 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -205,7 +205,7 @@ Each wake-up: 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). 6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. -7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. Its persisted message must belong to that root and have `createdBy=system` and `creationSource=server`, and every continuation's message must have `createdBy=agent` and `creationSource=server` (only the server's restart and usage-limit workers produce that; clients are forced to `user`). The id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. Its persisted message must belong to that root and have `createdBy=system` and `creationSource=server`, and every continuation's message must have `createdBy=agent` and `creationSource=server`. Other paths can produce that provenance (server notification wakes, and client-written scheduled tasks, whose upsert accepts `createdBy`/`creationSource`), but none of them sets a restart or usage-limit continuation link: client dispatches that set one are forced to `createdBy=user`, so only the server's recovery workers produce both. The id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. Launch/send do not expose a preassigned run id or pre-execution callback; early tool calls before the runner records `run_id` are allowed under the lowest-ordinal root rule. Known input-integrity gap: clients can steer an active wake-up with `steer_active`, and other agents can use `t3_thread_send` with `steer` or `auto`. No new run is created, so run authority does not exclude that injected text. Steered text is untrusted input; the lead brief treats non-owner text as quoted, not instructions. CHAT-11 (backend) owns refusal of non-owner steer and restart into lead threads at the orchestration access boundary; Kin chat authorization alone cannot guard these upstream entry points. Clients can also reuse a visible `kin-wakeup-msg:` message id, which overwrites the projected message row and makes the running wake-up fail closed (denial, not authority); CHAT-11 also reserves that prefix at client message intake. From 22023174e56bfe30884ef0a07872ba2807660d81 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:24:40 +0530 Subject: [PATCH 11/12] fix(kin): integrate chat tools with lead wakeup lifecycle --- .../src/kin/chat/KinChatService.test.ts | 8 +++ .../src/kin/chat/WakeupAuthority.testkit.ts | 33 +++++---- apps/server/src/kin/layer.ts | 4 +- apps/server/src/kin/lead/WakeupRunner.test.ts | 68 +++++++++++++------ apps/server/src/kin/lead/WakeupRunner.ts | 5 +- .../server/src/kin/mcp/KinChatToolkit.test.ts | 13 +++- apps/server/src/kin/mcp/layer.test.ts | 35 ++++++++-- apps/server/src/kin/mcp/layer.ts | 10 ++- 8 files changed, 133 insertions(+), 43 deletions(-) diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index e041d6f9d488e..d04c9e8a168d5 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -477,6 +477,12 @@ describe("KinChatService", () => { agentId: otherId, threadId: otherThread, }); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${otherId}, ${person.orgId}, 'Other', 0, '{}', '[]', ${chat.chatId})`; + const otherRun = yield* startWakeupRun(otherThread, "other-wakeup"); + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, run_id, model, started_at, status, event_ids) + VALUES ('other-wakeup', ${person.orgId}, ${otherId}, ${otherThread}, ${otherRun.id}, 'test', 0, 'running', '[]')`; const other = yield* identity.resolveAgent({ ...caller, thread: { ...caller.thread!, threadId: otherThread }, @@ -494,6 +500,8 @@ describe("KinChatService", () => { .pipe(Effect.flip), { reason: "not_allowed" }, ); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'other-wakeup'`; + yield* sql`UPDATE kin_wakeups SET status = 'running' WHERE wakeup_id = 'service-wakeup'`; for (const update of [{}, { revision: 1, replyTo: question.messageId }]) assert.include( yield* service diff --git a/apps/server/src/kin/chat/WakeupAuthority.testkit.ts b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts index 78a23a3e2013e..f700879c5b0c7 100644 --- a/apps/server/src/kin/chat/WakeupAuthority.testkit.ts +++ b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts @@ -17,19 +17,26 @@ import * as Sqlite from "../../persistence/Sqlite.ts"; const database = KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); const instanceId = ProviderInstanceId.make("codex"); -export const authorityLayer = ProviderReplayHarness.layerWithRegistry( - { name: "wakeup-authority" }, - ProviderAdapterRegistry.layerFromAdapters([ - { - instanceId, - driver: ProviderDriverKind.make("codex"), - getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), - planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), - openSession: () => Effect.die("Authority tests do not start provider processes"), - }, - ]), - { databaseLayer: database, runEffectWorker: false }, -).pipe(Layer.provideMerge(database)); +export const authorityLayerWithDatabase = ( + databaseLayer: NonNullable< + NonNullable[2]>["databaseLayer"] + >, +) => + ProviderReplayHarness.layerWithRegistry( + { name: "wakeup-authority" }, + ProviderAdapterRegistry.layerFromAdapters([ + { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: () => Effect.die("Authority tests do not start provider processes"), + }, + ]), + { databaseLayer, runEffectWorker: false }, + ).pipe(Layer.provideMerge(databaseLayer)); + +export const authorityLayer = authorityLayerWithDatabase(database); export const startWakeupRun = Effect.fn("test.startWakeupRun")(function* ( threadId: ThreadId, diff --git a/apps/server/src/kin/layer.ts b/apps/server/src/kin/layer.ts index 5c884f7e09583..51de323ea25c0 100644 --- a/apps/server/src/kin/layer.ts +++ b/apps/server/src/kin/layer.ts @@ -4,7 +4,6 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as HttpRouter from "effect/http/HttpRouter"; import * as Option from "effect/Option"; -import * as KinLead from "./lead/layer.ts"; import * as KinChat from "./chat/layer.ts"; import { KinChatHub } from "./chat/KinChatHub.ts"; @@ -14,7 +13,6 @@ import type * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as KinMcp from "./mcp/layer.ts"; -const lead = KinLead.layerServer; const enabled = Layer.effect( KinServer, Effect.gen(function* () { @@ -25,7 +23,7 @@ const enabled = Layer.effect( subscribeForViewer: Option.some(hub.subscribeForViewer), }; }), -).pipe(Layer.provide(lead.pipe(Layer.provideMerge(KinChat.layer)))); +).pipe(Layer.provide(KinChat.layer)); const disabled = Layer.succeed(KinServer, { services: Option.none(), subscribeForViewer: Option.none(), diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 0a11df76a8cac..6b49150d6331f 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -2,6 +2,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { HttpServer } from "effect/http"; import * as NetAddress from "effect/net/NetAddress"; import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import * as ProviderSessionManager from "../../orchestration-v2/ProviderSessionManager.ts"; import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; import * as McpSessionRegistryTestkit from "../../mcp/McpSessionRegistry.testkit.ts"; import { assert, describe, it } from "@effect/vitest"; @@ -553,25 +554,54 @@ describe("WakeupRunner", () => { it.effect("expires retired leads' queued events and ends their durable thread registration", () => Effect.gen(function* () { - yield* seed; - const queue = yield* KinEventQueue.KinEventQueue; - const runner = yield* WakeupRunner.WakeupRunner; - const sql = yield* SqlClient.SqlClient; - yield* queue.enqueue(event("first")); - yield* Fiber.join(yield* run()); - yield* queue.enqueue(event("retired")); - yield* sql`UPDATE kin_agents SET retired_at = '2026-11-08T00:00:00.000Z' WHERE org_id = ${orgId} AND agent_id = ${agentId}`; - yield* runner.runOnce(orgId, agentId); - assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'retired'`, [ - { state: "expired" }, - ]); - assert.strictEqual( - (yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL AND org_id = ${orgId}`) - .length, - 0, - ); - assert.strictEqual((yield* sql`SELECT 1 FROM kin_wakeups`).length, 1); - }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + const active = yield* run(); + yield* Deferred.await(entered); + yield* queue.enqueue(event("retired")); + yield* sql`UPDATE kin_agents SET retired_at = '2026-11-08T00:00:00.000Z' WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + const identity = yield* KinIdentity.KinIdentity; + const threads = yield* ThreadManagement.ThreadManagementService; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const retiringRunner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(KinIdentity.KinIdentity, { + ...identity, + endAgentThread: Effect.fn("test.endAgentThread")(function* ( + input: Parameters[0], + ) { + const projection = yield* threads + .getThreadProjection(input.threadId) + .pipe(Effect.orDie); + assert.isTrue(identity.isAgentThread(input.threadId)); + for (const providerThread of projection.providerThreads) + if (providerThread.providerSessionId !== null) + assert.isFalse(yield* sessions.isLive(providerThread.providerSessionId)); + return yield* identity.endAgentThread(input); + }), + }), + ); + yield* retiringRunner + .runOnce(orgId, agentId) + .pipe(Effect.ensuring(Deferred.succeed(hold, undefined))); + yield* Fiber.join(active); + assert.deepStrictEqual( + yield* sql`SELECT state FROM kin_events WHERE event_id = 'retired'`, + [{ state: "expired" }], + ); + assert.strictEqual( + (yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL AND org_id = ${orgId}`) + .length, + 0, + ); + assert.strictEqual((yield* sql`SELECT 1 FROM kin_wakeups`).length, 1); + }).pipe(Effect.provide(harness({ entered, hold }))); + }).pipe(provideDeterministicTestRuntime), ); it.effect( diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 30367709c26ef..9bbb4fb85daf2 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -130,8 +130,11 @@ const make = Effect.gen(function* () { const retire = Effect.fn("WakeupRunner.retire")(function* (record: typeof LeadRow.Type) { if (retired.has(record.agentId)) return; yield* queue.expireAgent(record.orgId, record.agentId); - if (record.threadId !== null) + if (record.threadId !== null) { + const stopped = yield* threads.forceStopThread({ threadId: record.threadId }); + if (stopped.outcome === "releasing" || stopped.outcome === "shared") return; yield* identity.endAgentThread({ orgId: record.orgId, threadId: record.threadId }); + } retired.add(record.agentId); }); const capture = Effect.fn("WakeupRunner.capture")(function* (row: typeof RunningRow.Type) { diff --git a/apps/server/src/kin/mcp/KinChatToolkit.test.ts b/apps/server/src/kin/mcp/KinChatToolkit.test.ts index 1d5d596802df0..450e213f9448c 100644 --- a/apps/server/src/kin/mcp/KinChatToolkit.test.ts +++ b/apps/server/src/kin/mcp/KinChatToolkit.test.ts @@ -1,3 +1,4 @@ +import { authorityLayer, startWakeupRun } from "../chat/WakeupAuthority.testkit.ts"; import * as FileSystem from "effect/FileSystem"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -97,7 +98,7 @@ const liveThreads = Layer.mock(ThreadManagement.ThreadManagementService)({ }); const layer = Layer.merge(registration, KinMcp.registry(fakeRegistry)).pipe( Layer.provideMerge(runtime({ KIN_ENABLED: true })), - Layer.provideMerge(Sqlite.layerMemory), + Layer.provideMerge(authorityLayer), Layer.provideMerge(liveThreads), ); const visible = Effect.gen(function* () { @@ -149,6 +150,11 @@ const seed = Effect.gen(function* () { ); yield* identity.registerAgentThread({ orgId: person.orgId, agentId, threadId }); const chat = yield* chatService.createDirect(person, { orgId: person.orgId, agentId }); + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; + const run = yield* startWakeupRun(threadId, "toolkit-wakeup"); + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, run_id, model, started_at, status, event_ids) + VALUES ('toolkit-wakeup', ${person.orgId}, ${agentId}, ${threadId}, ${run.id}, 'test', 0, 'running', '[]')`; return { identity, chatService, person, chat, sql }; }); @@ -338,7 +344,7 @@ it.effect.each([{}, { KIN_ENABLED: false }, { KIN_ENABLED: "maybe" }])( it.effect("admits only active provider agents at /kin/mcp", () => Effect.scoped( Effect.gen(function* () { - const { identity, person } = yield* seed; + const { identity, person, sql } = yield* seed; const kin = yield* Kin.KinServer; const providerScope = { ...scope, thread }; const authRegistry = Layer.mock(McpSessions.McpSessionRegistry)({ @@ -382,6 +388,9 @@ it.effect("admits only active provider agents at /kin/mcp", () => ); assert.strictEqual((yield* request("missing")).status, 401); assert.strictEqual((yield* request("ordinary")).status, 403); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'toolkit-wakeup'`; + assert.strictEqual((yield* request("agent")).status, 403); + yield* sql`UPDATE kin_wakeups SET status = 'running' WHERE wakeup_id = 'toolkit-wakeup'`; assert.strictEqual((yield* request("agent")).status, 200); yield* identity.endAgentThread({ orgId: person.orgId, threadId }); assert.strictEqual((yield* request("agent")).status, 403); diff --git a/apps/server/src/kin/mcp/layer.test.ts b/apps/server/src/kin/mcp/layer.test.ts index 44aab4ded732c..3eb1fd9f9ec48 100644 --- a/apps/server/src/kin/mcp/layer.test.ts +++ b/apps/server/src/kin/mcp/layer.test.ts @@ -1,3 +1,5 @@ +import { authorityLayerWithDatabase, startWakeupRun } from "../chat/WakeupAuthority.testkit.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; import { McpProviderSessions } from "@t3tools/provider-core/server/McpProviderSessions"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -31,6 +33,7 @@ import * as NetAddress from "effect/net/NetAddress"; import { liveThreadShell } from "../../mcp/McpToolAccess.testkit.ts"; import * as McpSessions from "../../mcp/McpSessionRegistry.ts"; import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as VcsProcess from "../../vcs/VcsProcess.ts"; import * as Kin from "../layer.ts"; import { KinIdentity } from "../chat/KinIdentity.ts"; import { KinChatService } from "../chat/KinChatService.ts"; @@ -66,7 +69,9 @@ import { GitVcsDriver } from "../../vcs/GitVcsDriver.ts"; import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; import { DocumentText } from "../../provider/DocumentText.ts"; import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; +const serverConfig = ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-context-budget-" }); const toolHandlerServices = Layer.mergeAll( + VcsProcess.layer, Layer.mock(AssetSigner)({}), Layer.mock(DeviceService)({}), Layer.mock(DocumentText)({}), @@ -90,7 +95,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(RemoteDelegation)({}), Layer.mock(ScheduledTaskService)({}), Layer.mock(SecretRequests)({}), - ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-context-budget-" }), + serverConfig, Layer.mock(ServerEnvironment)({ getEnvironmentId: Effect.succeed(EnvironmentId.make("environment")), }), @@ -112,18 +117,28 @@ it.effect.each([true, false])( Effect.gen(function* () { const ready = yield* Deferred.make< - Context.Context + Context.Context< + Kin.KinServer | McpSessions.McpSessionRegistry | SqlClient.SqlClient | OrchestratorV2 + > >(); const routes = Layer.merge( Kin.mcpLayer(McpHttpServer.layer), Layer.effectDiscard( Effect.context< - Kin.KinServer | McpSessions.McpSessionRegistry | SqlClient.SqlClient + Kin.KinServer | McpSessions.McpSessionRegistry | SqlClient.SqlClient | OrchestratorV2 >().pipe(Effect.flatMap((context) => Deferred.succeed(ready, context))), ), ).pipe( Layer.provideMerge(Kin.mcpRegistryLayer(McpSessions.layer)), - Layer.provideMerge(Sqlite.layerConfig), + Layer.provideMerge( + authorityLayerWithDatabase( + KinPersistence.layer.pipe( + Layer.provideMerge(Sqlite.layerConfig.pipe(Layer.orDie)), + Layer.provide(serverConfig), + Layer.provide(NodeServices.layer), + ), + ), + ), Layer.provide(toolHandlerServices), Layer.provide( Layer.mock(HttpServer.HttpServer)({ @@ -216,6 +231,18 @@ it.effect.each([true, false])( providerInstanceId: ProviderInstanceId.make("codex"), }); assert.strictEqual(issued.config.endpoint, "http://127.0.0.1:43123/kin/mcp"); + assert.strictEqual( + (yield* request("/kin/mcp", issued.config.authorizationHeader, "initialize", initialize)) + .status, + 403, + ); + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; + const run = yield* startWakeupRun(threadId, "composition-wakeup").pipe( + Effect.provide(services), + ); + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, run_id, model, started_at, status, event_ids) + VALUES ('composition-wakeup', ${person.orgId}, ${agentId}, ${threadId}, ${run.id}, 'test', 0, 'running', '[]')`; const initialized = yield* request( "/kin/mcp", issued.config.authorizationHeader, diff --git a/apps/server/src/kin/mcp/layer.ts b/apps/server/src/kin/mcp/layer.ts index cef6c3435c854..3100b36981cf9 100644 --- a/apps/server/src/kin/mcp/layer.ts +++ b/apps/server/src/kin/mcp/layer.ts @@ -11,6 +11,8 @@ import { McpInvocationContext } from "../../mcp/McpInvocationContext.ts"; import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; import { KinServer } from "../KinServer.ts"; import { KinIdentity } from "../chat/KinIdentity.ts"; +import * as KinLead from "../lead/layer.ts"; +import * as SqlitePersistence from "../../persistence/Sqlite.ts"; import { KinChatToolkit, handlers } from "./KinChatToolkit.ts"; const registration = McpHttpServer.toolkitRegistration(KinChatToolkit, handlers); @@ -67,7 +69,13 @@ export const transport = (base: typeof McpHttpServer.layer) => const kin = yield* KinServer; // The isolated registry also doubles HtmlRender's per-registry headless Chrome cap. return Option.isSome(kin.services) - ? endpoint(Layer.merge(McpHttpServer.layerTools, registration)) + ? Layer.merge( + endpoint(Layer.merge(McpHttpServer.layerTools, registration)), + KinLead.layerServer.pipe( + Layer.provide(Layer.succeedContext(kin.services.value)), + Layer.provide(SqlitePersistence.layerConfig), + ), + ) : Layer.empty; }), ), From c9b71d1f62c1aa99e0713a6204cbf81a21a1a3b8 Mon Sep 17 00:00:00 2001 From: GLDRoger <130458023+GLDRoger@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:40:09 +0530 Subject: [PATCH 12/12] fix(kin): compose lead runtime outside MCP transport --- apps/server/src/kin/layer.ts | 16 +- apps/server/src/kin/lead/WakeupRunner.test.ts | 157 ++++++++++++------ .../src/kin/lead/WakeupRunner.testkit.ts | 21 +++ apps/server/src/kin/lead/WakeupRunner.ts | 2 +- apps/server/src/kin/mcp/layer.test.ts | 23 ++- apps/server/src/kin/mcp/layer.ts | 10 +- 6 files changed, 169 insertions(+), 60 deletions(-) diff --git a/apps/server/src/kin/layer.ts b/apps/server/src/kin/layer.ts index 51de323ea25c0..0a82e33ca87f2 100644 --- a/apps/server/src/kin/layer.ts +++ b/apps/server/src/kin/layer.ts @@ -12,6 +12,7 @@ export { KinServer } from "./KinServer.ts"; import type * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as KinMcp from "./mcp/layer.ts"; +import * as KinLead from "./lead/layer.ts"; const enabled = Layer.effect( KinServer, @@ -66,7 +67,20 @@ const middleware = HttpRouter.middleware<{ provides: KinServer }>()( export const layer = middleware.pipe(Layer.provideMerge(runtime)); -export const mcpLayer = (base: Parameters[0]) => KinMcp.transport(base); +const leadRuntime = Layer.unwrap( + Effect.gen(function* () { + const kin = yield* KinServer; + return Option.isSome(kin.services) + ? KinLead.layerServer.pipe( + Layer.provide(Layer.succeedContext(kin.services.value)), + Layer.provide(SqlitePersistence.layerConfig), + ) + : Layer.empty; + }), +); + +export const mcpLayer = (base: Parameters[0]) => + Layer.merge(KinMcp.transport(base), leadRuntime); export const mcpRegistryLayer = ( base: Layer.Layer, ) => diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 6b49150d6331f..8bbee9125bf2b 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -552,56 +552,117 @@ describe("WakeupRunner", () => { }).pipe(provideDeterministicTestRuntime), ); - it.effect("expires retired leads' queued events and ends their durable thread registration", () => - Effect.gen(function* () { - const entered = yield* Deferred.make(); - const hold = yield* Deferred.make(); - yield* Effect.gen(function* () { - yield* seed; - const queue = yield* KinEventQueue.KinEventQueue; - const sql = yield* SqlClient.SqlClient; - yield* queue.enqueue(event("first")); - const active = yield* run(); - yield* Deferred.await(entered); - yield* queue.enqueue(event("retired")); - yield* sql`UPDATE kin_agents SET retired_at = '2026-11-08T00:00:00.000Z' WHERE org_id = ${orgId} AND agent_id = ${agentId}`; - const identity = yield* KinIdentity.KinIdentity; - const threads = yield* ThreadManagement.ThreadManagementService; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const retiringRunner = yield* WakeupRunner.WakeupRunner.pipe( - Effect.provide(Layer.fresh(WakeupRunner.layer)), - Effect.provideService(KinIdentity.KinIdentity, { - ...identity, - endAgentThread: Effect.fn("test.endAgentThread")(function* ( - input: Parameters[0], - ) { - const projection = yield* threads - .getThreadProjection(input.threadId) - .pipe(Effect.orDie); - assert.isTrue(identity.isAgentThread(input.threadId)); - for (const providerThread of projection.providerThreads) - if (providerThread.providerSessionId !== null) - assert.isFalse(yield* sessions.isLive(providerThread.providerSessionId)); - return yield* identity.endAgentThread(input); + it.effect.each(["released", "shared", "releasing"] as const)( + "expires retired leads' queued events and ends their durable thread registration (%s)", + (outcome) => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + const releaseEntered = yield* Deferred.make(); + const releaseHold = yield* Deferred.make(); + const releaseDone = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + const active = yield* run(); + yield* Deferred.await(entered); + yield* queue.enqueue(event("retired")); + const identity = yield* KinIdentity.KinIdentity; + const threads = yield* ThreadManagement.ThreadManagementService; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const [lead] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const threadId = ThreadId.make(lead!.thread_id); + const projection = yield* threads.getThreadProjection(threadId); + const credentials = yield* McpSessionRegistry.McpSessionRegistry; + const issued = yield* credentials.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), + }); + const token = issued.config.authorizationHeader.slice(7); + const caller = yield* credentials.resolve(token); + assert.isDefined(caller); + yield* identity.resolveAgent(caller!); + yield* sql`UPDATE kin_agents SET retired_at = '2026-11-08T00:00:00.000Z' WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + let stops = 0; + const retiringRunner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + forceStopThread: Effect.fn("test.forceStopThread")(function* (input) { + stops++; + const result = yield* threads.forceStopThread(input); + assert.strictEqual(result.outcome, stops === 1 ? outcome : "released"); + return result; + }), }), - }), - ); - yield* retiringRunner - .runOnce(orgId, agentId) - .pipe(Effect.ensuring(Deferred.succeed(hold, undefined))); - yield* Fiber.join(active); - assert.deepStrictEqual( - yield* sql`SELECT state FROM kin_events WHERE event_id = 'retired'`, - [{ state: "expired" }], - ); - assert.strictEqual( - (yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL AND org_id = ${orgId}`) - .length, - 0, + Effect.provideService(KinIdentity.KinIdentity, { + ...identity, + endAgentThread: Effect.fn("test.endAgentThread")(function* ( + input: Parameters[0], + ) { + const projection = yield* threads + .getThreadProjection(input.threadId) + .pipe(Effect.orDie); + assert.isTrue(identity.isAgentThread(input.threadId)); + for (const providerThread of projection.providerThreads) + if (providerThread.providerSessionId !== null) + assert.strictEqual( + yield* sessions.isLive(providerThread.providerSessionId), + outcome === "shared", + ); + return yield* identity.endAgentThread(input); + }), + }), + ); + const firstPass = yield* retiringRunner.runOnce(orgId, agentId).pipe(Effect.forkChild); + if (outcome === "releasing") { + yield* Deferred.await(releaseEntered); + yield* TestClock.adjust("10 seconds"); + } + yield* Fiber.join(firstPass); + assert.strictEqual(identity.isAgentThread(threadId), outcome === "releasing"); + if (outcome === "releasing") { + assert.isTrue(identity.isAgentThread(threadId)); + assert.lengthOf(yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL`, 1); + yield* Deferred.succeed(releaseHold, undefined); + yield* Deferred.await(releaseDone); + } + yield* retiringRunner.runOnce(orgId, agentId); + assert.strictEqual(stops, outcome === "releasing" ? 2 : 1); + if (outcome === "shared") { + assert.isTrue( + yield* sessions.isLive(projection.providerThreads[0]!.providerSessionId!), + ); + assert.isDefined(yield* credentials.resolve(token)); + const denied = yield* identity.resolveAgent(caller!).pipe(Effect.flip); + assert.propertyVal(denied, "reason", "not_allowed"); + assert.strictEqual(denied.message, "Kin agent thread is not active."); + } + yield* Deferred.succeed(hold, undefined); + yield* Fiber.join(active); + assert.deepStrictEqual( + yield* sql`SELECT state FROM kin_events WHERE event_id = 'retired'`, + [{ state: "expired" }], + ); + assert.strictEqual( + (yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL AND org_id = ${orgId}`) + .length, + 0, + ); + assert.strictEqual((yield* sql`SELECT 1 FROM kin_wakeups`).length, 1); + }).pipe( + Effect.provide( + harness({ + entered, + hold, + ...(outcome === "shared" ? { sharedWith: [ThreadId.make("sibling")] } : {}), + ...(outcome === "releasing" ? { releaseEntered, releaseHold, releaseDone } : {}), + }), + ), ); - assert.strictEqual((yield* sql`SELECT 1 FROM kin_wakeups`).length, 1); - }).pipe(Effect.provide(harness({ entered, hold }))); - }).pipe(provideDeterministicTestRuntime), + }).pipe(provideDeterministicTestRuntime), ); it.effect( diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts index df4baa1205da3..5cee69733a6cb 100644 --- a/apps/server/src/kin/lead/WakeupRunner.testkit.ts +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -1,3 +1,5 @@ +import * as ProviderSessionManager from "../../orchestration-v2/ProviderSessionManager.ts"; +import type { ThreadId } from "@t3tools/contracts"; import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; @@ -100,6 +102,10 @@ export const event = (id: string) => export function harness( options: { + readonly sharedWith?: ReadonlyArray; + readonly releaseEntered?: Deferred.Deferred; + readonly releaseHold?: Deferred.Deferred; + readonly releaseDone?: Deferred.Deferred; readonly hold?: Deferred.Deferred; readonly entered?: Deferred.Deferred; readonly prepare?: Memory.Memory["Service"]["prepareWakeup"]; @@ -209,7 +215,22 @@ export function harness( ); const settings = ServerSettings.layerTest(); const receipts = CommandReceiptStore.layer.pipe(Layer.provide(database)); + const sharedWith = options.sharedWith; const management = ThreadManagement.layer.pipe( + Layer.updateService(ProviderSessionManager.ProviderSessionManagerV2, (sessions) => ({ + ...sessions, + otherWorkingThreads: + sharedWith === undefined ? sessions.otherWorkingThreads : () => Effect.succeed(sharedWith), + releaseUnlessBusy: Effect.fn("test.releaseUnlessBusy")(function* (input) { + if (options.releaseEntered !== undefined) + yield* Deferred.succeed(options.releaseEntered, undefined); + if (options.releaseHold !== undefined) yield* Deferred.await(options.releaseHold); + const released = yield* sessions.releaseUnlessBusy(input); + if (options.releaseDone !== undefined) + yield* Deferred.succeed(options.releaseDone, undefined); + return released; + }), + })), Layer.provide(Layer.mergeAll(orchestration, settings, receipts)), ); const projectLayer = Layer.mock(ProjectService.ProjectService)({ diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index 9bbb4fb85daf2..b229772b1db69 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -132,7 +132,7 @@ const make = Effect.gen(function* () { yield* queue.expireAgent(record.orgId, record.agentId); if (record.threadId !== null) { const stopped = yield* threads.forceStopThread({ threadId: record.threadId }); - if (stopped.outcome === "releasing" || stopped.outcome === "shared") return; + if (stopped.outcome === "releasing") return; yield* identity.endAgentThread({ orgId: record.orgId, threadId: record.threadId }); } retired.add(record.agentId); diff --git a/apps/server/src/kin/mcp/layer.test.ts b/apps/server/src/kin/mcp/layer.test.ts index 3eb1fd9f9ec48..8a344c93809bf 100644 --- a/apps/server/src/kin/mcp/layer.test.ts +++ b/apps/server/src/kin/mcp/layer.test.ts @@ -9,6 +9,7 @@ import { AuthStandardClientScopes, EnvironmentId, KinAgentId, + KinOrgId, ProviderInstanceId, ThreadId, } from "@t3tools/contracts"; @@ -35,6 +36,7 @@ import * as McpSessions from "../../mcp/McpSessionRegistry.ts"; import * as Sqlite from "../../persistence/Sqlite.ts"; import * as VcsProcess from "../../vcs/VcsProcess.ts"; import * as Kin from "../layer.ts"; +import { ServerActivation } from "../../serverActivation.ts"; import { KinIdentity } from "../chat/KinIdentity.ts"; import { KinChatService } from "../chat/KinChatService.ts"; import { AssetSigner } from "../../assets/AssetSigner.ts"; @@ -115,6 +117,7 @@ it.effect.each([true, false])( (enabled) => Effect.scoped( Effect.gen(function* () { + let runnerStarts = 0; const ready = yield* Deferred.make< Context.Context< @@ -145,9 +148,27 @@ it.effect.each([true, false])( address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123), }), ), - Layer.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({ KIN_ENABLED: enabled }))), + Layer.provide( + Layer.succeed( + ServerActivation, + Effect.sync(() => { + runnerStarts++; + }).pipe(Effect.andThen(Effect.never)), + ), + ), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + KIN_ENABLED: enabled, + KIN_MEMORY_REPOSITORIES: JSON.stringify([ + { orgId: KinOrgId.make("fixture"), repoUrl: "fixture" }, + ]), + }), + ), + ), ); const http = yield* HttpRouter.toHttpEffect(routes); + assert.strictEqual(runnerStarts, enabled ? 1 : 0); const services = yield* Deferred.await(ready); const kin = Context.get(services, Kin.KinServer); const registry = Context.get(services, McpSessions.McpSessionRegistry); diff --git a/apps/server/src/kin/mcp/layer.ts b/apps/server/src/kin/mcp/layer.ts index 3100b36981cf9..cef6c3435c854 100644 --- a/apps/server/src/kin/mcp/layer.ts +++ b/apps/server/src/kin/mcp/layer.ts @@ -11,8 +11,6 @@ import { McpInvocationContext } from "../../mcp/McpInvocationContext.ts"; import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; import { KinServer } from "../KinServer.ts"; import { KinIdentity } from "../chat/KinIdentity.ts"; -import * as KinLead from "../lead/layer.ts"; -import * as SqlitePersistence from "../../persistence/Sqlite.ts"; import { KinChatToolkit, handlers } from "./KinChatToolkit.ts"; const registration = McpHttpServer.toolkitRegistration(KinChatToolkit, handlers); @@ -69,13 +67,7 @@ export const transport = (base: typeof McpHttpServer.layer) => const kin = yield* KinServer; // The isolated registry also doubles HtmlRender's per-registry headless Chrome cap. return Option.isSome(kin.services) - ? Layer.merge( - endpoint(Layer.merge(McpHttpServer.layerTools, registration)), - KinLead.layerServer.pipe( - Layer.provide(Layer.succeedContext(kin.services.value)), - Layer.provide(SqlitePersistence.layerConfig), - ), - ) + ? endpoint(Layer.merge(McpHttpServer.layerTools, registration)) : Layer.empty; }), ),