diff --git a/apps/server/src/kin/chat/KinChatRepository.test.ts b/apps/server/src/kin/chat/KinChatRepository.test.ts index 3786d5274f331..27e1c6b5652c2 100644 --- a/apps/server/src/kin/chat/KinChatRepository.test.ts +++ b/apps/server/src/kin/chat/KinChatRepository.test.ts @@ -1,3 +1,10 @@ +import * as EventSink from "../../orchestration-v2/EventSink.ts"; +import * as DateTime from "effect/DateTime"; +import { EventId } from "@t3tools/contracts"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import { CommandId, MessageId } from "@t3tools/contracts"; +import { startWakeupRun, authorityLayer } from "./WakeupAuthority.testkit.ts"; +import type { McpThreadCaller } from "../../mcp/McpInvocationContext.ts"; import { assert, describe, it } from "@effect/vitest"; import { KinOrgId, @@ -14,14 +21,13 @@ import { KinWidget, ThreadId, TurnItemId, + ProviderInstanceId, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; -import * as Sqlite from "../../persistence/Sqlite.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Repository from "./KinChatRepository.ts"; const decodeMessage = Schema.decodeSync(KinMessage); @@ -75,8 +81,14 @@ const agentMember: KinChatMember = { principal: { _tag: "agent", agentId }, role: "member", }; +const caller: McpThreadCaller = { + threadId, + providerSessionId: "provider-session", + providerInstanceId: ProviderInstanceId.make("codex"), +}; const message = (id: string, overrides: Partial> = {}) => ({ + caller, orgId, chatId, messageId: decodeMessageId(id), @@ -95,7 +107,10 @@ const message = (id: string, overrides: Partial> = {}) = editedAt: null, ...overrides, clientMessageId: { kind: "send", id }, - }) satisfies Omit & { clientMessageId: { kind: "send" | "act"; id: string } }; + }) satisfies Omit & { + caller: McpThreadCaller; + clientMessageId: { kind: "send" | "act"; id: string }; + }; const widget: KinWidget = { orgId, chatId, @@ -112,7 +127,7 @@ const widget: KinWidget = { closedAt: null, updatedAt: timestamp, }; -const database = Layer.provideMerge(KinPersistence.layer, Sqlite.layerMemory); +const database = authorityLayer; const layer = Layer.provideMerge(Repository.layer, database); const seed = Effect.gen(function* () { const repo = yield* Repository.KinChatRepository; @@ -217,6 +232,20 @@ describe("KinChatRepository", () => { }), { orgId, agentId, threadId, startedAt: timestamp, endedAt: null }, ); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_webhook_sources (source_id, org_id, repo, scheduled_task_id, inbox_thread_id, bot_login, status) VALUES ('source', ${orgId}, 'owner/repo', 'task', 'webhook-inbox', 'bot', 'active')`; + const inboxError = yield* repo + .registerAgentThread({ + orgId, + agentId, + threadId: decodeThreadId("webhook-inbox"), + startedAt: timestamp, + endedAt: null, + }) + .pipe(Effect.flip); + assert.strictEqual(inboxError._tag, "KinChatError"); + if (inboxError._tag === "KinChatError") + assert.strictEqual(inboxError.reason, "not_allowed"); const secondPerson = decodePersonId("second-person"); yield* repo.insertPerson({ orgId, @@ -472,7 +501,6 @@ describe("KinChatRepository", () => { assert.strictEqual(endedRetry._tag, "KinChatError"); if (endedRetry._tag === "KinChatError") assert.strictEqual(endedRetry.reason, "not_allowed"); - const sql = yield* SqlClient.SqlClient; assert.deepStrictEqual( yield* sql`SELECT typeof(seq) AS type FROM kin_messages WHERE org_id = ${orgId}`, [{ type: "integer" }, { type: "integer" }], @@ -977,9 +1005,228 @@ describe("KinChatRepository", () => { }).pipe(Effect.provide(layer)), ); + it.effect("rejects a lower-ordinal client forgery before the runner sends", () => + Effect.gen(function* () { + const repo = yield* seed; + const sql = yield* SqlClient.SqlClient; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const bootstrap = yield* startWakeupRun(threadId, "bootstrap"); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-bootstrap"), + threadId, + runId: bootstrap.id, + }); + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('pre-forged', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("pre-forged"), + threadId, + messageId: MessageId.make("kin-wakeup-msg:pre-forged"), + text: "Forged", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + const scope = { orgId, agentId, threadId, caller }; + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + const forged = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* sql`UPDATE kin_wakeups SET run_id = ${forged.id} WHERE wakeup_id = 'pre-forged'`; + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET run_id = NULL WHERE wakeup_id = 'pre-forged'`; + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-forged"), + threadId, + runId: forged.id, + }); + const real = yield* startWakeupRun(threadId, "pre-forged"); + assert.isBelow(forged.ordinal, real.ordinal); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET run_id = ${real.id} WHERE wakeup_id = 'pre-forged'`; + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + }).pipe(Effect.provide(layer)), + ); + + it.effect.each([ + ["restartContinuationOfRunId", "agent"], + ["usageLimitContinuationOfRunId", "agent"], + ["restartContinuationOfRunId", "user"], + ["usageLimitContinuationOfRunId", "user"], + ] as const)( + "authorizes only the wake-up active run across reused ids and %s created by %s", + ([continuationField, continuedBy]) => + Effect.gen(function* () { + const repo = yield* seed; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('active-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const wakeup = yield* startWakeupRun(threadId, "active-wakeup"); + const scope = { orgId, agentId, threadId, caller }; + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + const dispatch = (id: string, messageId = id) => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(id), + threadId, + messageId: MessageId.make(messageId), + text: "Queued user message", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + yield* dispatch("queued-user"); + assert.isTrue(yield* repo.hasRunningWakeup(scope)); + yield* repo.insertMessage({ + ...message("during-queue", { + author: { + _tag: "agent", + agentId, + ownerPersonId: personId, + threadId, + memberId: agentMemberId, + }, + authorRole: "member", + intent: "answer", + }), + caller, + }); + yield* dispatch("forged-wakeup", "kin-wakeup-msg:active-wakeup"); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-wakeup"), + threadId, + runId: wakeup.id, + holdQueue: true, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* orchestrator.dispatch({ + type: "queued-run.start", + commandId: CommandId.make("release-user"), + threadId, + runId: queued.id, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET run_id = ${wakeup.id} WHERE wakeup_id = 'active-wakeup'`; + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET run_id = NULL WHERE wakeup_id = 'active-wakeup'`; + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-user"), + threadId, + runId: queued.id, + holdQueue: true, + }); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-thread"), + threadId, + }); + yield* dispatch("held-before-wakeup"); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* dispatch("held-queued-ahead"); + const next = yield* startWakeupRun(threadId, "active-wakeup-next"); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-existing-queue"), + threadId, + }); + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + yield* sql`UPDATE kin_wakeups SET wakeup_id = 'active-wakeup-next' WHERE org_id = ${orgId} AND wakeup_id = 'active-wakeup'`; + assert.isFalse(yield* repo.hasRunningWakeup(scope)); + const held = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (r) => r.userMessageId === "held-queued-ahead", + )!; + assert.strictEqual(held.status, "queued"); + yield* orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("cancel-wakeup"), + threadId, + runId: next.id, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart-wakeup"), + threadId, + messageId: MessageId.make("restart-message"), + text: "Continue", + attachments: [], + dispatchMode: { type: "start_immediately" }, + restartContinuationOfRunId: next.id, + createdBy: continuedBy, + creationSource: "server", + }); + let continuation = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + if (continuationField === "usageLimitContinuationOfRunId") { + continuation = { + ...continuation, + restartContinuationOfRunId: undefined, + usageLimitContinuationOfRunId: next.id, + }; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("usage-continuation"), + type: "run.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: continuation, + }, + ], + }); + } + assert.strictEqual(continuation[continuationField], next.id); + if (continuation.status === "queued") + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("start-continuation"), + threadId, + }); + assert.strictEqual(yield* repo.hasRunningWakeup(scope), continuedBy === "agent"); + assert.notEqual(next.id, wakeup.id); + }).pipe(Effect.provide(layer)), + ); + it.effect("advances read state monotonically and counts only others' posted messages", () => Effect.gen(function* () { const repo = yield* seed; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Lead', 0, '{}', '[]', ${chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('chat-wakeup', ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + yield* startWakeupRun(threadId, "chat-wakeup"); + for (const rejectedCaller of [ + { ...caller, threadId: ThreadId.make("other-thread") }, + { ...caller, subagent: true }, + ]) { + assert.include( + yield* repo + .insertMessage({ + ...message("unauthorized-session", { + author: { + _tag: "agent", + agentId, + memberId: agentMemberId, + ownerPersonId: personId, + threadId, + }, + }), + caller: rejectedCaller, + }) + .pipe(Effect.flip), + { reason: "not_allowed" }, + ); + } const agentAuthor = { _tag: "agent", agentId, @@ -996,6 +1243,19 @@ describe("KinChatRepository", () => { ), ]) yield* repo.insertMessage(input); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'chat-wakeup'`; + const rejected = yield* repo + .insertMessage( + message("outside-wakeup", { + author: agentAuthor, + authorRole: "member", + intent: "answer", + }), + ) + .pipe(Effect.flip); + assert.include(rejected, { reason: "not_allowed" }); + assert.strictEqual(Option.getOrThrow(yield* repo.getChat(scope)).lastSeq, 6); + yield* sql`UPDATE kin_wakeups SET status = 'running' WHERE wakeup_id = 'chat-wakeup'`; const resumedThread = decodeThreadId("resumed-thread"); yield* repo.registerAgentThread({ orgId, diff --git a/apps/server/src/kin/chat/KinChatRepository.ts b/apps/server/src/kin/chat/KinChatRepository.ts index d4daf2a6b6d25..c9d8ca62f0a79 100644 --- a/apps/server/src/kin/chat/KinChatRepository.ts +++ b/apps/server/src/kin/chat/KinChatRepository.ts @@ -1,3 +1,5 @@ +import { ACTIVE_RUN_STATUSES } from "../../orchestration-v2/ThreadManagementService.ts"; +import type { McpThreadCaller } from "../../mcp/McpInvocationContext.ts"; import { KinOrg, KinOrgId, @@ -58,6 +60,7 @@ type Scope = { readonly orgId: KinOrgId }; type ChatScope = Scope & { readonly chatId: KinChatId }; type MemberScope = ChatScope & { readonly memberId: KinMemberId }; type InsertMessage = Omit & { + readonly caller?: McpThreadCaller | undefined; readonly clientMessageId: { readonly kind: "send" | "act"; readonly id: typeof TrimmedNonEmptyString.Type; @@ -181,6 +184,13 @@ export class KinChatRepository extends Context.Service< readonly ownerDisplayName: string; readonly createdAt: IsoDateTime; }) => Effect.Effect; + readonly hasRunningWakeup: ( + input: Scope & { + readonly agentId: KinAgentId; + readonly threadId: ThreadId; + readonly caller: McpThreadCaller; + }, + ) => Effect.Effect; readonly findAgentThread: ( threadId: ThreadId, ) => Effect.Effect, RepositoryError>; @@ -407,6 +417,52 @@ const make = Effect.gen(function* () { }, Effect.mapError(toRepositoryError("KinChatRepository.getAgentThread")), ); + const hasRunningWakeup = Effect.fn("KinChatRepository.hasRunningWakeup")( + function* ( + input: Scope & { + readonly agentId: KinAgentId; + readonly threadId: ThreadId; + readonly caller: McpThreadCaller; + }, + ) { + if (input.caller.subagent === true || input.caller.threadId !== input.threadId) return false; + const rows = + yield* sql`WITH RECURSIVE ancestors(run_id, thread_id, ordinal, payload_json) AS ( + SELECT run_id, thread_id, ordinal, payload_json + FROM orchestration_v2_projection_runs INDEXED BY orchestration_v2_projection_runs_thread_status_idx + WHERE thread_id = ${input.threadId} AND status IN ${sql.in(ACTIVE_RUN_STATUSES)} + UNION + SELECT p.run_id, p.thread_id, p.ordinal, p.payload_json + FROM ancestors child JOIN orchestration_v2_projection_runs p + ON p.thread_id = child.thread_id + AND (p.run_id = json_extract(child.payload_json, '$.restartContinuationOfRunId') + OR p.run_id = json_extract(child.payload_json, '$.usageLimitContinuationOfRunId')) + -- Continuation links plus agent/server provenance come only from the restart and usage-limit workers. + JOIN orchestration_v2_projection_messages cm + ON cm.message_id = json_extract(child.payload_json, '$.userMessageId') + AND cm.thread_id = child.thread_id AND cm.run_id = child.run_id + AND json_extract(cm.payload_json, '$.createdBy') = 'agent' + AND json_extract(cm.payload_json, '$.creationSource') = 'server' + ) + SELECT 1 FROM kin_wakeups w JOIN ancestors root ON root.thread_id = w.thread_id + JOIN orchestration_v2_projection_messages m + ON m.message_id = json_extract(root.payload_json, '$.userMessageId') + AND m.thread_id = root.thread_id AND m.run_id = root.run_id + WHERE w.org_id = ${input.orgId} AND w.agent_id = ${input.agentId} + AND w.thread_id = ${input.threadId} AND w.status = 'running' + AND json_extract(root.payload_json, '$.userMessageId') = 'kin-wakeup-msg:' || w.wakeup_id + AND json_extract(m.payload_json, '$.createdBy') = 'system' + AND json_extract(m.payload_json, '$.creationSource') = 'server' + AND root.run_id = COALESCE(w.run_id, ( + SELECT first.run_id FROM orchestration_v2_projection_runs first + WHERE first.thread_id = w.thread_id + AND json_extract(first.payload_json, '$.userMessageId') = 'kin-wakeup-msg:' || w.wakeup_id + ORDER BY first.ordinal LIMIT 1 + )) LIMIT 1`; + return rows.length > 0; + }, + Effect.mapError(toRepositoryError("KinChatRepository.hasRunningWakeup")), + ); const validateWidgetReferences = Effect.fn("KinChatRepository.validateWidgetReferences")( function* (input: Pick) { for (const personId of input.awaiting) { @@ -528,6 +584,7 @@ const make = Effect.gen(function* () { Effect.mapError(toRepositoryError("KinChatRepository.addWaiting")), ); return KinChatRepository.of({ + hasRunningWakeup, getOrg: orgOne, getPerson: personOne, getAgent: agentOne, @@ -900,6 +957,13 @@ const make = Effect.gen(function* () { reason: "not_allowed", message: "Retired Kin agents cannot register threads.", }); + const inbox = + yield* sql`SELECT 1 FROM kin_webhook_sources WHERE inbox_thread_id = ${input.threadId}`; + if (inbox.length > 0) + return yield* new KinChatError({ + reason: "not_allowed", + message: "Webhook inbox threads cannot be Kin agents.", + }); const existing = yield* SqlSchema.findOneOption({ Request: Schema.Unknown, Result: AgentThreadRow, @@ -1061,6 +1125,20 @@ const make = Effect.gen(function* () { message: "Kin record not found.", }); } + if ( + author._tag === "agent" && + (input.caller === undefined || + !(yield* hasRunningWakeup({ + ...input, + agentId: author.agentId, + threadId: author.threadId, + caller: input.caller, + }))) + ) + return yield* new KinChatError({ + reason: "not_allowed", + message: "Kin wake-up is not running.", + }); if (input.replyTo !== null) { const reply = yield* requireRecord( messageOne({ orgId: input.orgId, messageId: input.replyTo }), diff --git a/apps/server/src/kin/chat/KinChatService.test.ts b/apps/server/src/kin/chat/KinChatService.test.ts index 2043f89f9de9b..d04c9e8a168d5 100644 --- a/apps/server/src/kin/chat/KinChatService.test.ts +++ b/apps/server/src/kin/chat/KinChatService.test.ts @@ -1,3 +1,6 @@ +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import { CommandId } from "@t3tools/contracts"; +import { startWakeupRun, authorityLayer } from "./WakeupAuthority.testkit.ts"; import { assert, describe, it } from "@effect/vitest"; import { AuthSessionId, @@ -25,8 +28,6 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; import type { McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; -import * as Sqlite from "../../persistence/Sqlite.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Hub from "./KinChatHub.ts"; import * as Repository from "./KinChatRepository.ts"; import * as Identity from "./KinIdentity.ts"; @@ -39,7 +40,7 @@ const decodeKinChatId = Schema.decodeSync(KinChatId); const decodeKinMemberId = Schema.decodeSync(KinMemberId); const decodeThreadId = Schema.decodeSync(ThreadId); const decodeSend = Schema.decodeUnknownEffect(KinMessagesSendInput); -const database = Layer.provideMerge(KinPersistence.layer, Sqlite.layerMemory); +const database = authorityLayer; const repositories = Layer.provideMerge(Repository.layer, database); const dependencies = Layer.provideMerge(Hub.layer, repositories); const layer = Layer.provideMerge(Service.layer, dependencies); @@ -100,8 +101,15 @@ const seed = Effect.gen(function* () { createdAt: timestamp, }); yield* identity.registerAgentThread({ orgId: person.orgId, agentId, threadId }); - const agent = yield* identity.resolveAgent(caller); const chat = yield* service.createDirect(person, { orgId: person.orgId, agentId }); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('service-wakeup', ${person.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + const run = yield* startWakeupRun(threadId, "service-wakeup"); + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE wakeup_id = 'service-wakeup'`; + const agent = yield* identity.resolveAgent(caller); return { repo, service, identity, person, agent, chat }; }); @@ -206,6 +214,44 @@ describe("KinChatService", () => { .messages, [posted], ); + const widget = yield* service.upsertWidget(agent, { + chatId: chat.chatId, + type: "task", + state, + }); + const sql = yield* SqlClient.SqlClient; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const run = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("end-service-wakeup"), + threadId, + runId: run.id, + }); + assert.include( + yield* service + .upsertWidget(agent, { + chatId: chat.chatId, + widgetId: widget.widgetId, + type: "task", + state, + }) + .pipe(Effect.flip), + { reason: "not_allowed" }, + ); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; + assert.include( + yield* service + .upsertWidget(agent, { + chatId: chat.chatId, + widgetId: widget.widgetId, + revision: widget.revision, + type: "task", + state, + }) + .pipe(Effect.flip), + { reason: "not_allowed" }, + ); }).pipe(Effect.provide(layer)), ); @@ -431,6 +477,12 @@ describe("KinChatService", () => { agentId: otherId, threadId: otherThread, }); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'service-wakeup'`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${otherId}, ${person.orgId}, 'Other', 0, '{}', '[]', ${chat.chatId})`; + const otherRun = yield* startWakeupRun(otherThread, "other-wakeup"); + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, run_id, model, started_at, status, event_ids) + VALUES ('other-wakeup', ${person.orgId}, ${otherId}, ${otherThread}, ${otherRun.id}, 'test', 0, 'running', '[]')`; const other = yield* identity.resolveAgent({ ...caller, thread: { ...caller.thread!, threadId: otherThread }, @@ -448,6 +500,8 @@ describe("KinChatService", () => { .pipe(Effect.flip), { reason: "not_allowed" }, ); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'other-wakeup'`; + yield* sql`UPDATE kin_wakeups SET status = 'running' WHERE wakeup_id = 'service-wakeup'`; for (const update of [{}, { revision: 1, replyTo: question.messageId }]) assert.include( yield* service diff --git a/apps/server/src/kin/chat/KinChatService.ts b/apps/server/src/kin/chat/KinChatService.ts index 80ce46fdd23d9..fc8fa38e6e165 100644 --- a/apps/server/src/kin/chat/KinChatService.ts +++ b/apps/server/src/kin/chat/KinChatService.ts @@ -229,7 +229,8 @@ const make = Effect.gen(function* () { agent.retiredAt !== null || thread.endedAt !== null || thread.agentId !== agent.agentId || - agent.ownerPersonId !== identity.ownerPersonId + agent.ownerPersonId !== identity.ownerPersonId || + !(yield* repo.hasRunningWakeup(identity)) ) return yield* notAllowed; } @@ -309,6 +310,7 @@ const make = Effect.gen(function* () { const messageId = yield* decodeMessageId(SecureRandom.uuidV4Unsafe()).pipe(Effect.orDie); const message = yield* repo.insertMessage({ orgId: identity.orgId, + caller: identity._tag === "agent" ? identity.caller : undefined, chatId: input.chatId, messageId, author: authorFor(identity, member), diff --git a/apps/server/src/kin/chat/KinIdentity.test.ts b/apps/server/src/kin/chat/KinIdentity.test.ts index 072b3e916be38..1ec0016613c4e 100644 --- a/apps/server/src/kin/chat/KinIdentity.test.ts +++ b/apps/server/src/kin/chat/KinIdentity.test.ts @@ -23,13 +23,12 @@ import * as Scope from "effect/Scope"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; -import type { McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; -import * as Sqlite from "../../persistence/Sqlite.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; +import type { McpThreadInvocationScope } from "../../mcp/McpInvocationContext.ts"; import * as Repository from "./KinChatRepository.ts"; import * as Identity from "./KinIdentity.ts"; +import { startWakeupRun, authorityLayer } from "./WakeupAuthority.testkit.ts"; -const database = Layer.provideMerge(KinPersistence.layer, Sqlite.layerMemory); +const database = authorityLayer; const repositoryLayer = Layer.provideMerge(Repository.layer, database); const config = { orgName: "Example", ownerHandle: "owner", ownerDisplayName: "Example Owner" }; const identityLayer = Identity.KinIdentity.layer.pipe( @@ -54,7 +53,7 @@ const session: AuthenticatedSession = { }; const environmentId = Schema.decodeSync(EnvironmentId)("test-environment"); const providerInstanceId = Schema.decodeSync(ProviderInstanceId)("provider"); -const caller = (thread = threadId, subagent = false): McpInvocationScope => ({ +const caller = (thread = threadId, subagent = false): McpThreadInvocationScope => ({ environmentId, capabilities: new Set(), issuedAt: 0, @@ -308,8 +307,21 @@ describe("KinIdentity", () => { yield* identity.registerAgentThread(input); assert.deepStrictEqual(yield* repository.getAgentThread(input), original); assert.isTrue(identity.isAgentThread(threadId)); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) VALUES ('identity-chat', ${owner.orgId}, 'team_group', 0, '1970-01-01T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${owner.orgId}, 'Lead', 0, '{}', '[]', 'identity-chat')`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids) + VALUES ('identity-wakeup', ${owner.orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]')`; + assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller()))); + yield* startWakeupRun(threadId, "identity-wakeup"); + assert.isTrue(yield* Effect.isSuccess(identity.resolveAgent(caller()))); + assert.isFalse( + yield* Effect.isSuccess(identity.resolveAgent(caller(decodeThreadId("other-thread")))), + ); assert.deepStrictEqual(yield* identity.resolveAgent(caller()), { _tag: "agent", + caller: caller().thread, ...input, ownerPersonId: owner.personId, }); @@ -338,6 +350,7 @@ describe("KinIdentity", () => { ); const secondThreadId = decodeThreadId("second-thread"); yield* identity.registerAgentThread({ ...input, threadId: secondThreadId }); + assert.isFalse(yield* Effect.isSuccess(identity.resolveAgent(caller(secondThreadId)))); const otherOrgId = decodeOrgId("other-org"); const otherPersonId = decodePersonId("other-owner"); const otherAgentId = decodeAgentId("other-agent"); @@ -383,9 +396,15 @@ describe("KinIdentity", () => { assert.isTrue(restarted.isAgentThread(threadId)); assert.deepStrictEqual(yield* restarted.resolveAgent(caller()), { _tag: "agent", + caller: caller().thread, ...input, ownerPersonId: owner.personId, }); + yield* sql`UPDATE kin_wakeups SET status = 'completed', ended_at = 1000 WHERE wakeup_id = 'identity-wakeup'`; + assert.include(yield* restarted.resolveAgent(caller()).pipe(Effect.flip), { + reason: "not_allowed", + }); + assert.isTrue(restarted.isAgentThread(threadId)); assert.deepStrictEqual(yield* restarted.resolve(session), [owner]); assert.isTrue(yield* restarted.endAgentThread(input)); assert.isFalse(restarted.isAgentThread(threadId)); @@ -406,8 +425,11 @@ describe("KinIdentity", () => { const afterEnd = yield* start; assert.isFalse(afterEnd.isAgentThread(threadId)); assert.isTrue(afterEnd.isAgentThread(secondThreadId)); + yield* startWakeupRun(secondThreadId, "identity-wakeup"); + yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL, thread_id = ${secondThreadId} WHERE wakeup_id = 'identity-wakeup'`; assert.deepStrictEqual(yield* afterEnd.resolveAgent(caller(secondThreadId)), { _tag: "agent", + caller: caller(secondThreadId).thread, ...input, threadId: secondThreadId, ownerPersonId: owner.personId, diff --git a/apps/server/src/kin/chat/KinIdentity.ts b/apps/server/src/kin/chat/KinIdentity.ts index 4bd7f2aa94663..0ebd03cccca45 100644 --- a/apps/server/src/kin/chat/KinIdentity.ts +++ b/apps/server/src/kin/chat/KinIdentity.ts @@ -16,7 +16,7 @@ import * as Option from "effect/Option"; import * as Semaphore from "effect/Semaphore"; import * as SqlClient from "effect/sql/SqlClient"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; -import type { McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; +import type { McpThreadCaller, McpInvocationScope } from "../../mcp/McpInvocationContext.ts"; import * as KinChatRepository from "./KinChatRepository.ts"; export class KinIdentityConfig extends Context.Service< @@ -48,6 +48,7 @@ export type PersonIdentity = { readonly personId: KinPersonId; }; export type AgentIdentity = { + readonly caller: McpThreadCaller; readonly _tag: "agent"; readonly orgId: KinOrgId; readonly agentId: KinAgentId; @@ -114,6 +115,11 @@ export class KinIdentity extends Context.Service< message: "Kin agent thread is not active.", }); } + if (!(yield* repository.hasRunningWakeup({ ...row.value, caller }))) + return yield* new KinChatError({ + reason: "not_allowed", + message: "Kin wake-up is not running.", + }); const agent = yield* repository.getAgent(row.value); if (Option.isNone(agent) || agent.value.retiredAt !== null) return yield* new KinChatError({ @@ -122,6 +128,7 @@ export class KinIdentity extends Context.Service< }); return { _tag: "agent", + caller, orgId: row.value.orgId, agentId: agent.value.agentId, ownerPersonId: agent.value.ownerPersonId, diff --git a/apps/server/src/kin/chat/WakeupAuthority.testkit.ts b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts new file mode 100644 index 0000000000000..f700879c5b0c7 --- /dev/null +++ b/apps/server/src/kin/chat/WakeupAuthority.testkit.ts @@ -0,0 +1,74 @@ +import { + CommandId, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "../../orchestration-v2/testkit/ProviderReplayHarness.ts"; +import { CodexProviderCapabilitiesV2 } from "../../orchestration-v2/Adapters/CodexAdapterV2.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as Sqlite from "../../persistence/Sqlite.ts"; + +const database = KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); +const instanceId = ProviderInstanceId.make("codex"); +export const authorityLayerWithDatabase = ( + databaseLayer: NonNullable< + NonNullable[2]>["databaseLayer"] + >, +) => + ProviderReplayHarness.layerWithRegistry( + { name: "wakeup-authority" }, + ProviderAdapterRegistry.layerFromAdapters([ + { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: () => Effect.die("Authority tests do not start provider processes"), + }, + ]), + { databaseLayer, runEffectWorker: false }, + ).pipe(Layer.provideMerge(databaseLayer)); + +export const authorityLayer = authorityLayerWithDatabase(database); + +export const startWakeupRun = Effect.fn("test.startWakeupRun")(function* ( + threadId: ThreadId, + wakeupId: string, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projection = yield* orchestrator.getThreadProjection(threadId).pipe(Effect.option); + if (projection._tag === "None") + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create:${threadId}`), + threadId, + projectId: ProjectId.make("authority-project"), + title: "Lead", + modelSelection: { instanceId, model: "gpt-5.1-codex" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "system", + creationSource: "server", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`dispatch:${threadId}:${wakeupId}`), + threadId, + messageId: MessageId.make(`kin-wakeup-msg:${wakeupId}`), + text: "Wake up", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "system", + creationSource: "server", + }); + return (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; +}); diff --git a/apps/server/src/kin/chat/rpcHandlers.test.ts b/apps/server/src/kin/chat/rpcHandlers.test.ts index ed3944a4e798b..b96e43938ca82 100644 --- a/apps/server/src/kin/chat/rpcHandlers.test.ts +++ b/apps/server/src/kin/chat/rpcHandlers.test.ts @@ -1,3 +1,4 @@ +import { dormantLeadDependencies } from "../lead/layer.testkit.ts"; import { assert, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Hub from "./KinChatHub.ts"; @@ -59,7 +60,11 @@ const session = { scopes: AuthStandardClientScopes, }; const configured = ConfigProvider.layer(ConfigProvider.fromUnknown({ KIN_ENABLED: true })); -const root = Kin.layer.pipe(Layer.provideMerge(Chat.layer), Layer.provide(configured)); +const root = Kin.layer.pipe( + Layer.provide(dormantLeadDependencies), + Layer.provideMerge(Chat.layer), + Layer.provide(configured), +); const probe = WsRpcGroup.omit( ...[...WsRpcGroup.requests.keys()].filter( ( diff --git a/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts b/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts index 3f806ad84f3cd..ef3f262fab51b 100644 --- a/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts +++ b/apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts @@ -64,6 +64,16 @@ it.each([ it.effect("preserves the upstream database columns Kin reads", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + for (const [table, columns] of [ + ["orchestration_v2_projection_runs", ["run_id", "thread_id", "status", "payload_json"]], + ["orchestration_v2_projection_provider_turns", ["thread_id", "completed_at"]], + ] as const) { + const rows = yield* sql`PRAGMA table_info(${sql.literal(table)})`; + assert.includeMembers( + rows.map((row) => row.name), + [...columns], + ); + } const tasks = yield* sql`PRAGMA table_info(scheduled_tasks)`; const deliveries = yield* sql`PRAGMA table_info(scheduled_task_webhook_deliveries)`; assert.includeMembers( diff --git a/apps/server/src/kin/layer.test.ts b/apps/server/src/kin/layer.test.ts index 87db879d28828..bcc9d489269a6 100644 --- a/apps/server/src/kin/layer.test.ts +++ b/apps/server/src/kin/layer.test.ts @@ -1,3 +1,4 @@ +import { dormantLeadDependencies } from "./lead/layer.testkit.ts"; import * as ThreadLaunch from "../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as KinTriggerSink from "./triggers/KinTriggerSink.ts"; @@ -66,6 +67,7 @@ it.effect.each(["identity", "defect", "unconfigured"] as const)( const kin = yield* Kin.KinServer.pipe( Effect.provide( Kin.layer.pipe( + Layer.provide(dormantLeadDependencies), Layer.provide( ConfigProvider.layer( ConfigProvider.fromUnknown({ KIN_ENABLED: mode !== "unconfigured" }), @@ -155,7 +157,7 @@ it.effect.each(["disabled", "invalid", "failed"] as const)( }, }); const runtime = Layer.mergeAll( - Kin.layer, + Kin.layer.pipe(Layer.provide(dormantLeadDependencies)), KinTriggerSink.wrapTaskDispatch( Layer.mock(ThreadLaunch.ThreadLaunchService)({ launch }), Layer.mock(ThreadManagement.ThreadManagementService)({ sendToThread }), diff --git a/apps/server/src/kin/layer.ts b/apps/server/src/kin/layer.ts index ba72e1cf3a641..0a82e33ca87f2 100644 --- a/apps/server/src/kin/layer.ts +++ b/apps/server/src/kin/layer.ts @@ -12,9 +12,8 @@ export { KinServer } from "./KinServer.ts"; import type * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as KinMcp from "./mcp/layer.ts"; +import * as KinLead from "./lead/layer.ts"; -// LEAD-5 supplies its runner here, using this root's identity rather than building another. -const lead: Layer.Layer = Layer.empty; const enabled = Layer.effect( KinServer, Effect.gen(function* () { @@ -25,7 +24,7 @@ const enabled = Layer.effect( subscribeForViewer: Option.some(hub.subscribeForViewer), }; }), -).pipe(Layer.provide(lead.pipe(Layer.provideMerge(KinChat.layer)))); +).pipe(Layer.provide(KinChat.layer)); const disabled = Layer.succeed(KinServer, { services: Option.none(), subscribeForViewer: Option.none(), @@ -68,7 +67,20 @@ const middleware = HttpRouter.middleware<{ provides: KinServer }>()( export const layer = middleware.pipe(Layer.provideMerge(runtime)); -export const mcpLayer = (base: Parameters[0]) => KinMcp.transport(base); +const leadRuntime = Layer.unwrap( + Effect.gen(function* () { + const kin = yield* KinServer; + return Option.isSome(kin.services) + ? KinLead.layerServer.pipe( + Layer.provide(Layer.succeedContext(kin.services.value)), + Layer.provide(SqlitePersistence.layerConfig), + ) + : Layer.empty; + }), +); + +export const mcpLayer = (base: Parameters[0]) => + Layer.merge(KinMcp.transport(base), leadRuntime); export const mcpRegistryLayer = ( base: Layer.Layer, ) => diff --git a/apps/server/src/kin/lead/KinEventQueue.test.ts b/apps/server/src/kin/lead/KinEventQueue.test.ts index 8e8465668fb18..e20021ce4a87d 100644 --- a/apps/server/src/kin/lead/KinEventQueue.test.ts +++ b/apps/server/src/kin/lead/KinEventQueue.test.ts @@ -108,6 +108,12 @@ describe("KinEventQueue", () => { yield* seed(); const queue = yield* KinEventQueue.KinEventQueue; const sql = yield* SqlClient.SqlClient; + for (const agentId of ["unknown", "foreign-lead"]) { + const error = yield* queue + .enqueue({ ...(yield* event("missing")), agentId: KinAgentId.make(agentId) }) + .pipe(Effect.flip); + assert.strictEqual(error._tag, "KinEventQueueUnknownAgent"); + } yield* queue.enqueue({ ...(yield* event("first")), receivedAt: DateTime.makeUnsafe(-600_000), @@ -123,6 +129,11 @@ describe("KinEventQueue", () => { ["first", "second"], ); yield* finish("first-batch"); + yield* queue.enqueue(yield* event("colliding")); + yield* TestClock.adjust("20 seconds"); + const collision = yield* Effect.scoped(queue.claim(input("first-batch"))).pipe(Effect.flip); + assert.strictEqual(collision._tag, "KinEventQueueWakeupConflict"); + yield* sql`UPDATE kin_events SET state = 'expired' WHERE event_id = 'colliding'`; yield* queue.enqueue(yield* event("later")); const second = yield* Effect.scoped(queue.claim(input("second-batch"))).pipe( Effect.forkChild, @@ -210,7 +221,13 @@ describe("KinEventQueue", () => { WHERE member_id = 'agent-member-lead-a'`; yield* sql`UPDATE kin_agents SET retired_at = ${id === "retired" ? "2026-11-07T00:00:00.000Z" : null} WHERE agent_id = 'lead-a'`; - yield* queue.enqueue({ ...owner, eventId: KinEventId.make(id), dedupeKey: id }); + const enqueue = queue.enqueue({ ...owner, eventId: KinEventId.make(id), dedupeKey: id }); + if (id === "retired") + assert.strictEqual( + (yield* enqueue.pipe(Effect.flip))._tag, + "KinEventQueueRetiredAgent", + ); + else yield* enqueue; } yield* sql`UPDATE kin_agents SET retired_at = NULL WHERE agent_id = 'lead-a'`; yield* sql`UPDATE kin_chat_members SET left_at = NULL WHERE member_id = 'agent-member-lead-a'`; @@ -240,6 +257,31 @@ describe("KinEventQueue", () => { }).pipe(Effect.provide(layer)), ); + it.effect("selects a ready lead ahead of a deferred lead and notifies retirement", () => + Effect.gen(function* () { + yield* seed(); + yield* seed("a", "lead-a2"); + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const orgId = KinOrgId.make("a"); + yield* queue.enqueue(yield* event("deferred")); + yield* queue.enqueue(yield* event("ready", "routine", "a", "lead-a2")); + yield* sql`UPDATE kin_events SET not_before = 300000 WHERE event_id = 'deferred'`; + yield* TestClock.adjust("20 seconds"); + assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a2"); + const changed = yield* queue.changes(orgId); + const signal = yield* changed.pipe(Effect.forkChild); + yield* sql`UPDATE kin_agents SET retired_at = '2026-11-07T00:00:00.000Z' WHERE agent_id = 'lead-a2'`; + yield* queue.expireAgent(orgId, KinAgentId.make("lead-a2")); + yield* Fiber.join(signal); + yield* queue.expireAgent(orgId, KinAgentId.make("lead-a2")); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'ready'`, [ + { state: "expired" }, + ]); + assert.strictEqual((yield* queue.nextReady(orgId))?.agentId, "lead-a"); + }).pipe(Effect.provide(layer)), + ); + it.effect( "dedupes per organization, normalizes github deliveries, and surfaces other constraints", () => @@ -275,7 +317,10 @@ describe("KinEventQueue", () => { }, ]) { const conflict = yield* queue.enqueue(changed).pipe(Effect.flip); - assert.instanceOf(conflict, KinEventQueue.KinEventQueueConflict); + assert.strictEqual( + conflict._tag, + changed.agentId === "lead-b" ? "KinEventQueueUnknownAgent" : "KinEventQueueConflict", + ); } for (const redelivery of [ { @@ -400,20 +445,20 @@ describe("KinEventQueue", () => { assert.strictEqual(rejected._tag, "PersistenceDecodeError"); } assert.strictEqual( - yield* queue.release({ + (yield* queue.release({ ...input("admission", "b"), countAttempt: false, notBefore: 60_000, - }), + })).released, 0, ); assert.strictEqual( - yield* queue.release({ + (yield* queue.release({ ...input("admission"), eventIds: [KinEventId.make("deferred")], notBefore: 60_000, countAttempt: false, - }), + })).released, 1, ); assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY wakeup_id`, [ @@ -471,8 +516,12 @@ describe("KinEventQueue", () => { mode === "release" ? queue.release({ ...claimInput, countAttempt: true }) : queue.finish({ ...claimInput, status: mode }); - yield* failed; - yield* failed; + const terminal = yield* failed; + assert.deepStrictEqual( + terminal.failedEventIds, + attempt === 3 ? [KinEventId.make(id)] : [], + ); + assert.deepStrictEqual((yield* failed).failedEventIds, []); const now = yield* Clock.currentTimeMillis; assert.deepStrictEqual( yield* sql`SELECT state, attempts, not_before FROM kin_events WHERE event_id = ${id}`, diff --git a/apps/server/src/kin/lead/KinEventQueue.ts b/apps/server/src/kin/lead/KinEventQueue.ts index 259e9c3b5c043..48443ef05ef7f 100644 --- a/apps/server/src/kin/lead/KinEventQueue.ts +++ b/apps/server/src/kin/lead/KinEventQueue.ts @@ -1,3 +1,4 @@ +import { ACTIVE_RUN_STATUSES } from "../../orchestration-v2/ThreadManagementService.ts"; import { KinEvent, KinEventId, @@ -74,10 +75,25 @@ export class KinEventQueueUnknownEvent extends Schema.TaggedError()( + "KinEventQueueUnknownAgent", + { orgId: KinOrgId, agentId: KinAgentId }, +) {} +export class KinEventQueueRetiredAgent extends Schema.TaggedError()( + "KinEventQueueRetiredAgent", + { orgId: KinOrgId, agentId: KinAgentId }, +) {} +const isRetiredAgent = Schema.is(KinEventQueueRetiredAgent); +export class KinEventQueueWakeupConflict extends Schema.TaggedError()( + "KinEventQueueWakeupConflict", + { orgId: KinOrgId, wakeupId: KinWakeupId }, +) {} export class KinEventQueueBusy extends Schema.TaggedError()( "KinEventQueueBusy", { orgId: KinOrgId }, ) {} +const isUnknownAgent = Schema.is(KinEventQueueUnknownAgent); +const isWakeupConflict = Schema.is(KinEventQueueWakeupConflict); const isConflict = Schema.is(KinEventQueueConflict); const isUnknownEvent = Schema.is(KinEventQueueUnknownEvent); const isBusy = Schema.is(KinEventQueueBusy); @@ -101,21 +117,46 @@ export class KinEventQueue extends Context.Service< | PersistenceDecodeError | KinEventQueueConflict | KinEventQueueUnknownEvent + | KinEventQueueUnknownAgent + | KinEventQueueRetiredAgent + >; + readonly changes: (orgId: KinOrgId) => Effect.Effect>; + readonly nextReady: ( + orgId: KinOrgId, + ) => Effect.Effect< + { readonly agentId: KinAgentId; readonly readyAt: number } | undefined, + PersistenceSqlError | PersistenceDecodeError + >; + readonly expireAgent: ( + orgId: KinOrgId, + agentId: KinAgentId, + ) => Effect.Effect< + void, + PersistenceSqlError | PersistenceDecodeError | KinEventQueueUnknownAgent >; - // Scope closure releases only the permit; LEAD-5 must finish(timed_out | failed) in its own onExit. + // Scope closure releases only the permit; the runner owns wake-up finalization. readonly claim: ( input: ClaimInput, ) => Effect.Effect< ReadonlyArray, - PersistenceSqlError | PersistenceDecodeError | KinEventQueueBusy, + | PersistenceSqlError + | PersistenceDecodeError + | KinEventQueueBusy + | KinEventQueueWakeupConflict, Scope.Scope >; readonly release: ( input: typeof ReleaseInput.Type, - ) => Effect.Effect; + ) => Effect.Effect< + { readonly released: number; readonly failedEventIds: ReadonlyArray }, + PersistenceSqlError | PersistenceDecodeError + >; readonly finish: ( input: typeof FinishInput.Type, - ) => Effect.Effect; + ) => Effect.Effect< + { readonly failedEventIds: ReadonlyArray }, + PersistenceSqlError | PersistenceDecodeError + >; // LEAD-5 startup recovery calls this only after confirming the provider run stopped. readonly requeueUnfinished: ( orgId: KinOrgId, @@ -128,13 +169,13 @@ const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const organizations = new Map< KinOrgId, - { gate: Semaphore.Semaphore; changed: Deferred.Deferred; users: number } + { gate: Semaphore.Semaphore; changed: Deferred.Deferred } >(); const organization = (orgId: KinOrgId) => { const existing = organizations.get(orgId); if (existing !== undefined) return existing; - const entry = { gate: Semaphore.makeUnsafe(1), changed: Deferred.makeUnsafe(), users: 0 }; + const entry = { gate: Semaphore.makeUnsafe(1), changed: Deferred.makeUnsafe() }; organizations.set(orgId, entry); return entry; }; @@ -157,24 +198,41 @@ const make = Effect.gen(function* () { ) { return yield* new KinEventQueueConflict({ orgId: input.orgId, eventId: input.eventId }); } - const event = input._tag === "github" ? { ...input, dedupeKey: input.deliveryId } : input; - let priority: "P0" | "P1" | "P2" | "P3" = "P2"; - switch (event._tag) { - case "resume": - priority = "P0"; - break; - case "deferral.due": - case "action.resolved": - case "worker.report": - priority = "P1"; - break; - case "watchdog": - priority = "P3"; - break; - case "chat.message": - case "widget.action": - if (event.provenance._tag === "human") { - const owners = yield* sql` + return yield* sql.withTransaction( + Effect.gen(function* () { + const agents = yield* sql<{ + retired_at: string | null; + }>`SELECT a.retired_at FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${input.orgId} AND l.agent_id = ${input.agentId}`; + if (agents.length === 0) + return yield* new KinEventQueueUnknownAgent({ + orgId: input.orgId, + agentId: input.agentId, + }); + if (agents[0]?.retired_at !== null) + return yield* new KinEventQueueRetiredAgent({ + orgId: input.orgId, + agentId: input.agentId, + }); + const event = input._tag === "github" ? { ...input, dedupeKey: input.deliveryId } : input; + let priority: "P0" | "P1" | "P2" | "P3" = "P2"; + switch (event._tag) { + case "resume": + priority = "P0"; + break; + case "deferral.due": + case "action.resolved": + case "worker.report": + priority = "P1"; + break; + case "watchdog": + priority = "P3"; + break; + case "chat.message": + case "widget.action": + if (event.provenance._tag === "human") { + const owners = yield* sql` SELECT 1 FROM kin_chat_members WHERE org_id = ${event.orgId} AND chat_id = ${event.chatId} AND member_id = ${event.provenance.memberId} AND person_id = ${event.provenance.personId} @@ -186,46 +244,54 @@ const make = Effect.gen(function* () { WHERE org_id = ${event.orgId} AND chat_id = ${event.chatId} AND principal_kind = 'agent' AND agent_id = ${event.agentId} AND left_at IS NULL) `; - if (owners.length > 0) priority = "P0"; + if (owners.length > 0) priority = "P0"; + } + break; } - break; - } - const json = yield* encodeEvent(event); - const now = yield* Clock.currentTimeMillis; - const inserted = yield* sql` + const json = yield* encodeEvent(event); + const now = yield* Clock.currentTimeMillis; + const inserted = yield* sql` INSERT INTO kin_events (event_id, org_id, agent_id, tag, priority, json, dedupe_key, received_at, queued_at) VALUES (${event.eventId}, ${event.orgId}, ${event.agentId}, ${event._tag}, ${priority}, ${json}, ${event.dedupeKey}, ${DateTime.toEpochMillis(event.receivedAt)}, ${now}) ON CONFLICT DO NOTHING RETURNING event_id `; - if (inserted.length > 0) { - yield* notify(event.orgId); - return { event, duplicate: false }; - } - const rows = yield* sql<{ json: string }>`SELECT json FROM kin_events + if (inserted.length > 0) { + yield* sql.afterCommit(notify(event.orgId)); + return { event, duplicate: false }; + } + const rows = yield* sql<{ json: string }>`SELECT json FROM kin_events WHERE org_id = ${event.orgId} AND (event_id = ${event.eventId} OR (dedupe_key IS NOT NULL AND dedupe_key = ${event.dedupeKey})) ORDER BY event_id`; - let stored: KinEvent | undefined; - for (const row of rows) { - const candidate = yield* decodeEvent(row.json); - if ( - candidate.eventId === event.eventId - ? (yield* encodeEvent(candidate)) !== json - : candidate.agentId !== event.agentId || candidate._tag !== event._tag - ) { - return yield* new KinEventQueueConflict({ orgId: event.orgId, eventId: event.eventId }); - } - stored = candidate; - } - if (stored === undefined) { - return yield* new KinEventQueueUnknownEvent({ orgId: event.orgId, eventId: event.eventId }); - } - return { event: stored, duplicate: true }; + let stored: KinEvent | undefined; + for (const row of rows) { + const candidate = yield* decodeEvent(row.json); + if ( + candidate.eventId === event.eventId + ? (yield* encodeEvent(candidate)) !== json + : candidate.agentId !== event.agentId || candidate._tag !== event._tag + ) { + return yield* new KinEventQueueConflict({ + orgId: event.orgId, + eventId: event.eventId, + }); + } + stored = candidate; + } + if (stored === undefined) { + return yield* new KinEventQueueUnknownEvent({ + orgId: event.orgId, + eventId: event.eventId, + }); + } + return { event: stored, duplicate: true }; + }), + ); }, Effect.mapError((cause) => - isConflict(cause) || isUnknownEvent(cause) + isConflict(cause) || isUnknownEvent(cause) || isUnknownAgent(cause) || isRetiredAgent(cause) ? cause : toQueuePersistenceError("KinEventQueue.enqueue")(cause), ), @@ -233,7 +299,7 @@ const make = Effect.gen(function* () { ); const readiness = Effect.fn("KinEventQueue.readiness")(function* ( - input: ClaimInput, + input: Pick, now: number, ) { const [window] = yield* sql<{ @@ -265,17 +331,7 @@ const make = Effect.gen(function* () { const claim = Effect.fn("KinEventQueue.claim")( function* (input: ClaimInput) { yield* decodeClaimInput(input); - const entry = yield* Effect.acquireRelease( - Effect.sync(() => { - const entry = organization(input.orgId); - entry.users++; - return entry; - }), - (entry) => - Effect.sync(() => { - if (--entry.users === 0) organizations.delete(input.orgId); - }), - ); + const entry = organization(input.orgId); return yield* Effect.acquireRelease( Effect.interruptible( Effect.uninterruptibleMask((restore) => @@ -301,6 +357,13 @@ const make = Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; if ((yield* readiness(input, now)).readyAt > now) return yield* new NoClaimableEvents({}); + const collision = + yield* sql`SELECT 1 FROM kin_wakeups WHERE wakeup_id = ${input.wakeupId}`; + if (collision.length > 0) + return yield* new KinEventQueueWakeupConflict({ + orgId: input.orgId, + wakeupId: input.wakeupId, + }); // Cross-process exclusion rests on idx_kin_wakeups_running_org. const started = yield* sql` INSERT INTO kin_wakeups (org_id, wakeup_id, agent_id, model, started_at, status, event_ids) @@ -349,20 +412,22 @@ const make = Effect.gen(function* () { ); }, Effect.mapError((cause) => - isBusy(cause) ? cause : toQueuePersistenceError("KinEventQueue.claim")(cause), + isBusy(cause) || isWakeupConflict(cause) + ? cause + : toQueuePersistenceError("KinEventQueue.claim")(cause), ), ); const release = Effect.fn("KinEventQueue.release")( function* (input: typeof ReleaseInput.Type) { yield* decodeReleaseInput(input); - const count = yield* sql.withTransaction( + const result = yield* sql.withTransaction( Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; const notBefore = input.notBefore ?? now + 300_000; const ids = input.eventIds === undefined ? undefined : yield* encodeEventIds(input.eventIds); - const rows = yield* sql` + const rows = yield* sql<{ event_id: string; state: string }>` UPDATE kin_events SET attempts = attempts + ${input.countAttempt ? 1 : 0}, state = CASE WHEN attempts + ${input.countAttempt ? 1 : 0} >= 3 THEN 'failed' ELSE 'queued' END, wakeup_id = NULL, not_before = ${notBefore} @@ -370,16 +435,22 @@ const make = Effect.gen(function* () { AND (${ids ?? null} IS NULL OR event_id IN (SELECT value FROM json_each(${ids ?? null}))) AND EXISTS (SELECT 1 FROM kin_wakeups WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND status = 'running') - RETURNING event_id`; + RETURNING event_id, state`; yield* sql`UPDATE kin_wakeups SET status = ${input.countAttempt ? "failed" : "released"}, ended_at = ${now} WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND status = 'running' AND NOT EXISTS (SELECT 1 FROM kin_events WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed')`; - return rows.length; + return { + released: rows.length, + failedEventIds: rows + .filter((row) => row.state === "failed") + .map((row) => KinEventId.make(row.event_id)) + .sort(), + }; }), ); - if (count > 0) yield* notify(input.orgId); - return count; + if (result.released > 0) yield* notify(input.orgId); + return result; }, Effect.mapError(toQueuePersistenceError("KinEventQueue.release")), Effect.uninterruptible, @@ -388,26 +459,35 @@ const make = Effect.gen(function* () { const finish = Effect.fn("KinEventQueue.finish")( function* (input: typeof FinishInput.Type) { yield* decodeFinishInput(input); - yield* sql.withTransaction( + const failedEventIds = yield* sql.withTransaction( Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; const closed = yield* sql`UPDATE kin_wakeups SET status = ${input.status}, ended_at = ${now} WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND status = 'running' RETURNING wakeup_id`; - if (closed.length === 0) return; + if (closed.length === 0) return []; if (input.status === "completed") { yield* sql`UPDATE kin_events SET state = 'done' WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed'`; } else { - yield* sql`UPDATE kin_events SET attempts = attempts + 1, + const rows = yield* sql<{ + event_id: string; + state: string; + }>`UPDATE kin_events SET attempts = attempts + 1, state = CASE WHEN attempts + 1 >= 3 THEN 'failed' ELSE 'queued' END, wakeup_id = NULL, not_before = ${now + 300_000} - WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed'`; + WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed' RETURNING event_id, state`; + return rows + .filter((row) => row.state === "failed") + .map((row) => KinEventId.make(row.event_id)) + .sort(); } + return []; }), ); yield* notify(input.orgId); + return { failedEventIds }; }, Effect.mapError(toQueuePersistenceError("KinEventQueue.finish")), Effect.uninterruptible, @@ -440,7 +520,70 @@ const make = Effect.gen(function* () { Effect.mapError(toQueuePersistenceError("KinEventQueue.requeueUnfinished")), ); - return KinEventQueue.of({ enqueue, claim, release, finish, requeueUnfinished }); + const changes = Effect.fn("KinEventQueue.changes")((orgId: KinOrgId) => + Effect.sync(() => Deferred.await(organization(orgId).changed)), + ); + const expireAgent = Effect.fn("KinEventQueue.expireAgent")( + function* (orgId: KinOrgId, agentId: KinAgentId) { + yield* sql.withTransaction( + Effect.gen(function* () { + const leads = + yield* sql`SELECT 1 FROM kin_leads WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + if (leads.length === 0) return yield* new KinEventQueueUnknownAgent({ orgId, agentId }); + yield* sql`UPDATE kin_events SET state = 'expired' WHERE org_id = ${orgId} AND agent_id = ${agentId} AND state = 'queued'`; + }), + ); + yield* notify(orgId); + }, + Effect.mapError((cause) => + isUnknownAgent(cause) ? cause : toQueuePersistenceError("KinEventQueue.expireAgent")(cause), + ), + ); + const nextReady = Effect.fn("KinEventQueue.nextReady")( + function* (orgId: KinOrgId) { + const now = yield* Clock.currentTimeMillis; + const rows = yield* sql<{ agent_id: string }>`SELECT e.agent_id + FROM kin_events e JOIN kin_agents a ON a.org_id = e.org_id AND a.agent_id = e.agent_id + WHERE e.org_id = ${orgId} AND e.state = 'queued' AND a.retired_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM kin_leads l + WHERE l.org_id = e.org_id AND l.agent_id = e.agent_id AND ( + EXISTS (SELECT 1 FROM orchestration_v2_projection_runs r + WHERE r.thread_id = l.thread_id AND r.status IN ${sql.in(["queued", ...ACTIVE_RUN_STATUSES])}) + OR EXISTS (SELECT 1 FROM orchestration_v2_projection_provider_turns t + WHERE t.thread_id = l.thread_id AND t.completed_at IS NULL) + ) + ) + GROUP BY e.agent_id ORDER BY e.agent_id`; + const candidates = yield* Effect.forEach(rows, (row) => + Effect.gen(function* () { + const input = { + orgId, + agentId: KinAgentId.make(row.agent_id), + }; + const window = yield* readiness(input, now); + return { + agentId: input.agentId, + readyAt: Math.min(window.readyAt, window.next > now ? window.next : Infinity), + }; + }), + ); + return candidates.toSorted( + (a, b) => a.readyAt - b.readyAt || a.agentId.localeCompare(b.agentId), + )[0]; + }, + Effect.mapError(toQueuePersistenceError("KinEventQueue.nextReady")), + ); + return KinEventQueue.of({ + enqueue, + claim, + release, + finish, + requeueUnfinished, + changes, + expireAgent, + nextReady, + }); }); export const layer = Layer.effect(KinEventQueue, make); diff --git a/apps/server/src/kin/lead/KinLeadRegistry.ts b/apps/server/src/kin/lead/KinLeadRegistry.ts new file mode 100644 index 0000000000000..7167e40dc2e86 --- /dev/null +++ b/apps/server/src/kin/lead/KinLeadRegistry.ts @@ -0,0 +1,46 @@ +import { KinAgentId, KinLevel, KinOrgId, KinWakeupId, ThreadId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; + +const Caller = Schema.Struct({ + orgId: KinOrgId, + agentId: KinAgentId, + wakeupId: KinWakeupId, + level: KinLevel, +}); +export class KinLeadRegistryConflict extends Schema.TaggedError()( + "KinLeadRegistryConflict", + { threadId: ThreadId }, +) {} + +export class KinLeadRegistry extends Context.Service< + KinLeadRegistry, + { + readonly register: ( + threadId: ThreadId, + caller: typeof Caller.Type, + ) => Effect.Effect; + } +>()("t3/kin/lead/KinLeadRegistry") {} + +export const layer = Layer.sync(KinLeadRegistry, () => { + const callers = new Map(); + return KinLeadRegistry.of({ + register: Effect.fn("KinLeadRegistry.register")((threadId, caller) => + Effect.acquireRelease( + Effect.suspend(() => { + if (callers.has(threadId)) return Effect.fail(new KinLeadRegistryConflict({ threadId })); + callers.set(threadId, caller); + return Effect.void; + }), + () => + Effect.sync(() => { + callers.delete(threadId); + }), + ), + ), + }); +}); diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index 8bf2bbcf15878..1c8c5d3d9a5ab 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -7,6 +7,7 @@ import { KinMessageId, KinOrgId, KinPersonId, + ProviderInstanceId, KinWakeupId, ThreadId, type KinAuthor, @@ -18,9 +19,8 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; -import * as Sqlite from "../../persistence/Sqlite.ts"; import * as Chat from "../chat/KinChatRepository.ts"; -import * as KinPersistence from "../persistence/KinPersistence.ts"; +import { startWakeupRun, authorityLayer } from "../chat/WakeupAuthority.testkit.ts"; import * as Memory from "../memory/Memory.ts"; import { KinLeadPrompt, type KinLeadPromptInput } from "./LeadPrompt.ts"; @@ -54,7 +54,7 @@ const event = (messageId: string) => dedupeKey: null, provenance: { _tag: "system" }, }); -const database = KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); +const database = authorityLayer; const repository = Chat.layer.pipe(Layer.provide(database)); const memory = Layer.succeed( Memory.Memory, @@ -163,6 +163,11 @@ it.effect.each([ const { role, tag, label } = row; const repo = yield* seed; const threadId = ThreadId.make("fixture-thread"); + const caller = { + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), + providerSessionId: "provider-session", + }; if (tag === "agent") { yield* repo.insertAgent({ orgId, @@ -179,6 +184,14 @@ it.effect.each([ startedAt: timestamp, endedAt: null, }); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_leads + (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${orgId}, 'Fixture lead', 0, '{}', '[]', ${chatId})`; + const run = yield* startWakeupRun(threadId, input.wakeupId); + yield* sql`INSERT INTO kin_wakeups + (wakeup_id, org_id, agent_id, thread_id, model, started_at, status, event_ids, run_id) + VALUES (${input.wakeupId}, ${orgId}, ${agentId}, ${threadId}, 'test', 0, 'running', '[]', ${run.id})`; } yield* repo.addMember({ orgId, @@ -204,6 +217,7 @@ it.effect.each([ chatId, messageId: KinMessageId.make("message"), author, + caller: tag === "agent" ? caller : undefined, kind: "text", body, intent: null, diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts new file mode 100644 index 0000000000000..8bbee9125bf2b --- /dev/null +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -0,0 +1,851 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { HttpServer } from "effect/http"; +import * as NetAddress from "effect/net/NetAddress"; +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import * as ProviderSessionManager from "../../orchestration-v2/ProviderSessionManager.ts"; +import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; +import * as McpSessionRegistryTestkit from "../../mcp/McpSessionRegistry.testkit.ts"; +import { assert, describe, it } from "@effect/vitest"; +import { + CommandId, + EventId, + EnvironmentId, + MessageId, + KinWakeupId, + KinAgentId, + ThreadId, + RunId, + RunAttemptId, + ProviderTurnId, + ProjectId, + ProviderInstanceId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as KinIdentity from "../chat/KinIdentity.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryConfig from "../memory/MemoryConfig.ts"; +import * as VcsProcess from "../../vcs/VcsProcess.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as KinLead from "./layer.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as WakeupRunner from "./WakeupRunner.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as DateTime from "effect/DateTime"; +import * as EventSink from "../../orchestration-v2/EventSink.ts"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; +import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; +import { harness, seed, orgId, agentId, event } from "./WakeupRunner.testkit.ts"; + +const decodeUsage = Schema.decodeEffect( + Schema.fromJsonString( + Schema.Array( + Schema.Struct({ + inputTokens: Schema.Number, + cachedInputTokens: Schema.Number, + outputTokens: Schema.Number, + }), + ), + ), +); +const resolveAgent = Effect.fn("test.resolveAgent")(function* (threadId: ThreadId) { + const identity = yield* KinIdentity.KinIdentity; + const sessions = yield* McpSessionRegistry.McpSessionRegistry; + const issued = yield* sessions.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), + }); + const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); + const scope = yield* sessions.resolve(token); + assert.isDefined(scope); + return yield* identity.resolveAgent(scope!); +}); +const run = Effect.fn("test.run")(function* () { + const runner = yield* WakeupRunner.WakeupRunner; + const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); + return fiber; +}); + +describe("WakeupRunner", () => { + it.effect( + "runs two wake-ups on one T3 thread with distinct provider threads and captures only each run's usage", + () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + for (const id of ["first", "second"]) { + yield* queue.enqueue(event(id)); + yield* Fiber.join(yield* run()); + } + const rows = yield* sql<{ + thread_id: string; + tokens_json: string; + wakeup_id: string; + }>`SELECT thread_id, tokens_json, wakeup_id FROM kin_wakeups ORDER BY started_at`; + assert.strictEqual(rows.length, 2); + assert.strictEqual(rows[0]?.thread_id, rows[1]?.thread_id); + assert.match(rows[0]!.wakeup_id, /^[0-7][0-9A-HJKMNP-TV-Z]{25}$/); + const threads = yield* ThreadManagement.ThreadManagementService; + const projection = yield* threads.getThreadProjection(ThreadId.make(rows[0]!.thread_id)); + assert.strictEqual(projection.runs.length, 2); + assert.notEqual(projection.runs[0]?.providerThreadId, projection.runs[1]?.providerThreadId); + assert.strictEqual(projection.providerThreads.length, 2); + for (const row of rows) { + const usage = yield* decodeUsage(row.tokens_json); + assert.deepStrictEqual(usage, [ + { inputTokens: 20, cachedInputTokens: 5, outputTokens: 10 }, + ]); + } + assert.include(yield* resolveAgent(projection.thread.id).pipe(Effect.flip), { + reason: "not_allowed", + }); + assert.isTrue((yield* KinIdentity.KinIdentity).isAgentThread(projection.thread.id)); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events ORDER BY event_id`, [ + { state: "done" }, + { state: "done" }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + it.effect("interrupts mid-wake-up without counting an attempt and admits the next claim", () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("interrupt")); + const fiber = yield* run(); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY wakeup_id`, [ + { status: "interrupted" }, + ]); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 0 }, + ]); + yield* queue.enqueue(event("next")); + yield* TestClock.adjust("20 seconds"); + const events = yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("next"), model: "test" }), + ); + assert.deepStrictEqual( + events.map((item) => item.eventId), + [event("interrupt").eventId, event("next").eventId], + ); + yield* queue.finish({ orgId, wakeupId: KinWakeupId.make("next"), status: "completed" }); + assert.isDefined(runner); + }).pipe( + Effect.provide( + harness({ + prepare: () => Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect("backs off a cross-process busy claim on TestClock instead of spinning", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("blocker")); + yield* TestClock.adjust("20 seconds"); + yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("blocker"), model: "test" }), + ); + yield* queue.enqueue(event("waiting")); + const fiber = yield* run(); + yield* queue.finish({ orgId, wakeupId: KinWakeupId.make("blocker"), status: "completed" }); + yield* TestClock.adjust("4 seconds"); + assert.isUndefined(fiber.pollUnsafe()); + assert.strictEqual((yield* sql`SELECT * FROM kin_wakeups`).length, 1); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(fiber); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events ORDER BY event_id`, [ + { state: "done" }, + { state: "done" }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect( + "times out and interrupts a held provider run without leaving the organization busy", + () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const sessions = yield* McpSessionRegistry.__testing.make().pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("test")), + }), + ), + ), + ); + const launch = yield* ThreadLaunch.ThreadLaunchService; + const runner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(ThreadLaunch.ThreadLaunchService, { + ...launch, + launch: (input) => + launch.launch(input).pipe( + Effect.tap((result) => + Effect.gen(function* () { + assert.strictEqual( + (yield* resolveAgent(result.projection.thread.id)).agentId, + agentId, + ); + }), + ), + ), + }), + ); + yield* queue.enqueue(event("timeout")); + const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("20 seconds"); + yield* Deferred.await(entered); + const [lead] = yield* sql<{ + thread_id: string; + }>`SELECT thread_id FROM kin_leads WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + const threadId = ThreadId.make(lead!.thread_id); + const issued = yield* sessions.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), + }); + const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); + assert.isDefined(yield* sessions.resolve(token)); + assert.strictEqual( + (yield* (yield* KinIdentity.KinIdentity).resolveAgent( + (yield* sessions.resolve(token))!, + )).agentId, + agentId, + ); + yield* TestClock.adjust("15 minutes"); + yield* TestClock.adjust("30 seconds"); + yield* Fiber.join(fiber); + assert.isDefined(yield* sessions.resolve(token)); + assert.include(yield* resolveAgent(threadId).pipe(Effect.flip), { + reason: "not_allowed", + }); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [ + { status: "timed_out" }, + ]); + yield* Deferred.succeed(hold, undefined); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + }).pipe(Effect.provide(harness({ entered, hold }))); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect( + "recovers abandoned and completed wake-ups once after restart without relaunching", + () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("abandoned")); + yield* TestClock.adjust("20 seconds"); + yield* Effect.scoped( + queue.claim({ orgId, agentId, wakeupId: KinWakeupId.make("abandoned"), model: "test" }), + ); + const missingThread = ThreadId.make("provisional-thread"); + yield* (yield* KinIdentity.KinIdentity).registerAgentThread({ + orgId, + agentId, + threadId: missingThread, + }); + yield* sql`UPDATE kin_leads SET thread_id = ${missingThread} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* sql`UPDATE kin_wakeups SET thread_id = ${missingThread} WHERE org_id = ${orgId} AND wakeup_id = 'abandoned'`; + yield* runner.recover; + assert.deepStrictEqual(yield* sql`SELECT thread_id FROM kin_leads`, [{ thread_id: null }]); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [ + { status: "interrupted" }, + ]); + yield* Fiber.join(yield* run()); + yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL WHERE status = 'completed'`; + yield* sql`UPDATE kin_events SET state = 'claimed' WHERE state = 'done'`; + const threads = yield* ThreadManagement.ThreadManagementService; + const recoveredRunner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + getThreadProjection: (threadId) => + threads.getThreadProjection(threadId).pipe( + Effect.map((projection) => { + const run = projection.runs[0]; + const attempt = projection.attempts[0]; + const turn = projection.providerTurns[0]; + assert.isDefined(run); + assert.isDefined(attempt); + assert.isDefined(turn); + const continuedRunId = RunId.make("restart-continuation"); + const continuedAttemptId = RunAttemptId.make("restart-attempt"); + return { + ...projection, + runs: [ + ...projection.runs, + { + ...run!, + id: continuedRunId, + ordinal: run!.ordinal + 1, + restartContinuationOfRunId: run!.id, + }, + ], + attempts: [ + ...projection.attempts, + { ...attempt!, id: continuedAttemptId, runId: continuedRunId }, + ], + providerTurns: [ + ...projection.providerTurns, + { + ...turn!, + id: ProviderTurnId.make("restart-turn"), + runAttemptId: continuedAttemptId, + }, + ], + }; + }), + ), + }), + ); + yield* recoveredRunner.recover; + yield* recoveredRunner.recover; + const [usageRow] = yield* sql<{ + tokens_json: string; + }>`SELECT tokens_json FROM kin_wakeups WHERE status = 'completed'`; + assert.strictEqual((yield* decodeUsage(usageRow!.tokens_json)).length, 2); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY status`, [ + { status: "completed" }, + { status: "interrupted" }, + ]); + const [row] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const projection = + yield* (yield* ThreadManagement.ThreadManagementService).getThreadProjection( + ThreadId.make(row!.thread_id), + ); + assert.strictEqual(projection.runs.length, 1); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "done", attempts: 0 }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect.each(["projection", "decode"] as const)( + "finalizes recovery after a %s failure instead of retrying forever", + (failure) => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("recovery-failure")); + yield* Fiber.join(yield* run()); + yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL`; + yield* sql`UPDATE kin_events SET state = 'claimed'`; + if (failure === "decode") yield* sql`UPDATE kin_wakeups SET run_id = ''`; + const threads = yield* ThreadManagement.ThreadManagementService; + const runner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + getThreadProjection: () => Effect.die("projection unavailable"), + }), + ); + yield* runner.recover; + yield* runner.recover; + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [{ status: "failed" }]); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 1 }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect("wakes another lead while the first has a held queued run", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + yield* Fiber.join(yield* run()); + const [lead] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const threadId = ThreadId.make(lead!.thread_id); + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold"), + threadId, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("held-message"), + threadId, + messageId: MessageId.make("held-message"), + text: "Held", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("held-second"), + threadId, + messageId: MessageId.make("held-second"), + text: "Held", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("hold-queue"), + threadId, + }); + assert.strictEqual( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!.status, + "queued", + ); + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + SELECT 'lead-z', org_id, display_name, role, owner_person_id, created_at FROM kin_agents WHERE agent_id = 'lead'`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + SELECT 'lead-z', org_id, name, level_since, models_json, worker_projects_json, owner_chat_id FROM kin_leads WHERE agent_id = 'lead'`; + yield* queue.enqueue(event("busy")); + yield* queue.enqueue({ ...event("other"), agentId: KinAgentId.make("lead-z") }); + yield* TestClock.adjust("20 seconds"); + const next = yield* queue.nextReady(orgId); + assert.strictEqual(next?.agentId, "lead-z"); + yield* Fiber.join( + yield* (yield* WakeupRunner.WakeupRunner) + .runOnce(orgId, next!.agentId) + .pipe(Effect.forkChild), + ); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'other'`, [ + { state: "done" }, + ]); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events WHERE event_id = 'busy'`, [ + { state: "queued" }, + ]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect("defers claims while an interrupted run's provider turn is unfinished", () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + yield* Fiber.join(yield* run()); + const threads = yield* ThreadManagement.ThreadManagementService; + const [lead] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const projection = yield* threads.getThreadProjection(ThreadId.make(lead!.thread_id)); + const sink = yield* EventSink.EventSinkV2; + const turn = projection.providerTurns[0]!; + const occurredAt = yield* DateTime.now; + yield* sink.write({ + events: [ + { + id: EventId.make("unfinished-run"), + type: "run.updated", + threadId: projection.thread.id, + occurredAt, + payload: { ...projection.runs[0]!, status: "interrupted" }, + }, + { + id: EventId.make("unfinished-turn"), + type: "provider-turn.updated", + threadId: projection.thread.id, + occurredAt, + payload: { ...turn, status: "running", completedAt: null }, + }, + ], + }); + yield* queue.enqueue(event("next")); + const deferred = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("60 seconds"); + yield* Fiber.join(deferred); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [{ status: "completed" }]); + assert.deepStrictEqual( + yield* sql`SELECT state, attempts FROM kin_events WHERE event_id = 'next'`, + [{ state: "queued", attempts: 0 }], + ); + yield* sink.write({ + events: [ + { + id: EventId.make("finished-turn"), + type: "provider-turn.updated", + threadId: projection.thread.id, + occurredAt: yield* DateTime.now, + payload: turn, + }, + ], + }); + yield* Fiber.join(yield* run()); + assert.lengthOf(yield* sql`SELECT 1 FROM kin_wakeups`, 2); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect.each(["MemorySyncConflict", "MemoryCheckoutInvalid"] as const)( + "fails and backs off %s without launching a provider", + (tag) => + Effect.gen(function* () { + const error = + tag === "MemorySyncConflict" + ? new Memory.MemorySyncConflict({ orgId, operation: "sync", reason: "fixture" }) + : new Memory.MemoryCheckoutInvalid({ orgId, operation: "checkout", reason: "fixture" }); + let notified = 0; + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + yield* queue.enqueue(event("memory-failure")); + const result = yield* Fiber.await(yield* run()); + assert.strictEqual(result._tag, "Failure"); + const sql = yield* SqlClient.SqlClient; + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 1 }, + ]); + assert.deepStrictEqual(yield* sql`SELECT status, thread_id FROM kin_wakeups`, [ + { status: "failed", thread_id: null }, + ]); + assert.strictEqual(notified, tag === "MemoryCheckoutInvalid" ? 1 : 0); + }).pipe( + Effect.provide( + harness({ prepare: () => Effect.fail(error) }).pipe( + Layer.provide( + Layer.succeed(WakeupRunner.KinWakeupHooks, { + memoryCheckoutInvalid: () => + Effect.sync(() => { + notified++; + }), + }), + ), + ), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect.each(["released", "shared", "releasing"] as const)( + "expires retired leads' queued events and ends their durable thread registration (%s)", + (outcome) => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + const releaseEntered = yield* Deferred.make(); + const releaseHold = yield* Deferred.make(); + const releaseDone = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("first")); + const active = yield* run(); + yield* Deferred.await(entered); + yield* queue.enqueue(event("retired")); + const identity = yield* KinIdentity.KinIdentity; + const threads = yield* ThreadManagement.ThreadManagementService; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const [lead] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_leads`; + const threadId = ThreadId.make(lead!.thread_id); + const projection = yield* threads.getThreadProjection(threadId); + const credentials = yield* McpSessionRegistry.McpSessionRegistry; + const issued = yield* credentials.issue({ + threadId, + providerInstanceId: ProviderInstanceId.make("codex"), + }); + const token = issued.config.authorizationHeader.slice(7); + const caller = yield* credentials.resolve(token); + assert.isDefined(caller); + yield* identity.resolveAgent(caller!); + yield* sql`UPDATE kin_agents SET retired_at = '2026-11-08T00:00:00.000Z' WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + let stops = 0; + const retiringRunner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ...threads, + forceStopThread: Effect.fn("test.forceStopThread")(function* (input) { + stops++; + const result = yield* threads.forceStopThread(input); + assert.strictEqual(result.outcome, stops === 1 ? outcome : "released"); + return result; + }), + }), + Effect.provideService(KinIdentity.KinIdentity, { + ...identity, + endAgentThread: Effect.fn("test.endAgentThread")(function* ( + input: Parameters[0], + ) { + const projection = yield* threads + .getThreadProjection(input.threadId) + .pipe(Effect.orDie); + assert.isTrue(identity.isAgentThread(input.threadId)); + for (const providerThread of projection.providerThreads) + if (providerThread.providerSessionId !== null) + assert.strictEqual( + yield* sessions.isLive(providerThread.providerSessionId), + outcome === "shared", + ); + return yield* identity.endAgentThread(input); + }), + }), + ); + const firstPass = yield* retiringRunner.runOnce(orgId, agentId).pipe(Effect.forkChild); + if (outcome === "releasing") { + yield* Deferred.await(releaseEntered); + yield* TestClock.adjust("10 seconds"); + } + yield* Fiber.join(firstPass); + assert.strictEqual(identity.isAgentThread(threadId), outcome === "releasing"); + if (outcome === "releasing") { + assert.isTrue(identity.isAgentThread(threadId)); + assert.lengthOf(yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL`, 1); + yield* Deferred.succeed(releaseHold, undefined); + yield* Deferred.await(releaseDone); + } + yield* retiringRunner.runOnce(orgId, agentId); + assert.strictEqual(stops, outcome === "releasing" ? 2 : 1); + if (outcome === "shared") { + assert.isTrue( + yield* sessions.isLive(projection.providerThreads[0]!.providerSessionId!), + ); + assert.isDefined(yield* credentials.resolve(token)); + const denied = yield* identity.resolveAgent(caller!).pipe(Effect.flip); + assert.propertyVal(denied, "reason", "not_allowed"); + assert.strictEqual(denied.message, "Kin agent thread is not active."); + } + yield* Deferred.succeed(hold, undefined); + yield* Fiber.join(active); + assert.deepStrictEqual( + yield* sql`SELECT state FROM kin_events WHERE event_id = 'retired'`, + [{ state: "expired" }], + ); + assert.strictEqual( + (yield* sql`SELECT 1 FROM kin_agent_threads WHERE ended_at IS NULL AND org_id = ${orgId}`) + .length, + 0, + ); + assert.strictEqual((yield* sql`SELECT 1 FROM kin_wakeups`).length, 1); + }).pipe( + Effect.provide( + harness({ + entered, + hold, + ...(outcome === "shared" ? { sharedWith: [ThreadId.make("sibling")] } : {}), + ...(outcome === "releasing" ? { releaseEntered, releaseHold, releaseDone } : {}), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect( + "recovers other organizations while keeping a recovered active run claimed until completion", + () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + const wakeupId = KinWakeupId.make("recovery-active"); + yield* queue.enqueue(event("active")); + yield* TestClock.adjust("20 seconds"); + yield* Effect.scoped(queue.claim({ orgId, agentId, wakeupId, model: "test" })); + const launch = yield* ThreadLaunch.ThreadLaunchService; + const sent = yield* launch.launch({ + commandId: CommandId.make(`kin-wakeup:${wakeupId}`), + projectId: ProjectId.make("kin:org"), + title: "Lead", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5.1-codex", + }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { + messageId: MessageId.make(`kin-wakeup-msg:${wakeupId}`), + text: "Wake up", + attachments: [], + }, + createdBy: "system", + creationSource: "server", + }); + yield* sql`UPDATE kin_leads SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* sql`UPDATE kin_wakeups SET thread_id = ${sent.threadId} WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId}`; + yield* Deferred.await(entered); + yield* sql`INSERT INTO kin_orgs VALUES ('other-org', 'Other', '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) + VALUES ('other-owner', 'other-org', 'owner', 'Owner', '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) + VALUES ('other-lead', 'other-org', 'Lead', 'lead', 'other-owner', '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) + VALUES ('other-chat', 'other-org', 'team_group', 0, '2026-11-08T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + SELECT 'other-lead', 'other-org', 'Lead', 0, models_json, '[]', 'other-chat' FROM kin_leads WHERE agent_id = ${agentId}`; + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, model, started_at, status, event_ids) + VALUES ('other-abandoned', 'other-org', 'other-lead', 'test', 0, 'running', '[]')`; + const reconciled = yield* Deferred.make(); + const runner = yield* WakeupRunner.WakeupRunner.pipe( + Effect.provide(Layer.fresh(WakeupRunner.layer)), + Effect.provideService(KinEventQueue.KinEventQueue, { + ...queue, + requeueUnfinished: Effect.fn("test.requeueUnfinished")(function* (org, wakeup) { + const count = yield* queue.requeueUnfinished(org, wakeup); + yield* Deferred.succeed(reconciled, undefined); + return count; + }), + }), + ); + const recovered = yield* runner.recover.pipe(Effect.forkChild); + yield* Deferred.await(reconciled); + assert.deepStrictEqual( + yield* sql`SELECT status FROM kin_wakeups WHERE wakeup_id = 'other-abandoned'`, + [{ status: "interrupted" }], + ); + yield* sql`DELETE FROM kin_wakeups WHERE wakeup_id = 'other-abandoned'`; + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events`, [{ state: "claimed" }]); + assert.isUndefined(recovered.pollUnsafe()); + yield* Deferred.succeed(hold, undefined); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + yield* Fiber.join(recovered); + assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups`, [ + { status: "completed" }, + ]); + yield* runner.recover; + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events`, [{ state: "done" }]); + }).pipe(Effect.provide(harness({ entered, hold }))); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect( + "releases an organization paused during the quiet window without consuming an attempt", + () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + const sql = yield* SqlClient.SqlClient; + yield* queue.enqueue(event("paused")); + const fiber = yield* runner.runOnce(orgId, agentId).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* sql`INSERT INTO kin_org_state (org_id, paused, paused_at, paused_by) VALUES (${orgId}, 1, 1000, 'owner')`; + yield* TestClock.adjust("19 seconds"); + yield* Fiber.join(fiber); + assert.deepStrictEqual(yield* sql`SELECT state, attempts FROM kin_events`, [ + { state: "queued", attempts: 0 }, + ]); + assert.deepStrictEqual(yield* sql`SELECT status, thread_id FROM kin_wakeups`, [ + { status: "released", thread_id: null }, + ]); + yield* sql`UPDATE kin_org_state SET paused = 0, paused_at = NULL, paused_by = NULL WHERE org_id = ${orgId}`; + yield* Fiber.join(yield* run()); + assert.deepStrictEqual(yield* sql`SELECT state FROM kin_events`, [{ state: "done" }]); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + + it.effect("keeps an org loop alive after a one-shot defect", () => + Effect.gen(function* () { + const failed = yield* Deferred.make(); + const continued = yield* Deferred.make(); + let calls = 0; + yield* Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const runner = yield* WakeupRunner.WakeupRunner; + yield* runner.start; + yield* queue.enqueue(event("defect")); + yield* TestClock.adjust("20 seconds"); + yield* Deferred.await(failed); + yield* queue.enqueue(event("after-defect")); + yield* TestClock.adjust("21 seconds"); + yield* Deferred.await(continued); + assert.strictEqual(calls, 2); + }).pipe( + Effect.provide( + harness({ + prepare: () => + Effect.gen(function* () { + if (++calls === 1) { + yield* Deferred.succeed(failed, undefined); + return yield* Effect.die("one-shot defect"); + } + yield* Deferred.succeed(continued, undefined); + return yield* Effect.never; + }), + }), + ), + ); + }).pipe(provideDeterministicTestRuntime), + ); + + it.effect("contains Kin startup failures while building the non-Kin server runtime", () => { + class Core extends Context.Service()( + "t3/kin/lead/WakeupRunner.test/Core", + ) {} + return Effect.gen(function* () { + yield* seed; + const failing = KinLead.layer.pipe( + Layer.provide(Layer.mock(KinIdentity.KinIdentity)({ isAgentThread: () => false })), + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadLaunch.ThreadLaunchService)({}), + Layer.mock(ThreadManagement.ThreadManagementService)({}), + Layer.mock(ProjectService.ProjectService)({}), + Layer.succeed(KinLeadPrompt, { build: () => Effect.succeed("unused") }), + Layer.effect(MemoryConfig.MemoryConfig, Effect.die("invalid memory config")), + Layer.mock(VcsProcess.VcsProcess)({}), + NodeServices.layer, + ), + ), + ); + assert.strictEqual((yield* Layer.build(failing).pipe(Effect.exit))._tag, "Failure"); + const runtime = Layer.mergeAll( + Layer.succeed(Core, { ready: true }), + KinLead.containStartup(failing), + ); + const context = yield* Layer.build(runtime); + assert.isTrue(Context.get(context, Core).ready); + const defect = Layer.mergeAll( + Layer.succeed(Core, { ready: true }), + KinLead.containStartup(Layer.effectDiscard(Effect.die("Kin layer defect"))), + ); + assert.isTrue(Context.get(yield* Layer.build(defect), Core).ready); + }).pipe( + Effect.provide( + Layer.mergeAll( + McpSessionRegistryTestkit.layer, + KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)), + ), + ), + ); + }); +}); diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts new file mode 100644 index 0000000000000..5cee69733a6cb --- /dev/null +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -0,0 +1,305 @@ +import * as ProviderSessionManager from "../../orchestration-v2/ProviderSessionManager.ts"; +import type { ThreadId } from "@t3tools/contracts"; +import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { HttpServer } from "effect/http"; +import * as NetAddress from "effect/net/NetAddress"; +import { EnvironmentId } from "@t3tools/contracts"; +import * as ServerEnvironment from "../../environment/ServerEnvironment.ts"; +import { + KinAgentId, + KinEvent, + KinOrgId, + ProjectId, + ProviderInstanceId, + ProviderDriverKind, + ProviderThreadId, + ProviderTurnId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; + +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as KinChatRepository from "../chat/KinChatRepository.ts"; +import * as KinIdentity from "../chat/KinIdentity.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as WakeupRunner from "./WakeupRunner.ts"; +import * as KinLeadRegistry from "./KinLeadRegistry.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; +import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as ProjectSetupScriptRunner from "../../project/ProjectSetupScriptRunner.ts"; +import * as ProjectCloneTracker from "../../project/ProjectCloneTracker.ts"; +import * as WorktreeSetupTracker from "../../project/WorktreeSetupTracker.ts"; +import * as ManagedProjectFolders from "../../project/ManagedProjectFolders.ts"; +import * as TerminalManager from "../../terminal/Manager.ts"; +import * as GitWorkflow from "../../git/GitWorkflowService.ts"; +import * as TextGeneration from "../../textGeneration/TextGeneration.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import * as ProviderRegistryMock from "../../provider/testUtils/providerRegistryMock.ts"; +import * as CommandReceiptStore from "../../orchestration-v2/CommandReceiptStore.ts"; +import * as ProviderAdapterRegistry from "../../orchestration-v2/ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "../../orchestration-v2/testkit/ProviderReplayHarness.ts"; +import { CodexProviderCapabilitiesV2 } from "../../orchestration-v2/Adapters/CodexAdapterV2.ts"; + +export const orgId = KinOrgId.make("org"); +export const agentId = KinAgentId.make("lead"); +const timestamp = "2026-11-08T00:00:00.000Z"; +const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const decodeEvent = Schema.decodeSync(KinEvent); +const selection = { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.1-codex" }; +const project = { + id: ProjectId.make("kin:org"), + title: "Kin · Org", + workspaceRoot: "/private/tmp", + repositoryIdentity: null, + faviconPath: null, + defaultModelSelection: selection, + defaultThreadEnvMode: null, + scripts: [], + pinnedAt: null, + archivedAt: null, + createdAt: timestamp, + updatedAt: timestamp, + deletedAt: null, +}; +const database = KinPersistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); +export const seed = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_orgs VALUES ('org', 'Org', ${timestamp})`; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) VALUES ('owner', 'org', 'owner', 'Owner', ${timestamp})`; + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) VALUES ('lead', 'org', 'Lead', 'lead', 'owner', ${timestamp})`; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) VALUES ('owner-chat', 'org', 'team_group', 0, ${timestamp})`; + const models = yield* encodeJson({ + strong: selection, + cheap: selection, + }); + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) VALUES ('lead', 'org', 'Lead', 0, ${models}, '[]', 'owner-chat')`; +}); +export const event = (id: string) => + decodeEvent({ + _tag: "routine", + orgId, + agentId, + eventId: id, + receivedAt: timestamp, + dedupeKey: id, + provenance: { _tag: "system" }, + routine: "owner_check", + }); + +export function harness( + options: { + readonly sharedWith?: ReadonlyArray; + readonly releaseEntered?: Deferred.Deferred; + readonly releaseHold?: Deferred.Deferred; + readonly releaseDone?: Deferred.Deferred; + readonly hold?: Deferred.Deferred; + readonly entered?: Deferred.Deferred; + readonly prepare?: Memory.Memory["Service"]["prepareWakeup"]; + } = {}, +) { + let nativeOrdinal = 0; + const driver = ProviderDriverKind.make("codex"); + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId: selection.instanceId, + driver, + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (session) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const events = yield* PubSub.unbounded(); + const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { + instanceId: selection.instanceId, + driver, + providerSessionId: session.providerSessionId, + providerSession: { + id: session.providerSessionId, + driver, + providerInstanceId: selection.instanceId, + status: "ready", + cwd: "/private/tmp", + model: selection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromPubSub(events), + ensureThread: (input) => + Effect.sync(() => ({ + id: ProviderThreadId.make(`native-${++nativeOrdinal}`), + driver, + providerInstanceId: selection.instanceId, + providerSessionId: session.providerSessionId, + appThreadId: input.threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: `native-${nativeOrdinal}`, strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + contextUsage: null, + forkedFrom: null, + createdAt: now, + updatedAt: now, + })), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: Effect.fn("test.startTurn")(function* (input) { + if (options.entered !== undefined) yield* Deferred.succeed(options.entered, undefined); + if (options.hold !== undefined) yield* Deferred.await(options.hold); + const time = yield* DateTime.now; + const id = ProviderTurnId.make(`turn-${input.runOrdinal}`); + yield* PubSub.publish(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id, + providerThreadId: input.providerThread.id, + nodeId: input.rootNodeId, + runAttemptId: input.attemptId, + nativeTurnRef: null, + ordinal: input.runOrdinal, + status: "completed", + startedAt: time, + completedAt: time, + tokenUsage: { + usedTokens: 30, + inputTokens: 20, + cachedInputTokens: 5, + outputTokens: 10, + updatedAt: DateTime.formatIso(time), + }, + }, + }); + yield* PubSub.publish(events, { + type: "turn.terminal", + driver, + providerThreadId: input.providerThread.id, + providerTurnId: id, + runOrdinal: input.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + }), + compactThread: (input) => runtime.startTurn(input), + steerTurn: () => Effect.void, + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused snapshot"), + rollbackThread: () => Effect.die("unused rollback"), + forkThread: () => Effect.die("unused fork"), + }; + return runtime; + }), + }; + const orchestration = ProviderReplayHarness.layerWithRegistry( + { name: "kin-wakeup" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { databaseLayer: database }, + ); + const settings = ServerSettings.layerTest(); + const receipts = CommandReceiptStore.layer.pipe(Layer.provide(database)); + const sharedWith = options.sharedWith; + const management = ThreadManagement.layer.pipe( + Layer.updateService(ProviderSessionManager.ProviderSessionManagerV2, (sessions) => ({ + ...sessions, + otherWorkingThreads: + sharedWith === undefined ? sessions.otherWorkingThreads : () => Effect.succeed(sharedWith), + releaseUnlessBusy: Effect.fn("test.releaseUnlessBusy")(function* (input) { + if (options.releaseEntered !== undefined) + yield* Deferred.succeed(options.releaseEntered, undefined); + if (options.releaseHold !== undefined) yield* Deferred.await(options.releaseHold); + const released = yield* sessions.releaseUnlessBusy(input); + if (options.releaseDone !== undefined) + yield* Deferred.succeed(options.releaseDone, undefined); + return released; + }), + })), + Layer.provide(Layer.mergeAll(orchestration, settings, receipts)), + ); + const projectLayer = Layer.mock(ProjectService.ProjectService)({ + bootstrap: () => Effect.succeed({ project, created: true }), + getById: () => Effect.succeedSome(project), + getByWorkspaceRoot: () => Effect.succeedSome(project), + }); + const external = Layer.mergeAll( + projectLayer, + WorktreeSetupTracker.layer, + Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({ get: () => Effect.succeed(null) }), + Layer.mock(TerminalManager.TerminalManager)({ close: () => Effect.void }), + Layer.mock(GitWorkflow.GitWorkflowService)({}), + Layer.mock(TextGeneration.TextGeneration)({}), + Layer.mock(ProjectSetupScriptRunner.ProjectSetupScriptRunner)({ + runForThread: () => Effect.succeed({ status: "no-script" }), + }), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/private/tmp", + isScratchProject: () => Effect.succeed(false), + folderForThread: () => Effect.succeedNone, + }), + ProviderRegistryMock.layer(), + settings, + ); + const launch = ThreadLaunch.layer.pipe( + Layer.provide(Layer.mergeAll(external, management, receipts, IdAllocator.layer)), + ); + const kin = Layer.mergeAll( + KinEventQueue.layer, + KinLeadRegistry.layer, + KinIdentity.KinIdentity.layer.pipe( + Layer.provide(KinChatRepository.layer), + Layer.provide( + Layer.succeed(KinIdentity.KinIdentityConfig, { + orgName: "Org", + ownerHandle: "owner", + ownerDisplayName: "Owner", + }), + ), + ), + Layer.succeed(KinLeadPrompt, { build: () => Effect.succeed("Wake up") }), + Layer.mock(Memory.Memory)({ + prepareWakeup: + options.prepare ?? + (() => Effect.succeed({ head: "abc", indexAsOf: DateTime.makeUnsafe(timestamp) })), + ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), + }), + ); + const services = Layer.mergeAll( + kin, + launch, + management, + projectLayer, + orchestration, + McpSessionRegistry.layer.pipe( + Layer.provide( + Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("test")), + }), + ), + ), + ), + ).pipe(Layer.provideMerge(database)); + return WakeupRunner.layer.pipe(Layer.provideMerge(services)); +} diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts new file mode 100644 index 0000000000000..b229772b1db69 --- /dev/null +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -0,0 +1,612 @@ +import { + CommandId, + KinAgentId, + KinLead, + KinOrgId, + KinWakeupId, + MessageId, + ModelSelection, + ProjectId, + RunId, + ThreadId, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Random from "effect/Random"; +import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as SqlClient from "effect/sql/SqlClient"; +import { isSqlError } from "effect/sql/SqlError"; + +import * as ThreadLaunchService from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagementService from "../../orchestration-v2/ThreadManagementService.ts"; +import { toPersistenceSqlError } from "../../persistence/Errors.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as KinIdentity from "../chat/KinIdentity.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinLeadRegistry from "./KinLeadRegistry.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; + +export class KinWakeupError extends Schema.TaggedError()("KinWakeupError", { + orgId: KinOrgId, + agentId: KinAgentId, + cause: Schema.Defect(), +}) {} + +export class KinWakeupHooks extends Context.Service< + KinWakeupHooks, + { + readonly memoryCheckoutInvalid: (error: Memory.MemoryCheckoutInvalid) => Effect.Effect; + } +>()("t3/kin/lead/WakeupRunner/KinWakeupHooks") {} + +export class WakeupRunner extends Context.Service< + WakeupRunner, + { + readonly runOnce: (orgId: KinOrgId, agentId: KinAgentId) => Effect.Effect; + readonly recover: Effect.Effect; + readonly start: Effect.Effect; + } +>()("t3/kin/lead/WakeupRunner") {} + +const runnerCause = (cause: unknown) => + isSqlError(cause) ? toPersistenceSqlError("WakeupRunner")(cause) : cause; +const isWakeupError = Schema.is(KinWakeupError); +const LeadRow = Schema.Struct({ + orgId: KinOrgId, + agentId: KinAgentId, + name: Schema.String, + level: KinLead.fields.level, + threadId: Schema.NullOr(ThreadId), + models: Schema.fromJsonString(Schema.Struct({ strong: ModelSelection, cheap: ModelSelection })), + orgName: Schema.String, + retiredAt: Schema.NullOr(Schema.String), +}); +const RunningRow = Schema.Struct({ + orgId: KinOrgId, + agentId: KinAgentId, + wakeupId: KinWakeupId, + threadId: Schema.NullOr(ThreadId), + runId: Schema.NullOr(RunId), + startedAt: Schema.Number, +}); +const encodeUsage = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const decodeLead = Schema.decodeUnknownEffect(LeadRow); +const decodeRunning = Schema.decodeUnknownEffect(RunningRow); +const ulid = Effect.gen(function* () { + const alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; + let timestamp = yield* Clock.currentTimeMillis; + let value = ""; + for (let i = 0; i < 10; i++) { + value = alphabet.charAt(timestamp % 32) + value; + timestamp = Math.floor(timestamp / 32); + } + for (let i = 0; i < 16; i++) value += alphabet.charAt(yield* Random.nextIntBetween(0, 31)); + return value; +}); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const queue = yield* KinEventQueue.KinEventQueue; + const memory = yield* Memory.Memory; + const hooks = yield* Effect.serviceOption(KinWakeupHooks); + const prompt = yield* KinLeadPrompt; + const identity = yield* KinIdentity.KinIdentity; + const registry = yield* KinLeadRegistry.KinLeadRegistry; + const projects = yield* ProjectService.ProjectService; + const launch = yield* ThreadLaunchService.ThreadLaunchService; + const threads = yield* ThreadManagementService.ThreadManagementService; + const threadActivity = Effect.fn("WakeupRunner.threadActivity")(function* (orgId: KinOrgId) { + const rows = yield* sql<{ + threadId: string; + }>`SELECT thread_id AS "threadId" FROM kin_leads WHERE org_id = ${orgId} AND thread_id IS NOT NULL ORDER BY thread_id`; + const leadThreads = new Set(rows.map((row) => row.threadId)); + return yield* threads.streamDomainEvents.pipe( + Stream.filter( + (event) => + leadThreads.has(event.threadId) && + (event.type === "run.updated" || + (event.type === "provider-turn.updated" && event.payload.completedAt !== null)), + ), + Stream.runHead, + ); + }); + + const lead = Effect.fn("WakeupRunner.lead")(function* (orgId: KinOrgId, agentId: KinAgentId) { + const [row] = yield* sql`SELECT l.org_id AS "orgId", l.agent_id AS "agentId", l.name, + l.level, l.thread_id AS "threadId", l.models_json AS models, o.name AS "orgName", a.retired_at AS "retiredAt" + FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + JOIN kin_orgs o ON o.org_id = l.org_id WHERE l.org_id = ${orgId} AND l.agent_id = ${agentId}`; + return yield* decodeLead(row); + }); + const retired = new Set(); + const retire = Effect.fn("WakeupRunner.retire")(function* (record: typeof LeadRow.Type) { + if (retired.has(record.agentId)) return; + yield* queue.expireAgent(record.orgId, record.agentId); + if (record.threadId !== null) { + const stopped = yield* threads.forceStopThread({ threadId: record.threadId }); + if (stopped.outcome === "releasing") return; + yield* identity.endAgentThread({ orgId: record.orgId, threadId: record.threadId }); + } + retired.add(record.agentId); + }); + const capture = Effect.fn("WakeupRunner.capture")(function* (row: typeof RunningRow.Type) { + if (row.threadId === null || row.runId === null) return; + const projection = yield* threads.getThreadProjection(row.threadId); + const runIds = new Set([row.runId]); + for (const run of projection.runs.toSorted((a, b) => a.ordinal - b.ordinal)) { + if ( + (run.restartContinuationOfRunId !== undefined && + runIds.has(run.restartContinuationOfRunId)) || + (run.usageLimitContinuationOfRunId !== undefined && + runIds.has(run.usageLimitContinuationOfRunId)) + ) + runIds.add(run.id); + } + const attempts = new Set( + projection.attempts + .filter((attempt) => runIds.has(attempt.runId)) + .map((attempt) => attempt.id), + ); + const turns = projection.providerTurns.filter( + (turn) => turn.runAttemptId !== null && attempts.has(turn.runAttemptId), + ); + const usage = turns.flatMap((turn) => + turn.tokenUsage === undefined ? [] : [{ providerTurnId: turn.id, ...turn.tokenUsage }], + ); + const tokens = yield* encodeUsage(usage); + yield* sql`UPDATE kin_wakeups SET tokens_json = ${tokens} + WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + }); + const wait = Effect.fn("WakeupRunner.wait")(function* ( + row: typeof RunningRow.Type, + projectId: ProjectId, + ) { + if (row.threadId === null || row.runId === null) return; + const threadId = row.threadId; + const runId = row.runId; + const remaining = Math.max(0, row.startedAt + 900_000 - (yield* Clock.currentTimeMillis)); + const result = yield* threads.waitForThread({ + projectId, + threadId: row.threadId, + runId: row.runId, + timeoutMs: remaining, + }); + const status = result.timedOut + ? "timed_out" + : result.run?.status === "completed" + ? "completed" + : "failed"; + yield* Effect.gen(function* () { + if (result.timedOut) { + yield* threads.interruptThread({ + projectId, + threadId, + runId: result.run?.id ?? runId, + commandId: CommandId.make(`kin-timeout:${row.wakeupId}`), + }); + const settled = yield* threads.waitForThread({ + projectId, + threadId, + runId, + timeoutMs: 30_000, + }); + if (settled.timedOut) + yield* Effect.logWarning("Kin timed-out run may continue after interrupt", { + orgId: row.orgId, + wakeupId: row.wakeupId, + runId, + }); + } + yield* capture(row); + yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status }); + }).pipe( + Effect.onExit((exit) => + result.timedOut && Exit.isFailure(exit) + ? queue + .finish({ orgId: row.orgId, wakeupId: row.wakeupId, status: "timed_out" }) + .pipe(Effect.ignore) + : Effect.void, + ), + ); + if (status === "completed") + yield* threads.settleThread({ + threadId: row.threadId, + commandId: CommandId.make(`kin-settle:${row.wakeupId}`), + byOwnAgent: true, + }); + }); + const clearMissingThread = Effect.fn("WakeupRunner.clearMissingThread")(function* ( + row: typeof RunningRow.Type, + ) { + if (row.threadId === null) return; + const threadId = row.threadId; + yield* sql.withTransaction( + Effect.gen(function* () { + const existing = + yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${threadId}`; + if (existing.length > 0) return; + const registered = + yield* sql`SELECT 1 FROM kin_agent_threads WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND thread_id = ${threadId} AND ended_at IS NULL`; + if (registered.length > 0) yield* identity.endAgentThread({ orgId: row.orgId, threadId }); + yield* sql`UPDATE kin_leads SET thread_id = NULL WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND thread_id = ${threadId}`; + }), + ); + }); + const cleanup = Effect.fn("WakeupRunner.cleanup")(function* ( + row: typeof RunningRow.Type, + interrupted = false, + ) { + const [running] = + yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${row.orgId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + if (running === undefined) return; + if (row.threadId !== null) { + const projection = yield* threads.getThreadProjection(row.threadId).pipe(Effect.option); + if (Option.isSome(projection)) + yield* threads + .interruptThread({ + projectId: projection.value.thread.projectId, + threadId: row.threadId, + commandId: CommandId.make(`kin-exit:${row.wakeupId}`), + }) + .pipe(Effect.ignore); + } + yield* clearMissingThread(row).pipe(Effect.ignore); + yield* capture(row).pipe(Effect.ignore); + if (interrupted) yield* queue.requeueUnfinished(row.orgId, row.wakeupId); + else yield* queue.finish({ orgId: row.orgId, wakeupId: row.wakeupId, status: "failed" }); + }); + + const runOnce = Effect.fn("WakeupRunner.runOnce")( + function* (orgId: KinOrgId, agentId: KinAgentId) { + const record = yield* lead(orgId, agentId); + if (record.retiredAt !== null) return yield* retire(record); + const paused = yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; + if (paused.length > 0) return; + if (record.threadId !== null) { + const unfinished = yield* sql`SELECT 1 FROM orchestration_v2_projection_provider_turns + WHERE thread_id = ${record.threadId} AND completed_at IS NULL LIMIT 1`; + const busy = yield* sql`SELECT 1 FROM orchestration_v2_projection_runs + WHERE thread_id = ${record.threadId} + AND status IN ${sql.in(["queued", ...ThreadManagementService.ACTIVE_RUN_STATUSES])} LIMIT 1`; + if (unfinished.length > 0 || busy.length > 0) { + yield* Effect.raceFirst(yield* queue.changes(orgId), threadActivity(orgId)).pipe( + Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), + ); + return; + } + } + const wakeupId = KinWakeupId.make(yield* ulid); + const startedAt = yield* Clock.currentTimeMillis; + let row: typeof RunningRow.Type = { + orgId, + agentId, + wakeupId, + threadId: null, + runId: null, + startedAt, + }; + yield* Effect.scoped( + Effect.gen(function* () { + const events = yield* Effect.acquireRelease( + queue.claim({ orgId, agentId, wakeupId, model: record.models.strong.model }), + (_, exit) => + cleanup(row, Exit.hasInterrupts(exit)).pipe( + Effect.catch((error) => + Effect.logError("Kin wake-up finalizer failed", { + orgId, + wakeupId, + errorTag: error._tag, + }), + ), + ), + ); + row = { ...row, startedAt: yield* Clock.currentTimeMillis }; + yield* Effect.gen(function* () { + const paused = + yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; + if (paused.length > 0) { + yield* queue.release({ + orgId, + wakeupId, + countAttempt: false, + notBefore: (yield* Clock.currentTimeMillis) + 5_000, + }); + return; + } + const current = yield* lead(orgId, agentId); + if (current.retiredAt !== null) { + yield* queue.finish({ orgId, wakeupId, status: "failed" }); + return yield* retire(current); + } + const pinned = yield* memory.prepareWakeup(orgId).pipe( + Effect.tapError((error) => + Effect.logError("Kin memory preparation failed", { + orgId, + wakeupId, + errorTag: error._tag, + }), + ), + Effect.tapError((error) => + error._tag === "MemoryCheckoutInvalid" && Option.isSome(hooks) + ? hooks.value.memoryCheckoutInvalid(error) + : Effect.void, + ), + ); + const paths = yield* memory.ensureCheckouts(orgId); + const { project } = yield* projects.bootstrap({ + commandId: CommandId.make(`kin-project:${orgId}`), + projectId: ProjectId.make(`kin:${orgId}`), + workspaceRoot: paths.read, + title: `Kin · ${record.orgName}`, + }); + const text = yield* prompt.build({ + orgId, + agentId, + wakeupId, + events, + memoryHead: pinned.head, + }); + const urgent = + yield* sql`SELECT 1 FROM kin_events WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND priority IN ('P0', 'P1') LIMIT 1`; + const modelSelection = + urgent.length > 0 || events.some((event) => event._tag === "routine") + ? current.models.strong + : current.models.cheap; + const threadId = current.threadId ?? ThreadId.make(`kin:${yield* ulid}`); + row = { ...row, threadId }; + yield* sql.withTransaction( + Effect.gen(function* () { + const [active] = + yield* sql`SELECT 1 FROM kin_agents WHERE org_id = ${orgId} AND agent_id = ${agentId} AND retired_at IS NULL`; + if (active === undefined) + return yield* new KinWakeupError({ orgId, agentId, cause: "retired" }); + yield* identity.registerAgentThread({ orgId, agentId, threadId }); + yield* sql`UPDATE kin_leads SET thread_id = ${threadId} WHERE org_id = ${orgId} AND agent_id = ${agentId}`; + yield* sql`UPDATE kin_wakeups SET thread_id = ${threadId}, model = ${modelSelection.model} + WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND agent_id = ${agentId} AND status = 'running'`; + }), + ); + yield* registry.register(threadId, { orgId, agentId, wakeupId, level: current.level }); + const commandId = CommandId.make(`kin-wakeup:${wakeupId}`); + const messageId = MessageId.make(`kin-wakeup-msg:${wakeupId}`); + const sent = + current.threadId === null + ? (yield* launch.launch({ + commandId, + threadId, + projectId: project.id, + title: current.name, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + initialMessage: { messageId, text, attachments: [] }, + createdBy: "system", + creationSource: "server", + })).projection + : (yield* threads.sendToThread({ + commandId, + projectId: project.id, + threadId, + messageId, + text, + attachments: [], + mode: "queue", + freshContext: true, + modelSelection, + createdBy: "system", + creationSource: "server", + })).projection; + const run = sent.runs.findLast((run) => run.userMessageId === messageId); + if (run === undefined) + return yield* new KinWakeupError({ orgId, agentId, cause: "missing_run" }); + row = { ...row, runId: run.id }; + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND agent_id = ${agentId} AND status = 'running'`; + yield* wait(row, project.id); + }).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? cleanup(row, Exit.hasInterrupts(exit)).pipe( + Effect.catch((error) => + Effect.logError("Kin wake-up cleanup failed", { + orgId, + wakeupId, + errorTag: error._tag, + }), + ), + ) + : Effect.void, + ), + ); + }), + ).pipe( + Effect.retry({ + while: (error) => error._tag === "KinEventQueueBusy", + schedule: Schedule.spaced("5 seconds"), + }), + ); + }, + (effect, orgId, agentId) => + effect.pipe( + Effect.mapError( + (cause) => new KinWakeupError({ orgId, agentId, cause: runnerCause(cause) }), + ), + ), + ); + const recoverOrg = Effect.fn("WakeupRunner.recoverOrg")( + function* (orgId: KinOrgId) { + const rows = yield* sql<{ + wakeupId: string; + }>`SELECT org_id AS "orgId", agent_id AS "agentId", wakeup_id AS "wakeupId", thread_id AS "threadId", run_id AS "runId", started_at AS "startedAt" FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running' ORDER BY started_at, wakeup_id`; + for (const raw of rows) { + yield* Effect.scoped( + Effect.gen(function* () { + yield* Effect.acquireRelease(Effect.void, () => + queue + .finish({ orgId, wakeupId: KinWakeupId.make(raw.wakeupId), status: "failed" }) + .pipe(Effect.ignore), + ); + let row = yield* decodeRunning(raw); + yield* Effect.acquireRelease(Effect.void, (_, exit) => + cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore), + ); + const existing = + row.threadId === null + ? [] + : yield* sql`SELECT 1 FROM orchestration_v2_projection_threads WHERE thread_id = ${row.threadId}`; + const projection = + row.threadId === null || existing.length === 0 + ? undefined + : yield* threads.getThreadProjection(row.threadId); + const run = projection?.runs.find((run) => + row.runId === null + ? run.userMessageId === `kin-wakeup-msg:${row.wakeupId}` + : run.id === row.runId, + ); + if (projection === undefined || run === undefined) { + if (projection === undefined) yield* clearMissingThread(row); + yield* queue.requeueUnfinished(row.orgId, row.wakeupId); + return; + } + row = { ...row, runId: run.id }; + yield* sql`UPDATE kin_wakeups SET run_id = ${run.id} WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + const record = yield* lead(row.orgId, row.agentId); + if (record.retiredAt !== null) { + yield* cleanup(row); + return yield* retire(record); + } + yield* identity.registerAgentThread({ + orgId: row.orgId, + agentId: row.agentId, + threadId: projection.thread.id, + }); + yield* registry.register(projection.thread.id, { ...row, level: record.level }); + yield* wait(row, projection.thread.projectId).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? cleanup(row, Exit.hasInterrupts(exit)).pipe(Effect.ignore) + : Effect.void, + ), + ); + }), + ).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin wake-up recovery failed", { + orgId, + wakeupId: raw.wakeupId, + cause, + }), + ), + ); + } + }, + Effect.mapError((cause) => + isWakeupError(cause) + ? cause + : new KinWakeupError({ + orgId: KinOrgId.make("startup"), + agentId: KinAgentId.make("startup"), + cause: runnerCause(cause), + }), + ), + ); + + const orgIds = Effect.fn("WakeupRunner.orgIds")(function* () { + const rows = yield* sql<{ + orgId: string; + }>`SELECT org_id AS "orgId" FROM kin_leads GROUP BY org_id ORDER BY org_id`; + return rows.map((row) => KinOrgId.make(row.orgId)); + }); + const recover = Effect.scoped( + Effect.gen(function* () { + yield* Effect.forEach( + yield* orgIds(), + (orgId) => + recoverOrg(orgId).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin org recovery failed", { orgId, cause }), + ), + ), + { + concurrency: "unbounded", + discard: true, + }, + ); + }), + ).pipe( + Effect.mapError( + (cause) => + new KinWakeupError({ + orgId: KinOrgId.make("startup"), + agentId: KinAgentId.make("startup"), + cause: runnerCause(cause), + }), + ), + ); + const start = Effect.gen(function* () { + const organizations = new Set(); + yield* Effect.gen(function* () { + for (const orgId of yield* orgIds()) { + if (organizations.has(orgId)) continue; + organizations.add(orgId); + yield* Effect.gen(function* () { + const changed = yield* queue.changes(orgId); + yield* recoverOrg(orgId); + const unfinished = + yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${orgId} AND status = 'running'`; + if (unfinished.length > 0) { + yield* Effect.raceFirst(changed, threadActivity(orgId)).pipe( + Effect.timeoutOrElse({ duration: 60_000, orElse: () => Effect.void }), + ); + return; + } + const rows = yield* sql<{ + agentId: string; + }>`SELECT l.agent_id AS "agentId" FROM kin_leads l + JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = ${orgId} AND a.retired_at IS NOT NULL ORDER BY l.agent_id`; + for (const row of rows) yield* retire(yield* lead(orgId, KinAgentId.make(row.agentId))); + const paused = + yield* sql`SELECT 1 FROM kin_org_state WHERE org_id = ${orgId} AND paused = 1`; + const next = paused.length === 0 ? yield* queue.nextReady(orgId) : undefined; + const now = yield* Clock.currentTimeMillis; + if (next !== undefined && next.readyAt <= now) { + yield* runOnce(orgId, next.agentId); + return; + } + yield* Effect.raceFirst(changed, threadActivity(orgId)).pipe( + Effect.timeoutOrElse({ + duration: Math.min(60_000, Math.max(1, (next?.readyAt ?? Infinity) - now)), + orElse: () => Effect.void, + }), + ); + }).pipe( + Effect.catchCause((cause) => + Effect.logError("Kin runner iteration failed", { orgId, cause }).pipe( + Effect.andThen(Effect.sleep("1 second")), + ), + ), + Effect.forever, + Effect.catchCause((cause) => Effect.logError("Kin org loop failed", { orgId, cause })), + Effect.ensuring(Effect.sync(() => organizations.delete(orgId))), + Effect.forkScoped, + ); + } + }).pipe( + Effect.catchCause((cause) => Effect.logError("Kin runner discovery failed", { cause })), + Effect.repeat(Schedule.spaced("60 seconds")), + Effect.forkScoped, + ); + }).pipe(Effect.catchCause((cause) => Effect.logError("Kin runner unavailable", { cause }))); + return WakeupRunner.of({ runOnce, recover, start }); +}); + +export const layer = Layer.effect(WakeupRunner, make); diff --git a/apps/server/src/kin/lead/layer.test.ts b/apps/server/src/kin/lead/layer.test.ts new file mode 100644 index 0000000000000..21c669120637d --- /dev/null +++ b/apps/server/src/kin/lead/layer.test.ts @@ -0,0 +1,109 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { ThreadId } from "@t3tools/contracts"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; +import { vi } from "vite-plus/test"; +import * as VcsProcess from "../../vcs/VcsProcess.ts"; +import * as ServerConfig from "../../config.ts"; +import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinLead from "./layer.ts"; +import { event, harness, seed, orgId } from "./WakeupRunner.testkit.ts"; + +const external = Layer.mergeAll( + ServerConfig.layerTest(process.cwd(), { prefix: "kin-lead-layer-" }), + Layer.mock(VcsProcess.VcsProcess)({}), +).pipe(Layer.provideMerge(NodeServices.layer)); + +vi.mock("../memory/Memory.ts", async (importOriginal) => { + const original = await importOriginal(); + const Effect = await import("effect/Effect"); + const Layer = await import("effect/Layer"); + const DateTime = await import("effect/DateTime"); + return { + ...original, + layer: Layer.sync(original.Memory, () => { + let prepared = false; + return original.Memory.of({ + ensureCheckouts: () => Effect.succeed({ writer: "/private/tmp", read: "/private/tmp" }), + prepareWakeup: () => + Effect.sync(() => { + prepared = true; + return { head: "fixture-head", indexAsOf: DateTime.makeUnsafe(0) }; + }), + readPromptSnapshot: (_orgId, head) => + prepared && head === "fixture-head" + ? Effect.succeed({ rules: "Fixture standing rules", index: "Fixture memory index" }) + : Effect.die("Prompt and runner must share pinned memory"), + readIndex: () => Effect.die("unused"), + readRules: () => Effect.die("unused"), + query: () => Effect.die("unused"), + readItem: () => Effect.die("unused"), + rebuildIndex: () => Effect.die("unused"), + rebuildCache: () => Effect.die("unused"), + }); + }), + }; +}); + +it.effect("starts configured Kin with the real prompt and the runner's pinned memory", () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const hold = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* seed; + yield* (yield* KinEventQueue.KinEventQueue).enqueue(event("real-prompt")); + yield* Layer.build( + Layer.fresh(KinLead.layerServer).pipe( + Layer.provide(external), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + KIN_MEMORY_REPOSITORIES: JSON.stringify([{ orgId, repoUrl: "fixture" }]), + }), + ), + ), + ), + ); + yield* TestClock.adjust("20 seconds"); + yield* Deferred.await(entered); + const sql = yield* SqlClient.SqlClient; + const [wakeup] = yield* sql<{ thread_id: string }>`SELECT thread_id FROM kin_wakeups`; + assert.isDefined(wakeup); + const projection = + yield* (yield* ThreadManagement.ThreadManagementService).getThreadProjection( + ThreadId.make(wakeup!.thread_id), + ); + assert.strictEqual(projection.messages.length, 1); + assert.include(projection.messages[0]!.text, "instructions-version: 1"); + assert.include(projection.messages[0]!.text, "Fixture standing rules"); + assert.include(projection.messages[0]!.text, "Fixture memory index"); + yield* Deferred.succeed(hold, undefined); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + }).pipe(Effect.provide(harness({ entered, hold }))); + }).pipe(provideDeterministicTestRuntime), +); + +it.effect("leaves unconfigured Kin silent and inert without building its dependencies", () => + Effect.gen(function* () { + const entries: Array = []; + yield* Layer.build(KinLead.layerServer).pipe( + Effect.provide( + Layer.mergeAll( + ConfigProvider.layer(ConfigProvider.fromUnknown({})), + Logger.layer([Logger.make((entry) => entries.push(entry))]), + external, + ), + ), + ); + assert.deepStrictEqual(entries, []); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), +); diff --git a/apps/server/src/kin/lead/layer.testkit.ts b/apps/server/src/kin/lead/layer.testkit.ts new file mode 100644 index 0000000000000..f93d64ade578e --- /dev/null +++ b/apps/server/src/kin/lead/layer.testkit.ts @@ -0,0 +1,16 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Layer from "effect/Layer"; +import * as ServerConfig from "../../config.ts"; +import * as ThreadLaunch from "../../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as ProjectService from "../../project/ProjectService.ts"; +import * as VcsProcess from "../../vcs/VcsProcess.ts"; + +// Unconfigured lead runners must not call any of these external services. +export const dormantLeadDependencies = Layer.mergeAll( + Layer.mock(ThreadLaunch.ThreadLaunchService)({}), + Layer.mock(ThreadManagement.ThreadManagementService)({}), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(VcsProcess.VcsProcess)({}), + ServerConfig.layerTest(process.cwd(), { prefix: "kin-dormant-lead-" }), +).pipe(Layer.provideMerge(NodeServices.layer)); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts new file mode 100644 index 0000000000000..7f01a7671b1c7 --- /dev/null +++ b/apps/server/src/kin/lead/layer.ts @@ -0,0 +1,67 @@ +import { forkParked } from "../../serverActivation.ts"; +import { KinOrgId } from "@t3tools/contracts"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import * as KinChatRepository from "../chat/KinChatRepository.ts"; +import * as Memory from "../memory/Memory.ts"; +import * as MemoryConfig from "../memory/MemoryConfig.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; +import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinLeadRegistry from "./KinLeadRegistry.ts"; +import { KinLeadPrompt } from "./LeadPrompt.ts"; +import * as WakeupRunner from "./WakeupRunner.ts"; + +export class KinLeadStartupError extends Schema.TaggedError()( + "KinLeadStartupError", + { + stage: Schema.Literal("startup"), + cause: Schema.optional(Schema.Defect()), + }, +) {} + +export function containStartup(layer: Layer.Layer) { + return Layer.effectDiscard(Effect.void).pipe( + Layer.provide(layer), + Layer.catchCause((cause) => + Layer.effectDiscard( + Effect.logError( + "Kin startup failed; Kin disabled", + new KinLeadStartupError({ stage: "startup", cause }), + ), + ), + ), + ); +} + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const runner = yield* WakeupRunner.WakeupRunner; + yield* runner.start.pipe(forkParked); + }), +).pipe( + Layer.provide(WakeupRunner.layer), + Layer.provide( + Layer.mergeAll(KinLeadRegistry.layer, KinEventQueue.layer, KinLeadPrompt.layer).pipe( + Layer.provideMerge(Layer.mergeAll(Memory.layer, KinChatRepository.layer)), + ), + ), + Layer.provide(KinPersistence.layer), +); + +const Repositories = Schema.fromJsonString( + Schema.Array(Schema.Struct({ orgId: KinOrgId, repoUrl: Schema.String })), +); +export const layerServer = containStartup( + Layer.unwrap( + Effect.gen(function* () { + const repositories = yield* Config.schema(Repositories, "KIN_MEMORY_REPOSITORIES").pipe( + Config.withDefault([]), + ); + if (repositories.length === 0) return Layer.empty; + return layer.pipe(Layer.provide(MemoryConfig.layer(repositories))); + }), + ), +); diff --git a/apps/server/src/kin/mcp/KinChatToolkit.test.ts b/apps/server/src/kin/mcp/KinChatToolkit.test.ts index 1d5d596802df0..450e213f9448c 100644 --- a/apps/server/src/kin/mcp/KinChatToolkit.test.ts +++ b/apps/server/src/kin/mcp/KinChatToolkit.test.ts @@ -1,3 +1,4 @@ +import { authorityLayer, startWakeupRun } from "../chat/WakeupAuthority.testkit.ts"; import * as FileSystem from "effect/FileSystem"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -97,7 +98,7 @@ const liveThreads = Layer.mock(ThreadManagement.ThreadManagementService)({ }); const layer = Layer.merge(registration, KinMcp.registry(fakeRegistry)).pipe( Layer.provideMerge(runtime({ KIN_ENABLED: true })), - Layer.provideMerge(Sqlite.layerMemory), + Layer.provideMerge(authorityLayer), Layer.provideMerge(liveThreads), ); const visible = Effect.gen(function* () { @@ -149,6 +150,11 @@ const seed = Effect.gen(function* () { ); yield* identity.registerAgentThread({ orgId: person.orgId, agentId, threadId }); const chat = yield* chatService.createDirect(person, { orgId: person.orgId, agentId }); + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; + const run = yield* startWakeupRun(threadId, "toolkit-wakeup"); + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, run_id, model, started_at, status, event_ids) + VALUES ('toolkit-wakeup', ${person.orgId}, ${agentId}, ${threadId}, ${run.id}, 'test', 0, 'running', '[]')`; return { identity, chatService, person, chat, sql }; }); @@ -338,7 +344,7 @@ it.effect.each([{}, { KIN_ENABLED: false }, { KIN_ENABLED: "maybe" }])( it.effect("admits only active provider agents at /kin/mcp", () => Effect.scoped( Effect.gen(function* () { - const { identity, person } = yield* seed; + const { identity, person, sql } = yield* seed; const kin = yield* Kin.KinServer; const providerScope = { ...scope, thread }; const authRegistry = Layer.mock(McpSessions.McpSessionRegistry)({ @@ -382,6 +388,9 @@ it.effect("admits only active provider agents at /kin/mcp", () => ); assert.strictEqual((yield* request("missing")).status, 401); assert.strictEqual((yield* request("ordinary")).status, 403); + yield* sql`UPDATE kin_wakeups SET status = 'completed' WHERE wakeup_id = 'toolkit-wakeup'`; + assert.strictEqual((yield* request("agent")).status, 403); + yield* sql`UPDATE kin_wakeups SET status = 'running' WHERE wakeup_id = 'toolkit-wakeup'`; assert.strictEqual((yield* request("agent")).status, 200); yield* identity.endAgentThread({ orgId: person.orgId, threadId }); assert.strictEqual((yield* request("agent")).status, 403); diff --git a/apps/server/src/kin/mcp/layer.test.ts b/apps/server/src/kin/mcp/layer.test.ts index 44aab4ded732c..8a344c93809bf 100644 --- a/apps/server/src/kin/mcp/layer.test.ts +++ b/apps/server/src/kin/mcp/layer.test.ts @@ -1,3 +1,5 @@ +import { authorityLayerWithDatabase, startWakeupRun } from "../chat/WakeupAuthority.testkit.ts"; +import * as KinPersistence from "../persistence/KinPersistence.ts"; import { McpProviderSessions } from "@t3tools/provider-core/server/McpProviderSessions"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -7,6 +9,7 @@ import { AuthStandardClientScopes, EnvironmentId, KinAgentId, + KinOrgId, ProviderInstanceId, ThreadId, } from "@t3tools/contracts"; @@ -31,7 +34,9 @@ import * as NetAddress from "effect/net/NetAddress"; import { liveThreadShell } from "../../mcp/McpToolAccess.testkit.ts"; import * as McpSessions from "../../mcp/McpSessionRegistry.ts"; import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as VcsProcess from "../../vcs/VcsProcess.ts"; import * as Kin from "../layer.ts"; +import { ServerActivation } from "../../serverActivation.ts"; import { KinIdentity } from "../chat/KinIdentity.ts"; import { KinChatService } from "../chat/KinChatService.ts"; import { AssetSigner } from "../../assets/AssetSigner.ts"; @@ -66,7 +71,9 @@ import { GitVcsDriver } from "../../vcs/GitVcsDriver.ts"; import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; import { DocumentText } from "../../provider/DocumentText.ts"; import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; +const serverConfig = ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-context-budget-" }); const toolHandlerServices = Layer.mergeAll( + VcsProcess.layer, Layer.mock(AssetSigner)({}), Layer.mock(DeviceService)({}), Layer.mock(DocumentText)({}), @@ -90,7 +97,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(RemoteDelegation)({}), Layer.mock(ScheduledTaskService)({}), Layer.mock(SecretRequests)({}), - ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-context-budget-" }), + serverConfig, Layer.mock(ServerEnvironment)({ getEnvironmentId: Effect.succeed(EnvironmentId.make("environment")), }), @@ -110,29 +117,58 @@ it.effect.each([true, false])( (enabled) => Effect.scoped( Effect.gen(function* () { + let runnerStarts = 0; const ready = yield* Deferred.make< - Context.Context + Context.Context< + Kin.KinServer | McpSessions.McpSessionRegistry | SqlClient.SqlClient | OrchestratorV2 + > >(); const routes = Layer.merge( Kin.mcpLayer(McpHttpServer.layer), Layer.effectDiscard( Effect.context< - Kin.KinServer | McpSessions.McpSessionRegistry | SqlClient.SqlClient + Kin.KinServer | McpSessions.McpSessionRegistry | SqlClient.SqlClient | OrchestratorV2 >().pipe(Effect.flatMap((context) => Deferred.succeed(ready, context))), ), ).pipe( Layer.provideMerge(Kin.mcpRegistryLayer(McpSessions.layer)), - Layer.provideMerge(Sqlite.layerConfig), + Layer.provideMerge( + authorityLayerWithDatabase( + KinPersistence.layer.pipe( + Layer.provideMerge(Sqlite.layerConfig.pipe(Layer.orDie)), + Layer.provide(serverConfig), + Layer.provide(NodeServices.layer), + ), + ), + ), Layer.provide(toolHandlerServices), Layer.provide( Layer.mock(HttpServer.HttpServer)({ address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 43123), }), ), - Layer.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({ KIN_ENABLED: enabled }))), + Layer.provide( + Layer.succeed( + ServerActivation, + Effect.sync(() => { + runnerStarts++; + }).pipe(Effect.andThen(Effect.never)), + ), + ), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + KIN_ENABLED: enabled, + KIN_MEMORY_REPOSITORIES: JSON.stringify([ + { orgId: KinOrgId.make("fixture"), repoUrl: "fixture" }, + ]), + }), + ), + ), ); const http = yield* HttpRouter.toHttpEffect(routes); + assert.strictEqual(runnerStarts, enabled ? 1 : 0); const services = yield* Deferred.await(ready); const kin = Context.get(services, Kin.KinServer); const registry = Context.get(services, McpSessions.McpSessionRegistry); @@ -216,6 +252,18 @@ it.effect.each([true, false])( providerInstanceId: ProviderInstanceId.make("codex"), }); assert.strictEqual(issued.config.endpoint, "http://127.0.0.1:43123/kin/mcp"); + assert.strictEqual( + (yield* request("/kin/mcp", issued.config.authorizationHeader, "initialize", initialize)) + .status, + 403, + ); + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) + VALUES (${agentId}, ${person.orgId}, 'Lead', 0, '{}', '[]', ${chat.chatId})`; + const run = yield* startWakeupRun(threadId, "composition-wakeup").pipe( + Effect.provide(services), + ); + yield* sql`INSERT INTO kin_wakeups (wakeup_id, org_id, agent_id, thread_id, run_id, model, started_at, status, event_ids) + VALUES ('composition-wakeup', ${person.orgId}, ${agentId}, ${threadId}, ${run.id}, 'test', 0, 'running', '[]')`; const initialized = yield* request( "/kin/mcp", issued.config.authorizationHeader, diff --git a/apps/server/src/kin/memory/Memory.test.ts b/apps/server/src/kin/memory/Memory.test.ts index de52fda44e423..fb48bf9091937 100644 --- a/apps/server/src/kin/memory/Memory.test.ts +++ b/apps/server/src/kin/memory/Memory.test.ts @@ -45,6 +45,7 @@ const fakeRepository = () => { } return sha; }; + let validations = 0; let treeReads = 0; let blobReads = 0; let revision = 0; @@ -63,6 +64,7 @@ const fakeRepository = () => { : `fixture-head-${revision}`; break; case "remote": + validations++; stdout = "unused"; break; case "ls-tree": { @@ -118,7 +120,7 @@ const fakeRepository = () => { commit: () => { revision++; }, - counts: () => ({ treeReads, blobReads }), + counts: () => ({ treeReads, blobReads, validations }), corruptBatch: () => { corruptNextBatch = true; }, diff --git a/apps/server/src/kin/memory/Memory.ts b/apps/server/src/kin/memory/Memory.ts index 284fee200fb22..fdb05b0ab5aaa 100644 --- a/apps/server/src/kin/memory/Memory.ts +++ b/apps/server/src/kin/memory/Memory.ts @@ -176,6 +176,7 @@ const make = Effect.gen(function* () { const cache = yield* MemoryCache.MemoryCache; const locks = new Map(); const pinnedHeads = new Map(); + const initializedCheckouts = new Set(); const duplicateWarnings = new Map>(); const snapshots = new Map(); const blobCaches = new Map(); @@ -339,8 +340,13 @@ const make = Effect.gen(function* () { ); }); - const ensureCheckouts = Effect.fn("Memory.ensureCheckouts")(function* (orgId: KinOrgId) { + const ensureCheckouts = Effect.fn("Memory.ensureCheckouts")(function* ( + orgId: KinOrgId, + reusePin = false, + ) { const paths = yield* pathsFor(orgId); + if (reusePin && initializedCheckouts.has(orgId) && pinnedHeads.has(orgId)) + return { writer: paths.writer, read: paths.read }; yield* io(orgId, "mkdir", fs.makeDirectory(paths.root, { recursive: true })); const gitDirectory = path.join(paths.writer, ".git"); if (yield* io(orgId, "exists", fs.exists(paths.writer))) { @@ -396,14 +402,15 @@ const make = Effect.gen(function* () { yield* git(orgId, paths.writer, ["worktree", "prune"]); yield* git(orgId, paths.writer, ["worktree", "add", "--detach", paths.read, "HEAD"]); } + initializedCheckouts.add(orgId); return { writer: paths.writer, read: paths.read }; }); const load = Effect.fn("Memory.load")(function* (orgId: KinOrgId, force = false) { const paths = yield* pathsFor(orgId); - yield* validateWriter(orgId, paths); const head = pinnedHeads.get(orgId) ?? (yield* git(orgId, paths.writer, ["rev-parse", "HEAD"])).trim(); + if (pinnedHeads.get(orgId) !== head) yield* validateWriter(orgId, paths); pinnedHeads.set(orgId, head); const previous = snapshots.get(orgId); if (!force && previous?.head === head) return previous; @@ -790,7 +797,7 @@ const make = Effect.gen(function* () { return Memory.of({ ensureCheckouts: Effect.fn("Memory.ensureCheckoutsLocked")((orgId) => - locked(orgId, ensureCheckouts(orgId)), + locked(orgId, ensureCheckouts(orgId, true)), ), prepareWakeup: Effect.fn("Memory.prepareWakeup")((orgId) => locked(orgId, prepare(orgId))), readPromptSnapshot: Effect.fn("Memory.readPromptSnapshot")((orgId, expectedHead) => diff --git a/apps/server/src/kin/triggers/KinTriggerSink.test.ts b/apps/server/src/kin/triggers/KinTriggerSink.test.ts index 3bfd4f82e3dfe..38ec337580ceb 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.test.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.test.ts @@ -11,6 +11,9 @@ import { KinWebhookSource, OrchestrationV2ThreadProjectionJson, ScheduledTaskUpsertInput, + ProjectId, + ProviderInstanceId, + ScheduledTaskId, } from "@t3tools/contracts"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Deferred from "effect/Deferred"; @@ -488,6 +491,14 @@ describe("KinTriggerSink", () => { Effect.gen(function* () { yield* service.upsert(yield* upsert("task-org-a")); const a = source(orgA); + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_agent_threads (org_id, agent_id, thread_id, started_at) + VALUES (${orgA}, ${`agent-${orgA}`}, ${a.inboxThreadId}, ${timestamp})`; + assert.strictEqual( + (yield* store.registerSource(a).pipe(Effect.flip))._tag, + "KinWebhookSourceConflict", + ); + yield* sql`DELETE FROM kin_agent_threads WHERE thread_id = ${a.inboxThreadId}`; yield* store.registerSource(a); for (const candidate of [ { ...a, orgId: orgB }, @@ -554,3 +565,129 @@ describe("KinTriggerSink", () => { }).pipe(Effect.provide(services.pipe(Layer.provideMerge(database)))), ); }); + +it.effect("builds the runtime after a Kin migration failure and refuses only webhook turns", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_orgs (broken TEXT)`; + for (const [id, type] of [ + ["task", "webhook"], + ["interval", "interval"], + ]) { + yield* sql`INSERT INTO scheduled_tasks (task_id, title, prompt, enabled, schedule_json, project_id, + workspace_strategy_json, model_selection_json, runtime_mode, interaction_mode, created_by, + creation_source, created_at, updated_at, last_run_status, run_count) + VALUES (${id}, 'Task', 'Prompt', 1, ${JSON.stringify({ type })}, 'project', '{}', '{}', + 'full-access', 'default', 'system', 'server', ${timestamp}, ${timestamp}, 'never', 0)`; + } + const sends: string[] = []; + const managementLayer = Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Effect.gen(function* () { + sends.push(input.scheduledTaskId ?? "ordinary"); + return yield* new OrchestratorCommandRejectedError({ + commandId: input.commandId, + commandType: "test.inner_send", + }); + }), + }); + const launches: ThreadLaunchService.ThreadLaunchInput[] = []; + const inner = Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + launch: (input) => + Effect.sync(() => { + launches.push(input); + return { + threadId: projection(orgA).thread.id, + projection: projection(orgA), + resumed: false, + }; + }), + }); + yield* Effect.gen(function* () { + const launch = yield* ThreadLaunchService.ThreadLaunchService; + const input: ThreadLaunchService.ThreadLaunchInput = { + commandId: CommandId.make("ordinary"), + projectId: ProjectId.make("project"), + title: "Ordinary", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "test" }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + createdBy: "system", + creationSource: "server", + }; + yield* launch.launch(input); + yield* launch.launch({ + ...input, + commandId: CommandId.make("scheduled-task:interval:interval:1"), + initialMessage: { + text: "Interval", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("interval"), + }, + }); + const blocked = yield* launch + .launch({ + ...input, + commandId: CommandId.make("scheduled-task:task:webhook:delivery"), + initialMessage: { + text: "Scheduled", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("task"), + }, + }) + .pipe(Effect.flip); + assert.strictEqual(blocked._tag, "ThreadLaunchError"); + const management = yield* ThreadManagementService.ThreadManagementService; + const passed = yield* management + .sendToThread({ + projectId: input.projectId, + commandId: input.commandId, + threadId: projection(orgA).thread.id, + messageId: MessageId.make("interval-message"), + text: "Interval", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("interval"), + mode: "queue", + createdBy: "system", + creationSource: "server", + }) + .pipe(Effect.flip); + assert.include(passed, { commandType: "test.inner_send" }); + assert.deepStrictEqual(sends, ["interval"]); + const bound = yield* management + .sendToThread({ + projectId: input.projectId, + commandId: input.commandId, + threadId: projection(orgA).thread.id, + messageId: MessageId.make("message"), + text: "Bound", + attachments: [], + scheduledTaskId: ScheduledTaskId.make("task"), + mode: "queue", + createdBy: "system", + creationSource: "server", + }) + .pipe(Effect.flip); + assert.strictEqual(bound._tag, "OrchestratorCommandRejectedError"); + assert.strictEqual(launches.length, 2); + }).pipe( + Effect.provide( + KinTriggerSink.wrapTaskDispatch(inner, managementLayer).pipe( + Layer.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({ KIN_ENABLED: true }))), + ), + ), + ); + }).pipe(Effect.provide(Sqlite.layerMemory)), +); + +it.effect("does not relabel a core dispatch layer failure as a Kin startup failure", () => + Effect.gen(function* () { + const failure = new Error("core dispatch unavailable"); + const inner = Layer.effect(ThreadLaunchService.ThreadLaunchService, Effect.fail(failure)); + const result = yield* Layer.build(KinTriggerSink.wrapTaskDispatch(inner, threads)).pipe( + Effect.flip, + ); + assert.strictEqual(result, failure); + }).pipe(Effect.provide(Sqlite.layerMemory)), +); diff --git a/apps/server/src/kin/triggers/KinTriggerSink.ts b/apps/server/src/kin/triggers/KinTriggerSink.ts index 2e91c2d0e7c25..29a8f99a2b32b 100644 --- a/apps/server/src/kin/triggers/KinTriggerSink.ts +++ b/apps/server/src/kin/triggers/KinTriggerSink.ts @@ -181,8 +181,63 @@ export function wrapTaskDispatch( }), ), ).pipe( + Layer.provide(Layer.mergeAll(layer, KinTriggerStore.layer)), + Layer.catchCause((cause) => + Layer.mergeAll( + Layer.effectDiscard( + KinEnabled.KinEnabled.pipe(Effect.flatMap((config) => config.reportFailure(cause))), + ), + Layer.effect( + ThreadLaunchService.ThreadLaunchService, + Effect.gen(function* () { + const service = yield* ThreadLaunchService.ThreadLaunchService; + return ThreadLaunchService.ThreadLaunchService.of({ + ...service, + launch: Effect.fn("KinTriggerSink.disabledLaunch")(function* (input) { + const taskId = input.initialMessage?.scheduledTaskId; + if ( + taskId === undefined || + !input.commandId.startsWith(`scheduled-task:${taskId}:webhook:`) + ) + return yield* service.launch(input); + return yield* new ThreadLaunchService.ThreadLaunchError({ + operation: "dispatch-message", + commandId: input.commandId, + projectId: input.projectId, + cause: new Error("Kin is unavailable; webhook dispatch refused.", { cause }), + }); + }), + }); + }), + ), + Layer.effect( + ThreadManagementService.ThreadManagementService, + Effect.gen(function* () { + const service = yield* ThreadManagementService.ThreadManagementService; + const sql = yield* SqlClient.SqlClient; + return ThreadManagementService.ThreadManagementService.of({ + ...service, + sendToThread: Effect.fn("KinTriggerSink.disabledSend")(function* (input) { + if (input.scheduledTaskId === undefined) return yield* service.sendToThread(input); + const webhook = + yield* sql`SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.scheduledTaskId} + AND json_extract(schedule_json, '$.type') = 'webhook'`.pipe( + Effect.map((rows) => rows.length > 0), + Effect.catchCause(() => Effect.succeed(true)), + ); + if (!webhook) return yield* service.sendToThread(input); + return yield* new OrchestratorCommandRejectedError({ + commandId: input.commandId, + commandType: "kin.trigger.disabled", + cause, + }); + }), + }); + }), + ), + ), + ), Layer.provide(Layer.mergeAll(inner, threads)), - Layer.provide(Layer.mergeAll(layer, KinTriggerStore.layer).pipe(Layer.provide(threads))), ); const degraded = Layer.mergeAll( Layer.effect( diff --git a/apps/server/src/kin/triggers/KinTriggerStore.ts b/apps/server/src/kin/triggers/KinTriggerStore.ts index c8c57f0b5c3cc..efa4c8be7028a 100644 --- a/apps/server/src/kin/triggers/KinTriggerStore.ts +++ b/apps/server/src/kin/triggers/KinTriggerStore.ts @@ -195,6 +195,7 @@ const make = Effect.gen(function* () { AND t.thread_id IS NULL AND t.thread_mode = 'new_thread' AND json_extract(t.schedule_json, '$.type') = 'webhook' AND EXISTS (SELECT 1 FROM kin_orgs WHERE org_id = ${value.orgId}) + AND NOT EXISTS (SELECT 1 FROM kin_agent_threads WHERE thread_id = ${value.inboxThreadId}) AND NOT EXISTS ( SELECT 1 FROM kin_webhook_sources WHERE org_id <> ${value.orgId} diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index 742616f27bd3e..f7c2bb91ad0a9 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -198,75 +198,78 @@ it.effect( }, ); -it.effect("waitForThread follows a restart continuation instead of the cut run", () => - Effect.gen(function* () { - const projectId = ProjectId.make("project:thread-management:wait-restart"); - const threadId = ThreadId.make("thread:thread-management:wait-restart"); - const runId = RunId.make("run:thread-management:wait-restart"); - const continuationId = RunId.make("run:thread-management:wait-restart:2"); - const subscribed = yield* Deferred.make(); - const events = yield* Queue.unbounded(); - let runs: Array<{ - id: RunId; - status: OrchestrationV2Run["status"]; - restartContinuationOfRunId?: RunId; - }> = [{ id: runId, status: "cancelled" }]; - let pending = true; - const stored = (sequence: number, event: object) => - ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; - const layerTest = ThreadManagementService.layer.pipe( - Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), - Layer.provide( - Layer.mock(Orchestrator.OrchestratorV2)({ - getThreadEventSequence: () => Effect.succeed(0), - restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId), - awaitRestartContinuation: () => Effect.never, - streamStoredEventsFrom: (input) => - input?.eventType === "run.updated" - ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( - Stream.drain, - Stream.concat(Stream.fromQueue(events)), - ) - : Stream.never, - getThreadRecords: () => - Effect.sync( - () => - ({ - thread: { id: threadId, projectId, deletedAt: null }, - runs, - }) as unknown as OrchestrationV2ThreadProjection, - ), - }), - ), - Layer.provide(ServerSettings.layerTest()), - Layer.provide(layerReceiptsTest), - ); - const service = yield* ThreadManagementService.ThreadManagementService.pipe( - Effect.provide(layerTest), - ); - const fiber = yield* service - .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) - .pipe(Effect.forkChild); - yield* Deferred.await(subscribed); - const continuation = { id: continuationId, restartContinuationOfRunId: runId }; - runs = [...runs, { ...continuation, status: "running" }]; - pending = false; - yield* Queue.offer( - events, - stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }), - ); - runs = [runs[0]!, { ...continuation, status: "completed" }]; - yield* Queue.offer( - events, - stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }), - ); - const result = yield* Fiber.join(fiber); +it.effect.each(["restartContinuationOfRunId", "usageLimitContinuationOfRunId"] as const)( + "waitForThread follows %s instead of the cut run", + (continuationField) => + Effect.gen(function* () { + const projectId = ProjectId.make("project:thread-management:wait-restart"); + const threadId = ThreadId.make("thread:thread-management:wait-restart"); + const runId = RunId.make("run:thread-management:wait-restart"); + const continuationId = RunId.make("run:thread-management:wait-restart:2"); + const subscribed = yield* Deferred.make(); + const events = yield* Queue.unbounded(); + let runs: Array<{ + id: RunId; + status: OrchestrationV2Run["status"]; + restartContinuationOfRunId?: RunId; + usageLimitContinuationOfRunId?: RunId; + }> = [{ id: runId, status: "cancelled" }]; + let pending = true; + const stored = (sequence: number, event: object) => + ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadEventSequence: () => Effect.succeed(0), + restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId), + awaitRestartContinuation: () => Effect.never, + streamStoredEventsFrom: (input) => + input?.eventType === "run.updated" + ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.fromQueue(events)), + ) + : Stream.never, + getThreadRecords: () => + Effect.sync( + () => + ({ + thread: { id: threadId, projectId, deletedAt: null }, + runs, + }) as unknown as OrchestrationV2ThreadProjection, + ), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + const service = yield* ThreadManagementService.ThreadManagementService.pipe( + Effect.provide(layerTest), + ); + const fiber = yield* service + .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) + .pipe(Effect.forkChild); + yield* Deferred.await(subscribed); + const continuation = { id: continuationId, [continuationField]: runId }; + runs = [...runs, { ...continuation, status: "running" }]; + pending = false; + yield* Queue.offer( + events, + stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }), + ); + runs = [runs[0]!, { ...continuation, status: "completed" }]; + yield* Queue.offer( + events, + stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }), + ); + const result = yield* Fiber.join(fiber); - expect(result).toMatchObject({ - timedOut: false, - run: { id: continuationId, status: "completed" }, - }); - }), + expect(result).toMatchObject({ + timedOut: false, + run: { id: continuationId, status: "completed" }, + }); + }), ); it.effect("waitForThread can wait past an ended run for the thread's delegated work", () => diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 46ea454310554..d7ce5554b4605 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -503,13 +503,9 @@ export class ThreadManagementService extends Context.Service< ThreadManagementServiceShape >()("t3/orchestration-v2/ThreadManagementService") {} +export const ACTIVE_RUN_STATUSES = ["preparing", "starting", "running", "waiting"] as const; export function isActiveRun(run: OrchestrationV2Run): boolean { - return ( - run.status === "preparing" || - run.status === "starting" || - run.status === "running" || - run.status === "waiting" - ); + return ACTIVE_RUN_STATUSES.some((status) => status === run.status); } export function isTerminalRunStatus( @@ -530,13 +526,17 @@ export function latestRun( return projection.runs.toSorted((left, right) => right.ordinal - left.ordinal)[0]; } -function latestRestartContinuation( +function latestContinuation( runs: ReadonlyArray, run: OrchestrationV2Run, ): OrchestrationV2Run { let current = run; for (let hops = 0; hops < runs.length; hops++) { - const next = runs.find((candidate) => candidate.restartContinuationOfRunId === current.id); + const next = runs.find( + (candidate) => + candidate.restartContinuationOfRunId === current.id || + candidate.usageLimitContinuationOfRunId === current.id, + ); if (next === undefined) break; current = next; } @@ -930,7 +930,7 @@ const make = Effect.gen(function* () { if (selectedRun === undefined) { return { threadId: input.threadId, run: null, timedOut: false }; } - // A restart cuts a run and continues it as a new run. Follow that chain, + // Restarts and usage-limit recovery continue as new runs. Follow both chains, // so a cut run is not the answer while its continuation is pending. // Runs per matching event and is already covered by the waitForThread span. const outcome = Effect.fnUntraced(function* (runs: ReadonlyArray) { @@ -941,7 +941,7 @@ const make = Effect.gen(function* () { runId: selectedRun.id, }); } - const run = latestRestartContinuation(runs, current); + const run = latestContinuation(runs, current); const pending = isTerminalRunStatus(run.status) && (yield* orchestrator.restartContinuationPending(run)); return { run, pending, settled: isTerminalRunStatus(run.status) && !pending }; @@ -1015,7 +1015,8 @@ const make = Effect.gen(function* () { untilDelegatedWorkSettled || stored.event.type !== "run.updated" || stored.event.payload.id === selectedRun.id || - stored.event.payload.restartContinuationOfRunId !== undefined, + stored.event.payload.restartContinuationOfRunId !== undefined || + stored.event.payload.usageLimitContinuationOfRunId !== undefined, ), ), initial.pending diff --git a/docs/kin/specs/chat.md b/docs/kin/specs/chat.md index bfd1b19054a99..5cc4550881f78 100644 --- a/docs/kin/specs/chat.md +++ b/docs/kin/specs/chat.md @@ -48,7 +48,7 @@ Every row carries `orgId`. Phase 1 has one org, created on first start. Bootstra - **Session to person** (`KinIdentity.resolve(session)`): - Phase 1: after the RPC scope check, a `/ws` session holding every scope in `AuthStandardClientScopes` resolves to the default org owner. A narrower session resolves to an empty list, never to a person inferred from `subject`. The result is a list of `{ orgId, personId }` entries so phase-2 bindings do not change callers. Only the owner may pair devices before CHAT-11. - Phase 2: a row in `kin_session_bindings (session_id, org_id, person_id)` decides. A session without a binding is the owner only if it holds `AuthStandardClientScopes`. Members get sessions with the new `kin:chat` scope only (CHAT-11), so their device never gets the owner's terminal or files. -- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead session's thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` when the session ends; an ended thread cannot be registered again. Retiring an agent ends its threads through `KinIdentity`; active-thread discovery also excludes retired agents. +- **Agent thread to agent** (`KinIdentity.resolveAgent(mcpScope)`): the lead spec registers each lead's durable thread with `registerAgentThread({ orgId, agentId, threadId })`. Kin MCP tools look up `McpInvocationScope.thread.threadId`. An absent caller thread, no registry row, an ended row, or `subagent: true` means `not_allowed`. Resolution also requires a persisted `kin_wakeups` row with `status = running` and the same org, agent and thread. The thread's active run must belong to the wake-up, including restart and usage-limit continuations. Durable registration alone grants no authority between wake-ups; `insertMessage` rechecks the running row inside its write transaction. Resolution reads the database, not the tool-visibility cache, and derives `{ orgId, agentId, ownerPersonId, threadId }` only from stored rows. Retired agents are refused with `not_allowed` by both resolution and registration. Workers never post; they report to their lead. The lead calls `endAgentThread({ orgId, threadId })` on retirement, not between fresh provider sessions on the same T3 thread; an ended thread cannot be registered again. Retiring an agent ends its threads through `KinIdentity`; active-thread discovery also excludes retired agents. ### Chats, members, roles @@ -483,7 +483,7 @@ Upstream files touched, hooks only: ## Interfaces to other specs -- **Lead spec:** calls `KinIdentity.registerAgentThread` / `endAgentThread` around each lead session. Subscribes to `KinChatHub` for `message.posted` and `widget_action` to build `KinEvent`s. Uses the Kin MCP tools to post. Owns `KinAgent` config beyond identity, the `deferral` card state, levels per chat (its own table keyed by `chatId`), and the brief builder that quotes non-owner text. Receives `floor.offer` wake requests from the floor manager. +- **Lead spec:** calls `KinIdentity.registerAgentThread` before the first lead session and `endAgentThread` on retirement. Subscribes to `KinChatHub` for `message.posted` and `widget_action` to build `KinEvent`s. Uses the Kin MCP tools to post. Owns `KinAgent` config beyond identity, the `deferral` card state, levels per chat (its own table keyed by `chatId`), and the brief builder that quotes non-owner text. Receives `floor.offer` wake requests from the floor manager. - **Approvals spec:** creates `approval` widgets server-side from `KinPendingAction` and creates approval waiting rows through the server-only repository `addApprovalWaiting` in its transaction, never the public chat service or a transport. It resolves rows through its signed-grant flow; agents cannot create or resolve them. Reads `KinMessage.author` and `authorRole` as provenance for ceilings. Its signed-grant RPC is the only way to approve; `kin.widgets.act` refuses approvals. Shows "Waiting on Nishchay" to members from `KinWidget.awaiting`. - **Triggers spec:** deferral and routine posts arrive as lead notices; nothing here. Its upstream contract tests should cover what this spec depends on: `Migrator` `table` option, `WsRpcGroup` and `RPC_REQUIRED_SCOPES` shapes, `layerWsRpc` session argument, `McpInvocationScope.thread`, `toolkitRegistration`, `getThreadShell` and `streamDomainEvents`, `OrchestratorMcpDelegateTaskResult`. - **Mobile spec:** consumes `kin.*` RPCs and both streams through the normal WS client; draws native cards from `KinWidget.type` and `state`; renders generated widgets with `ThreadMcpApp`; opens task threads with `formatThreadLink`; folds runs by `floorRunId`; avatars from `KinAuthor._tag` (rule 7). diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index 93abf960e157c..762499473c67d 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -32,7 +32,7 @@ Only one wake-up runs per organization at a time. Every wake-up counts against a | Test harness | `DeterministicRuntime`, `OrchestratorScenario`, replay fixtures | `apps/server/src/orchestration-v2/testkit/` | | Advertised tool budget | Measures every tool in `layerTools` | `apps/server/src/provider/AgentContextBudget.test.ts` | -`apps/server/src/kin/` does not exist yet. +Kin services live in `apps/server/src/kin/`. ## Design @@ -177,11 +177,13 @@ Coalescing: - It then claims every queued event for that lead whose `not_before` has passed, ordered by `priority, received_at`, in one transaction, and starts one wake-up for the batch. The claim index is `(org_id, agent_id, state, priority, received_at, not_before)`. Twenty webhook events from one push become one wake-up. - A batch holds at most 50 events. Overflow stays queued for the next batch; no delivery references are dropped or marked handled. Same-repo summaries belong to the prompt builder, not the durable queue. - Events that arrive while a wake-up runs wait for the next one. The runner never starts a second session for the same organization. This is the one-writer rule: memory is written only inside a wake-up, and wake-ups are serial. -- Implementation: an organization permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 must call `finish(timed_out | failed)` in its own `onExit`. Idle semaphore entries are evicted. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. +- Implementation: an organization permit is held for the whole wake-up, but never while waiting for that lead’s quiet window. `claim({ orgId, agentId, wakeupId, model })` waits interruptibly, takes the permit, rechecks readiness in the transaction and atomically claims up to 50 events, recording their ordered ids without incrementing `attempts`. A stale claim rolls back and waits again; an empty wake-up is never committed. The caller keeps its Effect scope open through the provider run and terminal database write; scope closure releases only the permit. LEAD-5 calls `finish(timed_out | failed)` for real failures and timeouts; graceful scope interruption uses `requeueUnfinished` without consuming an attempt. Organization signal entries stay available for idle runner subscriptions. Cross-process exclusion rests on SQLite’s running-wake-up unique index; a loser receives `KinEventQueueBusy`. -The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval`, so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. +The runner owns stop/budget admission and provider execution; all event transitions go through the queue. `release({ orgId, wakeupId, eventIds?, notBefore?, countAttempt })` returns the selected events (all by default) to the queue. Pause and budget deferrals use `countAttempt: false`, which requires `notBefore`; LEAD-11 pause handling must supply `notBefore = now + pause poll interval`, so a paused organization waits instead of repeatedly claiming and releasing. Once all claimed events are released, the wake-up closes as `released` (or `failed` for a counted failure). A partial release leaves it `running` for the admitted events. `finish({ orgId, wakeupId, status })` closes the wake-up as `completed`, `failed` or `timed_out`. Completion marks the remaining events `done`; failure increments attempts and requeues them for five minutes, marking them `failed` at three attempts. `release` with `countAttempt: true` applies the same failure limit and defaults to the same five-minute delay. `release` returns `{ released, failedEventIds }` and `finish` returns `{ failedEventIds }`; the ids are the events that transitioned to terminal `failed` in that call, ordered by id. Unknown or foreign lead ids fail with `KinEventQueueUnknownAgent`, and reused wake-up ids fail with `KinEventQueueWakeupConflict`, not SQL constraint errors. Enqueue validates the lead and computes owner priority inside the insert transaction. Identical enqueue retries return `{ event, duplicate: true }` with the stored event; conflicting retries fail with `KinEventQueueConflict`. -Restart: the runner first reconciles each unfinished wake-up with its provider run (see failure cases). It must not requeue events belonging to a still-active run. For an abandoned run only, `requeueUnfinished(orgId, wakeupId)` atomically marks the running wake-up interrupted and returns its claimed events to `queued`, clearing their `wakeup_id` while preserving `not_before` and `attempts`. It is idempotent and organization-scoped; `idx_kin_events_wakeup` supports the update. Completed runs are finished by the runner instead. Layer construction never blindly requeues active work. +Ownership: T3's `serverOwnership.ts` takes `server-owner.sqlite` before opening the application database, guaranteeing one server per T3 home. Kin relies on that ownership and has no sidecar lock. + +Restart: recovery runs independently in each organization loop and logs failures per row. The runner first reconciles each unfinished wake-up with its provider run (see failure cases). It must not requeue events belonging to a still-active run. For an abandoned run only, `requeueUnfinished(orgId, wakeupId)` atomically marks the running wake-up interrupted and returns its claimed events to `queued`, clearing their `wakeup_id` while preserving `not_before` and `attempts`. It is idempotent and organization-scoped; `idx_kin_events_wakeup` supports the update. Completed runs are finished by the runner instead. Layer construction never blindly requeues active work. Recovery also checks the deterministic first-message id if launch committed before `run_id` was saved. A provisional lead thread with no upstream thread is ended and cleared before requeue, so the next attempt launches rather than sending to a nonexistent thread. ### A wake-up (one fresh lead session) @@ -197,18 +199,28 @@ Setup once per organization (`Memory.ensureCheckouts` owns the checkouts; the wa Each wake-up: -1. `KinStop.isPaused(orgId)` → leave events queued and return. +1. `KinStop.isPaused(orgId)` → leave events queued and return. Before claiming on an existing thread, defer while any provider turn remains unfinished. 2. `KinBudget.admit(orgId, batchPriority)` → if refused, leave lower-priority events `queued` with `not_before` = next budget reset and run what remains. 3. Pick the model: `models.strong` if the batch holds a P0 or P1 event, a contradiction question or a `routine`; otherwise `models.cheap`. 4. `Memory.prepareWakeup(orgId)`: fetch then `git pull --rebase` in the writer checkout (preserving unpushed commits), move the read checkout to the new HEAD (`checkout --force --detach`, `reset --hard`, `clean -fdx`), and rebuild `INDEX.md` first if its "As of" line is older than 36 hours. If fetching fails, log a warning and pin the local writer HEAD for this wake-up. If rebasing conflicts, abort the rebase to restore the writer and fail with `MemorySyncConflict`. 5. Build the prompt (below). -6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every run is a new provider session. -7. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, an in-memory map the tools use. It is filled before the send and cleared after the run ends. -8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread` and mark `timed_out`. +6. First wake-up: `ThreadLaunchService.launch({ commandId: "kin-wakeup:", projectId, title: "", modelSelection, runtimeMode: "full-access", interactionMode: "default", workspaceStrategy: { type: "root" }, initialMessage: { text: prompt, attachments: [] }, createdBy: "system", creationSource: "server" })`, then store `threadId` on `KinLead`. Later wake-ups: `ThreadManagementService.sendToThread({ commandId, threadId, messageId: "kin-wakeup-msg:", text: prompt, mode: "queue", freshContext: true, modelSelection, createdBy: "system", creationSource: "server" })`. One thread holds the lead's history for people to read; every wake-up starts a fresh provider process with its cwd set to the read clone, as LEAD-3 requires. +7. Register the durable lead thread through `KinIdentity.registerAgentThread` before launching or sending; keep it registered between wake-ups and call `endAgentThread` on retirement. `KinIdentity.resolveAgent` and the `insertMessage` write transaction additionally require a persisted running wake-up matching the org, agent and server-resolved caller thread, whose active run belongs to the wake-up (including restart and usage-limit continuations), whose root is exactly `kin_wakeups.run_id` when recorded, otherwise the lowest-ordinal run on that thread with `kin-wakeup-msg:`. Its persisted message must belong to that root and have `createdBy=system` and `creationSource=server`, and every continuation's message must have `createdBy=agent` and `creationSource=server`. Other paths can produce that provenance (server notification wakes, and client-written scheduled tasks, whose upsert accepts `createdBy`/`creationSource`), but none of them sets a restart or usage-limit continuation link: client dispatches that set one are forced to `createdBy=user`, so only the server's recovery workers produce both. The id itself is client-suppliable, and durable registration is not authority outside a wake-up. Webhook inbox threads (`kin_webhook_sources.inbox_thread_id`) cannot be registered, even across organizations. Register `threadId → { orgId, agentId, wakeupId, level }` in `KinLeadRegistry`, a scoped overlap guard, not a tool authority source. It is filled before the first launch or later send and cleared after the run ends, including interruption. Retired leads never launch: their queued events become `expired` and their durable thread registration is ended. + Launch/send do not expose a preassigned run id or pre-execution callback; early tool calls before the runner records `run_id` are allowed under the lowest-ordinal root rule. + +Known input-integrity gap: clients can steer an active wake-up with `steer_active`, and other agents can use `t3_thread_send` with `steer` or `auto`. No new run is created, so run authority does not exclude that injected text. Steered text is untrusted input; the lead brief treats non-owner text as quoted, not instructions. CHAT-11 (backend) owns refusal of non-owner steer and restart into lead threads at the orchestration access boundary; Kin chat authorization alone cannot guard these upstream entry points. Clients can also reuse a visible `kin-wakeup-msg:` message id, which overwrites the projected message row and makes the running wake-up fail closed (denial, not authority); CHAT-11 also reserves that prefix at client message intake. + +8. `waitForThread({ threadId, runId, timeoutMs: 15 min })`. On timeout, `interruptThread`, wait up to 30 seconds for the run to settle, then capture usage and mark `timed_out`. If it has not settled, log that it may continue; captured usage is a snapshot, not a final total. Closing the runner scope interrupts and requeues unfinished wake-ups without counting an attempt. 9. Finish: `Memory.commitWakeup` (one commit holding the staged item writes and the journal entry), reset the read checkout, record token usage from the run's provider turns into `kin_wakeups.tokens_json` and `KinBudget`, mark events `done`, and `settleThread`. If the lead writes files by hand in the read checkout, step 9's reset throws the changes away. A non-empty `git status --porcelain` before the reset is logged in the journal as "Lead edited memory files directly; discarded." +Kin boot is optional: configuration, migrations, identity bootstrap and runner startup failures are logged with typed errors and disable Kin, never the core T3 runtime. The runner is constructed in `kin/layer.ts` when `KIN_MEMORY_REPOSITORIES` is non-empty, with the real `KinLeadPrompt` implementation. Memory repositories are supplied through `KIN_MEMORY_REPOSITORIES` (a JSON array of `{ orgId, repoUrl }`); no private repository is assumed by source code. + +The scheduled-task dispatch wrapper also contains Kin migration failures. When Kin persistence is unavailable, launch refusal is limited to `scheduled-task::webhook:` command ids. Sends classify the task using upstream `scheduled_tasks.schedule_json` and refuse only webhooks; an unreadable upstream row fails closed for that send only. Cron, interval, manual and ordinary dispatch remain available. Restoring the Kin database and restarting restores webhook dispatch. + +A `MemorySyncConflict` fails the wake-up and leaves the events queued with the queue's five-minute counted-failure backoff. `MemoryCheckoutInvalid` does the same and logs an error; `KinWakeupHooks.memoryCheckoutInvalid` is the typed notification hook for LEAD-3. Writer commits/journal/reset and budget charging remain LEAD-3 and LEAD-10 integrations, not fabricated runner implementations. Until the writer lands, `memory_commit` remains null rather than claiming a writer commit. `tokens_json` contains each provider turn's recorded token usage and provider-turn id, limited to the wake-up run's attempts, on success and failure. + ### The prompt and caching (gap 2) The CLIs own their system prompt and tool list, and Kin cannot set cache breakpoints through them. Kin can keep the start of the first message byte-identical between wake-ups, so the providers' automatic prefix caching hits during bursts. Order, most stable first: @@ -238,7 +250,7 @@ No wake-up timestamps or ids go above the divider (dates and item ids already in ### Lead tools (`KinLeadToolkit`) -**New** `apps/server/src/kin/lead/mcp/tools.ts` and `handlers.ts`, served on Kin's own `/kin/mcp` endpoint (set up in CHAT-7 and shared with the approvals spec), not in upstream's `layerTools`. Only lead threads are given that endpoint, and each call is checked with `KinLeadRegistry.isLeadCaller`, which reads `McpInvocationContext.thread.threadId`. Other threads never see these tools, so `AgentContextBudget.test.ts` stays unchanged. Each handler resolves `{ orgId, agentId, wakeupId }` from the registry, never from tool input, so a lead can't act for another organization. +**New** `apps/server/src/kin/lead/mcp/tools.ts` and `handlers.ts`, served on Kin's own `/kin/mcp` endpoint (set up in CHAT-7 and shared with the approvals spec), not in upstream's `layerTools`. Only lead threads are given that endpoint, and each call is checked with `KinIdentity.resolveAgent`, binding the server-resolved `McpInvocationContext.thread.threadId` to the persisted running wake-up's current non-terminal run, rejecting subagents. Other threads never see these tools, so `AgentContextBudget.test.ts` stays unchanged. Each handler derives `{ orgId, agentId }` from that identity and `wakeupId` from its matching running database row, never from tool input, so a lead can't act for another organization. | Tool | Input (sketch) | What it does | | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- |